From d74279f2ec28ca778adfa3ecf5e4f9eb6c75f867 Mon Sep 17 00:00:00 2001 From: Revanth Reddy Airre Date: Thu, 20 Aug 2026 22:40:38 -0700 Subject: [PATCH 1/6] fix: update llm-d EPP image (#86) Signed-off-by: Revanth Reddy Airre Co-authored-by: Brent Salisbury --- .../grid-llmd-pool-metrics/forge.yaml | 12 +++--- .../resources/pool-a/epp-deployment-mtls.yaml | 2 +- .../resources/pool-a/epp-deployment.yaml | 2 +- .../resources/pool-b/epp-deployment-mtls.yaml | 2 +- .../resources/pool-b/epp-deployment.yaml | 2 +- xtask/src/env/llmd_pool_metrics_demo.rs | 42 ++++++++++++------- 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml index ad34e5f..49d2f83 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml @@ -22,15 +22,15 @@ spec: siteName: pool-a gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" - eppImage: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + eppImage: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" gatewayImageTag: "v0.1.3" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" operatorImageTag: "v0.1.3" - eppImageRepo: "ghcr.io/llm-d/llm-d-inference-scheduler" - eppImageTag: "v0.8.0" + eppImageRepo: "ghcr.io/llm-d/llm-d-router-endpoint-picker" + eppImageTag: "v0.9.0" vcrImageRepo: "ghcr.io/neuralmagic/vllm-vcr" vcrImageTag: "vllm0.23" poolName: pool-a @@ -43,15 +43,15 @@ spec: siteName: pool-b gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" - eppImage: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + eppImage: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" gatewayImageTag: "v0.1.3" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" operatorImageTag: "v0.1.3" - eppImageRepo: "ghcr.io/llm-d/llm-d-inference-scheduler" - eppImageTag: "v0.8.0" + eppImageRepo: "ghcr.io/llm-d/llm-d-router-endpoint-picker" + eppImageTag: "v0.9.0" vcrImageRepo: "ghcr.io/neuralmagic/vllm-vcr" vcrImageTag: "vllm0.23" poolName: pool-b diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml index 5737956..31c0781 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-a diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml index 294929f..4fc1949 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-a diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml index 62ad34c..02172d3 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-b diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml index 231b14f..8e9b8e4 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-b diff --git a/xtask/src/env/llmd_pool_metrics_demo.rs b/xtask/src/env/llmd_pool_metrics_demo.rs index 11f148f..6c23245 100644 --- a/xtask/src/env/llmd_pool_metrics_demo.rs +++ b/xtask/src/env/llmd_pool_metrics_demo.rs @@ -125,7 +125,7 @@ const DEFAULT_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" const DEFAULT_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.3"; /// Default EPP image reference required by this demo. -const DEFAULT_EPP_IMAGE: &str = "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0"; +const DEFAULT_EPP_IMAGE: &str = "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0"; /// Default vllm-vcr image reference required by this demo. const DEFAULT_VCR_IMAGE: &str = "ghcr.io/neuralmagic/vllm-vcr:vllm0.23"; @@ -836,11 +836,14 @@ fn proof_provenance(mtls: bool) -> ProofResult { let deadline = Instant::now() + Duration::from_secs(30); while Instant::now() < deadline { if let Ok(metrics_text) = kubectl_exec_epp_metrics(cluster, mtls) { - let has_kv = metrics_text.contains("inference_pool_average_kv_cache_utilization") + let has_kv = metrics_text.contains("llm_d_epp_average_kv_cache_utilization") + || metrics_text.contains("inference_pool_average_kv_cache_utilization") || metrics_text.contains("llm_d_router_epp_average_kv_cache_utilization"); - let has_queue = metrics_text.contains("inference_pool_average_queue_size") + let has_queue = metrics_text.contains("llm_d_epp_average_queue_size") + || metrics_text.contains("inference_pool_average_queue_size") || metrics_text.contains("llm_d_router_epp_average_queue_size"); - let has_ready = metrics_text.contains("inference_pool_ready_pods") + let has_ready = metrics_text.contains("llm_d_epp_ready_endpoints") + || metrics_text.contains("inference_pool_ready_pods") || metrics_text.contains("llm_d_router_epp_ready_endpoints"); if has_kv && has_queue && has_ready { observations.push(format!("{cluster}: all 3 EPP pool metrics present")); @@ -1392,17 +1395,16 @@ fn scrape_epp_metrics(cluster: &str, mtls: bool) -> EppMetrics { /// [`scrape_epp_metrics`], separated out so metric-name-fallback behavior is /// unit-testable without a live EPP). /// -/// Both `queue_size` and `kv_cache` fall back from the `inference_pool_*` -/// series to the `llm_d_router_epp_*` series symmetrically -- an EPP build -/// that only exposes the latter must not silently read `kv_cache` as a -/// permanent 0.0, which would make a `kvCachePressure` run's pressure phase -/// never announce despite real KV pressure driving the flip. +/// Both `queue_size` and `kv_cache` prefer the canonical `llm_d_epp_*` +/// series and fall back symmetrically to the legacy metric names. fn parse_epp_metrics(text: &str) -> EppMetrics { EppMetrics { - queue_size: extract_prom_value(text, "inference_pool_average_queue_size") + queue_size: extract_prom_value(text, "llm_d_epp_average_queue_size") + .or_else(|| extract_prom_value(text, "inference_pool_average_queue_size")) .or_else(|| extract_prom_value(text, "llm_d_router_epp_average_queue_size")) .unwrap_or(0.0), - kv_cache: extract_prom_value(text, "inference_pool_average_kv_cache_utilization") + kv_cache: extract_prom_value(text, "llm_d_epp_average_kv_cache_utilization") + .or_else(|| extract_prom_value(text, "inference_pool_average_kv_cache_utilization")) .or_else(|| extract_prom_value(text, "llm_d_router_epp_average_kv_cache_utilization")) .unwrap_or(0.0), } @@ -3838,14 +3840,26 @@ inference_pool_average_kv_cache_utilization{name="pool-a"} 0.35 #[test] #[expect(clippy::float_cmp, reason = "exact literal round-trips in test assertions")] - fn parse_epp_metrics_prefers_primary_metric_names() { - let text = "inference_pool_average_queue_size{name=\"pool-a\"} 4.5\n\ - inference_pool_average_kv_cache_utilization{name=\"pool-a\"} 0.35\n"; + fn parse_epp_metrics_prefers_llm_d_epp_metric_names() { + let text = "llm_d_epp_average_queue_size{name=\"pool-a\"} 4.5\n\ + llm_d_epp_average_kv_cache_utilization{name=\"pool-a\"} 0.35\n\ + inference_pool_average_queue_size{name=\"pool-a\"} 7.0\n\ + inference_pool_average_kv_cache_utilization{name=\"pool-a\"} 0.70\n"; let epp = parse_epp_metrics(text); assert_eq!(epp.queue_size, 4.5); assert_eq!(epp.kv_cache, 0.35); } + #[test] + #[expect(clippy::float_cmp, reason = "exact literal round-trips in test assertions")] + fn parse_epp_metrics_falls_back_to_inference_pool_metric_names() { + let text = "inference_pool_average_queue_size{name=\"pool-a\"} 5.0\n\ + inference_pool_average_kv_cache_utilization{name=\"pool-a\"} 0.40\n"; + let epp = parse_epp_metrics(text); + assert_eq!(epp.queue_size, 5.0); + assert_eq!(epp.kv_cache, 0.40); + } + #[test] #[expect(clippy::float_cmp, reason = "exact literal round-trips in test assertions")] fn parse_epp_metrics_falls_back_to_llm_d_router_metric_names() { From 919f3d579e854f09399378aabc74313aef1e453e Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Thu, 20 Aug 2026 06:02:57 +0000 Subject: [PATCH 2/6] feat(routing): publish provider selection modes Add the provider-selection contract to GridNetwork and routing overlays. Support deterministic, random, and round-robin selection while keeping request-time choice in Praxis AI. Carry selection groups and policy fields through semantic digests, overlay-sync validation, gateway configuration, CRDs, and Helm. Keep admission, locality, freshness, and scoring separate from request selection. Add focused equal-selection coverage and document session affinity, multi-consumer behavior, failure boundaries, and upgrade considerations. Signed-off-by: Brent Salisbury --- charts/grid-operator/crds/gridnetwork.yaml | 18 + charts/grid-site/templates/gridnetwork.yaml | 17 + charts/grid-site/tests/gridnetwork_test.yaml | 9 + charts/grid-site/values.schema.json | 11 + deploy/crds/gridnetwork.yaml | 18 + docs/README.md | 3 + .../provider-selection-and-load-balancing.md | 442 +++ docs/architecture/routing.md | 12 +- docs/architecture/scoring.md | 7 +- operator/src/controller/grid_network.rs | 1 + operator/src/crd/grid_network.rs | 65 +- operator/src/resources/consumer_config.rs | 58 +- operator/src/resources/overlay_bridge.rs | 2 + operator/src/resources/overlay_envelope.rs | 19 +- operator/src/resources/routing_overlay.rs | 283 +- overlay-sync/src/types.rs | 30 + overlay-sync/src/validation.rs | 165 +- overlay-sync/src/watcher.rs | 2 + .../topologies/grid-combined-site/forge.yaml | 15 + .../common/vcr-provider-workload.yaml | 2 + .../configs/provider/praxis.yaml | 60 + .../v1/valid-selection-policy.json | 56 + xtask/src/env.rs | 17 + xtask/src/env/combined_site_demo.rs | 31 - xtask/src/env/image_overrides.rs | 11 + xtask/src/env/provider_traffic_demo.rs | 3047 +++++++++++++++++ 26 files changed, 4336 insertions(+), 65 deletions(-) create mode 100644 docs/architecture/provider-selection-and-load-balancing.md create mode 100644 tests/e2e/topologies/grid-provider-traffic/configs/provider/praxis.yaml create mode 100644 tests/fixtures/overlay-contract/v1/valid-selection-policy.json create mode 100644 xtask/src/env/provider_traffic_demo.rs diff --git a/charts/grid-operator/crds/gridnetwork.yaml b/charts/grid-operator/crds/gridnetwork.yaml index 7822f0d..4007ed5 100644 --- a/charts/grid-operator/crds/gridnetwork.yaml +++ b/charts/grid-operator/crds/gridnetwork.yaml @@ -381,6 +381,24 @@ spec: items: type: string type: array + selectionPolicy: + description: |- + Local request distribution policy for the active selection group. + + This is independent of scoring. When absent, the overlay carries no + selection override and Praxis uses deterministic selection. + nullable: true + properties: + mode: + description: Local selection mode used by the data-plane gateway. + enum: + - deterministic + - roundRobin + - random + type: string + required: + - mode + type: object staleCandidateTtlSeconds: description: |- Maximum age in seconds before a stale (`fresh=false`) remote routing diff --git a/charts/grid-site/templates/gridnetwork.yaml b/charts/grid-site/templates/gridnetwork.yaml index 379f297..bd5cf48 100644 --- a/charts/grid-site/templates/gridnetwork.yaml +++ b/charts/grid-site/templates/gridnetwork.yaml @@ -1,4 +1,15 @@ {{- if .Values.gridNetwork.name }} +{{- $existing := lookup "grid.praxis-proxy.io/v1alpha1" "GridNetwork" .Release.Namespace .Values.gridNetwork.name }} +{{- $configuredSelectionPolicy := .Values.gridNetwork.selectionPolicy }} +{{- $existingSelectionPolicy := dig "spec" "selectionPolicy" nil $existing }} +{{- $selectionPolicy := $configuredSelectionPolicy }} +{{- if not $selectionPolicy }} + {{- if $existingSelectionPolicy }} + {{- $selectionPolicy = $existingSelectionPolicy }} + {{- else if not $existing }} + {{- $selectionPolicy = dict "mode" "roundRobin" }} + {{- end }} +{{- end }} apiVersion: grid.praxis-proxy.io/v1alpha1 kind: GridNetwork metadata: @@ -33,6 +44,12 @@ spec: admissionPolicy: {{- toYaml . | nindent 4 }} {{- end }} + {{- with $selectionPolicy }} + {{- if .mode }} + selectionPolicy: + mode: {{ .mode | quote }} + {{- end }} + {{- end }} {{- with .Values.gridNetwork.metricsRefreshInterval }} {{- if . }} metricsRefreshInterval: {{ . | quote }} diff --git a/charts/grid-site/tests/gridnetwork_test.yaml b/charts/grid-site/tests/gridnetwork_test.yaml index fbe5df9..7ad37ae 100644 --- a/charts/grid-site/tests/gridnetwork_test.yaml +++ b/charts/grid-site/tests/gridnetwork_test.yaml @@ -32,6 +32,15 @@ tests: path: spec.gridId value: "" + - it: defaults a new GridNetwork to roundRobin selection + set: + gridNetwork.name: prod-grid + gridSite.name: prod-site + asserts: + - equal: + path: spec.selectionPolicy.mode + value: roundRobin + - it: renders budgetPolicy.tenants verbatim when set set: gridNetwork.name: prod-grid diff --git a/charts/grid-site/values.schema.json b/charts/grid-site/values.schema.json index 19b825b..ac1b416 100644 --- a/charts/grid-site/values.schema.json +++ b/charts/grid-site/values.schema.json @@ -71,6 +71,17 @@ } } }, + "selectionPolicy": { + "type": "object", + "additionalProperties": false, + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "enum": ["deterministic", "roundRobin", "random"] + } + } + }, "metricsRefreshInterval": { "type": "string", "pattern": "^([1-9][0-9]*s|[1-9][0-9]{3,}ms)$" diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 7822f0d..4007ed5 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -381,6 +381,24 @@ spec: items: type: string type: array + selectionPolicy: + description: |- + Local request distribution policy for the active selection group. + + This is independent of scoring. When absent, the overlay carries no + selection override and Praxis uses deterministic selection. + nullable: true + properties: + mode: + description: Local selection mode used by the data-plane gateway. + enum: + - deterministic + - roundRobin + - random + type: string + required: + - mode + type: object staleCandidateTtlSeconds: description: |- Maximum age in seconds before a stale (`fresh=false`) remote routing diff --git a/docs/README.md b/docs/README.md index 79fe397..b528df1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,9 @@ - [Routing](architecture/routing.md) — versioned overlay contract, revision lifecycle, candidate ordering, `intelligent_route`, `peer_identity_trust`, and provider-side request forwarding. +- [Provider Selection and Load Balancing](architecture/provider-selection-and-load-balancing.md) — + eligibility, routing groups, scoring, selection modes, affinity, and + overlay lifecycle. - [Scoring](architecture/scoring.md) — operator-side candidate scoring, metrics input, and request-time scoring boundaries. - [Auth and Policy](architecture/auth.md) — provider authentication strategies, diff --git a/docs/architecture/provider-selection-and-load-balancing.md b/docs/architecture/provider-selection-and-load-balancing.md new file mode 100644 index 0000000..7576617 --- /dev/null +++ b/docs/architecture/provider-selection-and-load-balancing.md @@ -0,0 +1,442 @@ +# Provider Selection and Load Balancing + +Grid and Praxis divide provider routing into two parts: + +- Grid observes provider state asynchronously, decides which providers are + eligible, orders them, and publishes a versioned routing overlay. +- Praxis consumes that overlay and makes the final request-time choice locally + in the `intelligent_route` filter. + +This separation keeps Kubernetes, Grid reconciliation, EPP metrics, and remote +coordination out of the request hot path. + +```text +Client + | + v +Consumer gateway + | + v +intelligent_route + | + v +First viable selection group + +--> Provider gateway A + +--> Provider gateway B + `--> Provider gateway C + +Lower-priority group + `--> Provider gateway D +``` + +A, B, and C can share active traffic when the selected policy permits it. D is +fallback capacity and is not selected while the earlier group remains viable. +A selection group is a priority and resilience boundary; it is not a score +bucket and does not represent a traffic percentage. + +This layer balances requests across provider gateways. After a provider gateway +is selected, its local serving stack can make a separate backend-level decision. +For example, an llm-d provider gateway can delegate endpoint selection to EPP. +Grid does not use round-robin to choose individual inference replicas hidden +behind one provider gateway. + +## Choose a configuration + +Use these starting points, then adjust them to match the deployment's routing +goal: + +| Goal | Routing policy | Scoring strategy | Selection mode | +|---|---|---|---| +| Share traffic across nearby generic providers | `geographyFirst` | `noMetrics` | `roundRobin` | +| Keep nearby providers active and remote providers as fallback | `geographyFirst` | Any | `roundRobin` | +| Send new traffic to the highest-ranked provider | `geographyFirst` or `scoreFirst` | Any | `deterministic` | +| Share traffic across sites without inference metrics | `scoreFirst` | `noMetrics` | `roundRobin` | +| Randomize selection inside the preferred provider group | Either | Any | `random` | + +The three policy fields answer different questions: + +- `routingPolicy`: Which providers belong in the same active or fallback group? +- `scoringPolicy`: How should providers be ranked using available metrics? +- `selectionPolicy`: How should Praxis choose within the active group? + +## The decision sequence + +```text +Request for a capability + | + v +Eligibility and admission + | + v +Routing policy orders candidates and creates priority groups + | + v +Session-affinity lookup + +-- permitted existing binding -> reuse its provider + `-- no usable binding -> find the first viable group + | + v + apply the configured selection mode + +-- deterministic mode + +-- roundRobin mode + `-- random mode +``` + +Scores contribute to candidate ordering. They do not create groups. The +selection mode operates only inside the first group that can serve the request. + +## Eligibility and admission + +Before ordering, Grid builds candidates for the requested capability. The +overlay already reflects capability matching, authorization and trust, +provider health, freshness, and provider availability. Admission is a hard +boundary: + +| Admission state | New requests | Existing sessions | +|---|---:|---:| +| `newAndExisting` | Allowed | Allowed | +| `existingOnly` | Not selected | Allowed when the binding is permitted | +| `excluded` | Not selected | Not selected | + +An `existingOnly` provider can finish work for a session that is already bound +to it, but it does not receive new bindings. An excluded provider cannot be +selected. Neither scoring nor a selection mode can override these states. + +## Routing policy and groups + +`spec.routingPolicy` controls candidate ordering and hard priority boundaries. +The supported values are `geographyFirst` and `scoreFirst`. + +### `geographyFirst` + +Candidates are ordered by admission, locality tier, freshness, score, and +deterministic identity tie-breakers. Groups are separated by admission, +locality tier, and freshness. Scores order candidates within a group but do +not split that group. + +```text +Closest healthy and fresh tier: A, B, C <- active selection +More distant healthy tier: D, E <- fallback +``` + +In plain language: balance within the closest healthy provider tier and use +more distant capacity as fallback. A remote provider does not join local +active traffic merely because its score is higher. + +### `scoreFirst` + +Candidates are ordered by admission, freshness, score, locality, and +deterministic identity tie-breakers. Groups are separated by admission and +freshness only. Fresh admitted providers from different sites can therefore +share one active group. Score differences affect order, not group membership. + +In plain language: allow fresh admitted providers across sites to participate +in the same active traffic group. + +## Scoring policy + +`spec.scoringPolicy.strategy` selects the provider-level signal used for score +calculation. It is independent of request-time selection: + +- `noMetrics` requires no EPP, Prometheus, or inference-specific metrics. + Dynamic score contributions are zero, while health, admission, freshness, + authorization, locality, affinity, and selection policy still apply. This is + the normal choice for generic or heterogeneous provider gateways. Use it when + llm-d EPP metrics are unavailable or are not comparable across providers. +- `queueDepth` uses asynchronously observed, normalized provider-pool queue + pressure. Lower pressure produces a higher preference score. It requires + comparable queue metrics and a meaningful queue capacity. For an llm-d + provider, Grid retrieves this signal from the configured EPP metrics endpoint. +- `kvCachePressure` uses provider-level KV-cache utilization as a capacity + pressure signal. Lower utilization produces a higher score. It is not + request-specific prefix-cache affinity; that decision belongs inside the + inference scheduler. For an llm-d provider, Grid retrieves this signal from + the configured EPP metrics endpoint. + +Grid currently uses one explicitly selected strategy rather than blending +unrelated signals into an opaque total. Missing local samples can use the +implementation's neutral fallback values, and a recent local sample can be +reused while it remains within `staleMetricsSeconds`. Deployments using a +metric strategy should provide fresh, comparable telemetry for every competing +provider. + +The important rule is: + +```text +score != traffic weight +``` + +Scores are preference and observability signals. They do not turn a score of +`0.8` versus `0.4` into a 2:1 traffic split. With `roundRobin`, candidates in +the active group receive equal turns regardless of their scores. + +For detailed metric input and normalization, see [Provider Scoring](scoring.md). + +## Selection policy + +`spec.selectionPolicy.mode` controls request-time selection inside the first +viable group. The selection mode is applied from an accepted in-memory +snapshot by Praxis; Grid is not called for each request. + +### `deterministic` + +Selects the first viable candidate in the active group. This is strict +preference behavior: Grid's ordering determines which provider receives new +unbound traffic. It is useful when locality, score, primary/standby order, or +predictability should dominate. When `selectionPolicy` is absent from an +overlay, Praxis uses `deterministic`. + +### `roundRobin` + +Takes equal turns across viable candidates in the active group. It does not +require inference metrics and does not distribute across lower-priority groups +while the active group is viable. It balances selections, not necessarily +tokens, latency, request cost, or concurrent work. Session affinity is checked +before this mode runs. + +### `random` + +Selects uniformly from viable candidates in the active group. It follows the +same admission, group, and affinity rules as round-robin. Random state is local +to the gateway process and is not a global coordinator. + +## Policy matrix + +| Routing policy | Selection policy | Effective behavior | +|---|---|---| +| `geographyFirst` | `deterministic` | Strict preference for the highest-ranked provider in the closest viable tier | +| `geographyFirst` | `roundRobin` | Equal selection in the closest viable tier; remote tiers are fallback | +| `geographyFirst` | `random` | Uniform selection in the closest viable tier | +| `scoreFirst` | `deterministic` | Strict preference for the highest-ranked fresh admitted provider across sites | +| `scoreFirst` | `roundRobin` | Equal selection across fresh admitted providers in the active group | +| `scoreFirst` | `random` | Uniform selection across fresh admitted providers in the active group | + +The scoring strategy changes ordering, not the selection mode: + +| Scoring strategy | Metrics required | Deterministic | Round-robin | +|---|---|---|---| +| `noMetrics` | No | Ordering and deterministic tie-breaks decide | Equal selection inside the active group | +| `queueDepth` | Compatible queue metrics | Highest queue-based preference is first | Scores remain visible; selection remains equal | +| `kvCachePressure` | Compatible KV metrics | Highest available-capacity preference is first | Scores remain visible; selection remains equal | + +## Configuration examples + +### Generic provider-gateway balancing + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: provider-grid +spec: + routingPolicy: geographyFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin +``` + +Providers in the nearest viable group share selections equally. No inference +metrics are required, and remote groups remain available for fallback. + +### Strict metric preference + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: inference-grid +spec: + routingPolicy: scoreFirst + scoringPolicy: + strategy: queueDepth + selectionPolicy: + mode: deterministic + metricsRefreshInterval: "10s" +``` + +Grid refreshes the selected signal asynchronously. Deterministic selection +uses the resulting ordering; it does not query EPP during a request. + +### Cross-site active/active selection + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: active-active-grid +spec: + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin +``` + +Fresh, admitted providers from multiple sites can share the active group. + +### Random selection + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: random-provider-grid +spec: + routingPolicy: geographyFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: random +``` + +Random selection is uniform within the active group. It is useful when an +equal probabilistic distribution is sufficient and a repeating sequence is not +required. + +## Request-time behavior and affinity + +For a new request, Praxis reads the accepted snapshot, resolves the requested +capability, checks affinity, finds the first viable group, applies the selection mode, +and records a binding when the selection succeeds. + +```text +Client + | + | request + v +Consumer gateway / intelligent_route + | 1. Read accepted in-memory overlay + | 2. Resolve capability and eligibility + | 3. Check session affinity + | 4. Find the first viable group + | 5. Apply the configured selection mode + | 6. Record the successful binding + v +Selected provider gateway + | + v +Provider backend +``` + +For an existing permitted binding, no new selection mode is applied: + +```text +Client with an existing session + | + v +intelligent_route + | + | permitted affinity binding found + v +Previously selected provider +``` + +Round-robin does not move an established session just to improve aggregate +balance. The observed request distribution can therefore differ from an exact +split when sessions generate different amounts of traffic. + +## Multiple consumer gateways + +The design supports multiple consumer gateways. Each gateway receives an +accepted overlay snapshot and keeps its own local selection state: + +```text + Grid overlay + / \ + v v + Consumer gateway 1 Consumer gateway 2 + local counter local counter + A -> B -> C A -> B -> C +``` + +Counters are not coordinated globally. Each gateway can produce a balanced +local sequence, while aggregate traffic depends on request rates, affinity, +restarts, and snapshot replacement. A globally synchronized quota would need +a different coordination design and would add hot-path trade-offs. + +## Overlay lifecycle and re-ranking + +```text +Provider health and optional EPP metrics + | + v +Grid operator reconciliation + | eligibility, admission, ordering, scores, groups, selection policy + v +Content-addressed routing overlay + | + v +overlay-sync validation and publication + | + v +Praxis validates and atomically loads a snapshot + | precomputed group index and local selection state + v +Request-time selection from memory +``` + +Reconciliation is triggered by watched provider, site, and network changes, +remote Grid state, and the periodic `metricsRefreshInterval`. The default +periodic cadence is 300 seconds for plaintext metrics; TLS-protected metrics +use a 60-second safety cap. A configured interval must be at least one second. +The interval controls observation and overlay publication, not request-path +latency. A demo or operator can cause an earlier reconcile through a real +watched resource change. + +After an overlay is accepted, requests do not call Grid, Kubernetes, +ConfigMaps, EPP, Prometheus, or a remote scoring service. An unchanged semantic +revision should not continually rebuild selection state. An accepted semantic +change may create fresh snapshot-scoped state. + +## Failure and fallback + +Grid observes health, admission, freshness, and optional metrics +asynchronously. Praxis serves from the last accepted snapshot until a valid +new overlay is delivered. An unavailable or excluded provider cannot receive +new selections, and the first viable group is preferred. Later groups are +fallback capacity, not part of normal active distribution. + +Failover is therefore bounded by observation, reconciliation, overlay +distribution, and snapshot acceptance. It is not an immediate request-time +call to Grid. A request already sent upstream can fail before a newer snapshot +is accepted; do not assume automatic retry unless the gateway configuration +explicitly provides it. + +## Overlay contract and future weighting + +`selectionPolicy` is optional in both the Grid API and the overlay. An omitted +field remains omitted, and Praxis interprets it as deterministic selection. +The Helm chart explicitly renders `roundRobin` by default. Users applying a +`GridNetwork` directly can either set the selection mode explicitly or omit the policy +to select `deterministic`. + +`selection_group` and `selection_policy` are part of the semantic digest when +present. Group numbers are zero-based and contiguous per capability. Unknown +mode values and malformed policy structures are rejected. Overlays without +the optional selection fields remain valid and use deterministic selection. + +The provider ordering used to construct groups is also part of the routing +contract. In particular, the updated `geographyFirst` ordering evaluates +freshness before score after locality. Existing resources that omit +`selectionPolicy` still use deterministic selection, but an upgrade can change +which provider is first when candidates differ in freshness. Deployments that +require a fixed primary provider should set their routing and selection policy +explicitly and validate the resulting overlay during upgrade. + +Weighted selection is a future extension, not part of the current API. A +future mode such as `weightedRandom` would need an explicit overlay weight, +normalization, capacity semantics, missing-metric behavior, bounds, and +stability controls. It must not be inferred from score, rank, metric presence, +or candidate count, and it must not change admission, locality, +authorization, freshness, or group boundaries. + +## Demonstration reference + +The [Grid provider-selection research spike](https://github.com/praxis-proxy/grid/issues/31) +describes the focused provider-traffic demonstration: one consumer gateway, +three provider gateways, one active group, `noMetrics`, `roundRobin`, 60 +successful requests, exact 20/20/20 attribution, and a stable overlay during +the measured window. That proof demonstrates equal selection, not weighted +routing, coordinated round-robin across multiple consumers, retry behavior, +or fallback groups unless separate evidence is provided. diff --git a/docs/architecture/routing.md b/docs/architecture/routing.md index 5be5d35..438ca3b 100644 --- a/docs/architecture/routing.md +++ b/docs/architecture/routing.md @@ -32,6 +32,12 @@ llm-d / EPP / inference backend Grid does not proxy traffic. It writes the overlay used by Praxis filters. +For the complete provider-selection model, including selection groups and the +`deterministic`, `roundRobin`, and `random` modes, see [Provider Selection +and Load Balancing](provider-selection-and-load-balancing.md). Routing policy +defines candidate ordering and hard group boundaries; selection policy controls +request distribution within the first viable group. + ## Control-plane rendering path For each `GridNetwork` and gateway reference, the operator: @@ -363,8 +369,8 @@ The operator orders candidates before writing the overlay. Ordering proceeds in two phases: 1. **Scoring.** Each provider is scored by `scoring::score_backends` using - the weights selected by `GridNetwork.spec.scoringPolicy.strategy`, - optional live metrics, and optional CRDT-propagated provider metrics. + the signal selected by `GridNetwork.spec.scoringPolicy.strategy`, optional + live metrics, and optional CRDT-propagated provider metrics. Providers with no live metrics use neutral metric scores. See [Scoring](scoring.md) for the available strategies. @@ -375,7 +381,7 @@ in two phases: | Policy | Order | |---|---| - | `geographyFirst` (default) | admission, locality, score descending, freshness, deterministic tie-break | + | `geographyFirst` (default) | admission, locality, freshness, score descending, deterministic tie-break | | `scoreFirst` | admission, freshness, score descending, locality, deterministic tie-break | The deterministic tie-break is `(site, name, cluster)`. After diff --git a/docs/architecture/scoring.md b/docs/architecture/scoring.md index c9796a0..3f25bb4 100644 --- a/docs/architecture/scoring.md +++ b/docs/architecture/scoring.md @@ -1,5 +1,10 @@ # Provider Scoring +For the complete relationship between scoring, routing groups, and request-time +selection, see [Provider Selection and Load Balancing](provider-selection-and-load-balancing.md). +Scores influence candidate ordering; they are not traffic weights and do not +split selection groups. + Grid scores provider pools when the operator renders a Praxis routing overlay. Praxis reads that overlay from memory at request time; it does not call Grid, Kubernetes, the operator, or an EPP metrics endpoint on the request path. @@ -53,7 +58,7 @@ spec: Omitting `scoringPolicy` has the same effect. Every admitted candidate receives zero dynamic score. "No metrics" does not mean "no policy": health, admission, freshness, model compatibility, geography, selection tiers, session affinity, -and Praxis picker policy continue to apply. +and Praxis selection policy continue to apply. Use this strategy for heterogeneous grids, external APIs such as OpenAI, Anthropic, or Bedrock, and providers that do not expose comparable pool diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index e64b206..776fcd4 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -4392,6 +4392,7 @@ mod tests { network: "net".to_owned(), local_site: "site".to_owned(), candidates: Vec::new(), + selection_policy: None, generated_at: Some("2026-07-29T01:00:00Z".to_owned()), }, }, diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index c340642..09c4570 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -60,7 +60,7 @@ pub enum ScoringStrategy { /// Do not prefer providers using dynamic metrics. /// /// All score contributions are zero. Health, admission, freshness, - /// geography, selection tiers, session affinity, and request-time picker + /// geography, selection tiers, session affinity, and request-time selection /// policy still apply. This is the generic default. #[default] NoMetrics, @@ -143,6 +143,28 @@ pub struct ScoringPolicyConfig { pub strategy: ScoringStrategy, } +/// Local request-selection mode applied by Praxis inside the active group. +#[derive(Clone, Copy, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum SelectionMode { + /// Select the first admitted candidate in the active group. + #[default] + Deterministic, + /// Distribute new requests equally in the active group. + RoundRobin, + /// Distribute new requests randomly in the active group. + Random, +} + +/// Request selection policy published in the routing overlay. +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct SelectionPolicyConfig { + /// Local selection mode used by the data-plane gateway. + pub mode: SelectionMode, +} + /// Resolve the effective [`scoring::ScoringWeights`] from a scoring policy. /// /// The public API selects one scorer. The weight adapter is internal and @@ -517,7 +539,8 @@ pub struct GridNetworkSpec { /// Admission state (`newAndExisting` before `existingOnly`) always /// outranks both geography and score in either mode. In `scoreFirst` /// mode, freshness also outranks both; in `geographyFirst` mode, - /// freshness is a tiebreaker below geography and score. + /// freshness is below locality but above score so it cannot interleave + /// candidates across freshness-based selection groups. #[serde(default, skip_serializing_if = "Option::is_none")] pub routing_policy: Option, @@ -540,6 +563,13 @@ pub struct GridNetworkSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_policy: Option, + /// Local request distribution policy for the active selection group. + /// + /// This is independent of scoring. When absent, the overlay carries no + /// selection override and Praxis uses deterministic selection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selection_policy: Option, + /// Maximum time between metric refreshes and score/ranking recalculation. /// /// This controls the `GridNetwork` reconcile cadence for provider metrics; @@ -1933,6 +1963,37 @@ mod tests { assert!(result.is_err(), "unknown strategy must be rejected"); } + #[test] + fn selection_policy_round_trips_and_rejects_unknown_mode() { + let spec: GridNetworkSpec = serde_json::from_value(serde_json::json!({ + "seeds": [], + "selectionPolicy": { "mode": "roundRobin" } + })) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!( + spec.selection_policy.map(|policy| policy.mode), + Some(SelectionMode::RoundRobin) + ); + + let result = serde_json::from_value::(serde_json::json!({ + "seeds": [], + "selectionPolicy": { "mode": "weightedRandom" } + })); + let Err(error) = result else { + std::process::abort(); + }; + assert!(error.to_string().contains("unknown variant")); + } + + #[test] + fn round_robin_mode_serializes_for_explicit_policy() { + let serialized = serde_json::to_value(&SelectionPolicyConfig { + mode: SelectionMode::RoundRobin, + }) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!(serialized, serde_json::json!({"mode": "roundRobin"})); + } + #[test] fn scoring_policy_rejects_removed_weights_field() { let json = serde_json::json!({ diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index ff2456d..16c392c 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -23,7 +23,7 @@ use std::collections::{BTreeMap, BTreeSet}; use k8s_openapi::api::core::v1::ConfigMap; use crate::{ - crd::grid_network::{ClusterEndpointConfig, TransportMode}, + crd::grid_network::{ClusterEndpointConfig, SelectionMode, TransportMode}, resources::routing_overlay::{RoutingCandidate, RoutingOverlay}, }; @@ -150,6 +150,7 @@ pub(crate) fn generate_consumer_praxis_config( } let candidates_yaml = render_candidates(&overlay.candidates); + let selection_policy_yaml = render_selection_policy(overlay.selection_policy.as_ref()); let local_site = yaml_scalar(&overlay.local_site)?; let credential_inject_section = render_credential_inject(&overlay.candidates, credential_mount_base); @@ -171,6 +172,7 @@ pub(crate) fn generate_consumer_praxis_config( \x20 - filter: intelligent_route\n\ \x20 local_site: {local_site}\n\ \x20 model_header: \"X-Model\"\n\ + {selection_policy_yaml}\ \x20 candidates:\n\ {candidates_yaml}" ); @@ -230,7 +232,24 @@ fn render_candidates(candidates: &[RoutingCandidate]) -> String { candidates.iter().map(render_candidate).collect::>().join("\n") } +/// Render the explicit Grid-owned request-selection policy. +fn render_selection_policy(policy: Option<&crate::crd::grid_network::SelectionPolicyConfig>) -> String { + let Some(policy) = policy else { + return String::new(); + }; + let mode = match policy.mode { + SelectionMode::Deterministic => "deterministic", + SelectionMode::RoundRobin => "roundRobin", + SelectionMode::Random => "random", + }; + format!(" selection_policy:\n mode: {mode}\n") +} + /// Render one `intelligent_route` candidate. +#[expect( + clippy::too_many_lines, + reason = "Candidate YAML fields are kept together to mirror the wire contract." +)] fn render_candidate(c: &RoutingCandidate) -> String { let mut lines = vec![ format!( @@ -251,6 +270,15 @@ fn render_candidate(c: &RoutingCandidate) -> String { ), format!(" fresh: {}", c.fresh), ]; + if let Some(admission) = c.admission_state { + lines.push(format!( + " admission_state: {}", + serde_json::to_string(&admission).unwrap_or_default() + )); + } + if let Some(group) = c.selection_group { + lines.push(format!(" selection_group: {group}")); + } if let Some(cred) = &c.credential { lines.extend(render_credential_reference(cred)); } @@ -525,6 +553,7 @@ mod tests { score: None, score_breakdown: None, rank: None, + selection_group: None, } } @@ -557,6 +586,7 @@ mod tests { score: None, score_breakdown: None, rank: None, + selection_group: None, } } @@ -565,6 +595,7 @@ mod tests { network: "test-net".to_owned(), local_site: "site-a".to_owned(), candidates, + selection_policy: None, generated_at: None, } } @@ -626,6 +657,25 @@ mod tests { assert!(yaml.contains("gateway-site-a"), "cluster must appear in load_balancer"); } + #[test] + fn explicit_selection_policy_reaches_intelligent_route_config() { + let mut overlay = simple_overlay(vec![plain_candidate( + "inference_model", + "model", + "site-a", + "cluster-a", + true, + )]); + overlay.selection_policy = Some(crate::crd::grid_network::SelectionPolicyConfig { + mode: SelectionMode::RoundRobin, + }); + let endpoints = endpoint_coverage(&overlay); + let config = generate_consumer_praxis_config(&overlay, MOUNT_BASE, &endpoints, "/run/tls", 8080) + .unwrap_or_else(|_| std::process::abort()); + assert!(config.contains("selection_policy:")); + assert!(config.contains("mode: roundRobin")); + } + #[test] fn generated_praxis_yaml_omits_operator_overlay_metadata() { let mut candidate = plain_candidate("inference_model", "model-a", "site-a", "gateway-site-a", true); @@ -633,6 +683,7 @@ mod tests { candidate.admission_state = Some(AdmissionState::NewAndExisting); candidate.selection_tier = Some(LocalityTier::SameSite); candidate.rank = Some(0); + candidate.selection_group = Some(0); let mut overlay = simple_overlay(vec![candidate]); overlay.generated_at = Some("2026-07-24T12:00:00Z".to_owned()); @@ -646,12 +697,14 @@ mod tests { ) .unwrap(); - for forbidden in ["stable_id", "admission_state", "selection_tier", "rank", "generated_at"] { + for forbidden in ["stable_id", "selection_tier", "rank", "generated_at"] { assert!( !yaml.contains(forbidden), "operator-only metadata field {forbidden} must not enter generated Praxis YAML" ); } + assert!(yaml.contains("admission_state: \"new_and_existing\"")); + assert!(yaml.contains("selection_group: 0")); } #[test] @@ -932,6 +985,7 @@ mod tests { network: "n".to_owned(), local_site: String::new(), candidates: vec![], + selection_policy: None, generated_at: None, }; assert!( diff --git a/operator/src/resources/overlay_bridge.rs b/operator/src/resources/overlay_bridge.rs index 9007d93..1442e3c 100644 --- a/operator/src/resources/overlay_bridge.rs +++ b/operator/src/resources/overlay_bridge.rs @@ -95,8 +95,10 @@ mod tests { score: None, score_breakdown: None, rank: None, + selection_group: None, }) .collect(), + selection_policy: None, generated_at: None, } } diff --git a/operator/src/resources/overlay_envelope.rs b/operator/src/resources/overlay_envelope.rs index f1d8939..9461dbe 100644 --- a/operator/src/resources/overlay_envelope.rs +++ b/operator/src/resources/overlay_envelope.rs @@ -4,9 +4,9 @@ //! //! Wraps a routing overlay in a versioned envelope with a SHA-256 digest //! computed over the [RFC 8785][] canonical form of the semantic payload. -//! The semantic payload includes only routing-relevant fields (`network`, -//! `local_site`, `candidates`), so the revision changes only when routing -//! behavior changes — timestamp and provenance changes are no-ops. +//! The semantic payload includes `network`, `local_site`, and `candidates`, +//! plus `selection_policy` when present, so the revision changes only when +//! routing behavior changes — timestamp and provenance changes are no-ops. //! //! The envelope is published as the `routing-overlay.json` key in the overlay //! `ConfigMap`, alongside the `routing-config.json` legacy key. @@ -182,11 +182,17 @@ pub struct EnvelopeBuildResult { /// [`ProjectedCredentialRef`]: super::routing_overlay::ProjectedCredentialRef pub fn compute_semantic_digest(overlay: &RoutingOverlay) -> Result { let candidates_value = serde_json::to_value(&overlay.candidates)?; - let semantic_payload = serde_json::json!({ + let mut semantic_payload = serde_json::json!({ "candidates": candidates_value, "local_site": overlay.local_site, "network": overlay.network, }); + if let Some(policy) = &overlay.selection_policy { + let Some(object) = semantic_payload.as_object_mut() else { + return Err(serde_json::Error::custom("semantic payload must be an object")); + }; + object.insert("selection_policy".to_owned(), serde_json::to_value(policy)?); + } let canonical = serde_json_canonicalizer::to_vec(&semantic_payload).map_err(serde_json::Error::custom)?; @@ -307,7 +313,9 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }], + selection_policy: None, generated_at: Some("2026-07-29T00:00:00Z".to_owned()), } } @@ -341,6 +349,7 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }, RoutingCandidate { kind: "inference_model".to_owned(), @@ -355,8 +364,10 @@ mod tests { score: None, score_breakdown: None, rank: Some(1), + selection_group: None, }, ], + selection_policy: None, generated_at: Some("2026-07-29T01:00:00Z".to_owned()), } } diff --git a/operator/src/resources/routing_overlay.rs b/operator/src/resources/routing_overlay.rs index f922bb0..5820277 100644 --- a/operator/src/resources/routing_overlay.rs +++ b/operator/src/resources/routing_overlay.rs @@ -269,6 +269,7 @@ pub(crate) fn remote_crdt_provider_to_candidates(provider: &crdt::ProviderState) score: None, score_breakdown: None, rank: None, + selection_group: None, }) .collect() } @@ -876,6 +877,13 @@ pub struct RoutingCandidate { /// Zero-based position in the final sorted overlay. #[serde(skip_serializing_if = "Option::is_none")] pub rank: Option, + + /// Zero-based active selection group for request distribution. + /// + /// Group metadata is additive and does not change the default ordered + /// routing behavior until the data plane explicitly enables it. + #[serde(skip_serializing_if = "Option::is_none")] + pub selection_group: Option, } /// The full routing overlay for a single [`GridNetwork`]. @@ -904,10 +912,15 @@ pub struct RoutingOverlay { /// higher than remote candidates. pub local_site: String, - /// Routing candidates, ordered by admission state, locality tier, score, - /// freshness, then alphabetical tiebreak. + /// Routing candidates, ordered by admission state, locality tier, + /// freshness, score, then alphabetical tiebreak. pub candidates: Vec, + /// Optional explicit local selection policy for Praxis. An absent field is + /// intentionally backward-compatible and means deterministic selection. + #[serde(skip_serializing_if = "Option::is_none")] + pub selection_policy: Option, + /// RFC 3339 timestamp of when this overlay was rendered. #[serde(skip_serializing_if = "Option::is_none")] pub generated_at: Option, @@ -1016,6 +1029,71 @@ fn locality_sort_key(tier: Option) -> u8 { } } +/// Assign deterministic priority groups after the existing candidate sort. +/// +/// Groups are hard eligibility/priority boundaries, not score buckets. The +/// current producer contract deliberately leaves metric differences inside a +/// group so a later traffic-weight field can distribute among eligible +/// candidates without changing the active priority boundary. +/// +/// Geography-first keeps admission, locality, and freshness as boundaries; +/// Score-first keeps admission and freshness as boundaries. Scores are +/// therefore never used to split a group, and floating-point equality is not +/// part of the grouping contract. +#[derive(Clone, Copy)] +struct GroupState { + /// Admission boundary for the previous candidate. + admission: AdmissionState, + /// Freshness boundary for the previous candidate. + fresh: bool, + /// Locality boundary for the previous candidate. + tier: LocalityTier, + /// Group assigned to the previous candidate. + group: u32, +} + +/// Assign zero-based contiguous groups for each capability. +#[expect( + clippy::too_many_lines, + reason = "The two routing policies are explicit at this contract boundary." +)] +fn assign_selection_groups(candidates: &mut [RoutingCandidate], policy: crate::crd::grid_network::RoutingPolicy) { + let mut last_by_capability: HashMap<(String, String), GroupState> = HashMap::new(); + for candidate in candidates { + let key = (candidate.kind.clone(), candidate.name.clone()); + let admission = candidate.admission_state.unwrap_or(AdmissionState::NewAndExisting); + let tier = candidate.selection_tier.unwrap_or(LocalityTier::Unknown); + let next = match last_by_capability.get(&key) { + Some(last) => { + let boundary = match policy { + crate::crd::grid_network::RoutingPolicy::GeographyFirst => { + admission != last.admission || tier != last.tier || candidate.fresh != last.fresh + }, + crate::crd::grid_network::RoutingPolicy::ScoreFirst => { + admission != last.admission || candidate.fresh != last.fresh + }, + }; + if boundary { + last.group.saturating_add(1) + } else { + last.group + } + }, + None => 0, + }; + candidate.selection_group = Some(next); + last_by_capability.insert( + key, + GroupState { + admission, + fresh: candidate.fresh, + tier, + group: next, + }, + ); + } +} + // --------------------------------------------------------------------------- // Renderer // --------------------------------------------------------------------------- @@ -1048,10 +1126,14 @@ fn locality_sort_key(tier: Option) -> u8 { /// **`GeographyFirst`** (default): /// 1. admission state: `new_and_existing` before `existing_only`; /// 2. geography tier: same site, same zone, same region, cross region; -/// 3. scoring engine score, descending; -/// 4. `fresh=true` before `fresh=false`; +/// 3. freshness: `fresh=true` before `fresh=false`; +/// 4. scoring engine score, descending; /// 5. deterministic `(site, name, cluster)` tiebreak. /// +/// Freshness intentionally precedes score in this policy. Group assignment +/// treats freshness as a hard boundary, so this ordering prevents a stale +/// candidate from interleaving with fresh candidates of the same capability. +/// /// **`ScoreFirst`**: /// 1. admission state: `new_and_existing` before `existing_only`; /// 2. `fresh=true` before `fresh=false`; @@ -1220,8 +1302,8 @@ pub fn render_routing_overlay_with_admission( admission_sort_key(a.admission_state) .cmp(&admission_sort_key(b.admission_state)) .then(locality_sort_key(a.selection_tier).cmp(&locality_sort_key(b.selection_tier))) - .then_with(|| score_of(&b.cluster).total_cmp(&score_of(&a.cluster))) .then(b.fresh.cmp(&a.fresh)) + .then_with(|| score_of(&b.cluster).total_cmp(&score_of(&a.cluster))) .then(a.site.cmp(&b.site)) .then(a.name.cmp(&b.name)) .then(a.cluster.cmp(&b.cluster)) @@ -1255,10 +1337,19 @@ pub fn render_routing_overlay_with_admission( } } + assign_selection_groups(&mut candidates, policy); + + // Preserve omission for existing GridNetwork resources. New Helm + // installations resolve their round-robin default in chart values, while + // an upgraded resource without this field must retain deterministic + // Praxis compatibility behavior. + let selection_policy = network.spec.selection_policy.clone(); + Ok(RoutingOverlay { network: network_name.to_owned(), local_site: local_site.to_owned(), candidates, + selection_policy, generated_at: generated_at.map(str::to_owned), }) } @@ -1454,6 +1545,7 @@ fn candidates_from_provider( score: None, score_breakdown: None, rank: None, + selection_group: None, }); } } @@ -1669,6 +1761,156 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + #[expect( + clippy::too_many_arguments, + reason = "Test fixture mirrors all grouping dimensions explicitly." + )] + fn group_candidate( + name: &str, + site: &str, + cluster: &str, + admission_state: AdmissionState, + tier: LocalityTier, + fresh: bool, + score: f64, + ) -> RoutingCandidate { + RoutingCandidate { + kind: CANDIDATE_KIND.to_owned(), + name: name.to_owned(), + site: site.to_owned(), + cluster: cluster.to_owned(), + fresh, + credential: None, + stable_id: None, + admission_state: Some(admission_state), + selection_tier: Some(tier), + score: Some(score), + score_breakdown: None, + rank: None, + selection_group: None, + } + } + + #[test] + fn equivalent_candidates_share_group_without_identity_comparison() { + let mut candidates = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.8, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.8, + ), + ]; + assign_selection_groups(&mut candidates, crate::crd::grid_network::RoutingPolicy::GeographyFirst); + assert_eq!(candidates[0].selection_group, Some(0)); + assert_eq!(candidates[1].selection_group, Some(0)); + } + + #[test] + fn changed_routing_properties_split_groups_deterministically() { + let mut candidates = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.8, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::CrossRegion, + true, + 0.8, + ), + group_candidate( + "model-a", + "site-c", + "cluster-c", + AdmissionState::ExistingOnly, + LocalityTier::CrossRegion, + false, + 0.2, + ), + ]; + assign_selection_groups(&mut candidates, crate::crd::grid_network::RoutingPolicy::GeographyFirst); + assert_eq!( + candidates.iter().map(|c| c.selection_group).collect::>(), + vec![Some(0), Some(1), Some(2)] + ); + } + + #[test] + fn metric_differences_stay_in_same_group_for_both_policies() { + let mut geography = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.9, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.1, + ), + ]; + assign_selection_groups(&mut geography, crate::crd::grid_network::RoutingPolicy::GeographyFirst); + assert_eq!( + geography.iter().map(|c| c.selection_group).collect::>(), + vec![Some(0), Some(0)] + ); + + let mut score_first = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.9, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.1, + ), + ]; + assign_selection_groups(&mut score_first, crate::crd::grid_network::RoutingPolicy::ScoreFirst); + assert_eq!( + score_first.iter().map(|c| c.selection_group).collect::>(), + vec![Some(0), Some(0)] + ); + } + fn test_site(name: &str, network: &str) -> GridSite { serde_json::from_value(serde_json::json!({ "apiVersion": "grid.praxis-proxy.io/v1alpha1", @@ -1713,6 +1955,27 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + #[test] + fn omitted_selection_policy_remains_omitted_from_overlay() { + let network = test_network_score_first("net"); + let provider = test_provider("provider", "net", &["model"]); + let overlay = render_routing_overlay( + &network, + &[], + &[provider], + &[], + "site-a", + None, + None, + &scoring::ScoringWeights::default(), + ) + .unwrap_or_else(|_| std::process::abort()); + assert!( + overlay.selection_policy.is_none(), + "an upgraded GridNetwork that omits selectionPolicy must not be migrated implicitly" + ); + } + fn test_provider_with_selector( name: &str, network: &str, @@ -2308,9 +2571,9 @@ mod tests { // provider is also unavailable, but its fresh=false signals that its // metrics are stale. // - // Default GeographyFirst sort: admission → locality → score → fresh → tiebreak. - // The degraded local (SameSite, fresh=false) outranks the fresh API - // (CrossRegion, fresh=true) because geography sorts above freshness. + // Default GeographyFirst sort: admission → locality → fresh → score → tiebreak. + // Freshness is evaluated before score after locality, so a fresh remote + // candidate can precede a degraded local candidate. let network = test_network("fallback-net"); let local_degraded = test_provider_with_backend_kind_and_phase("provider-local", "fallback-net", "local", "Degraded"); @@ -2346,8 +2609,8 @@ mod tests { assert!(api_c.fresh, "API provider with absent status must have fresh=true"); assert_eq!( overlay.candidates.first().map(|c| c.cluster.as_str()), - Some("provider-local"), - "Degraded local must rank before API (GeographyFirst: locality outranks freshness)" + Some("provider-api"), + "with equal unknown locality, GeographyFirst freshness precedes score" ); } diff --git a/overlay-sync/src/types.rs b/overlay-sync/src/types.rs index 3e7b459..57fb06f 100644 --- a/overlay-sync/src/types.rs +++ b/overlay-sync/src/types.rs @@ -116,11 +116,35 @@ pub(crate) struct RoutingOverlay { /// Routing candidates ordered by the operator's sort. pub(crate) candidates: Vec, + /// Optional local request-selection policy from the Grid operator. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) selection_policy: Option, + /// RFC 3339 timestamp of when this overlay was rendered. #[serde(skip_serializing_if = "Option::is_none")] pub(crate) generated_at: Option, } +/// Selection policy copied without interpretation by overlay-sync. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct SelectionPolicy { + /// Selection mode consumed by Praxis. + pub(crate) mode: SelectionMode, +} + +/// Wire-level mode validation. Overlay-sync does not choose or execute it. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum SelectionMode { + /// Ordered selection. + Deterministic, + /// Equal local rotation. + RoundRobin, + /// Local random selection. + Random, +} + /// A single routing candidate. #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct RoutingCandidate { @@ -166,11 +190,16 @@ pub(crate) struct RoutingCandidate { /// Zero-based position in the final sorted overlay. #[serde(skip_serializing_if = "Option::is_none")] pub(crate) rank: Option, + + /// Producer-assigned active selection group. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) selection_group: Option, } /// Credential reference projected alongside a routing candidate. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] pub(crate) struct ProjectedCredential { /// Authentication strategy. pub(crate) strategy: String, @@ -182,6 +211,7 @@ pub(crate) struct ProjectedCredential { /// A reference to a Kubernetes Secret holding a credential value. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] pub(crate) struct ProjectedCredentialRef { /// Secret name. pub(crate) name: String, diff --git a/overlay-sync/src/validation.rs b/overlay-sync/src/validation.rs index cff86cc..773a643 100644 --- a/overlay-sync/src/validation.rs +++ b/overlay-sync/src/validation.rs @@ -8,10 +8,13 @@ use std::{fmt, fmt::Write as _}; +#[cfg(test)] use serde::ser::Error as _; use sha2::{Digest as _, Sha256}; -use crate::types::{OverlayEnvelope, OverlayScope, RoutingOverlay}; +#[cfg(test)] +use crate::types::RoutingOverlay; +use crate::types::{OverlayEnvelope, OverlayScope}; // --------------------------------------------------------------------------- // Constants @@ -126,15 +129,12 @@ pub(crate) fn validate_envelope( }); } - let envelope: OverlayEnvelope = serde_json::from_slice(raw).map_err(|e| ValidationError { - reason: RejectionReason::Malformed, - detail: format!("JSON parse failed: {e}"), - })?; + let (raw_value, envelope) = parse_envelope(raw)?; validate_schema_version(&envelope.schema_version)?; validate_scope(&envelope.scope, expected_scope)?; validate_revision_digest_agreement(&envelope)?; - validate_content_digest(&envelope)?; + validate_content_digest(&raw_value, &envelope)?; let revision = envelope.revision.value; @@ -153,6 +153,20 @@ pub(crate) fn validate_envelope( }) } +/// Parse the raw envelope while converting both parser failures into the +/// bounded validation error type. +fn parse_envelope(raw: &[u8]) -> Result<(serde_json::Value, OverlayEnvelope), ValidationError> { + let raw_value: serde_json::Value = serde_json::from_slice(raw).map_err(|e| ValidationError { + reason: RejectionReason::Malformed, + detail: format!("JSON parse failed: {e}"), + })?; + let envelope: OverlayEnvelope = serde_json::from_value(raw_value.clone()).map_err(|e| ValidationError { + reason: RejectionReason::Malformed, + detail: format!("envelope structure invalid: {e}"), + })?; + Ok((raw_value, envelope)) +} + /// Check that the schema version is supported. fn validate_schema_version(version: &str) -> Result<(), ValidationError> { if version.starts_with(SUPPORTED_MAJOR) { @@ -224,8 +238,15 @@ fn validate_revision_digest_agreement(envelope: &OverlayEnvelope) -> Result<(), } /// Recompute the semantic digest and verify it matches the envelope. -fn validate_content_digest(envelope: &OverlayEnvelope) -> Result<(), ValidationError> { - let recomputed = compute_semantic_digest(&envelope.overlay).map_err(|e| ValidationError { +fn validate_content_digest( + raw_envelope: &serde_json::Value, + envelope: &OverlayEnvelope, +) -> Result<(), ValidationError> { + let raw_overlay = raw_envelope.get("overlay").ok_or_else(|| ValidationError { + reason: RejectionReason::Malformed, + detail: "overlay payload is missing".to_owned(), + })?; + let recomputed = compute_raw_semantic_digest(raw_overlay).map_err(|e| ValidationError { reason: RejectionReason::Malformed, detail: format!("cannot canonicalize overlay: {e}"), })?; @@ -239,21 +260,64 @@ fn validate_content_digest(envelope: &OverlayEnvelope) -> Result<(), ValidationE Ok(()) } +/// Compute the semantic digest from the raw overlay JSON. +/// +/// The operator's semantic payload includes the complete candidate objects. +/// Computing from raw JSON keeps additive routing metadata digest-significant +/// even when this sidecar does not understand the new field yet. +fn compute_raw_semantic_digest(overlay: &serde_json::Value) -> Result { + let object = overlay + .as_object() + .ok_or_else(|| "overlay must be a JSON object".to_owned())?; + let network = object + .get("network") + .ok_or_else(|| "overlay.network is missing".to_owned())?; + let local_site = object + .get("local_site") + .ok_or_else(|| "overlay.local_site is missing".to_owned())?; + let candidates = object + .get("candidates") + .ok_or_else(|| "overlay.candidates is missing".to_owned())?; + let mut semantic_payload = serde_json::json!({ + "candidates": candidates, + "local_site": local_site, + "network": network, + }); + if let Some(selection_policy) = object.get("selection_policy") { + semantic_payload + .as_object_mut() + .ok_or_else(|| "semantic payload is not an object".to_owned())? + .insert("selection_policy".to_owned(), selection_policy.clone()); + } + let canonical = serde_json_canonicalizer::to_vec(&semantic_payload) + .map_err(|e| format!("RFC 8785 canonicalization failed: {e}"))?; + let digest: [u8; 32] = Sha256::digest(&canonical).into(); + Ok(hex_encode(&digest)) +} + // --------------------------------------------------------------------------- // Digest computation (same algorithm as the operator) // --------------------------------------------------------------------------- /// Compute the SHA-256 digest of the RFC 8785 canonical semantic payload. /// -/// The semantic payload includes only `network`, `local_site`, and -/// `candidates` — the same fields used by the operator. +/// The semantic payload includes `network`, `local_site`, and `candidates`, +/// plus `selection_policy` when present — the same fields used by the +/// operator. +#[cfg(test)] fn compute_semantic_digest(overlay: &RoutingOverlay) -> Result { let candidates_value = serde_json::to_value(&overlay.candidates)?; - let semantic_payload = serde_json::json!({ + let mut semantic_payload = serde_json::json!({ "candidates": candidates_value, "local_site": overlay.local_site, "network": overlay.network, }); + if let Some(policy) = &overlay.selection_policy { + let Some(object) = semantic_payload.as_object_mut() else { + return Err(serde_json::Error::custom("semantic payload must be an object")); + }; + object.insert("selection_policy".to_owned(), serde_json::to_value(policy)?); + } let canonical = serde_json_canonicalizer::to_vec(&semantic_payload).map_err(serde_json::Error::custom)?; @@ -280,7 +344,7 @@ fn hex_encode(bytes: &[u8]) -> String { #[allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing, reason = "tests")] mod tests { use super::*; - use crate::types::{ContentDigest, ContentRevision, OverlayProvenance, RoutingCandidate}; + use crate::types::{ContentDigest, ContentRevision, OverlayProvenance, RoutingCandidate, RoutingOverlay}; fn test_scope() -> ExpectedScope { ExpectedScope { @@ -308,7 +372,9 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }], + selection_policy: None, generated_at: Some("2026-07-29T00:00:00Z".to_owned()), } } @@ -354,6 +420,50 @@ mod tests { result.unwrap(); } + #[test] + fn additive_candidate_field_is_digest_significant_and_accepted() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["candidates"][0]["future_field"] = serde_json::json!({"mode": "x"}); + let digest = compute_raw_semantic_digest(&value["overlay"]).unwrap(); + value["revision"]["value"] = serde_json::Value::String(digest.clone()); + value["content_digest"]["value"] = serde_json::Value::String(digest); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(result.is_ok(), "additive candidate metadata should survive validation"); + } + + #[test] + fn additive_candidate_field_mutation_without_digest_update_is_rejected() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["candidates"][0]["future_field"] = serde_json::json!({"mode": "x"}); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(matches!(result.unwrap_err().reason, RejectionReason::DigestMismatch)); + } + + #[test] + fn credential_value_fields_remain_rejected() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["candidates"][0]["credential"] = serde_json::json!({ + "strategy": "bearer_token", + "secretRef": { + "name": "credential", + "namespace": "ns", + "key": "token" + }, + "token": "must-not-enter-overlay" + }); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(matches!(result.unwrap_err().reason, RejectionReason::Malformed)); + } + #[test] fn oversized_payload_rejected() { let env = valid_envelope(); @@ -437,6 +547,37 @@ mod tests { ); } + #[test] + fn selection_policy_fixture_digest_matches_operator_contract() { + let fixture = include_str!("../../tests/fixtures/overlay-contract/v1/valid-selection-policy.json"); + let envelope: OverlayEnvelope = serde_json::from_str(fixture).unwrap(); + let recomputed = compute_semantic_digest(&envelope.overlay).unwrap(); + assert_eq!( + recomputed, envelope.revision.value, + "selection policy fixture must match the producer digest; computed={recomputed}" + ); + let scope = test_scope(); + validate_envelope(fixture.as_bytes(), &scope, 1_048_576, None) + .expect("selection policy fixture must pass transparent validation"); + } + + #[test] + fn unknown_selection_policy_field_is_rejected() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["selection_policy"] = serde_json::json!({ + "mode": "roundRobin", + "unexpected": true + }); + let digest = compute_raw_semantic_digest(&value["overlay"]).unwrap(); + value["revision"]["value"] = serde_json::Value::String(digest.clone()); + value["content_digest"]["value"] = serde_json::Value::String(digest); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(matches!(result.unwrap_err().reason, RejectionReason::Malformed)); + } + #[test] fn hex_encode_lowercase() { let bytes = [0xAB, 0xCD, 0x01, 0x23]; diff --git a/overlay-sync/src/watcher.rs b/overlay-sync/src/watcher.rs index 9a41e83..0205183 100644 --- a/overlay-sync/src/watcher.rs +++ b/overlay-sync/src/watcher.rs @@ -481,7 +481,9 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }], + selection_policy: None, generated_at: Some("2026-07-29T00:00:00Z".to_owned()), } } diff --git a/tests/e2e/topologies/grid-combined-site/forge.yaml b/tests/e2e/topologies/grid-combined-site/forge.yaml index 7686b30..a3bc1cb 100644 --- a/tests/e2e/topologies/grid-combined-site/forge.yaml +++ b/tests/e2e/topologies/grid-combined-site/forge.yaml @@ -210,6 +210,11 @@ spec: gridId: grid-combined-site-v1 region: west zone: west-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin swim: probeInterval: "5s" suspicionTimeout: "15s" @@ -268,6 +273,11 @@ spec: gridId: grid-combined-site-v1 region: central zone: central-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin swim: probeInterval: "5s" suspicionTimeout: "15s" @@ -326,6 +336,11 @@ spec: gridId: grid-combined-site-v1 region: east zone: east-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin swim: probeInterval: "5s" suspicionTimeout: "15s" diff --git a/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml b/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml index debf2d2..1a68c7f 100644 --- a/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml +++ b/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml @@ -36,6 +36,8 @@ spec: - ALL readOnlyRootFilesystem: false runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 env: - name: MODEL value: "Qwen/Qwen3-0.6B" diff --git a/tests/e2e/topologies/grid-provider-traffic/configs/provider/praxis.yaml b/tests/e2e/topologies/grid-provider-traffic/configs/provider/praxis.yaml new file mode 100644 index 0000000..aa41394 --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/configs/provider/praxis.yaml @@ -0,0 +1,60 @@ +# Provider gateway fixture used by the provider-traffic xtask tests. +listeners: + - name: provider + address: "0.0.0.0:8443" + filter_chains: + - provider-inference + tls: + certificates: + - cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + client_ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert_mode: require + +filter_chains: + - name: provider-inference + filters: + - filter: peer_identity_trust + trusted_peers: + - organization: ai-grid + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Provider-Traffic-Provider-Gateway + value: "SITE_PLACEHOLDER" + - filter: provider_route + provider_id: "SITE_PLACEHOLDER" + model_header: X-Model + emit_demo_attribution: true + routes: + - candidate_id: "CANDIDATE_ID_PLACEHOLDER" + model: Qwen/Qwen3-0.6B + paths: + - /v1/chat/completions + - /v1/responses + cluster: vcr-backend + credential: + strategy: bearer_token + secretRef: + name: vcr-inference-credential + namespace: grid-system + key: token + - filter: credential_inject + credentials: + - strategy: bearer_token + name: vcr-inference-credential + namespace: grid-system + key: token + file: /etc/praxis/credentials/vcr-inference/token + - filter: load_balancer + clusters: + - name: vcr-backend + endpoints: + - "vcr-inference-SITE_PLACEHOLDER.grid-system.svc.cluster.local:8000" + +admin: + address: "127.0.0.1:9901" +shutdown_timeout_secs: 5 diff --git a/tests/fixtures/overlay-contract/v1/valid-selection-policy.json b/tests/fixtures/overlay-contract/v1/valid-selection-policy.json new file mode 100644 index 0000000..1bd8f46 --- /dev/null +++ b/tests/fixtures/overlay-contract/v1/valid-selection-policy.json @@ -0,0 +1,56 @@ +{ + "schema_version": "1.0.0", + "revision": { + "kind": "content_addressed", + "algorithm": "sha256", + "value": "d3321586b5e414b8bd80c07a9d16ac10dce38d6810e3591a8526a6bdd0639007" + }, + "content_digest": { + "algorithm": "sha256", + "value": "d3321586b5e414b8bd80c07a9d16ac10dce38d6810e3591a8526a6bdd0639007" + }, + "scope": { + "network": "test-net", + "gateway": "gw", + "namespace": "ns", + "local_site": "site-a" + }, + "provenance": { + "producer": "grid-operator", + "producer_version": "0.1.3", + "source_name": "test-net", + "source_uid": "uid-selection-policy", + "source_generation": 1, + "rendered_at": "2026-08-12T00:00:00Z" + }, + "overlay": { + "network": "test-net", + "local_site": "site-a", + "selection_policy": { + "mode": "roundRobin" + }, + "candidates": [ + { + "kind": "inference_model", + "name": "model-a", + "site": "site-a", + "cluster": "cluster-a", + "fresh": true, + "stable_id": "abcd1234", + "rank": 0, + "selection_group": 0 + }, + { + "kind": "inference_model", + "name": "model-b", + "site": "site-b", + "cluster": "cluster-b", + "fresh": true, + "stable_id": "efgh5678", + "rank": 1, + "selection_group": 0 + } + ], + "generated_at": "2026-08-12T00:00:00Z" + } +} diff --git a/xtask/src/env.rs b/xtask/src/env.rs index 66ae5e3..39a5ee9 100644 --- a/xtask/src/env.rs +++ b/xtask/src/env.rs @@ -16,6 +16,7 @@ pub(crate) mod kubectl; pub(crate) mod llmd_pool_metrics_demo; pub(crate) mod operator; pub(crate) mod operator_overlay; +pub(crate) mod provider_traffic_demo; pub(crate) mod providers; pub(crate) mod trust; pub(crate) mod verify; @@ -974,6 +975,19 @@ pub(crate) enum Action { #[arg(long)] kv_cache: bool, }, + + /// Create the focused provider-gateway traffic demo, then prove equal + /// selection across its active provider group. + RunGridProviderTrafficDemo { + /// Path to the public or internal Forge environment config file. + /// This is required because the focused demo currently lives in the + /// public demos repository rather than the Grid source tree. + #[arg(long)] + forge_config: PathBuf, + /// Demo mode and teardown options. Only `--quick` is supported. + #[command(flatten)] + options: GlbDemoOptions, + }, } #[cfg(test)] @@ -1096,6 +1110,9 @@ pub(crate) fn run(action: &Action) -> Result<(), Box> { metrics_mtls, kv_cache, } => llmd_pool_metrics_demo::run(forge_config, options, *metrics_mtls, *kv_cache), + Action::RunGridProviderTrafficDemo { forge_config, options } => { + provider_traffic_demo::run(forge_config, options) + }, } } diff --git a/xtask/src/env/combined_site_demo.rs b/xtask/src/env/combined_site_demo.rs index 9b15b45..d98199f 100644 --- a/xtask/src/env/combined_site_demo.rs +++ b/xtask/src/env/combined_site_demo.rs @@ -3407,34 +3407,6 @@ fn require_local_image(image: &str) -> Result<(), Box> { .into()) } -/// Verify that `image` has a numeric USER (or UID:GID) so Kubernetes can -/// enforce `runAsNonRoot` without `runAsUser` in the pod spec. -fn require_numeric_image_user(image: &str) -> Result<(), Box> { - let output = Command::new("docker") - .args(["inspect", "--format", "{{.Config.User}}", image]) - .output()?; - if !output.status.success() { - return Err(format!( - "cannot inspect image {image:?}: {}", - String::from_utf8_lossy(&output.stderr).trim() - ) - .into()); - } - let user = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - if user.is_empty() { - return Err(format!("image {image:?} has no USER set; runAsNonRoot requires a numeric user").into()); - } - let valid = user.split(':').all(|part| part.parse::().is_ok()); - if !valid { - return Err(format!( - "image {image:?} has non-numeric USER {user:?}; \ - Kubernetes cannot verify runAsNonRoot with a non-numeric user" - ) - .into()); - } - Ok(()) -} - /// Load local container images into all Kind clusters. /// /// Reads image references from the `GRID_XTASK_*_IMAGE` environment variables @@ -3458,9 +3430,6 @@ fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result eprintln!(" verified local image: {image}"); } - require_numeric_image_user(&vcr)?; - eprintln!(" verified numeric USER for {vcr}"); - for cluster in CLUSTERS { for image in [&gateway, &operator, &vcr] { eprintln!(" loading {image} into {cluster}..."); diff --git a/xtask/src/env/image_overrides.rs b/xtask/src/env/image_overrides.rs index 21944a5..a8abb1a 100644 --- a/xtask/src/env/image_overrides.rs +++ b/xtask/src/env/image_overrides.rs @@ -29,6 +29,9 @@ const VCR_IMAGE_ENV: &str = "GRID_XTASK_VCR_IMAGE"; /// Environment variable to override the operator image. const OPERATOR_IMAGE_ENV: &str = "GRID_XTASK_OPERATOR_IMAGE"; +/// Environment variable to override the overlay-sync image. +const OVERLAY_SYNC_IMAGE_ENV: &str = "GRID_XTASK_OVERLAY_SYNC_IMAGE"; + /// Environment variable to override the image pull policy. const IMAGE_PULL_POLICY_ENV: &str = "GRID_XTASK_IMAGE_PULL_POLICY"; @@ -48,6 +51,9 @@ const DEFAULT_MOCK_PROVIDER_IMAGE: &str = "grid-mock-providers:latest"; /// Default operator image (matches operator.rs). const DEFAULT_OPERATOR_IMAGE: &str = "grid-operator:latest"; +/// Default overlay-sync image used by overlay-enabled demo gateways. +const DEFAULT_OVERLAY_SYNC_IMAGE: &str = "grid-overlay-sync:latest"; + /// Default gateway image used by the GLB demo. const DEFAULT_GLB_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3"; @@ -96,6 +102,11 @@ pub(crate) fn operator_image() -> String { env::var(OPERATOR_IMAGE_ENV).unwrap_or_else(|_| DEFAULT_OPERATOR_IMAGE.to_owned()) } +/// Get the overlay-sync image name, respecting environment overrides. +pub(crate) fn overlay_sync_image() -> String { + env::var(OVERLAY_SYNC_IMAGE_ENV).unwrap_or_else(|_| DEFAULT_OVERLAY_SYNC_IMAGE.to_owned()) +} + /// Get the VCR image name, respecting environment overrides. pub(crate) fn vcr_image() -> String { env::var(VCR_IMAGE_ENV).unwrap_or_else(|_| DEFAULT_VCR_IMAGE.to_owned()) diff --git a/xtask/src/env/provider_traffic_demo.rs b/xtask/src/env/provider_traffic_demo.rs new file mode 100644 index 0000000..7bd6057 --- /dev/null +++ b/xtask/src/env/provider_traffic_demo.rs @@ -0,0 +1,3047 @@ +//! Narrated, evidence-backed provider-traffic demo scenarios. +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::{Path, PathBuf}, + process::Command, + sync::atomic::{AtomicU64, Ordering}, + time::{Duration, Instant}, +}; + +use serde::Serialize; + +use super::{DemoMode, GlbDemoOptions, certs, glb, kubectl, operator, safe_truncate_str}; + +// ----------------------------------------------------------------------------- +// Constants +// ----------------------------------------------------------------------------- + +/// Directory where generated TLS certificates are stored. +const CERTS_DIR: &str = "tests/env/certs"; + +/// Ordered provider-site cluster names in the provider-traffic scenario. +/// +/// The consumer entrypoint is deployed only in `CONSUMER_SITE`; the other +/// clusters contain provider gateways and backends only. +const CLUSTERS: &[&str] = &["provider-a", "provider-b", "provider-c"]; + +/// The single consumer gateway used by the focused request-routing proof. +const CONSUMER_SITE: &str = "provider-a"; + +/// Consumer gateway TLS secret name (matches Helm `existingSecret` reference). +const CONSUMER_TLS_SECRET: &str = "consumer-gateway-tls"; + +/// Evidence JSON schema version. +const EVIDENCE_SCHEMA_VERSION: &str = "1"; + +/// Kubernetes namespace for all Grid components. +const GRID_SYSTEM_NS: &str = "grid-system"; + +/// Overlay `ConfigMap` name created by the Grid operator for consumer gateways. +const OVERLAY_CONFIGMAP: &str = "grid-overlay-grid-provider-traffic-consumer-gateway"; + +/// Provider credential secret name (matches Helm `credentials[0].name`). +const VCR_INFERENCE_CREDENTIAL: &str = "vcr-inference-credential"; + +/// Stable terminal separator that also remains readable in captured logs. +const OUTPUT_RULE: &str = "==============================================================================="; + +/// Provider gateway service name advertised via SWIM for cross-site discovery. +const PROVIDER_GATEWAY_SERVICE: &str = "provider-gateway"; + +/// Provider gateway port advertised via SWIM for cross-site discovery. +const PROVIDER_GATEWAY_PORT: &str = "8443"; + +/// Provider gateway TLS secret name (matches Helm `existingSecret` reference). +const PROVIDER_TLS_SECRET: &str = "provider-gateway-tls"; + +/// Same-CA client identity with an organization rejected by `peer_identity_trust`. +const WRONG_ORG_TLS_SECRET: &str = "wrong-org-client-tls"; + +/// Number of environment setup phases shown to the user. +const SETUP_PHASES: usize = 14; + +/// Makes retry probe names unique while retaining a recognizable prefix. +static PROBE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +// ----------------------------------------------------------------------------- +// Context +// ----------------------------------------------------------------------------- + +/// Provider-traffic demo execution context. +struct ProviderTrafficContext { + /// Canonical demo root directory for resolving configs, resources, and + /// other demo-relative assets. + demo_root: PathBuf, + /// Path to the resolved Forge config. + resolved_config: PathBuf, + /// Path to the forge binary. + forge_bin: PathBuf, +} + +// ----------------------------------------------------------------------------- +// Overlay State +// ----------------------------------------------------------------------------- + +/// Per-site overlay snapshot captured from the `ConfigMap`. +#[derive(Clone, Debug, serde::Deserialize, Serialize)] +struct OverlayData { + /// Kubernetes `ConfigMap` `resourceVersion` (per-cluster, never compared + /// across clusters). + resource_version: String, + /// Content-addressed semantic revision from the + /// `grid.praxis-proxy.io/overlay-revision` annotation. Changes only when + /// routing-relevant fields change; safe to compare across clusters. + semantic_revision: String, + /// Candidate name to stable ID mapping. + stable_ids: BTreeMap, + /// Full candidate details for evidence (kind, name, site, cluster, model). + candidates: Vec, +} + +/// Subset of `RoutingCandidate` fields kept for evidence and validation. +#[derive(Clone, Debug, serde::Deserialize, Serialize)] +struct OverlayCandidate { + /// Candidate kind (e.g. `inference_model`). + kind: String, + /// Model name. + name: String, + /// Site name. + site: String, + /// Upstream cluster identifier (used as the candidate key). + cluster: String, + /// Deterministic stable ID for session binding. + stable_id: String, + /// Whether the candidate's metrics snapshot is fresh. + fresh: Option, + /// Admission state emitted by the operator, when present. + admission_state: Option, + /// Explicit priority group emitted by the operator, when present. + selection_group: Option, +} + +/// Pre- and post-SWIM overlay snapshots for evidence. +#[derive(Clone, Debug, Default, serde::Deserialize, Serialize)] +struct OverlayState { + /// Per-site overlays before SWIM seeding (local candidates only). + pre_swim: BTreeMap, + /// Per-site overlays after global convergence. + post_swim: BTreeMap, +} + +// ----------------------------------------------------------------------------- +// Evidence +// ----------------------------------------------------------------------------- + +/// Evidence written to `results.json`. +#[derive(Debug, serde::Deserialize, Serialize)] +struct Evidence { + /// Evidence schema version. + schema_version: String, + /// Demo mode that was executed. + mode: String, + /// Topology name. + topology: String, + /// List of cluster names. + clusters: Vec, + /// Proof results for each assertion. + proof_results: BTreeMap, + /// Exact image references used. + images: BTreeMap, + /// Pre- and post-SWIM overlay snapshots. + overlay_state: OverlayState, + /// Cluster health status. + cluster_health: Vec, + /// Component deployment status. + components: Vec, + /// SWIM membership views. + swim_membership: Vec, + /// Provider response samples. + provider_responses: Vec, + /// Security assertion results. + security_results: Vec, + /// Teardown success. + teardown_success: bool, +} + +/// Evidence for one proof assertion. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ProofResult { + /// Whether the proof passed. + success: bool, + /// Human-readable reason. + reason: String, + /// Observed facts that support this result. + observed_facts: BTreeMap, + /// Duration of the assertion in milliseconds. + duration_ms: u64, +} + +/// Cluster health status. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ClusterHealth { + /// Cluster name. + name: String, + /// Whether the cluster is healthy. + healthy: bool, + /// API server response time in milliseconds. + api_response_ms: Option, + /// Number of ready nodes. + ready_nodes: u32, +} + +/// Component deployment status. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ComponentStatus { + /// Component name. + name: String, + /// Deployment namespace. + namespace: String, + /// Ready replicas. + ready_replicas: u32, + /// Desired replicas. + desired_replicas: u32, + /// Whether the component is ready. + ready: bool, +} + +/// SWIM membership view. +#[derive(Debug, serde::Deserialize, Serialize)] +struct SwimMembership { + /// Site name. + site: String, + /// Local node ID. + local_node: String, + /// List of known peers. + peers: Vec, + /// Membership convergence status. + converged: bool, +} + +/// Provider response metadata. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ProviderResponse { + /// Consumer site that made the request. + consumer_site: String, + /// Provider site that served the request. + provider_site: String, + /// Provider instance ID. + provider_instance: String, + /// Session ID. + session_id: String, + /// Serving revision. + serving_revision: String, + /// Response time in milliseconds. + response_time_ms: u64, + /// Whether the response was successful. + success: bool, +} + +/// Security assertion result. +#[derive(Debug, serde::Deserialize, Serialize)] +struct SecurityResult { + /// Type of security test. + test_type: String, + /// Expected result (allow/deny). + expected: String, + /// Actual result. + actual: String, + /// Whether the test passed. + passed: bool, + /// Additional context. + context: String, +} + +// ----------------------------------------------------------------------------- +// Utility Functions +// ----------------------------------------------------------------------------- + +/// Format current UTC timestamp for run IDs. +fn format_utc_timestamp() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or_else(|_| "unknown".to_owned(), |duration| format!("{}", duration.as_secs())) +} + +/// Format current UTC timestamp in ISO format. +fn format_utc_iso() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now().duration_since(UNIX_EPOCH).map_or_else( + |_| "unknown-utc".to_owned(), + |duration| format!("{}-utc", duration.as_secs()), + ) +} + +/// Resolve the evidence directory path. +fn resolve_evidence_dir( + forge_config: &Path, + options: &GlbDemoOptions, + run_id: &str, +) -> Result> { + if let Some(dir) = &options.evidence_dir { + Ok(dir.clone()) + } else { + let config_dir = forge_config + .parent() + .ok_or("forge config should have parent directory")?; + Ok(config_dir.join(format!("evidence-{run_id}"))) + } +} + +// ----------------------------------------------------------------------------- +// Assertion Framework +// ----------------------------------------------------------------------------- + +/// Result of a runtime assertion. +type AssertionResult = Result>; + +/// Create a successful proof result with observed facts. +fn proof_success(reason: &str, observed_facts: BTreeMap, duration: Duration) -> ProofResult { + ProofResult { + success: true, + reason: reason.to_owned(), + observed_facts, + duration_ms: u64::try_from(duration.as_millis().min(u128::from(u64::MAX))).unwrap_or(u64::MAX), + } +} + +/// Create a failed proof result with observed facts. +fn proof_failure(reason: &str, observed_facts: BTreeMap, duration: Duration) -> ProofResult { + ProofResult { + success: false, + reason: reason.to_owned(), + observed_facts, + duration_ms: u64::try_from(duration.as_millis().min(u128::from(u64::MAX))).unwrap_or(u64::MAX), + } +} + +/// Execute an assertion with timing and error handling. +fn run_assertion(name: &str, assertion_fn: F) -> AssertionResult +where + F: FnOnce() -> AssertionResult, +{ + let start = Instant::now(); + eprintln!(" [ASSERT] {name}"); + + let result = assertion_fn(); + let _duration = start.elapsed(); + + match &result { + Ok(proof) => { + if proof.success { + eprintln!(" [OK] {name}: {}", proof.reason); + } else { + eprintln!(" [FAIL] {name}: {}", proof.reason); + } + }, + Err(e) => { + eprintln!(" [ERROR] {name}: {e}"); + }, + } + + result.map_err(|e| { + // Convert assertion errors to proof failures + format!("Assertion {name} failed: {e}").into() + }) +} + +/// Poll for a condition with bounded retries. +fn poll_until(condition: F, timeout: Duration, interval: Duration) -> Result> +where + F: Fn() -> Result, Box>, +{ + let start = Instant::now(); + + loop { + match condition() { + Ok(Some(result)) => return Ok(result), + Ok(None) => { + if start.elapsed() >= timeout { + return Err("Timeout waiting for condition".into()); + } + std::thread::park_timeout(interval); + }, + Err(e) => return Err(e), + } + } +} + +/// Wait for a deployment to be ready. +fn wait_for_deployment(deployment: &str, namespace: &str, context: &str) -> Result<(), Box> { + kubectl::wait_for_rollout_ns(context, deployment, namespace, "deployment")?; + Ok(()) +} + +/// Pod-security overrides for ephemeral curl pods in restricted namespaces. +/// +/// `kubectl run` names the container after the pod, so the container name must +/// match. The curlimages/curl image runs as UID 100. +fn curl_pod_overrides(pod_name: &str, curl_args: &[&str]) -> String { + let args = curl_args.strip_prefix(&["curl"]).unwrap_or(curl_args); + serde_json::json!({ + "spec": { + "automountServiceAccountToken": false, + "securityContext": { + "runAsNonRoot": true, + "seccompProfile": { "type": "RuntimeDefault" } + }, + "containers": [{ + "name": pod_name, + "image": "curlimages/curl:8.12.1", + "command": ["curl"], + "args": args, + "securityContext": { + "runAsUser": 100, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { "drop": ["ALL"] } + } + }] + } + }) + .to_string() +} + +/// Run an ephemeral curl pod with restricted `PodSecurity` context. +fn run_curl_probe(context: &str, pod_name: &str, curl_args: &[&str]) -> Result { + let sequence = PROBE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let prefix = pod_name.get(..pod_name.len().min(40)).unwrap_or(pod_name); + let unique_pod_name = format!("{prefix}-{sequence}"); + let overrides = curl_pod_overrides(&unique_pod_name, curl_args); + Command::new("kubectl") + .args([ + "run", + &unique_pod_name, + "--image=curlimages/curl:8.12.1", + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "--rm", + "-i", + "--restart=Never", + "--overrides", + &overrides, + ]) + .output() +} + +/// Run an ephemeral curl pod with additional kubectl flags (e.g. `--labels`). +fn response_header(output: &[u8], name: &str) -> Option { + let expected = name.to_ascii_lowercase(); + String::from_utf8_lossy(output).lines().find_map(|line| { + let (header, value) = line.split_once(':')?; + header.eq_ignore_ascii_case(&expected).then(|| value.trim().to_owned()) + }) +} + +/// Wait for the demo environment to be ready. +fn wait_for_environment_ready() -> Result> { + // Wait for Grid operators to converge + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + eprintln!(" [WAIT] {cluster}: Grid operator convergence"); + + // Wait for deployment to be ready + wait_for_deployment("grid-operator", "grid-system", &context)?; + + // Every site has a provider gateway; only the designated ingress site + // has the consumer gateway used by the request proof. + if *cluster == CONSUMER_SITE { + wait_for_deployment("consumer-gateway", "grid-system", &context)?; + } + wait_for_deployment("provider-gateway", "grid-system", &context)?; + + eprintln!(" [OK] {cluster}: Gateways ready"); + } + + Ok("Three provider sites converged; one consumer entrypoint and all provider gateways are ready".to_owned()) +} + +// ----------------------------------------------------------------------------- +// Runtime Assertions +// ----------------------------------------------------------------------------- + +/// Assert exactly three provider clusters exist and are healthy. +#[expect( + clippy::too_many_lines, + reason = "The proof reports one bounded fact set per cluster." +)] +fn assert_cluster_health() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let mut cluster_health = Vec::new(); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + // Check API server responsiveness + let api_start = Instant::now(); + let output = Command::new("kubectl") + .args(["cluster-info", "--context", &context]) + .output()?; + + let api_response_ms = + u64::try_from(api_start.elapsed().as_millis().min(u128::from(u64::MAX))).unwrap_or(u64::MAX); + let healthy = output.status.success(); + + // Get node count + let nodes_output = Command::new("kubectl") + .args([ + "get", + "nodes", + "--context", + &context, + "-o", + "jsonpath={.items[*].status.conditions[?(@.type=='Ready')].status}", + ]) + .output()?; + + let ready_nodes = if nodes_output.status.success() { + u32::try_from( + String::from_utf8_lossy(&nodes_output.stdout) + .split_whitespace() + .filter(|s| s == &"True") + .count() + .min(u32::MAX as usize), + ) + .unwrap_or(u32::MAX) + } else { + 0 + }; + + cluster_health.push(ClusterHealth { + name: cluster.to_string(), + healthy, + api_response_ms: Some(api_response_ms), + ready_nodes, + }); + + observed_facts.insert(format!("{cluster}_healthy"), serde_json::Value::Bool(healthy)); + observed_facts.insert( + format!("{cluster}_ready_nodes"), + serde_json::Value::Number(ready_nodes.into()), + ); + } + + let all_healthy = cluster_health.iter().all(|c| c.healthy && c.ready_nodes > 0); + observed_facts.insert( + "total_clusters".to_owned(), + serde_json::Value::Number(CLUSTERS.len().into()), + ); + + if all_healthy { + Ok(proof_success( + &format!("All {} clusters are healthy with ready nodes", CLUSTERS.len()), + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more clusters are unhealthy", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert one provider stack per site and one consumer entrypoint. +#[expect( + clippy::too_many_lines, + reason = "The proof checks the required deployed components together." +)] +fn assert_component_deployment() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let mut components = Vec::new(); + + let static_components = ["grid-operator", "provider-gateway"]; + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + let mock_name = format!("vcr-inference-{cluster}"); + let cluster_components: Vec<&str> = static_components + .iter() + .copied() + .chain(std::iter::once(mock_name.as_str())) + .collect(); + let cluster_components = if *cluster == CONSUMER_SITE { + cluster_components + .into_iter() + .chain(std::iter::once("consumer-gateway")) + .collect::>() + } else { + cluster_components + }; + + for component in &cluster_components { + let output = Command::new("kubectl") + .args([ + "get", + "deployment", + component, + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "-o", + "jsonpath={.status.readyReplicas},{.status.replicas}", + ]) + .output()?; + + if output.status.success() { + let status_str = String::from_utf8_lossy(&output.stdout); + let parts: Vec<&str> = status_str.trim().split(',').collect(); + + let ready_replicas = parts.first().and_then(|s| s.parse::().ok()).unwrap_or(0); + let desired_replicas = parts.get(1).and_then(|s| s.parse::().ok()).unwrap_or(0); + let ready = ready_replicas > 0 && ready_replicas == desired_replicas; + + components.push(ComponentStatus { + name: format!("{cluster}-{component}"), + namespace: GRID_SYSTEM_NS.to_owned(), + ready_replicas, + desired_replicas, + ready, + }); + + observed_facts.insert(format!("{cluster}_{component}_ready"), serde_json::Value::Bool(ready)); + observed_facts.insert( + format!("{cluster}_{component}_replicas"), + serde_json::Value::Number(ready_replicas.into()), + ); + } else { + components.push(ComponentStatus { + name: format!("{cluster}-{component}"), + namespace: GRID_SYSTEM_NS.to_owned(), + ready_replicas: 0, + desired_replicas: 1, + ready: false, + }); + observed_facts.insert(format!("{cluster}_{component}_ready"), serde_json::Value::Bool(false)); + } + } + } + + let expected_count = components.len(); + let all_ready = components.len() == expected_count && components.iter().all(|c| c.ready); + observed_facts.insert( + "total_components".to_owned(), + serde_json::Value::Number(expected_count.into()), + ); + + if all_ready { + Ok(proof_success( + &format!( + "{} components are ready across {} provider sites with one consumer entrypoint", + expected_count, + CLUSTERS.len() + ), + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more components are not ready", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert every site's `GridNetwork` reports all three remote sites connected. +#[expect( + clippy::too_many_lines, + clippy::unnecessary_wraps, + reason = "The assertion framework requires a fallible, named proof boundary." +)] +fn assert_swim_convergence() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let expected_remote_sites = CLUSTERS.len() - 1; + let mut all_converged = true; + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + let status = poll_until( + || { + let output = Command::new("kubectl") + .args([ + "get", + "gridnetwork/grid-provider-traffic", + "--context", + &context, + "-o", + "jsonpath={.status.phase},{.status.connectedSites}", + ]) + .output()?; + if !output.status.success() { + return Ok(None); + } + let value = String::from_utf8_lossy(&output.stdout); + let Some((phase, connected)) = value.trim().split_once(',') else { + return Ok(None); + }; + let connected = connected.parse::().unwrap_or_default(); + Ok((phase == "Active" && connected == expected_remote_sites).then_some((phase.to_owned(), connected))) + }, + Duration::from_secs(90), + Duration::from_secs(3), + ); + + match status { + Ok((phase, connected)) => { + observed_facts.insert(format!("{cluster}_phase"), serde_json::Value::String(phase)); + observed_facts.insert( + format!("{cluster}_connected_sites"), + serde_json::Value::Number(connected.into()), + ); + }, + Err(error) => { + all_converged = false; + observed_facts.insert(format!("{cluster}_error"), serde_json::Value::String(error.to_string())); + }, + } + } + + observed_facts.insert( + "expected_remote_sites".to_owned(), + serde_json::Value::Number(expected_remote_sites.into()), + ); + + if all_converged { + Ok(proof_success( + "Every GridNetwork is Active with all three remote sites connected", + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more GridNetworks did not report all three remote sites connected", + observed_facts, + start.elapsed(), + )) + } +} + +/// Verify that both remote sites are discovered and routing-eligible. +/// +/// The locally declared placement site is not a remote SWIM discovery result +/// and is excluded from this assertion. +#[expect( + clippy::too_many_lines, + clippy::unnecessary_wraps, + reason = "The assertion framework requires a fallible, named proof boundary." +)] +fn assert_site_auto_discovery() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let expected_remote_count = CLUSTERS.len() - 1; + let mut all_ok = true; + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let result = poll_until( + || { + let output = Command::new("kubectl") + .args([ + "get", "gridsite", + "-l", "grid.praxis-proxy.io/auto-discovered=true", + "--context", &context, + "-n", GRID_SYSTEM_NS, + "-o", "jsonpath={range .items[*]}{.metadata.name}\t{.status.phase}\t{.status.reason}\t{.spec.egress.address}\t{.spec.egress.tls.serverName}\t{.spec.trust.canonicalFingerprints}\n{end}", + ]) + .output()?; + if !output.status.success() { + return Ok(None); + } + let body = String::from_utf8_lossy(&output.stdout); + let mut verified = Vec::new(); + let port_suffix = format!(":{PROVIDER_GATEWAY_PORT}"); + for line in body.lines() { + let fields: Vec<&str> = line.trim().split('\t').collect(); + let [name, phase, reason, addr, server_name, fingerprints, ..] = fields.as_slice() else { + continue; + }; + if *phase == "Active" + && *reason == "TlsVerified" + && addr.ends_with(&port_suffix) + && !server_name.is_empty() + && !fingerprints.is_empty() + { + verified.push(serde_json::json!({ + "name": name, + "phase": phase, + "reason": reason, + "egressAddress": addr, + "serverName": server_name, + "hasFingerprints": true, + })); + } + } + if verified.len() == expected_remote_count { + Ok(Some(verified)) + } else { + Ok(None) + } + }, + Duration::from_secs(120), + Duration::from_secs(5), + ); + + match result { + Ok(remotes) => { + observed_facts.insert(format!("{cluster}_remote_sites"), serde_json::Value::Array(remotes)); + }, + Err(error) => { + all_ok = false; + observed_facts.insert(format!("{cluster}_error"), serde_json::Value::String(error.to_string())); + }, + } + } + + observed_facts.insert( + "expected_remote_count".to_owned(), + serde_json::Value::Number(expected_remote_count.into()), + ); + + if all_ok { + Ok(proof_success( + "Every cluster has two Active/TlsVerified remote `GridSites` with :8443 addresses and trust configuration", + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more clusters lack Active auto-discovered remote GridSites", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert each consumer receives and accepts a versioned overlay. +/// +/// Validates per-site: `ConfigMap` exists with non-empty data, gateway deployment +/// is ready, and a request is routed through the accepted overlay. +#[expect( + clippy::too_many_lines, + reason = "The proof checks overlay, gateway, and routing acceptance together." +)] +fn assert_overlay_acceptance() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let mut all_accepted = true; + + { + let cluster = CONSUMER_SITE; + let context = format!("kind-grid-provider-traffic-{cluster}"); + + // Step 1: Overlay ConfigMap exists + let overlay_output = Command::new("kubectl") + .args([ + "get", + "configmap", + "grid-overlay-grid-provider-traffic-consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.metadata.resourceVersion}", + ]) + .output()?; + + let resource_version = String::from_utf8_lossy(&overlay_output.stdout).trim().to_owned(); + let overlay_exists = overlay_output.status.success() && !resource_version.is_empty(); + + // Step 2: Overlay ConfigMap has non-empty data + let overlay_data_output = Command::new("kubectl") + .args([ + "get", + "configmap", + "grid-overlay-grid-provider-traffic-consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.data}", + ]) + .output()?; + + let data_str = String::from_utf8_lossy(&overlay_data_output.stdout).trim().to_owned(); + let has_data = overlay_data_output.status.success() && !data_str.is_empty() && data_str != "{}"; + + // Step 3: Consumer gateway deployment is ready + let deploy_output = Command::new("kubectl") + .args([ + "get", + "deployment/consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.status.readyReplicas}", + ]) + .output()?; + + let gateway_ready = deploy_output.status.success() + && String::from_utf8_lossy(&deploy_output.stdout) + .trim() + .parse::() + .is_ok_and(|n| n > 0); + + // Step 4: A valid request proves that the accepted overlay is serving. + // Praxis health endpoints are exposed only on the pod-local admin listener. + let routing_output = run_curl_probe( + &context, + &format!("overlay-routing-{cluster}"), + &[ + "curl", + "--fail-with-body", + "--silent", + "--show-error", + "--max-time", + "10", + "--header", + "Content-Type: application/json", + "--header", + "Authorization: Bearer consumer-token", + "--data", + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"overlay probe"}],"max_tokens":16}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ], + )?; + + let routing_ok = routing_output.status.success(); + + let overlay_accepted = overlay_exists && has_data && gateway_ready && routing_ok; + if !overlay_accepted { + all_accepted = false; + } + + observed_facts.insert( + format!("{cluster}_overlay_configmap_exists"), + serde_json::Value::Bool(overlay_exists), + ); + observed_facts.insert(format!("{cluster}_overlay_has_data"), serde_json::Value::Bool(has_data)); + observed_facts.insert( + format!("{cluster}_gateway_ready"), + serde_json::Value::Bool(gateway_ready), + ); + observed_facts.insert( + format!("{cluster}_overlay_routing_ok"), + serde_json::Value::Bool(routing_ok), + ); + observed_facts.insert( + format!("{cluster}_resource_version"), + serde_json::Value::String(resource_version), + ); + } + + observed_facts.insert( + "all_overlays_accepted".to_owned(), + serde_json::Value::Bool(all_accepted), + ); + + if all_accepted { + Ok(proof_success( + "Consumer entrypoint overlay ConfigMap exists with data, gateway ready, and routing passes", + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more sites failed overlay acceptance (ConfigMap/data/ready/routing)", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert consumer and operator cannot access provider credentials. +fn require_local_image(image: &str) -> Result<(), Box> { + let status = Command::new("docker") + .args(["image", "inspect", image]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status()?; + if status.success() { + return Ok(()); + } + Err(format!( + "required local image {image:?} is absent; build it or set \ + GRID_XTASK_IMAGE_PULL_POLICY=IfNotPresent with registry image overrides" + ) + .into()) +} + +/// Load local container images into all Kind clusters. +/// +/// Reads image references from the `GRID_XTASK_*_IMAGE` environment variables +/// (same source as `apply_image_overrides`). When `imagePullPolicy` is not +/// `Never`, this is a no-op. +#[expect( + clippy::too_many_lines, + reason = "Image loading is one bounded setup operation across the three clusters." +)] +fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result<(), Box> { + let pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); + if pull_policy != "Never" { + eprintln!(" skipping Kind image loading (pull policy is {pull_policy})"); + return Ok(()); + } + + let gateway = + std::env::var("GRID_XTASK_GATEWAY_IMAGE").unwrap_or_else(|_| "praxis-ai:provider-traffic-demo".to_owned()); + let operator = + std::env::var("GRID_XTASK_OPERATOR_IMAGE").unwrap_or_else(|_| "grid-operator:provider-traffic-demo".to_owned()); + let overlay_sync = crate::env::image_overrides::overlay_sync_image(); + let vcr = crate::env::image_overrides::vcr_image(); + + for image in [&gateway, &operator, &overlay_sync, &vcr] { + require_local_image(image)?; + eprintln!(" verified local image: {image}"); + } + + for cluster in CLUSTERS { + for image in [&gateway, &operator, &overlay_sync, &vcr] { + eprintln!(" loading {image} into {cluster}..."); + let output = Command::new(forge_bin.as_os_str()) + .arg("--config") + .arg(resolved_config) + .args(["--non-interactive", "cluster", "load-image", cluster, image]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to load {image} into {cluster}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + } + eprintln!(" [OK] {cluster}: all images loaded"); + } + Ok(()) +} + +/// Generate TLS certificates for all provider-traffic identities. +/// +/// Must be called BEFORE `forge up` so the certificates exist on the host +/// when `install_provider_boundary` creates the Kubernetes Secrets. +fn stage_provider_boundary() -> Result<(), Box> { + let identities: Vec = CLUSTERS.iter().map(|c| (*c).to_owned()).collect(); + certs::generate_all(&identities)?; + + let wrong_ca = ::certs::generate_ca("Combined Site untrusted test CA")?; + fs::write(Path::new(CERTS_DIR).join("untrusted-ca.pem"), wrong_ca.cert_pem)?; + + eprintln!(" [OK] TLS certificates generated for provider-a, provider-b, provider-c"); + Ok(()) +} + +/// Create TLS and credential Secrets in every provider-traffic cluster. +/// +/// Must be called AFTER `forge up` since the clusters must exist. Gateway +/// deployments are restarted so pods pick up the new volume mounts. +fn install_provider_boundary() -> Result<(), Box> { + let certs_dir = Path::new(CERTS_DIR); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + apply_tls_secret(&context, cluster, CONSUMER_TLS_SECRET, certs_dir)?; + apply_tls_secret(&context, cluster, PROVIDER_TLS_SECRET, certs_dir)?; + apply_tls_secret(&context, "wrong-org-client", WRONG_ORG_TLS_SECRET, certs_dir)?; + + eprintln!(" [OK] {cluster}: TLS secrets installed"); + } + + Ok(()) +} + +/// Create a TLS secret from the generated cert, key, and CA files. +#[expect( + clippy::too_many_lines, + reason = "TLS secret creation is one bounded setup operation." +)] +fn apply_tls_secret( + context: &str, + identity: &str, + secret_name: &str, + certs_dir: &Path, +) -> Result<(), Box> { + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "create", + "secret", + "generic", + secret_name, + &format!( + "--from-file=tls.crt={}", + certs_dir.join(format!("{identity}-cert.pem")).display() + ), + &format!( + "--from-file=tls.key={}", + certs_dir.join(format!("{identity}-key.pem")).display() + ), + &format!("--from-file=ca.crt={}", certs_dir.join("ca.pem").display()), + "--dry-run=client", + "-o", + "yaml", + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to render {identity} Secret/{secret_name}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + kubectl::apply_manifest(context, &String::from_utf8(output.stdout)?) +} + +/// Generate a random 32-byte hex provider credential. +fn generate_provider_credential() -> Result> { + let output = Command::new("openssl").args(["rand", "-hex", "32"]).output()?; + if !output.status.success() { + return Err("openssl failed to generate provider credential".into()); + } + let token = String::from_utf8(output.stdout)?.trim().to_owned(); + if token.len() != 64 || !token.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err("openssl returned an invalid provider credential".into()); + } + Ok(token) +} + +/// Create an Opaque Secret with a `token` key. +fn apply_credential_secret(context: &str, secret_name: &str, token: &str) -> Result<(), Box> { + let manifest = format!( + r#"{{"apiVersion":"v1","kind":"Secret","metadata":{{"name":"{secret_name}","namespace":"{GRID_SYSTEM_NS}"}},"type":"Opaque","stringData":{{"token":"{token}"}}}}"#, + ); + kubectl::apply_manifest(context, &manifest) +} + +/// Data extracted from the operator-created overlay `ConfigMap`. +/// Read a single cluster's overlay `ConfigMap` and return structured data. +/// +/// Captures both the Kubernetes `resourceVersion` (per-cluster) and the +/// semantic revision from the `grid.praxis-proxy.io/overlay-revision` +/// annotation (content-addressed, safe to compare across clusters). +#[expect( + clippy::too_many_lines, + reason = "The reader validates and parses one Kubernetes ConfigMap response." +)] +fn read_cluster_overlay(cluster: &str) -> Result> { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + OVERLAY_CONFIGMAP, + "-o", + "json", + ]) + .output()?; + + if !output.status.success() { + return Err(format!("{cluster}: overlay ConfigMap not found").into()); + } + + let cm: serde_json::Value = + serde_json::from_slice(&output.stdout).map_err(|e| format!("{cluster}: overlay ConfigMap invalid: {e}"))?; + + let resource_version = cm + .pointer("/metadata/resourceVersion") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_owned(); + + let semantic_revision = cm + .pointer("/metadata/annotations/grid.praxis-proxy.io~1overlay-revision") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_owned(); + + let routing_json = cm + .pointer("/data/routing-config.json") + .and_then(|v| v.as_str()) + .ok_or_else(|| format!("{cluster}: overlay missing routing-config.json"))?; + + let parsed: serde_json::Value = + serde_json::from_str(routing_json).map_err(|e| format!("{cluster}: routing-config.json invalid: {e}"))?; + + let raw_candidates = parsed + .get("candidates") + .and_then(|v| v.as_array()) + .ok_or_else(|| format!("{cluster}: overlay missing candidates array"))?; + + let mut stable_ids = BTreeMap::new(); + let mut candidates = Vec::new(); + + for c in raw_candidates { + let cluster_field = c.get("cluster").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let stable_id = c.get("stable_id").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let kind = c.get("kind").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let name = c.get("name").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let site = c.get("site").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let fresh = c.get("fresh").and_then(serde_json::Value::as_bool); + let admission_state = c + .get("admission_state") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let selection_group = c + .get("selection_group") + .and_then(serde_json::Value::as_u64) + .and_then(|value| u32::try_from(value).ok()); + + if !cluster_field.is_empty() && !stable_id.is_empty() { + stable_ids.insert(cluster_field.clone(), stable_id.clone()); + } + + candidates.push(OverlayCandidate { + kind, + name, + site, + cluster: cluster_field, + stable_id, + fresh, + admission_state, + selection_group, + }); + } + + Ok(OverlayData { + resource_version, + semantic_revision, + stable_ids, + candidates, + }) +} + +/// Establish the non-traffic preconditions for the measured picker proof. +/// +/// This gate deliberately performs no request. It verifies one consumer +/// replica, the explicit round-robin policy, three fresh `NewAndExisting` +/// candidates in group zero, and three consecutive identical semantic +/// revisions. The measured request window starts only after this gate passes. +#[expect( + clippy::too_many_lines, + reason = "The readiness barrier checks all non-traffic serving invariants." +)] +fn wait_for_round_robin_readiness() -> Result, Box> { + let context = "kind-grid-provider-traffic-provider-a"; + let replicas = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "deployment/consumer-gateway", + "-o", + "jsonpath={.spec.replicas},{.status.replicas},{.status.readyReplicas}", + ]) + .output()?; + let replica_text = String::from_utf8_lossy(&replicas.stdout).trim().to_owned(); + if !replicas.status.success() || replica_text != "1,1,1" { + return Err(format!("consumer gateway is not exactly one ready replica: {replica_text}").into()); + } + + let mut stable_revision: Option = None; + let mut stable_signature: Option = None; + for observation in 1..=3 { + let overlay = read_cluster_overlay("provider-a")?; + let configmap = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + OVERLAY_CONFIGMAP, + "-o", + "json", + ]) + .output()?; + if !configmap.status.success() { + return Err("consumer overlay ConfigMap could not be read".into()); + } + let configmap_json: serde_json::Value = serde_json::from_slice(&configmap.stdout)?; + let routing_text = configmap_json + .pointer("/data/routing-config.json") + .and_then(serde_json::Value::as_str) + .ok_or("consumer overlay ConfigMap has no routing-config.json data")?; + let routing: serde_json::Value = serde_json::from_str(routing_text)?; + let mode = routing + .pointer("/selection_policy/mode") + .and_then(serde_json::Value::as_str) + .ok_or("overlay does not publish selection_policy.mode")?; + if mode != "roundRobin" { + return Err(format!("overlay selection mode is {mode}, expected roundRobin").into()); + } + + let mut candidate_signature = Vec::new(); + for candidate in &overlay.candidates { + if candidate.selection_group != Some(0) { + return Err(format!( + "candidate {} is not in group 0: {:?}", + candidate.cluster, candidate.selection_group + ) + .into()); + } + if candidate.fresh == Some(false) { + return Err(format!("candidate {} is stale", candidate.cluster).into()); + } + if candidate + .admission_state + .as_deref() + .is_some_and(|state| state != "new_and_existing") + { + return Err(format!( + "candidate {} is not NewAndExisting: {:?}", + candidate.cluster, candidate.admission_state + ) + .into()); + } + candidate_signature.push(format!( + "{}:{}:{:?}:{:?}", + candidate.cluster, candidate.stable_id, candidate.selection_group, candidate.admission_state + )); + } + candidate_signature.sort(); + let signature = candidate_signature.join("|"); + if overlay.candidates.len() != 3 { + return Err(format!("expected three candidates, found {}", overlay.candidates.len()).into()); + } + if stable_revision + .as_ref() + .is_some_and(|revision| revision != &overlay.semantic_revision) + || stable_signature.as_ref().is_some_and(|previous| previous != &signature) + { + return Err("overlay changed while establishing the measured proof precondition".into()); + } + stable_revision = Some(overlay.semantic_revision); + stable_signature = Some(signature); + if observation < 3 { + std::thread::park_timeout(Duration::from_secs(2)); + } + } + + let serving_revision = stable_revision + .as_deref() + .ok_or("round-robin readiness did not establish a semantic revision")?; + wait_for_consumer_gateway_revision(serving_revision)?; + + let mut facts = BTreeMap::new(); + facts.insert("consumer_gateway_replicas".to_owned(), serde_json::json!(replica_text)); + facts.insert("semantic_revision".to_owned(), serde_json::json!(stable_revision)); + facts.insert("candidate_signature".to_owned(), serde_json::json!(stable_signature)); + facts.insert("selection_policy".to_owned(), serde_json::json!("roundRobin")); + facts.insert("candidate_count".to_owned(), serde_json::json!(3)); + facts.insert( + "praxis_serving_revision".to_owned(), + serde_json::json!(serving_revision), + ); + Ok(facts) +} + +/// Read bounded logs from the single consumer gateway used by the measured +/// provider-traffic proof. +fn consumer_gateway_logs() -> Result> { + let output = Command::new("kubectl") + .args([ + "--context", + "kind-grid-provider-traffic-provider-a", + "-n", + GRID_SYSTEM_NS, + "logs", + "deployment/consumer-gateway", + "-c", + "praxis", + "--tail=300", + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to read consumer gateway logs: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 160) + ) + .into()); + } + Ok(strip_csi_sgr(&String::from_utf8_lossy(&output.stdout))) +} + +/// Strip the ANSI SGR sequences emitted by the gateway's tracing subscriber. +fn strip_csi_sgr(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(character) = chars.next() { + if character == '\x1b' { + if chars.next() == Some('[') { + for final_byte in chars.by_ref() { + if final_byte.is_ascii_alphabetic() { + break; + } + } + } + } else { + output.push(character); + } + } + output +} + +/// Wait until Praxis reports that the exact final overlay revision is serving. +/// +/// The `ConfigMap` and projected file can be ready before the gateway watcher +/// has accepted the file. Starting the measured window earlier would count +/// requests against the initial local-only snapshot and invalidate the +/// round-robin proof. +fn wait_for_consumer_gateway_revision(revision: &str) -> Result<(), Box> { + const TIMEOUT: Duration = Duration::from_secs(90); + const POLL: Duration = Duration::from_secs(2); + let deadline = Instant::now() + TIMEOUT; + loop { + let logs = consumer_gateway_logs()?; + let accepted = latest_log_field(&logs, "accepted_revision"); + let serving = latest_log_field(&logs, "serving_revision"); + if accepted.as_deref() == Some(revision) && serving.as_deref() == Some(revision) { + return Ok(()); + } + if Instant::now() >= deadline { + let accepted_summary = accepted + .as_deref() + .map_or_else(|| "none".to_owned(), |value| safe_truncate_str(value, 16)); + let serving_summary = serving + .as_deref() + .map_or_else(|| "none".to_owned(), |value| safe_truncate_str(value, 16)); + return Err(format!( + "consumer gateway did not report accepted/serving overlay revision {} within {TIMEOUT:?} (accepted={}, serving={})", + safe_truncate_str(revision, 16), + accepted_summary, + serving_summary + ) + .into()); + } + std::thread::park_timeout(POLL); + } +} + +/// Return the latest exact tracing field value from bounded gateway logs. +fn latest_log_field(logs: &str, field: &str) -> Option { + logs.lines() + .rev() + .find_map(|line| { + let prefix = format!("{field}="); + line.match_indices(&prefix).find_map(|(index, _)| { + let at_boundary = index == 0 + || line + .get(..index) + .and_then(|value| value.chars().next_back()) + .is_some_and(char::is_whitespace); + if !at_boundary { + return None; + } + let value = line.get(index + prefix.len()..)?; + Some( + value + .strip_prefix('"') + .map_or_else( + || value.split_whitespace().next().unwrap_or(""), + |value| value.split('"').next().unwrap_or(""), + ) + .to_owned(), + ) + }) + }) + .filter(|value| !value.is_empty()) +} + +/// Build the expected provider candidate name set. +fn expected_candidates() -> BTreeSet { + let mut expected = BTreeSet::new(); + for cluster in CLUSTERS { + expected.insert(format!("vcr-{cluster}-provider")); + } + + expected +} + +/// Wait for each cluster's operator to produce its expected local candidate. +/// +/// Pre-SWIM, each operator only knows its local `InferenceProvider` resources. +/// Global convergence (all candidates on every cluster) happens after SWIM +/// seeding in a separate phase. +#[expect( + clippy::too_many_lines, + reason = "The readiness poll checks each local cluster's overlay." +)] +fn wait_for_local_overlays() -> Result, Box> { + let timeout = Duration::from_secs(180); + let interval = Duration::from_secs(5); + let start = Instant::now(); + + eprintln!(" Polling for local overlay ConfigMaps (timeout {timeout:?})..."); + + while start.elapsed() < timeout { + let mut overlays = BTreeMap::new(); + let mut all_ready = true; + + for cluster in CLUSTERS { + let expected_local = format!("vcr-{cluster}-provider"); + match read_cluster_overlay(cluster) { + Ok(data) if data.stable_ids.contains_key(&expected_local) => { + eprintln!( + " {cluster}: {expected_local} present (semantic_rev={}, stable_id={})", + data.semantic_revision, + data.stable_ids.get(&expected_local).map_or("?", String::as_str) + ); + overlays.insert((*cluster).to_owned(), data); + }, + Ok(data) => { + eprintln!( + " {cluster}: overlay present but missing {expected_local} (has: {:?})", + data.stable_ids.keys().collect::>() + ); + all_ready = false; + }, + Err(_) => { + all_ready = false; + }, + } + } + + if all_ready && overlays.len() == CLUSTERS.len() { + return Ok(overlays); + } + + std::thread::park_timeout(interval); + } + + collect_overlay_diagnostics(); + Err(format!("Local overlay ConfigMaps not ready after {timeout:?}").into()) +} + +/// Wait until every provider-traffic cluster serves the same candidate set. +#[expect( + clippy::too_many_lines, + reason = "The convergence poll compares the bounded three-cluster overlay set." +)] +fn wait_for_global_overlay_convergence( + expected: &BTreeSet, +) -> Result, Box> { + let timeout = Duration::from_secs(180); + let interval = Duration::from_secs(5); + let start = Instant::now(); + + eprintln!( + " Waiting for global overlay convergence \ + ({} candidates on all clusters, timeout {timeout:?})...", + expected.len() + ); + + while start.elapsed() < timeout { + let mut overlays = BTreeMap::new(); + let mut all_converged = true; + + for cluster in CLUSTERS { + match read_cluster_overlay(cluster) { + Ok(data) => { + let missing: Vec<&String> = expected.iter().filter(|c| !data.stable_ids.contains_key(*c)).collect(); + if missing.is_empty() { + overlays.insert((*cluster).to_owned(), data); + } else { + eprintln!(" {cluster}: missing candidates {missing:?}"); + all_converged = false; + } + }, + Err(_) => { + all_converged = false; + }, + } + } + + if all_converged && overlays.len() == CLUSTERS.len() { + let reference_site = CLUSTERS.first().ok_or("CLUSTERS is empty")?; + let reference = overlays + .get(*reference_site) + .ok_or("reference site missing from overlays")?; + + for cluster in CLUSTERS.iter().skip(1) { + let site_data = overlays + .get(*cluster) + .ok_or_else(|| format!("{cluster} missing from overlays"))?; + for candidate in expected { + let ref_id = reference + .stable_ids + .get(candidate) + .ok_or_else(|| format!("{reference_site}: missing {candidate}"))?; + let site_id = site_data + .stable_ids + .get(candidate) + .ok_or_else(|| format!("{cluster}: missing {candidate}"))?; + if site_id != ref_id { + return Err(format!( + "stable_id mismatch for {candidate}: \ + {reference_site}={ref_id} vs {cluster}={site_id}" + ) + .into()); + } + } + } + + eprintln!( + " Global overlay converged: {} candidates on all {} clusters, stable_ids agree", + expected.len(), + CLUSTERS.len() + ); + return Ok(overlays); + } + + std::thread::park_timeout(interval); + } + + collect_overlay_diagnostics(); + Err(format!("Global overlay convergence failed after {timeout:?}").into()) +} + +/// Collect diagnostic information when the overlay `ConfigMap` fails to converge. +#[expect( + clippy::too_many_lines, + reason = "Diagnostics intentionally report each bounded control-plane boundary." +)] +fn collect_overlay_diagnostics() { + eprintln!(" [DIAG] Collecting overlay failure diagnostics...\n"); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + eprintln!(" ======== {cluster} ========"); + + eprintln!(" [DIAG] {cluster}: 1. Operator deployment SWIM env vars"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "deployment", + "grid-operator", + "-o", + "jsonpath={range .spec.template.spec.containers[0].env[*]}{.name}={.value}{'\\n'}{end}", + ]) + .status(), + ); + + eprintln!("\n [DIAG] {cluster}: 2. SWIM service details"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "svc", + "grid-operator-swim", + "-o", + "wide", + ]) + .status(), + ); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "endpoints", + "grid-operator-swim", + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 3. Operator pod status"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "pods", + "-l", + "app.kubernetes.io/name=grid-operator", + "-o", + "wide", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 4. Operator logs (last 50 lines, unfiltered)"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "logs", + "deployment/grid-operator", + "--tail=50", + ]) + .status(), + ); + + eprintln!("\n [DIAG] {cluster}: 5. GridNetwork CRD status"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 6. Overlay ConfigMap content"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + OVERLAY_CONFIGMAP, + "-o", + "json", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 7. InferenceProvider CRs"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "inferenceprovider", + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 8. Helm values for grid-operator"); + drop( + Command::new("helm") + .args([ + "get", + "values", + "grid-operator", + "--namespace", + GRID_SYSTEM_NS, + "--kube-context", + &context, + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 9. NetworkPolicy in grid-system"); + drop( + Command::new("kubectl") + .args(["--context", &context, "-n", GRID_SYSTEM_NS, "get", "networkpolicy"]) + .status(), + ); + + eprintln!(); + } + + eprintln!(" [DIAG] 10. Cross-cluster SWIM connectivity check"); + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + for target in CLUSTERS { + if *target == *cluster { + continue; + } + let target_context = format!("kind-grid-provider-traffic-{target}"); + if let Ok(output) = Command::new("kubectl") + .args([ + "--context", + &target_context, + "-n", + GRID_SYSTEM_NS, + "get", + "svc", + "grid-operator-swim", + "-o", + "jsonpath={.status.loadBalancer.ingress[0].ip}", + ]) + .output() + { + let ip = String::from_utf8_lossy(&output.stdout); + eprintln!(" [DIAG] {cluster} -> {target} (SWIM LB {ip}): testing TCP 7946"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "exec", + "deployment/grid-operator", + "--", + "sh", + "-c", + &format!( + "timeout 3 sh -c 'echo | nc -w 2 {ip} 7946' && echo REACHABLE || echo UNREACHABLE" + ), + ]) + .status(), + ); + } + } + } +} + +/// Materialize provider gateway configuration from the pre-SWIM overlay map. +/// +/// For each cluster, extracts the `stable_id` for the local +/// `vcr-{cluster}-provider` candidate and renders the provider praxis.yaml +/// template with: +/// - `SITE_PLACEHOLDER` → cluster name +/// - `CANDIDATE_ID_PLACEHOLDER` → stable ID from the overlay +/// +/// Creates the `provider-gateway-config` `ConfigMap` in each cluster. +#[expect( + clippy::too_many_lines, + reason = "Provider config materialization is one bounded setup phase." +)] +fn materialize_provider_config( + overlays: &BTreeMap, + demo_root: &Path, +) -> Result<(), Box> { + let template_path = demo_root.join("configs/provider/praxis.yaml"); + let template = + fs::read_to_string(template_path).map_err(|e| format!("failed to read provider config template: {e}"))?; + + for cluster in CLUSTERS { + let provider_name = format!("vcr-{cluster}-provider"); + let overlay = overlays + .get(*cluster) + .ok_or_else(|| format!("no overlay data for cluster {cluster}"))?; + let stable_id = overlay.stable_ids.get(&provider_name).ok_or_else(|| { + format!( + "{cluster}: no candidate {provider_name} in overlay (has: {:?})", + overlay.stable_ids.keys().collect::>() + ) + })?; + + eprintln!(" {cluster}: {provider_name} -> {stable_id}"); + + let rendered = template + .replace("SITE_PLACEHOLDER", cluster) + .replace("CANDIDATE_ID_PLACEHOLDER", stable_id); + + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let create = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "create", + "configmap", + "provider-gateway-config", + &format!("--from-literal=praxis.yaml={rendered}"), + "--dry-run=client", + "-o", + "yaml", + ]) + .output()?; + + if !create.status.success() { + return Err(format!( + "failed to render provider-gateway-config for {cluster}: {}", + String::from_utf8_lossy(&create.stderr).trim() + ) + .into()); + } + + kubectl::apply_manifest(&context, &String::from_utf8(create.stdout)?)?; + eprintln!(" [OK] {cluster}: provider-gateway-config created (stable_id={stable_id})"); + } + + Ok(()) +} + +/// Materialize the Forge configuration with image overrides. +fn materialize_config(source: &Path) -> Result> { + let content = fs::read_to_string(source)?; + let mut config: serde_yaml::Value = serde_yaml::from_str(&content)?; + apply_image_overrides(&mut config); + let rendered = serde_yaml::to_string(&config)?; + let parent = source.parent().ok_or("source config must have parent directory")?; + let output = parent.join(".forge.resolved.yaml"); + fs::write(&output, rendered)?; + Ok(output) +} + +/// Apply image overrides from environment variables to the Forge configuration. +#[expect( + clippy::too_many_lines, + clippy::collapsible_if, + reason = "Image override application with structured YAML manipulation; nested ifs follow YAML structure hierarchy" +)] +fn apply_image_overrides(config: &mut serde_yaml::Value) { + let gateway_image = + std::env::var("GRID_XTASK_GATEWAY_IMAGE").unwrap_or_else(|_| "praxis-ai:provider-traffic-demo".to_owned()); + let operator_image = + std::env::var("GRID_XTASK_OPERATOR_IMAGE").unwrap_or_else(|_| "grid-operator:provider-traffic-demo".to_owned()); + let overlay_sync_image = crate::env::image_overrides::overlay_sync_image(); + let vcr_image = crate::env::image_overrides::vcr_image(); + let image_pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); + + let (gateway_repo, gateway_tag) = parse_image_ref(&gateway_image); + let (operator_repo, operator_tag) = parse_image_ref(&operator_image); + let (overlay_sync_repo, overlay_sync_tag) = parse_image_ref(&overlay_sync_image); + + if let Some(spec) = config.get_mut("spec") { + if let Some(clusters) = spec.get_mut("clusters") { + if let Some(clusters_array) = clusters.as_sequence_mut() { + for cluster in clusters_array { + if let Some(properties) = cluster.get_mut("properties") { + if let Some(props_map) = properties.as_mapping_mut() { + let pairs = [ + ("gatewayImage", &gateway_image), + ("operatorImage", &operator_image), + ("vcrImage", &vcr_image), + ("imagePullPolicy", &image_pull_policy), + ("gatewayImageRepo", &gateway_repo), + ("gatewayImageTag", &gateway_tag), + ("operatorImageRepo", &operator_repo), + ("operatorImageTag", &operator_tag), + ("overlaySyncImage", &overlay_sync_image), + ("overlaySyncImageRepo", &overlay_sync_repo), + ("overlaySyncImageTag", &overlay_sync_tag), + ]; + for (key, val) in pairs { + props_map.insert( + serde_yaml::Value::String(key.to_owned()), + serde_yaml::Value::String(val.clone()), + ); + } + } + } + } + } + } + } +} + +/// Parse image reference into (repo, tag) components. +fn parse_image_ref(image: &str) -> (String, String) { + if let Some(colon_pos) = image.rfind(':') { + let (repo, tag) = image.split_at(colon_pos); + // Skip the ':' character + let tag = tag.strip_prefix(':').unwrap_or(tag); + (repo.to_owned(), tag.to_owned()) + } else { + (image.to_owned(), "latest".to_owned()) + } +} + +/// Prepare setup context from configuration. +fn prepare_setup(forge_config: &Path) -> Result> { + let root = super::demo_root(forge_config); + eprintln!("Forge config: {}", forge_config.display()); + eprintln!("Demo root: {}", root.display()); + let resolved_config = materialize_config(forge_config)?; + let forge_bin = glb::resolve_forge_binary() + .ok_or("praxis-forge binary not found")? + .into(); + + Ok(ProviderTrafficContext { + demo_root: root, + resolved_config, + forge_bin, + }) +} + +/// Authorize auto-discovered remote `GridSites` with identity trust material. +/// +/// For each local cluster, waits for the two remote auto-discovered `GridSites`, +/// verifies the SWIM-advertised certificate matches the staged identity, then +/// patches `spec.egress.tls.serverName` and `spec.trust.canonicalFingerprints`. +/// The controller transitions the site to Active naturally after the patch. +fn authorize_discovered_sites() -> Result<(), Box> { + const TRUST_TIMEOUT: Duration = Duration::from_secs(120); + const GRID_NETWORK: &str = "grid-provider-traffic"; + + for local in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{local}"); + eprintln!(); + eprintln!(" {local}: authorizing remote provider sites"); + for remote in CLUSTERS { + if *remote == *local { + continue; + } + let site_name = format!("{GRID_NETWORK}-{remote}"); + operator::wait_for_auto_gridsite(&context, &site_name, GRID_NETWORK, TRUST_TIMEOUT)?; + let canonical_fp = certs::site_certificate_fingerprint(remote)?; + operator::wait_for_expected_site_certificate(&context, &site_name, &canonical_fp, TRUST_TIMEOUT)?; + let server_name = format!("{remote}.grid.internal"); + operator::patch_gridsite_identity_trust(&context, &site_name, &canonical_fp, &server_name)?; + operator::wait_for_gridsite_phase(&context, &site_name, "Active", TRUST_TIMEOUT)?; + } + } + eprintln!(" [OK] All auto-discovered remote GridSites authorized and Active"); + Ok(()) +} + +/// Deploy the provider-traffic environment. +#[expect( + clippy::too_many_lines, + reason = "sequential setup steps: each step depends on the previous; splitting obscures the setup flow" +)] +fn deploy_setup(context: &ProviderTrafficContext) -> Result> { + let total_phases = SETUP_PHASES; + let mut phase = 0; + let mut next = || { + phase += 1; + phase + }; + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Resolving Forge config and building images", + next(), + total_phases + ); + + // Validate the resolved forge configuration + let output = Command::new(&context.forge_bin) + .args(["config", "validate", "--config"]) + .arg(&context.resolved_config) + .output()?; + + if !output.status.success() { + return Err(format!( + "Forge config validation failed: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + eprintln!(" [OK] Forge config resolved to {}", context.resolved_config.display()); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Generating TLS certificates for all sites", + next(), + total_phases + ); + + stage_provider_boundary()?; + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Creating three provider Kind clusters: provider-a, provider-b, provider-c", + next(), + total_phases + ); + + let status = Command::new(&context.forge_bin) + .args(["up", "--config"]) + .arg(&context.resolved_config) + .status()?; + + if !status.success() { + return Err("Failed to create provider-traffic clusters".into()); + } + + eprintln!(" [OK] All three provider clusters created"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Loading container images into Kind clusters", + next(), + total_phases + ); + + load_images_into_clusters(&context.forge_bin, &context.resolved_config)?; + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying infrastructure stacks", next(), total_phases); + + let apply_stack = |forge_bin: &Path, + resolved_config: &Path, + cluster: &str, + stack: &str| + -> Result<(), Box> { + eprintln!(" applying {stack} to {cluster}..."); + let status = Command::new(forge_bin) + .arg("--config") + .arg(resolved_config) + .args(["--non-interactive", "stack", "apply", cluster, stack]) + .status()?; + if !status.success() { + return Err(format!("Failed to apply {stack} to {cluster}").into()); + } + Ok(()) + }; + + for cluster in CLUSTERS { + apply_stack(&context.forge_bin, &context.resolved_config, cluster, "metallb")?; + } + for cluster in CLUSTERS { + let op_stack = format!("{cluster}-operator-base"); + apply_stack(&context.forge_bin, &context.resolved_config, cluster, &op_stack)?; + } + eprintln!(" [OK] Infrastructure stacks applied"); + + eprintln!(); + eprintln!("[SETUP {}/{}] Verifying Grid operators are ready", next(), total_phases); + + for cluster in CLUSTERS { + let ctx = format!("kind-grid-provider-traffic-{cluster}"); + wait_for_deployment("grid-operator", GRID_SYSTEM_NS, &ctx)?; + eprintln!(" [OK] {cluster}: Grid operator ready"); + } + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying VCR backends", next(), total_phases); + + for cluster in CLUSTERS { + let ctx = format!("kind-grid-provider-traffic-{cluster}"); + let credential = generate_provider_credential()?; + apply_credential_secret(&ctx, VCR_INFERENCE_CREDENTIAL, &credential)?; + eprintln!(" [OK] {cluster}: vcr-inference-credential created"); + apply_stack(&context.forge_bin, &context.resolved_config, cluster, "vcr-backend")?; + } + eprintln!(" [OK] VCR backends deployed"); + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying grid-site resources", next(), total_phases); + + for cluster in CLUSTERS { + let site_stack = format!("{cluster}-site"); + apply_stack(&context.forge_bin, &context.resolved_config, cluster, &site_stack)?; + } + eprintln!(" [OK] Grid site resources deployed"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Waiting for local overlay ConfigMaps", + next(), + total_phases + ); + + let pre_swim_overlays = wait_for_local_overlays()?; + eprintln!(" [OK] Local overlay ConfigMaps ready"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Materializing provider config and installing trust", + next(), + total_phases + ); + + install_provider_boundary()?; + + materialize_provider_config(&pre_swim_overlays, &context.demo_root)?; + eprintln!(" [OK] Provider config materialized, trust installed"); + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying provider gateways", next(), total_phases); + + for cluster in CLUSTERS { + apply_stack( + &context.forge_bin, + &context.resolved_config, + cluster, + "provider-gateway", + )?; + } + eprintln!(" [OK] Provider gateways deployed"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Deploying the single consumer gateway", + next(), + total_phases + ); + + apply_stack( + &context.forge_bin, + &context.resolved_config, + CONSUMER_SITE, + "consumer-gateway", + )?; + eprintln!(" [OK] Consumer gateway deployed in {CONSUMER_SITE}"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] SWIM discovery and trust authorization", + next(), + total_phases + ); + + configure_swim_peers(&context.forge_bin, &context.resolved_config)?; + authorize_discovered_sites()?; + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Waiting for global overlay convergence", + next(), + total_phases + ); + + let expected = expected_candidates(); + let post_swim_overlays = wait_for_global_overlay_convergence(&expected)?; + let environment_status = wait_for_environment_ready()?; + eprintln!(" [OK] {environment_status}"); + + Ok(OverlayState { + pre_swim: pre_swim_overlays, + post_swim: post_swim_overlays, + }) +} + +// ----------------------------------------------------------------------------- +// Demo Scenarios +// ----------------------------------------------------------------------------- + +/// Run the quick-mode proof scenarios using the assertion framework. +/// +/// Run the six focused provider-traffic proof scenarios. +#[expect( + clippy::too_many_lines, + reason = "The focused demo presents its six proof phases in order." +)] +fn run_quick_scenarios() -> BTreeMap { + let mut results = BTreeMap::new(); + let mut scenario_num: usize = 0; + let mut scenario = || { + scenario_num += 1; + scenario_num + }; + + eprintln!(); + eprintln!("=== QUICK MODE SCENARIOS ==="); + eprintln!(); + + eprintln!("[SCENARIO {}] Verify three provider clusters are healthy", scenario()); + run_and_insert(&mut results, "cluster_health", assert_cluster_health); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify component deployment", scenario()); + run_and_insert(&mut results, "component_deployment", assert_component_deployment); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify SWIM convergence", scenario()); + run_and_insert(&mut results, "swim_convergence", assert_swim_convergence); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify site auto-discovery", scenario()); + run_and_insert(&mut results, "site_auto_discovery", assert_site_auto_discovery); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify overlay acceptance", scenario()); + run_and_insert(&mut results, "overlay_acceptance", assert_overlay_acceptance); + + eprintln!(); + eprintln!(); + eprintln!( + "[SCENARIO {}] Prove equal round-robin across distinct provider gateways", + scenario() + ); + run_and_insert( + &mut results, + "provider_gateway_round_robin", + assert_provider_gateway_round_robin, + ); + + results +} + +/// Send serial, unbound requests through one consumer gateway and verify that +/// the active no-metrics round-robin picker distributes them across the +/// distinct provider gateways. This is intentionally a request-path proof: +/// the request itself is the only source of the attribution counts. +#[expect( + clippy::too_many_lines, + clippy::unnecessary_wraps, + reason = "The assertion framework requires a fallible, named traffic proof boundary." +)] +fn assert_provider_gateway_round_robin() -> AssertionResult { + let start = Instant::now(); + let context = "kind-grid-provider-traffic-provider-a"; + let mut counts: BTreeMap = BTreeMap::new(); + let mut sequence = Vec::new(); + let mut failures = Vec::new(); + let mut overlay_changes = Vec::new(); + + let readiness = match wait_for_round_robin_readiness() { + Ok(facts) => facts, + Err(error) => { + return Ok(proof_failure( + &format!("round-robin readiness gate failed: {error}"), + BTreeMap::from([(String::from("readiness_error"), serde_json::json!(error.to_string()))]), + start.elapsed(), + )); + }, + }; + let baseline_overlay = match read_cluster_overlay("provider-a") { + Ok(overlay) => overlay, + Err(error) => { + return Ok(proof_failure( + &format!("could not capture baseline overlay after readiness: {error}"), + BTreeMap::new(), + start.elapsed(), + )); + }, + }; + + for request_number in 1..=60 { + if let Ok(current_overlay) = read_cluster_overlay("provider-a") + && (current_overlay.resource_version != baseline_overlay.resource_version + || current_overlay.semantic_revision != baseline_overlay.semantic_revision) + { + overlay_changes.push(serde_json::json!({ + "request": request_number, + "resource_version": current_overlay.resource_version, + "semantic_revision": current_overlay.semantic_revision, + })); + } + let request_label = format!("provider-traffic-rr-{request_number:03}"); + let output = run_curl_probe( + context, + &format!("rr-{request_number}"), + &[ + "curl", + "--fail-with-body", + "--include", + "--silent", + "--show-error", + "--header", + "Content-Type: application/json", + "--header", + "Authorization: Bearer consumer-token", + "--data", + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"provider traffic proof"}],"max_tokens":8}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ], + ); + + match output { + Ok(output) if output.status.success() => { + let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway") + .or_else(|| response_header(&output.stdout, "x-grid-provider-traffic-provider-gateway")) + .or_else(|| response_header(&output.stdout, "x-grid-provider-gateway")); + if let Some(provider) = provider { + *counts.entry(provider.clone()).or_default() += 1; + sequence.push(provider); + } else { + failures.push(format!("{request_label}: provider attribution header missing")); + } + }, + Ok(output) => failures.push(format!( + "{request_label}: HTTP probe failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )), + Err(error) => failures.push(format!("{request_label}: probe execution failed: {error}")), + } + } + + let canonical = ["provider-a", "provider-b", "provider-c"]; + let exact_counts = canonical + .iter() + .all(|name| counts.get(*name).copied().unwrap_or(0) == 20); + let cycle_start = sequence + .first() + .and_then(|first| canonical.iter().position(|name| *name == first)); + let repeating_cycle = sequence.len() == 60 + && cycle_start.is_some_and(|start_index| { + sequence.iter().enumerate().all(|(index, provider)| { + canonical + .get((start_index + index) % canonical.len()) + .is_some_and(|expected| *expected == provider.as_str()) + }) + }); + let balanced = exact_counts && repeating_cycle && failures.is_empty(); + + let mut facts = BTreeMap::new(); + facts.insert("request_count".to_owned(), serde_json::json!(sequence.len())); + facts.insert("provider_counts".to_owned(), serde_json::json!(counts)); + facts.insert("ordered_provider_sequence".to_owned(), serde_json::json!(sequence)); + facts.insert("cycle_start_provider".to_owned(), serde_json::json!(sequence.first())); + facts.insert("exact_20_each".to_owned(), serde_json::json!(exact_counts)); + facts.insert( + "repeating_three_provider_cycle".to_owned(), + serde_json::json!(repeating_cycle), + ); + facts.insert("failures".to_owned(), serde_json::json!(failures)); + facts.insert("selection_policy".to_owned(), serde_json::json!("roundRobin")); + facts.insert("scoring_strategy".to_owned(), serde_json::json!("noMetrics")); + facts.insert("readiness".to_owned(), serde_json::json!(readiness)); + facts.insert( + "baseline_resource_version".to_owned(), + serde_json::json!(baseline_overlay.resource_version), + ); + facts.insert( + "baseline_semantic_revision".to_owned(), + serde_json::json!(baseline_overlay.semantic_revision), + ); + let overlay_stable = overlay_changes.is_empty(); + facts.insert( + "overlay_changes_during_requests".to_owned(), + serde_json::json!(overlay_changes), + ); + + if balanced && overlay_stable { + Ok(proof_success( + "60 serial requests distributed evenly across three distinct provider gateways", + facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + &format!( + "provider gateway distribution or overlay stability failed: counts={counts:?}, overlay_changes={overlay_changes:?}" + ), + facts, + start.elapsed(), + )) + } +} + +/// Run one proof assertion and retain failures as structured evidence. +fn run_and_insert(results: &mut BTreeMap, name: &str, assertion_fn: fn() -> AssertionResult) { + match run_assertion(name, assertion_fn) { + Ok(proof) => { + results.insert(name.to_owned(), proof); + }, + Err(error) => { + eprintln!(" [X] {name} failed: {error}"); + results.insert( + name.to_owned(), + proof_failure(&format!("{name} failed: {error}"), BTreeMap::new(), Duration::ZERO), + ); + }, + } +} + +/// Configure SWIM peer discovery by updating each operator with peer seed addresses. +fn configure_swim_peers(_forge_bin: &Path, _resolved_config: &Path) -> Result<(), Box> { + let mut swim_ips = BTreeMap::new(); + + for cluster in CLUSTERS { + let ip = read_swim_lb_ip(cluster)?; + eprintln!(" {cluster}: SWIM LB IP = {ip}"); + swim_ips.insert(*cluster, ip); + } + + for cluster in CLUSTERS { + let this_ip = swim_ips + .get(cluster) + .ok_or_else(|| format!("{cluster}: missing SWIM IP"))?; + let mut peer_parts = Vec::new(); + for c in CLUSTERS { + if *c != *cluster { + let ip = swim_ips.get(c).ok_or_else(|| format!("{c}: missing SWIM IP"))?; + peer_parts.push(format!("{ip}:7946")); + } + } + let peer_seeds = peer_parts.join(","); + update_operator_swim_config(cluster, this_ip, &peer_seeds)?; + } + + for cluster in CLUSTERS { + let ctx = format!("kind-grid-provider-traffic-{cluster}"); + wait_for_deployment("grid-operator", GRID_SYSTEM_NS, &ctx)?; + eprintln!(" [OK] {cluster}: operator restarted with SWIM config"); + } + + Ok(()) +} + +/// Read the SWIM `LoadBalancer` IP for a cluster directly from Kubernetes. +fn read_swim_lb_ip(cluster: &str) -> Result> { + let context = format!("kind-grid-provider-traffic-{cluster}"); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "svc", + "grid-operator-swim", + "-o", + "jsonpath={.status.loadBalancer.ingress[0].ip}", + ]) + .output()?; + + if !output.status.success() { + return Err(format!( + "{cluster}: cannot read SWIM service: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + + let ip = String::from_utf8(output.stdout)?.trim().to_owned(); + if ip.is_empty() { + return Err(format!("{cluster}: SWIM LoadBalancer has no ingress IP").into()); + } + + Ok(ip) +} + +/// Update a single operator's SWIM configuration with peer addresses. +#[expect( + clippy::too_many_lines, + reason = "The Helm upgrade carries the bounded SWIM configuration contract." +)] +fn update_operator_swim_config( + cluster: &str, + advertise_ip: &str, + seeds: &str, +) -> Result<(), Box> { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let operator_image = + std::env::var("GRID_XTASK_OPERATOR_IMAGE").unwrap_or_else(|_| "grid-operator:provider-traffic-demo".to_owned()); + let image_pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); + let (operator_repo, operator_tag) = parse_image_ref(&operator_image); + + let seeds_escaped = seeds.replace(',', "\\,"); + + let upgrade_output = Command::new("helm") + .args([ + "upgrade", + "grid-operator", + "charts/grid-operator", + "--version", + "0.1.0", + "--namespace", + "grid-system", + "--kube-context", + &context, + "--reuse-values", + "--set", + &format!("image.repository={operator_repo}"), + "--set", + &format!("image.tag={operator_tag}"), + "--set", + &format!("image.pullPolicy={image_pull_policy}"), + "--set", + &format!("swim.siteName={cluster}"), + "--set", + &format!("swim.advertiseAddress={advertise_ip}:7946"), + "--set", + &format!("swim.seeds={seeds_escaped}"), + "--set", + "swim.service.enabled=true", + "--set", + "swim.service.type=LoadBalancer", + "--set", + &format!("gateway.serviceName={PROVIDER_GATEWAY_SERVICE}"), + "--set-string", + &format!("gateway.port={PROVIDER_GATEWAY_PORT}"), + ]) + .output()?; + + if !upgrade_output.status.success() { + return Err(format!( + "Failed to update SWIM config for {cluster}: {}", + String::from_utf8_lossy(&upgrade_output.stderr) + ) + .into()); + } + + eprintln!(" [OK] {cluster}: SWIM peers configured"); + + assert_operator_gateway_env(&context, cluster)?; + + Ok(()) +} + +/// Post-upgrade assertion: the operator deployment must reflect the expected +/// gateway discovery contract after every Helm upgrade. +fn assert_operator_gateway_env(context: &str, cluster: &str) -> Result<(), Box> { + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "deployment/grid-operator", + "-o", + "jsonpath={.spec.template.spec.containers[0].env}", + ]) + .output()?; + + let env_json = String::from_utf8_lossy(&output.stdout); + + let has_service = env_json.contains(&format!( + "\"name\":\"GRID_GATEWAY_SERVICE_NAME\",\"value\":\"{PROVIDER_GATEWAY_SERVICE}\"" + )); + let has_port = env_json.contains(&format!( + "\"name\":\"GRID_GATEWAY_PORT\",\"value\":\"{PROVIDER_GATEWAY_PORT}\"" + )); + + if !has_service || !has_port { + return Err(format!( + "{cluster}: operator gateway env mismatch after helm upgrade \ + (expected GRID_GATEWAY_SERVICE_NAME={PROVIDER_GATEWAY_SERVICE}, \ + GRID_GATEWAY_PORT={PROVIDER_GATEWAY_PORT}); got: {env_json}" + ) + .into()); + } + + Ok(()) +} + +/// Tear down only the provider-traffic Forge environment. +fn teardown_environment(context: &ProviderTrafficContext) -> Result<(), Box> { + eprintln!(); + eprintln!("=== TEARDOWN ==="); + + let status = Command::new(&context.forge_bin) + .args(["down", "--config"]) + .arg(&context.resolved_config) + .status()?; + + if !status.success() { + return Err("failed to tear down provider-traffic environment".into()); + } + + eprintln!(" [OK] Environment torn down successfully"); + Ok(()) +} + +/// Run the focused provider-traffic demo. +#[expect( + clippy::too_many_lines, + reason = "The public demo entrypoint keeps setup, proof, evidence, and teardown visible." +)] +pub(crate) fn run(forge_config: &Path, options: &GlbDemoOptions) -> Result<(), Box> { + if options.mode() != DemoMode::Quick { + return Err("provider-traffic supports only the focused quick proof".into()); + } + let mode = DemoMode::Quick; + let run_id = format_utc_timestamp(); + let wall_start = Instant::now(); + let _started_at = format_utc_iso(); + + let evidence_dir = resolve_evidence_dir(forge_config, options, &run_id)?; + fs::create_dir_all(&evidence_dir)?; + + let setup_ctx = prepare_setup(forge_config); + let mut teardown_success = false; + let mut run_error = None; + let mut overlay_state = OverlayState::default(); + let mut images = BTreeMap::new(); + + let proof_results = match &setup_ctx { + Ok(context) => { + eprintln!("{OUTPUT_RULE}"); + eprintln!("Grid Provider Traffic Demo"); + eprintln!("Mode: {}", if mode == DemoMode::Quick { "quick" } else { "full" }); + eprintln!("Config: {}", forge_config.display()); + eprintln!("{OUTPUT_RULE}"); + + match deploy_setup(context) { + Ok(state) => { + overlay_state = state; + images = match collect_image_evidence() { + Ok(images) => images, + Err(error) => { + run_error = Some(format!("image evidence collection failed: {error}")); + BTreeMap::new() + }, + }; + eprintln!(); + eprintln!("{OUTPUT_RULE}"); + eprintln!("ENVIRONMENT READY - Starting proof scenarios"); + eprintln!("{OUTPUT_RULE}"); + + let scenario_results = run_quick_scenarios(); + + let failed_proofs: Vec<&str> = scenario_results + .iter() + .filter_map(|(name, proof)| (!proof.success).then_some(name.as_str())) + .collect(); + if !failed_proofs.is_empty() { + run_error = Some(format!("runtime proofs failed: {}", failed_proofs.join(", "))); + } + + // Teardown if requested + if options.teardown && (run_error.is_none() || !options.keep_on_failure) { + match teardown_environment(context) { + Ok(()) => teardown_success = true, + Err(error) => { + eprintln!("[WARN] Teardown failed: {error}"); + run_error = Some(match run_error { + Some(previous) => format!("{previous}; teardown failed: {error}"), + None => format!("teardown failed: {error}"), + }); + }, + } + } + + scenario_results + }, + Err(e) => { + eprintln!("[FAIL] Environment setup failed: {e}"); + run_error = Some(format!("environment setup failed: {e}")); + + if options.teardown && !options.keep_on_failure { + if let Err(cleanup_err) = teardown_environment(context) { + eprintln!("[WARN] Cleanup after setup failure also failed: {cleanup_err}"); + run_error = Some(format!("environment setup failed: {e}; cleanup failed: {cleanup_err}")); + } else { + teardown_success = true; + } + } + + BTreeMap::new() + }, + } + }, + Err(e) => { + eprintln!("[FAIL] Setup preparation failed: {e}"); + run_error = Some(format!("setup preparation failed: {e}")); + BTreeMap::new() + }, + }; + + let evidence = Evidence { + schema_version: EVIDENCE_SCHEMA_VERSION.to_owned(), + mode: if mode == DemoMode::Quick { "quick" } else { "full" }.to_owned(), + topology: "provider-traffic".to_owned(), + clusters: CLUSTERS.iter().map(|&s| s.to_owned()).collect(), + proof_results, + images, + overlay_state, + cluster_health: Vec::new(), // Will be populated during runtime assertions + components: Vec::new(), // Will be populated during runtime assertions + swim_membership: Vec::new(), // Will be populated during runtime assertions + provider_responses: Vec::new(), // Will be populated during runtime assertions + security_results: Vec::new(), // Will be populated during runtime assertions + teardown_success, + }; + + // Write evidence + let evidence_file = evidence_dir.join("results.json"); + let evidence_json = serde_json::to_string_pretty(&evidence)?; + fs::write(&evidence_file, evidence_json)?; + + eprintln!(); + eprintln!("{OUTPUT_RULE}"); + eprintln!("Demo completed in {:.1}s", wall_start.elapsed().as_secs_f64()); + eprintln!("Evidence: {}", evidence_file.display()); + eprintln!("{OUTPUT_RULE}"); + + match run_error { + Some(error) => Err(error.into()), + None => Ok(()), + } +} + +/// Collect actual image evidence from the deployed clusters. +#[expect( + clippy::too_many_lines, + reason = "Evidence collection queries the bounded set of deployed component images." +)] +fn collect_image_evidence() -> Result, Box> { + let mut images = BTreeMap::new(); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + // Get grid operator image + let output = Command::new("kubectl") + .args([ + "get", + "deployment/grid-operator", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if output.status.success() { + let operator_image = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + images.insert(format!("{cluster}_operator"), operator_image); + } + + // Get consumer gateway image + let output = Command::new("kubectl") + .args([ + "get", + "deployment/consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if output.status.success() { + let consumer_image = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + images.insert(format!("{cluster}_consumer_gateway"), consumer_image); + } + + // Get provider gateway image + let output = Command::new("kubectl") + .args([ + "get", + "deployment/provider-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if output.status.success() { + let provider_image = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + images.insert(format!("{cluster}_provider_gateway"), provider_image); + } + + // Get VCR inference image + let output = Command::new("kubectl") + .args([ + "get", + &format!("deployment/vcr-inference-{cluster}"), + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if output.status.success() { + let mock_image = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + images.insert(format!("{cluster}_vcr_inference"), mock_image); + } + } + + Ok(images) +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, time::Duration}; + + use super::*; + + #[test] + fn test_proof_success_creation() { + let mut facts = BTreeMap::new(); + facts.insert("cluster_count".to_owned(), serde_json::Value::Number(3.into())); + facts.insert("all_healthy".to_owned(), serde_json::Value::Bool(true)); + + let proof = proof_success("Test success", facts.clone(), Duration::from_millis(100)); + + assert!(proof.success); + assert_eq!(proof.reason, "Test success"); + assert_eq!(proof.duration_ms, 100); + assert_eq!(proof.observed_facts.len(), 2); + assert_eq!( + proof.observed_facts.get("all_healthy"), + Some(&serde_json::Value::Bool(true)) + ); + } + + #[test] + fn test_proof_failure_creation() { + let mut facts = BTreeMap::new(); + facts.insert("error_code".to_owned(), serde_json::Value::Number(500.into())); + + let proof = proof_failure("Test failure", facts.clone(), Duration::from_millis(50)); + + assert!(!proof.success); + assert_eq!(proof.reason, "Test failure"); + assert_eq!(proof.duration_ms, 50); + assert_eq!(proof.observed_facts.len(), 1); + assert_eq!( + proof.observed_facts.get("error_code"), + Some(&serde_json::Value::Number(500.into())) + ); + } + + #[test] + fn test_assertion_result_error_handling() { + let assertion_fn = || -> AssertionResult { Err("Simulated assertion failure".into()) }; + + let result = run_assertion("test_assertion", assertion_fn); + assert!(result.is_err()); + + let Err(error) = result else { + std::process::abort(); + }; + let error_msg = error.to_string(); + assert!(error_msg.contains("Assertion test_assertion failed")); + assert!(error_msg.contains("Simulated assertion failure")); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "This test exercises the complete evidence schema.")] + fn test_evidence_serialization() { + let evidence = Evidence { + schema_version: "test".to_owned(), + mode: "quick".to_owned(), + topology: "provider-traffic".to_owned(), + clusters: vec![ + "provider-a".to_owned(), + "provider-b".to_owned(), + "provider-c".to_owned(), + ], + proof_results: BTreeMap::new(), + images: BTreeMap::new(), + overlay_state: OverlayState::default(), + cluster_health: vec![ClusterHealth { + name: "provider-a".to_owned(), + healthy: true, + api_response_ms: Some(100), + ready_nodes: 1, + }], + components: vec![ComponentStatus { + name: "provider-a-grid-operator".to_owned(), + namespace: "grid-system".to_owned(), + ready_replicas: 1, + desired_replicas: 1, + ready: true, + }], + swim_membership: vec![SwimMembership { + site: "provider-a".to_owned(), + local_node: "provider-a-operator".to_owned(), + peers: vec!["provider-b-operator".to_owned(), "provider-c-operator".to_owned()], + converged: true, + }], + provider_responses: vec![], + security_results: vec![], + teardown_success: true, + }; + + let Ok(json) = serde_json::to_string(&evidence) else { + std::process::abort(); + }; + assert!(json.contains("\"schema_version\":\"test\"")); + assert!(json.contains("\"topology\":\"provider-traffic\"")); + assert!(json.contains("\"healthy\":true")); + assert!(json.contains("\"ready_replicas\":1")); + assert!(json.contains("\"converged\":true")); + + // Verify deserialization + let Ok(_deserialized) = serde_json::from_str::(&json) else { + std::process::abort(); + }; + } + + #[test] + fn test_proof_count_validation() { + let names = [ + "cluster_health", + "component_deployment", + "swim_convergence", + "site_auto_discovery", + "overlay_acceptance", + "provider_gateway_round_robin", + ]; + assert_eq!(names.len(), 6); + assert_eq!(names[0], "cluster_health"); + assert_eq!(names[5], "provider_gateway_round_robin"); + assert_eq!(CLUSTERS.len(), 3); + assert_eq!(CLUSTERS, &["provider-a", "provider-b", "provider-c"]); + } + + #[test] + fn test_evidence_schema_version() { + assert_eq!(EVIDENCE_SCHEMA_VERSION, "1"); + } + + #[test] + fn curl_pod_overrides_meets_restricted_pod_security() { + let json = curl_pod_overrides("test-probe", &["curl", "--fail", "http://example.test"]); + let actual: serde_json::Value = serde_json::from_str(&json).unwrap_or_else(|_| std::process::abort()); + assert_eq!( + actual, + serde_json::json!({ + "spec": { + "automountServiceAccountToken": false, + "securityContext": { + "runAsNonRoot": true, + "seccompProfile": { "type": "RuntimeDefault" } + }, + "containers": [{ + "name": "test-probe", + "image": "curlimages/curl:8.12.1", + "command": ["curl"], + "args": ["--fail", "http://example.test"], + "securityContext": { + "runAsUser": 100, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { "drop": ["ALL"] } + } + }] + } + }) + ); + } + + #[test] + fn provider_traffic_constants_describe_focused_topology() { + assert_eq!(CLUSTERS, &["provider-a", "provider-b", "provider-c"]); + assert_eq!(CONSUMER_SITE, "provider-a"); + assert_eq!(EVIDENCE_SCHEMA_VERSION, "1"); + } +} From 1c8181479c14a7b1d7369efa5323b53a53e6b540 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Fri, 21 Aug 2026 15:58:38 +0000 Subject: [PATCH 3/6] fix(overlay-sync): include selection policy in test digest Signed-off-by: Brent Salisbury --- overlay-sync/src/watcher.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/overlay-sync/src/watcher.rs b/overlay-sync/src/watcher.rs index 0205183..1316321 100644 --- a/overlay-sync/src/watcher.rs +++ b/overlay-sync/src/watcher.rs @@ -489,16 +489,39 @@ mod tests { } fn overlay_digest(overlay: &crate::types::RoutingOverlay) -> String { - let canonical = serde_json::json!({ + let mut semantic_payload = serde_json::json!({ "candidates": serde_json::to_value(&overlay.candidates).unwrap(), "local_site": overlay.local_site, "network": overlay.network, }); - let canonical_bytes = serde_json_canonicalizer::to_vec(&canonical).unwrap(); + if let Some(policy) = &overlay.selection_policy { + semantic_payload + .as_object_mut() + .unwrap() + .insert("selection_policy".to_owned(), serde_json::to_value(policy).unwrap()); + } + let canonical_bytes = serde_json_canonicalizer::to_vec(&semantic_payload).unwrap(); let digest: [u8; 32] = sha2::Sha256::new().chain_update(&canonical_bytes).finalize().into(); digest.iter().map(|b| format!("{b:02x}")).collect() } + #[test] + fn overlay_digest_includes_selection_policy() { + let scope = ExpectedScope { + network: "test-net".to_owned(), + gateway: "gw".to_owned(), + namespace: "ns".to_owned(), + local_site: "site-a".to_owned(), + }; + let without_policy = test_overlay(&scope); + let mut with_policy = without_policy.clone(); + with_policy.selection_policy = Some(crate::types::SelectionPolicy { + mode: crate::types::SelectionMode::RoundRobin, + }); + + assert_ne!(overlay_digest(&without_policy), overlay_digest(&with_policy)); + } + fn test_provenance() -> crate::types::OverlayProvenance { crate::types::OverlayProvenance { producer: "grid-operator".to_owned(), From 38acf8f1975e4de4dc648439c5ca05620583cda9 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Thu, 20 Aug 2026 05:26:02 +0000 Subject: [PATCH 4/6] feat(e2e): add distributed token quota topology Add a Forge topology for validating shared token quota enforcement across multiple Praxis consumer gateways while Grid independently distributes admitted traffic across provider gateways. Configure Secret-backed Valkey connectivity, restricted network access, provider trust, explicit image overrides, and readiness barriers. Keep quota admission and state ownership in Praxis AI. Grid continues to publish provider routing state and does not contact the quota backend. Signed-off-by: Brent Salisbury --- .../praxis-gateway/templates/deployment.yaml | 4 + charts/praxis-gateway/values.schema.json | 34 + charts/praxis-gateway/values.yaml | 4 + .../grid-token-rate-limit/.gitignore | 1 + .../grid-token-rate-limit/README.md | 29 + .../grid-token-rate-limit/configs/README.md | 13 + .../configs/consumer/README.md | 5 + .../configs/consumer/praxis-valkey-a.yaml | 134 +++ .../configs/consumer/praxis-valkey-b.yaml | 134 +++ .../configs/consumer/praxis.yaml | 130 +++ .../configs/provider/README.md | 5 + .../configs/provider/praxis.yaml | 72 ++ .../grid-token-rate-limit/forge.yaml | 770 ++++++++++++++++++ .../grid-token-rate-limit/resources/README.md | 9 + .../resources/central/README.md | 3 + .../resources/central/gridnetwork.yaml | 23 + .../resources/central/inference.yaml | 23 + .../resources/central/site.yaml | 10 + .../resources/common/README.md | 4 + .../common/backend-network-policy.yaml | 43 + .../common/grid-system-namespace.yaml | 9 + .../resources/common/valkey.yaml | 109 +++ .../common/vcr-provider-workload.yaml | 117 +++ .../resources/east/README.md | 3 + .../resources/east/gridnetwork.yaml | 23 + .../resources/east/inference.yaml | 23 + .../resources/east/site.yaml | 10 + .../resources/trust/central.yaml | 35 + .../resources/trust/east.yaml | 35 + .../resources/trust/west.yaml | 35 + .../resources/west/README.md | 3 + .../resources/west/gridnetwork.yaml | 23 + .../resources/west/inference.yaml | 23 + .../resources/west/site.yaml | 10 + 34 files changed, 1908 insertions(+) create mode 100644 tests/e2e/topologies/grid-token-rate-limit/.gitignore create mode 100644 tests/e2e/topologies/grid-token-rate-limit/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/configs/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/configs/consumer/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/configs/provider/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/configs/provider/praxis.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/forge.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/central/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/central/gridnetwork.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/central/inference.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/central/site.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/common/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/common/backend-network-policy.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/common/grid-system-namespace.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/common/valkey.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/common/vcr-provider-workload.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/east/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/east/gridnetwork.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/east/inference.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/east/site.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/trust/central.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/trust/east.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/trust/west.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/west/README.md create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/west/gridnetwork.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/west/inference.yaml create mode 100644 tests/e2e/topologies/grid-token-rate-limit/resources/west/site.yaml diff --git a/charts/praxis-gateway/templates/deployment.yaml b/charts/praxis-gateway/templates/deployment.yaml index 9c7073e..c38c362 100644 --- a/charts/praxis-gateway/templates/deployment.yaml +++ b/charts/praxis-gateway/templates/deployment.yaml @@ -147,6 +147,10 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} securityContext: runAsNonRoot: true allowPrivilegeEscalation: false diff --git a/charts/praxis-gateway/values.schema.json b/charts/praxis-gateway/values.schema.json index 279c190..a9519e1 100644 --- a/charts/praxis-gateway/values.schema.json +++ b/charts/praxis-gateway/values.schema.json @@ -100,6 +100,40 @@ "items": { "type": "string" }, "description": "Container arguments." }, + "env": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "string" }, + "valueFrom": { + "type": "object", + "additionalProperties": false, + "required": ["secretKeyRef"], + "properties": { + "secretKeyRef": { + "type": "object", + "additionalProperties": false, + "required": ["name", "key"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "key": { "type": "string", "minLength": 1 }, + "optional": { "type": "boolean" } + } + } + } + } + }, + "oneOf": [ + { "required": ["value"] }, + { "required": ["valueFrom"] } + ] + }, + "description": "Environment variables passed to the Praxis container." + }, "config": { "type": "object", "additionalProperties": false, diff --git a/charts/praxis-gateway/values.yaml b/charts/praxis-gateway/values.yaml index 0f55dfb..aca6f9e 100644 --- a/charts/praxis-gateway/values.yaml +++ b/charts/praxis-gateway/values.yaml @@ -38,6 +38,10 @@ args: - "--config" - "/etc/praxis/praxis.yaml" +# -- Environment variables passed to the Praxis container. Use valueFrom for +# -- credentials and other deployment-managed secrets. +env: [] + # -- Required Praxis configuration. config: # -- Name of an existing ConfigMap containing the Praxis configuration. diff --git a/tests/e2e/topologies/grid-token-rate-limit/.gitignore b/tests/e2e/topologies/grid-token-rate-limit/.gitignore new file mode 100644 index 0000000..068f980 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/.gitignore @@ -0,0 +1 @@ +.forge/ diff --git a/tests/e2e/topologies/grid-token-rate-limit/README.md b/tests/e2e/topologies/grid-token-rate-limit/README.md new file mode 100644 index 0000000..2046169 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/README.md @@ -0,0 +1,29 @@ +# Grid-aware token-rate-limit Stage 2 topology + +This Forge topology is the distributed quota proof for the token-rate-limit +POC. It is separate from the generic provider-traffic and llm-d demonstrations. + +The west cluster runs two independently addressable Praxis consumer deployments: +`consumer-gateway-a` and `consumer-gateway-b`. Both use the same authenticated, +cluster-private Valkey service and the same logical quota namespace, rule, and +canonical model. Central and east remain attributable provider sites with their +own VCR-backed provider gateways. + +The routing contract is Grid selection groups with `noMetrics` scoring and +`selection_policy.mode: roundRobin`. No llm-d, EPP, pressure generator, queue +metric, or KV-cache metric is part of this topology. + +Valkey owns shared quota state only. Provider round-robin counters remain local +to each consumer; this topology does not claim a globally synchronized provider +sequence, cross-cluster Valkey reachability, or Valkey high availability. + +The Valkey password and connection URL are delivered through the `valkey-auth` +Kubernetes Secret. Consumers receive the URL through a Secret-backed environment +variable; the Praxis configuration supports the exact `${ENV_VAR}` form for this +backend URL. The service is ClusterIP-only and its NetworkPolicy permits access +only from the two quota-client consumer pods. + +Before distributed measurement, the harness must wait for both consumers to +serve the same overlay revision and for the asynchronous Valkey reconciliation +worker completion metric to advance. Consumer restarts are not quota resets in +this topology. diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/README.md b/tests/e2e/topologies/grid-token-rate-limit/configs/README.md new file mode 100644 index 0000000..2270058 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/README.md @@ -0,0 +1,13 @@ +# Praxis Configuration + +This directory owns the consumer-gateway and provider-gateway Praxis +configuration used by the combined-site demo. + +Consumer configuration must contain the Grid-managed routing overlay mount and +must not contain provider credentials. Provider configuration must enforce +mTLS peer identity, provider-route authorization, and final-hop credential +injection before forwarding to a private inference endpoint. + +Site-specific addresses and identities should be rendered from structured +inputs. Do not maintain three hand-edited copies of otherwise identical Praxis +configuration. diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/README.md b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/README.md new file mode 100644 index 0000000..3cba79c --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/README.md @@ -0,0 +1,5 @@ +# Consumer Gateway + +Place the shared consumer-gateway Praxis configuration template here. Site +identity and local Service addresses must be rendered rather than copied into +separate static files. diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml new file mode 100644 index 0000000..912d0c7 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml @@ -0,0 +1,134 @@ +listeners: + - name: proxy + address: "0.0.0.0:8080" + filter_chains: + - main + +filter_chains: + - name: main + filters: + - filter: basic_auth + realm: "Grid token-rate-limit demo" + strip_authorization: true + credentials: + - username: alice + password: alice-secret + - username: bob + password: bob-secret + - filter: token_rate_limit + key: + principal: + source: metadata + name: identity.user_id + onMissing: reject + model: + source: header + name: x-model + onMissing: reject + allowedModels: + - Qwen/Qwen3-0.6B + reservationTimeout: 2m + backend: + kind: valkey + url: "${TOKEN_RATE_LIMIT_VALKEY_URL}" + namespace: praxis:token-rate-limit:v1 + limits: + maxKeys: 32 + maxKeyLength: 256 + maxActiveReservations: 16 + rules: + - name: alice + match: + metadata: + identity.user_id: alice + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 1m + capacity: 60 + - name: bob + match: + metadata: + identity.user_id: bob + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 5000 + - name: default + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 5000 + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Combined-Consumer-Gateway + value: "{{ cluster.name }}" + - filter: intelligent_route + overlay_file: /etc/praxis/routing/routing-overlay.json + model_header: X-Model + provider_hop_clusters: + - vcr-west-provider + - vcr-central-provider + - vcr-east-provider + expected_overlay_scope: + network: grid-token-rate-limit + gateway: consumer-gateway-a + namespace: grid-system + local_site: "{{ cluster.name }}" + reload: + enabled: true + debounce_ms: 500 + session_affinity: + enabled: true + header: X-Session-Id + ttl_secs: 3600 + - filter: token_count + provider: openai + - filter: load_balancer + clusters: + - name: vcr-west-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: west.grid.internal + verify: true + endpoints: + - "{{ captures.west.provider-gateway-ip }}:8443" + - name: vcr-central-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: central.grid.internal + verify: true + endpoints: + - "{{ captures.central.provider-gateway-ip }}:8443" + - name: vcr-east-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: east.grid.internal + verify: true + endpoints: + - "{{ captures.east.provider-gateway-ip }}:8443" + +admin: + address: "127.0.0.1:9901" + +shutdown_timeout_secs: 5 diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml new file mode 100644 index 0000000..28c3720 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml @@ -0,0 +1,134 @@ +listeners: + - name: proxy + address: "0.0.0.0:8080" + filter_chains: + - main + +filter_chains: + - name: main + filters: + - filter: basic_auth + realm: "Grid token-rate-limit demo" + strip_authorization: true + credentials: + - username: alice + password: alice-secret + - username: bob + password: bob-secret + - filter: token_rate_limit + key: + principal: + source: metadata + name: identity.user_id + onMissing: reject + model: + source: header + name: x-model + onMissing: reject + allowedModels: + - Qwen/Qwen3-0.6B + reservationTimeout: 2m + backend: + kind: valkey + url: "${TOKEN_RATE_LIMIT_VALKEY_URL}" + namespace: praxis:token-rate-limit:v1 + limits: + maxKeys: 32 + maxKeyLength: 256 + maxActiveReservations: 16 + rules: + - name: alice + match: + metadata: + identity.user_id: alice + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 1m + capacity: 60 + - name: bob + match: + metadata: + identity.user_id: bob + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 5000 + - name: default + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 5000 + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Combined-Consumer-Gateway + value: "{{ cluster.name }}" + - filter: intelligent_route + overlay_file: /etc/praxis/routing/routing-overlay.json + model_header: X-Model + provider_hop_clusters: + - vcr-west-provider + - vcr-central-provider + - vcr-east-provider + expected_overlay_scope: + network: grid-token-rate-limit + gateway: consumer-gateway-b + namespace: grid-system + local_site: "{{ cluster.name }}" + reload: + enabled: true + debounce_ms: 500 + session_affinity: + enabled: true + header: X-Session-Id + ttl_secs: 3600 + - filter: token_count + provider: openai + - filter: load_balancer + clusters: + - name: vcr-west-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: west.grid.internal + verify: true + endpoints: + - "{{ captures.west.provider-gateway-ip }}:8443" + - name: vcr-central-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: central.grid.internal + verify: true + endpoints: + - "{{ captures.central.provider-gateway-ip }}:8443" + - name: vcr-east-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: east.grid.internal + verify: true + endpoints: + - "{{ captures.east.provider-gateway-ip }}:8443" + +admin: + address: "127.0.0.1:9901" + +shutdown_timeout_secs: 5 diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis.yaml b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis.yaml new file mode 100644 index 0000000..b94f835 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis.yaml @@ -0,0 +1,130 @@ +listeners: + - name: proxy + address: "0.0.0.0:8080" + filter_chains: + - main + +filter_chains: + - name: main + filters: + - filter: basic_auth + realm: "Grid token-rate-limit demo" + strip_authorization: true + credentials: + - username: alice + password: alice-secret + - username: bob + password: bob-secret + - filter: token_rate_limit + key: + principal: + source: metadata + name: identity.user_id + onMissing: reject + model: + source: header + name: x-model + onMissing: reject + allowedModels: + - Qwen/Qwen3-0.6B + reservationTimeout: 2m + limits: + maxKeys: 32 + maxKeyLength: 256 + maxActiveReservations: 16 + rules: + - name: alice + match: + metadata: + identity.user_id: alice + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 1m + capacity: 60 + - name: bob + match: + metadata: + identity.user_id: bob + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 5000 + - name: default + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 5000 + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Combined-Consumer-Gateway + value: "{{ cluster.name }}" + - filter: intelligent_route + overlay_file: /etc/praxis/routing/routing-overlay.json + model_header: X-Model + provider_hop_clusters: + - vcr-west-provider + - vcr-central-provider + - vcr-east-provider + expected_overlay_scope: + network: grid-token-rate-limit + gateway: consumer-gateway + namespace: grid-system + local_site: "{{ cluster.name }}" + reload: + enabled: true + debounce_ms: 500 + session_affinity: + enabled: true + header: X-Session-Id + ttl_secs: 3600 + - filter: token_count + provider: openai + - filter: load_balancer + clusters: + - name: vcr-west-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: west.grid.internal + verify: true + endpoints: + - "{{ captures.west.provider-gateway-ip }}:8443" + - name: vcr-central-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: central.grid.internal + verify: true + endpoints: + - "{{ captures.central.provider-gateway-ip }}:8443" + - name: vcr-east-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: east.grid.internal + verify: true + endpoints: + - "{{ captures.east.provider-gateway-ip }}:8443" + +admin: + address: "127.0.0.1:9901" + +shutdown_timeout_secs: 5 diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/provider/README.md b/tests/e2e/topologies/grid-token-rate-limit/configs/provider/README.md new file mode 100644 index 0000000..9baf127 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/provider/README.md @@ -0,0 +1,5 @@ +# Provider Gateway + +Place the shared provider-gateway Praxis configuration template here. It owns +the authenticated provider boundary, provider-route authorization, and +final-hop credential injection. diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/provider/praxis.yaml b/tests/e2e/topologies/grid-token-rate-limit/configs/provider/praxis.yaml new file mode 100644 index 0000000..33d402a --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/provider/praxis.yaml @@ -0,0 +1,72 @@ +# Combined-site provider gateway: mTLS listener with Grid peer authentication. +# +# This listener requires downstream mTLS. `peer_identity_trust` authenticates +# and authorizes the consumer gateway before AI-owned x-ai-routing-* fields can +# influence provider-local routing. + +listeners: + - name: provider + address: "0.0.0.0:8443" + filter_chains: + - provider-inference + tls: + certificates: + - cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + client_ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert_mode: require + +filter_chains: + - name: provider-inference + filters: + # This security filter must remain first, unconditional, and fail closed. + - filter: peer_identity_trust + trusted_peers: + - organization: ai-grid + + # Parse the OpenAI-compatible request and promote model to X-Model. + - filter: json_body_field + field: model + header: X-Model + + - filter: headers + response_set: + - name: X-Grid-Combined-Provider-Gateway + value: "SITE_PLACEHOLDER" + + - filter: provider_route + provider_id: "SITE_PLACEHOLDER" + model_header: X-Model + emit_demo_attribution: true + routes: + - candidate_id: "CANDIDATE_ID_PLACEHOLDER" + model: Qwen/Qwen3-0.6B + paths: + - /v1/chat/completions + - /v1/responses + cluster: vcr-backend + credential: + strategy: bearer_token + secretRef: + name: vcr-inference-credential + namespace: grid-system + key: token + + - filter: credential_inject + credentials: + - strategy: bearer_token + name: vcr-inference-credential + namespace: grid-system + key: token + file: /etc/praxis/credentials/vcr-inference/token + + - filter: load_balancer + clusters: + - name: vcr-backend + endpoints: + - "vcr-inference-SITE_PLACEHOLDER.grid-system.svc.cluster.local:8000" + +admin: + address: "127.0.0.1:9901" +shutdown_timeout_secs: 5 diff --git a/tests/e2e/topologies/grid-token-rate-limit/forge.yaml b/tests/e2e/topologies/grid-token-rate-limit/forge.yaml new file mode 100644 index 0000000..124780b --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/forge.yaml @@ -0,0 +1,770 @@ +apiVersion: forge.praxis.dev/v1alpha1 +kind: Environment + +metadata: + name: grid-token-rate-limit + +spec: + runtime: + provider: docker + clusterPrefix: grid-token-rate-limit + + network: + crossCluster: true + dnsZone: grid-token-rate-limit.test + + clusters: + - name: west + stacks: [metallb, west-operator-base, vcr-backend, provider-boundary, west-trust-bootstrap, west-site, provider-gateway, valkey, consumer-west-a, consumer-west-b] + properties: + region: west + role: combined + siteName: west + gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" + vcrImagePullPolicy: IfNotPresent + imagePullPolicy: IfNotPresent + gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" + gatewayImageTag: "v0.1.3" + operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" + operatorImageTag: "v0.1.3" + overlaySyncImageRepo: "ghcr.io/praxis-proxy/grid-overlay-sync" + overlaySyncImageTag: "v0.1.3" + + - name: central + stacks: [metallb, central-operator-base, vcr-backend, provider-boundary, central-trust-bootstrap, central-site, provider-gateway] + properties: + region: central + role: combined + siteName: central + gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" + vcrImagePullPolicy: IfNotPresent + imagePullPolicy: IfNotPresent + gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" + gatewayImageTag: "v0.1.3" + operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" + operatorImageTag: "v0.1.3" + overlaySyncImageRepo: "ghcr.io/praxis-proxy/grid-overlay-sync" + overlaySyncImageTag: "v0.1.3" + + - name: east + stacks: [metallb, east-operator-base, vcr-backend, provider-boundary, east-trust-bootstrap, east-site, provider-gateway] + properties: + region: east + role: combined + siteName: east + gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" + vcrImagePullPolicy: IfNotPresent + imagePullPolicy: IfNotPresent + gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" + gatewayImageTag: "v0.1.3" + operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" + operatorImageTag: "v0.1.3" + overlaySyncImageRepo: "ghcr.io/praxis-proxy/grid-overlay-sync" + overlaySyncImageTag: "v0.1.3" + + stacks: + metallb: + description: MetalLB load balancer with auto-configured address pool + steps: + - type: url + url: https://raw.githubusercontent.com/metallb/metallb/v0.14.9/config/manifests/metallb-native.yaml + sha256: 951065e85692aa106f1bb5d5a487d9306154923a794ab1d82122881cbaf588e4 + - type: wait + resource: deployment/controller + namespace: metallb-system + condition: available + timeout: "120s" + - type: metallb-auto-pool + name: forge-pool + + west-operator-base: + description: West Grid operator initial deployment with SWIM LoadBalancer + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "west" + seeds: "" + advertiseAddress: "{{ captures.west.swim-lb-ip }}:7946" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/grid-operator-swim + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: swim-lb-ip + timeout: "60s" + interval: "2s" + + central-operator-base: + description: Central Grid operator initial deployment with SWIM LoadBalancer + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "central" + seeds: "" + advertiseAddress: "{{ captures.central.swim-lb-ip }}:7946" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/grid-operator-swim + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: swim-lb-ip + timeout: "60s" + interval: "2s" + + east-operator-base: + description: East Grid operator initial deployment with SWIM LoadBalancer + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "east" + seeds: "" + advertiseAddress: "{{ captures.east.swim-lb-ip }}:7946" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/grid-operator-swim + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: swim-lb-ip + timeout: "60s" + interval: "2s" + + vcr-backend: + description: vllm-vcr inference backend for demo scenarios + steps: + - type: manifest + path: resources/common/grid-system-namespace.yaml + - type: template-manifest + path: resources/common/vcr-provider-workload.yaml + - type: manifest + path: resources/common/backend-network-policy.yaml + - type: wait + resource: deployment/vcr-inference-{{ cluster.name }} + namespace: grid-system + condition: available + timeout: "120s" + + provider-boundary: + description: mTLS, provider credential, and provider gateway configuration bootstrap + steps: + - type: template-file + source: configs/provider/praxis.yaml + target: .forge/runtime/{{ cluster.name }}/provider/praxis.yaml + - type: exec + command: + - bash + - -c + - >- + context=kind-grid-token-rate-limit-{{ cluster.name }}; + kubectl --context "$context" -n grid-system create secret generic consumer-gateway-tls + --from-file=tls.crt=tests/env/certs/{{ cluster.name }}-cert.pem + --from-file=tls.key=tests/env/certs/{{ cluster.name }}-key.pem + --from-file=ca.crt=tests/env/certs/ca.pem --dry-run=client -o yaml | + kubectl --context "$context" apply -f -; + kubectl --context "$context" -n grid-system create secret generic provider-gateway-tls + --from-file=tls.crt=tests/env/certs/{{ cluster.name }}-cert.pem + --from-file=tls.key=tests/env/certs/{{ cluster.name }}-key.pem + --from-file=ca.crt=tests/env/certs/ca.pem --dry-run=client -o yaml | + kubectl --context "$context" apply -f -; + kubectl --context "$context" -n grid-system create secret generic vcr-inference-credential + --from-literal=token="$(openssl rand -hex 32)" --dry-run=client -o yaml | + kubectl --context "$context" apply -f -; + sed -i -e "s/SITE_PLACEHOLDER/{{ cluster.name }}/g" + -e "s/CANDIDATE_ID_PLACEHOLDER/$(case '{{ cluster.name }}' in west) echo f4c20a10;; central) echo c7fa24b0;; east) echo 9d876274;; esac)/g" + tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/provider/praxis.yaml; + kubectl --context "$context" -n grid-system create configmap provider-gateway-config + --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/provider/praxis.yaml --dry-run=client -o yaml | + kubectl --context "$context" apply -f - + + west-trust-bootstrap: + description: Pre-provision verified remote GridSite trust pins for west + steps: + - type: template-file + source: resources/trust/west.yaml + target: .forge/runtime/{{ cluster.name }}/trust.yaml + - type: exec + command: + - bash + - -c + - >- + set -euo pipefail; + runtime=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/trust.yaml; + central=$(kubectl --context kind-grid-token-rate-limit-central -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); + east=$(kubectl --context kind-grid-token-rate-limit-east -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); + sed -i "s/PEER_CENTRAL_FINGERPRINT/${central}/g; s/PEER_EAST_FINGERPRINT/${east}/g" "$runtime"; + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f "$runtime" + + central-trust-bootstrap: + description: Pre-provision verified remote GridSite trust pins for central + steps: + - type: template-file + source: resources/trust/central.yaml + target: .forge/runtime/{{ cluster.name }}/trust.yaml + - type: exec + command: + - bash + - -c + - >- + set -euo pipefail; + runtime=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/trust.yaml; + west=$(kubectl --context kind-grid-token-rate-limit-west -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); + east=$(kubectl --context kind-grid-token-rate-limit-east -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); + sed -i "s/PEER_WEST_FINGERPRINT/${west}/g; s/PEER_EAST_FINGERPRINT/${east}/g" "$runtime"; + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f "$runtime" + + east-trust-bootstrap: + description: Pre-provision verified remote GridSite trust pins for east + steps: + - type: template-file + source: resources/trust/east.yaml + target: .forge/runtime/{{ cluster.name }}/trust.yaml + - type: exec + command: + - bash + - -c + - >- + set -euo pipefail; + runtime=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/trust.yaml; + west=$(kubectl --context kind-grid-token-rate-limit-west -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); + central=$(kubectl --context kind-grid-token-rate-limit-central -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); + sed -i "s/PEER_WEST_FINGERPRINT/${west}/g; s/PEER_CENTRAL_FINGERPRINT/${central}/g" "$runtime"; + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f "$runtime" + + west-site: + description: West combined site with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site + chart: charts/grid-site + version: "0.1.0" + namespace: grid-system + values: + commonLabels: + grid.praxis-proxy.io/auto-discover-sites: "true" + gridNetwork: + name: grid-token-rate-limit + gridId: grid-token-rate-limit-v1 + region: west + zone: west-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + seeds: + - "{{ captures.central.swim-lb-ip }}:7946" + - "{{ captures.east.swim-lb-ip }}:7946" + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: consumer-gateway-a + namespace: grid-system + localSiteName: west + - name: consumer-gateway-b + namespace: grid-system + localSiteName: west + gridSite: + name: west + region: west + zone: west-1 + providerSiteLabel: west + inferenceProviders: + - name: vcr-west-provider + gridNetworkRef: grid-token-rate-limit + providerKind: vllm-vcr + backendKind: local + endpoint: "http://vcr-inference-west.grid-system.svc.cluster.local:8000" + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: west + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: "30s" + timeout: "5s" + + central-site: + description: Central combined site with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site + chart: charts/grid-site + version: "0.1.0" + namespace: grid-system + values: + commonLabels: + grid.praxis-proxy.io/auto-discover-sites: "true" + gridNetwork: + name: grid-token-rate-limit + gridId: grid-token-rate-limit-v1 + region: central + zone: central-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + seeds: + - "{{ captures.west.swim-lb-ip }}:7946" + - "{{ captures.east.swim-lb-ip }}:7946" + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: provider-gateway + namespace: grid-system + localSiteName: central + gridSite: + name: central + region: central + zone: central-1 + providerSiteLabel: central + inferenceProviders: + - name: vcr-central-provider + gridNetworkRef: grid-token-rate-limit + providerKind: vllm-vcr + backendKind: local + endpoint: "http://vcr-inference-central.grid-system.svc.cluster.local:8000" + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: central + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: "30s" + timeout: "5s" + + east-site: + description: East combined site with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site + chart: charts/grid-site + version: "0.1.0" + namespace: grid-system + values: + commonLabels: + grid.praxis-proxy.io/auto-discover-sites: "true" + gridNetwork: + name: grid-token-rate-limit + gridId: grid-token-rate-limit-v1 + region: east + zone: east-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + seeds: + - "{{ captures.west.swim-lb-ip }}:7946" + - "{{ captures.central.swim-lb-ip }}:7946" + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: provider-gateway + namespace: grid-system + localSiteName: east + gridSite: + name: east + region: east + zone: east-1 + providerSiteLabel: east + inferenceProviders: + - name: vcr-east-provider + gridNetworkRef: grid-token-rate-limit + providerKind: vllm-vcr + backendKind: local + endpoint: "http://vcr-inference-east.grid-system.svc.cluster.local:8000" + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: east + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: "30s" + timeout: "5s" + + provider-gateway: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: helm + release: provider-gateway + chart: charts/praxis-gateway + version: "0.1.0" + namespace: grid-system + values: + fullnameOverride: "provider-gateway" + image: + repository: "{{ cluster.properties.gatewayImageRepo }}" + tag: "{{ cluster.properties.gatewayImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: "provider-gateway-config" + port: + containerPort: 8443 + name: "https-mtls" + service: + type: "LoadBalancer" + port: 8443 + health: + readiness: + tcpSocket: + port: "https-mtls" + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: "https-mtls" + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: "provider-gateway-tls" + credentials: + - name: "vcr-inference-credential" + mountPath: "/etc/praxis/credentials/vcr-inference" + podLabels: + grid.praxis-proxy.io/backend-access: "provider-gateway" + grid.praxis-proxy.io/provider-site: "{{ cluster.name }}" + - type: wait + resource: deployment/provider-gateway + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/provider-gateway + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: provider-gateway-ip + timeout: "60s" + interval: "2s" + + consumer-gateway: + description: Praxis consumer gateway with operator-managed overlay + steps: + - type: template-file + source: configs/consumer/praxis.yaml + target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + - type: exec + command: + - bash + - -c + - >- + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} -n grid-system + create configmap consumer-gateway-config + --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + --dry-run=client -o yaml | + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway + chart: charts/praxis-gateway + version: "0.1.0" + namespace: grid-system + values: + fullnameOverride: "consumer-gateway" + image: + repository: "{{ cluster.properties.gatewayImageRepo }}" + tag: "{{ cluster.properties.gatewayImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: "consumer-gateway-config" + service: + type: "LoadBalancer" + overlay: + enabled: true + existingConfigMap: "grid-overlay-grid-token-rate-limit-consumer-gateway" + sidecar: + enabled: true + image: + repository: "{{ cluster.properties.overlaySyncImageRepo }}" + tag: "{{ cluster.properties.overlaySyncImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + expectedNetwork: "grid-token-rate-limit" + expectedLocalSite: "{{ cluster.name }}" + tls: + enabled: true + existingSecret: "consumer-gateway-tls" + podLabels: + grid.praxis-proxy.io/consumer-site: "{{ cluster.name }}" + - type: wait + resource: deployment/consumer-gateway + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/consumer-gateway + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: consumer-gateway-ip + timeout: "60s" + interval: "2s" + + valkey: + description: Private shared Valkey quota state for the west consumers + steps: + - type: manifest + path: resources/common/valkey.yaml + - type: wait + resource: deployment/valkey + namespace: grid-system + condition: available + timeout: "120s" + + consumer-west-a: + description: West consumer A using shared Valkey quota state + steps: + - type: template-file + source: configs/consumer/praxis-valkey-a.yaml + target: .forge/runtime/{{ cluster.name }}/consumer-a/praxis.yaml + - type: exec + command: + - bash + - -c + - >- + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} -n grid-system + create configmap consumer-gateway-a-config + --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/consumer-a/praxis.yaml + --dry-run=client -o yaml | + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway-a + chart: charts/praxis-gateway + version: "0.1.0" + namespace: grid-system + values: + fullnameOverride: "consumer-gateway-a" + image: + repository: "{{ cluster.properties.gatewayImageRepo }}" + tag: "{{ cluster.properties.gatewayImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: { cpu: 100m, memory: 64Mi } + limits: { cpu: 500m, memory: 256Mi } + config: + existingConfigMap: "consumer-gateway-a-config" + env: + - name: TOKEN_RATE_LIMIT_VALKEY_URL + valueFrom: + secretKeyRef: + name: valkey-auth + key: url + service: { type: LoadBalancer } + overlay: + enabled: true + existingConfigMap: "grid-overlay-grid-token-rate-limit-consumer-gateway-a" + sidecar: + enabled: true + image: + repository: "{{ cluster.properties.overlaySyncImageRepo }}" + tag: "{{ cluster.properties.overlaySyncImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + expectedNetwork: "grid-token-rate-limit" + expectedLocalSite: "{{ cluster.name }}" + tls: + enabled: true + existingSecret: consumer-gateway-tls + podLabels: + grid.praxis-proxy.io/consumer-site: "{{ cluster.name }}" + grid.praxis-proxy.io/quota-client: "true" + - type: wait + resource: deployment/consumer-gateway-a + namespace: grid-system + condition: available + timeout: "120s" + + consumer-west-b: + description: West consumer B using shared Valkey quota state + steps: + - type: template-file + source: configs/consumer/praxis-valkey-b.yaml + target: .forge/runtime/{{ cluster.name }}/consumer-b/praxis.yaml + - type: exec + command: + - bash + - -c + - >- + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} -n grid-system + create configmap consumer-gateway-b-config + --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/consumer-b/praxis.yaml + --dry-run=client -o yaml | + kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway-b + chart: charts/praxis-gateway + version: "0.1.0" + namespace: grid-system + values: + fullnameOverride: "consumer-gateway-b" + image: + repository: "{{ cluster.properties.gatewayImageRepo }}" + tag: "{{ cluster.properties.gatewayImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: { cpu: 100m, memory: 64Mi } + limits: { cpu: 500m, memory: 256Mi } + config: + existingConfigMap: "consumer-gateway-b-config" + env: + - name: TOKEN_RATE_LIMIT_VALKEY_URL + valueFrom: + secretKeyRef: + name: valkey-auth + key: url + service: { type: LoadBalancer } + overlay: + enabled: true + existingConfigMap: "grid-overlay-grid-token-rate-limit-consumer-gateway-b" + sidecar: + enabled: true + image: + repository: "{{ cluster.properties.overlaySyncImageRepo }}" + tag: "{{ cluster.properties.overlaySyncImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + expectedNetwork: "grid-token-rate-limit" + expectedLocalSite: "{{ cluster.name }}" + tls: + enabled: true + existingSecret: consumer-gateway-tls + podLabels: + grid.praxis-proxy.io/consumer-site: "{{ cluster.name }}" + grid.praxis-proxy.io/quota-client: "true" + - type: wait + resource: deployment/consumer-gateway-b + namespace: grid-system + condition: available + timeout: "120s" diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/README.md b/tests/e2e/topologies/grid-token-rate-limit/resources/README.md new file mode 100644 index 0000000..888f6a5 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/README.md @@ -0,0 +1,9 @@ +# Kubernetes Resources + +This directory owns Kubernetes resources that are not emitted directly by the +two Helm charts. Keep reusable resources under `common/` and site-specific +resources under `west/`, `central/`, and `east/`. + +Resources must preserve separate consumer and provider identities and prevent +consumer workloads from reaching private inference endpoints directly. Secret +manifests and credential values must never be committed. diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/central/README.md b/tests/e2e/topologies/grid-token-rate-limit/resources/central/README.md new file mode 100644 index 0000000..d7d9108 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/central/README.md @@ -0,0 +1,3 @@ +# Central Resources + +Place resources that differ specifically for the central Grid site here. diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/central/gridnetwork.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/central/gridnetwork.yaml new file mode 100644 index 0000000..d3ecbc9 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/central/gridnetwork.yaml @@ -0,0 +1,23 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: grid-combined-site + labels: + grid.praxis-proxy.io/auto-discover-sites: "true" +spec: + gridId: grid-combined-site-v1 + region: central + zone: central-1 + seeds: + - "{{ captures.west.swim-lb-ip }}:7946" + - "{{ captures.east.swim-lb-ip }}:7946" + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: null + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + swimKeyRef: null \ No newline at end of file diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/central/inference.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/central/inference.yaml new file mode 100644 index 0000000..91ed701 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/central/inference.yaml @@ -0,0 +1,23 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: vcr-central-provider +spec: + gridNetworkRef: grid-combined-site + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-central.grid-system.svc.cluster.local:8000 + models: + - name: Qwen/Qwen3-0.6B + capabilities: ["text_generation"] + contextWindow: 4096 + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: central + accessPolicy: + siteSelector: + matchLabels: {} + healthCheck: + path: /health + interval: "30s" + timeout: "5s" diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/central/site.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/central/site.yaml new file mode 100644 index 0000000..e2e1d74 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/central/site.yaml @@ -0,0 +1,10 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: central + labels: + grid.praxis-proxy.io/combined-site: central +spec: + gridNetworkRef: grid-combined-site + region: central + zone: central-1 \ No newline at end of file diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/common/README.md b/tests/e2e/topologies/grid-token-rate-limit/resources/common/README.md new file mode 100644 index 0000000..c965cf3 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/common/README.md @@ -0,0 +1,4 @@ +# Common Resources + +Place reusable namespace, policy, ServiceAccount, and simulated-backend +resources here when they are not emitted by a Helm chart. diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/common/backend-network-policy.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/common/backend-network-policy.yaml new file mode 100644 index 0000000..5ab38d5 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/common/backend-network-policy.yaml @@ -0,0 +1,43 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: vcr-inference-allow-provider-gateway-only + namespace: grid-system + labels: + app.kubernetes.io/part-of: grid-combined-site +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: vllm-vcr + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + grid.praxis-proxy.io/backend-access: "provider-gateway" + ports: + - protocol: TCP + port: 8000 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: vcr-inference-allow-grid-operator-health + namespace: grid-system + labels: + app.kubernetes.io/part-of: grid-combined-site +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: vllm-vcr + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: grid-operator + ports: + - protocol: TCP + port: 8000 diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/common/grid-system-namespace.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/common/grid-system-namespace.yaml new file mode 100644 index 0000000..43169c6 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/common/grid-system-namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: grid-system + labels: + name: grid-system + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted \ No newline at end of file diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/common/valkey.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/common/valkey.yaml new file mode 100644 index 0000000..48ad4a9 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/common/valkey.yaml @@ -0,0 +1,109 @@ +apiVersion: v1 +kind: Service +metadata: + name: valkey + namespace: grid-system + labels: + app.kubernetes.io/name: valkey + app.kubernetes.io/component: quota-state +spec: + selector: + app.kubernetes.io/name: valkey + ports: + - name: valkey + port: 6379 + targetPort: valkey + type: ClusterIP +--- +apiVersion: v1 +kind: Secret +metadata: + name: valkey-auth + namespace: grid-system +type: Opaque +stringData: + # Demo-only credential; consumers receive the URL through a Secret-backed env var. + password: token-rate-limit-demo-only + url: redis://:token-rate-limit-demo-only@valkey.grid-system.svc.cluster.local:6379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: valkey + namespace: grid-system + labels: + app.kubernetes.io/name: valkey + app.kubernetes.io/component: quota-state +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: valkey + template: + metadata: + labels: + app.kubernetes.io/name: valkey + app.kubernetes.io/component: quota-state + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: valkey + image: valkey/valkey@sha256:a038175878d66b9d274fbf8be73c0305e93798b83917647f167e18cef3c71eec + imagePullPolicy: IfNotPresent + args: ["valkey-server", "--appendonly", "no", "--requirepass", "$(VALKEY_PASSWORD)"] + env: + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-auth + key: password + ports: + - name: valkey + containerPort: 6379 + securityContext: + runAsUser: 999 + runAsGroup: 999 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readinessProbe: + tcpSocket: + port: valkey + periodSeconds: 2 + timeoutSeconds: 1 + livenessProbe: + tcpSocket: + port: valkey + periodSeconds: 5 + timeoutSeconds: 1 + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: valkey-ingress + namespace: grid-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: valkey + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + grid.praxis-proxy.io/quota-client: "true" + ports: + - protocol: TCP + port: 6379 diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/common/vcr-provider-workload.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/common/vcr-provider-workload.yaml new file mode 100644 index 0000000..3124032 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/common/vcr-provider-workload.yaml @@ -0,0 +1,117 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vcr-inference-{{ cluster.name }} + namespace: grid-system + labels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/part-of: grid-token-rate-limit +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/instance: "{{ cluster.name }}" + template: + metadata: + labels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/instance: "{{ cluster.name }}" + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: vcr + image: "{{ cluster.properties.vcrImage }}" + imagePullPolicy: "{{ cluster.properties.vcrImagePullPolicy }}" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + env: + - name: MODEL + value: "Qwen/Qwen3-0.6B" + - name: MOCK_PD_ROLE + value: "both" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: VLLM_PORT + value: "8000" + - name: MOCK_MAX_NUM_SEQS + value: "4" + - name: MOCK_KV_CACHE_SIZE + value: "64" + - name: MOCK_TOKENS_PER_BLOCK + value: "16" + - name: MOCK_MAX_MODEL_LEN + value: "512" + - name: MOCK_TTFT_MS + value: "50" + - name: MOCK_ITL_MS + value: "20" + - name: MOCK_TIME_FACTOR_UNDER_LOAD + value: "2.0" + ports: + - name: http + containerPort: 8000 + startupProbe: + httpGet: + path: /v1/models + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /health + port: 8000 + periodSeconds: 5 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /health + port: 8000 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + memory: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: vcr-inference-{{ cluster.name }} + namespace: grid-system + labels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/part-of: grid-token-rate-limit +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/instance: "{{ cluster.name }}" + ports: + - name: http + port: 8000 + targetPort: 8000 diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/east/README.md b/tests/e2e/topologies/grid-token-rate-limit/resources/east/README.md new file mode 100644 index 0000000..7710ec8 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/east/README.md @@ -0,0 +1,3 @@ +# East Resources + +Place resources that differ specifically for the east Grid site here. diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/east/gridnetwork.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/east/gridnetwork.yaml new file mode 100644 index 0000000..732a601 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/east/gridnetwork.yaml @@ -0,0 +1,23 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: grid-combined-site + labels: + grid.praxis-proxy.io/auto-discover-sites: "true" +spec: + gridId: grid-combined-site-v1 + region: east + zone: east-1 + seeds: + - "{{ captures.west.swim-lb-ip }}:7946" + - "{{ captures.central.swim-lb-ip }}:7946" + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: null + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + swimKeyRef: null \ No newline at end of file diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/east/inference.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/east/inference.yaml new file mode 100644 index 0000000..d3d7769 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/east/inference.yaml @@ -0,0 +1,23 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: vcr-east-provider +spec: + gridNetworkRef: grid-combined-site + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-east.grid-system.svc.cluster.local:8000 + models: + - name: Qwen/Qwen3-0.6B + capabilities: ["text_generation"] + contextWindow: 4096 + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: east + accessPolicy: + siteSelector: + matchLabels: {} + healthCheck: + path: /health + interval: "30s" + timeout: "5s" diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/east/site.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/east/site.yaml new file mode 100644 index 0000000..b657459 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/east/site.yaml @@ -0,0 +1,10 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: east + labels: + grid.praxis-proxy.io/combined-site: east +spec: + gridNetworkRef: grid-combined-site + region: east + zone: east-1 \ No newline at end of file diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/trust/central.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/trust/central.yaml new file mode 100644 index 0000000..edc88ff --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/trust/central.yaml @@ -0,0 +1,35 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: grid-token-rate-limit-west + labels: + grid.praxis-proxy.io/network: grid-token-rate-limit + grid.praxis-proxy.io/auto-discovered: "true" +spec: + gridNetworkRef: grid-token-rate-limit + egress: + address: "{{ captures.west.swim-lb-ip }}:8443" + tls: + mode: Mutual + serverName: west.grid.internal + trust: + canonicalFingerprints: + - PEER_WEST_FINGERPRINT +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: grid-token-rate-limit-east + labels: + grid.praxis-proxy.io/network: grid-token-rate-limit + grid.praxis-proxy.io/auto-discovered: "true" +spec: + gridNetworkRef: grid-token-rate-limit + egress: + address: "{{ captures.east.swim-lb-ip }}:8443" + tls: + mode: Mutual + serverName: east.grid.internal + trust: + canonicalFingerprints: + - PEER_EAST_FINGERPRINT diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/trust/east.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/trust/east.yaml new file mode 100644 index 0000000..9fa7b16 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/trust/east.yaml @@ -0,0 +1,35 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: grid-token-rate-limit-west + labels: + grid.praxis-proxy.io/network: grid-token-rate-limit + grid.praxis-proxy.io/auto-discovered: "true" +spec: + gridNetworkRef: grid-token-rate-limit + egress: + address: "{{ captures.west.swim-lb-ip }}:8443" + tls: + mode: Mutual + serverName: west.grid.internal + trust: + canonicalFingerprints: + - PEER_WEST_FINGERPRINT +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: grid-token-rate-limit-central + labels: + grid.praxis-proxy.io/network: grid-token-rate-limit + grid.praxis-proxy.io/auto-discovered: "true" +spec: + gridNetworkRef: grid-token-rate-limit + egress: + address: "{{ captures.central.swim-lb-ip }}:8443" + tls: + mode: Mutual + serverName: central.grid.internal + trust: + canonicalFingerprints: + - PEER_CENTRAL_FINGERPRINT diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/trust/west.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/trust/west.yaml new file mode 100644 index 0000000..c56f826 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/trust/west.yaml @@ -0,0 +1,35 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: grid-token-rate-limit-central + labels: + grid.praxis-proxy.io/network: grid-token-rate-limit + grid.praxis-proxy.io/auto-discovered: "true" +spec: + gridNetworkRef: grid-token-rate-limit + egress: + address: "{{ captures.central.swim-lb-ip }}:8443" + tls: + mode: Mutual + serverName: central.grid.internal + trust: + canonicalFingerprints: + - PEER_CENTRAL_FINGERPRINT +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: grid-token-rate-limit-east + labels: + grid.praxis-proxy.io/network: grid-token-rate-limit + grid.praxis-proxy.io/auto-discovered: "true" +spec: + gridNetworkRef: grid-token-rate-limit + egress: + address: "{{ captures.east.swim-lb-ip }}:8443" + tls: + mode: Mutual + serverName: east.grid.internal + trust: + canonicalFingerprints: + - PEER_EAST_FINGERPRINT diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/west/README.md b/tests/e2e/topologies/grid-token-rate-limit/resources/west/README.md new file mode 100644 index 0000000..a324e11 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/west/README.md @@ -0,0 +1,3 @@ +# West Resources + +Place resources that differ specifically for the west Grid site here. diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/west/gridnetwork.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/west/gridnetwork.yaml new file mode 100644 index 0000000..8fb6905 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/west/gridnetwork.yaml @@ -0,0 +1,23 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: grid-combined-site + labels: + grid.praxis-proxy.io/auto-discover-sites: "true" +spec: + gridId: grid-combined-site-v1 + region: west + zone: west-1 + seeds: + - "{{ captures.central.swim-lb-ip }}:7946" + - "{{ captures.east.swim-lb-ip }}:7946" + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: null + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + swimKeyRef: null \ No newline at end of file diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/west/inference.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/west/inference.yaml new file mode 100644 index 0000000..55827cc --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/west/inference.yaml @@ -0,0 +1,23 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: vcr-west-provider +spec: + gridNetworkRef: grid-combined-site + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-west.grid-system.svc.cluster.local:8000 + models: + - name: Qwen/Qwen3-0.6B + capabilities: ["text_generation"] + contextWindow: 4096 + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: west + accessPolicy: + siteSelector: + matchLabels: {} + healthCheck: + path: /health + interval: "30s" + timeout: "5s" diff --git a/tests/e2e/topologies/grid-token-rate-limit/resources/west/site.yaml b/tests/e2e/topologies/grid-token-rate-limit/resources/west/site.yaml new file mode 100644 index 0000000..f6208e2 --- /dev/null +++ b/tests/e2e/topologies/grid-token-rate-limit/resources/west/site.yaml @@ -0,0 +1,10 @@ +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridSite +metadata: + name: west + labels: + grid.praxis-proxy.io/combined-site: west +spec: + gridNetworkRef: grid-combined-site + region: west + zone: west-1 \ No newline at end of file From a4d2ae883d6774c6fdd66de0092b908c70f9dd08 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 22 Aug 2026 05:19:45 +0000 Subject: [PATCH 5/6] fix(e2e): make quota topology deployable Materialize explicit local image overrides and document the resolved Forge\nconfiguration used by Kind validation. Split operator bootstrap from remote SWIM\nseed application and apply providers, trust, Valkey, and consumers in dependency\norder so a fresh deployment can converge without hidden manual mutations. Signed-off-by: Brent Salisbury --- .dockerignore | 5 + charts/grid-site/templates/gridnetwork.yaml | 4 + charts/grid-site/templates/gridsite.yaml | 4 + charts/grid-site/values.schema.json | 12 ++ charts/grid-site/values.yaml | 2 + operator/src/controller/grid_network.rs | 45 ++++++ .../configs/consumer/praxis-valkey-a.yaml | 2 +- .../configs/consumer/praxis-valkey-b.yaml | 2 +- .../grid-token-rate-limit/forge.yaml | 130 +++++++++++++++--- xtask/src/env.rs | 16 +++ xtask/src/env/forge_config.rs | 108 +++++++++++++++ 11 files changed, 311 insertions(+), 19 deletions(-) create mode 100644 .dockerignore create mode 100644 xtask/src/env/forge_config.rs diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3865270 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +.forge +target +**/target +evidence diff --git a/charts/grid-site/templates/gridnetwork.yaml b/charts/grid-site/templates/gridnetwork.yaml index bd5cf48..96c3878 100644 --- a/charts/grid-site/templates/gridnetwork.yaml +++ b/charts/grid-site/templates/gridnetwork.yaml @@ -31,6 +31,10 @@ spec: {{- with .Values.gridNetwork.zone }} zone: {{ . | quote }} {{- end }} + {{- with .Values.gridNetwork.seeds }} + seeds: + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.gridNetwork.routingPolicy }} routingPolicy: {{ . | quote }} {{- end }} diff --git a/charts/grid-site/templates/gridsite.yaml b/charts/grid-site/templates/gridsite.yaml index ac05b00..39675fb 100644 --- a/charts/grid-site/templates/gridsite.yaml +++ b/charts/grid-site/templates/gridsite.yaml @@ -16,4 +16,8 @@ spec: {{- with .Values.gridSite.zone }} zone: {{ . | quote }} {{- end }} + {{- with .Values.gridSite.egress }} + egress: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/grid-site/values.schema.json b/charts/grid-site/values.schema.json index ac1b416..c4a63a3 100644 --- a/charts/grid-site/values.schema.json +++ b/charts/grid-site/values.schema.json @@ -21,6 +21,10 @@ "zone": { "type": "string" }, + "seeds": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, "swim": { "type": "object", "additionalProperties": true, @@ -136,6 +140,14 @@ }, "providerSiteLabel": { "type": "string" + }, + "egress": { + "type": "object", + "additionalProperties": true, + "properties": { + "address": { "type": "string", "minLength": 1 }, + "tls": { "type": "object", "additionalProperties": true } + } } } }, diff --git a/charts/grid-site/values.yaml b/charts/grid-site/values.yaml index f7241bf..81af658 100644 --- a/charts/grid-site/values.yaml +++ b/charts/grid-site/values.yaml @@ -3,6 +3,7 @@ gridNetwork: gridId: "" region: "" zone: "" + seeds: [] swim: {} tls: {} gatewayRefs: [] @@ -24,6 +25,7 @@ gridSite: region: "" zone: "" providerSiteLabel: "" + egress: {} inferenceProviders: [] diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 776fcd4..0e3de3e 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -442,12 +442,57 @@ pub async fn reconcile(network: Arc, ctx: Arc) -> Resu .is_some_and(|v| v == "true"); if auto_discover_enabled && let (Some(swim), Some(snapshot)) = (ctx.swim.as_ref(), membership.as_ref()) { let plaintext = network_uses_plaintext_egress(&network); + reconcile_local_site(name, swim.site_name(), client).await?; reconcile_discovered_sites(name, swim.site_name(), snapshot, client, plaintext).await?; } Ok(Action::requeue(requeue_interval)) } +/// Advance the explicitly configured local [`GridSite`] into discovery. +/// +/// Remote sites are created from SWIM membership, but the local site is +/// intentionally excluded from that discovery list. When a topology provides +/// its local `GridSite` declaratively, this controller must still hand it to the +/// `GridSite` controller for gateway probing. The status write is idempotent and +/// only applies while the site is still pending. +#[expect( + clippy::too_many_lines, + reason = "status patch keeps the local-site transition explicit" +)] +async fn reconcile_local_site(network_name: &str, local_site: &str, client: &Client) -> Result<(), OperatorError> { + let api: Api = Api::all(client.clone()); + let Ok(site) = api.get(local_site).await else { + return Ok(()); + }; + if site.spec.grid_network_ref != network_name { + return Ok(()); + } + let pending = site + .status + .as_ref() + .is_none_or(|status| status.phase == GridSitePhase::Pending); + if !pending { + return Ok(()); + } + let status_doc = serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "GridSite", + "status": { + "phase": "Discovered", + "reason": "SWIMDiscovered", + "message": "local site configured and ready for gateway probing" + } + }); + api.patch_status( + local_site, + &PatchParams::apply(FIELD_MANAGER).force(), + &Patch::Apply(&status_doc), + ) + .await?; + Ok(()) +} + /// Apply `spec.tls.swimKeyRef` before any reconcile-triggered SWIM send. /// /// A configured Secret reference is mandatory for that reconcile: missing diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml index 912d0c7..297922c 100644 --- a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-a.yaml @@ -70,7 +70,7 @@ filter_chains: - filter: headers response_set: - name: X-Grid-Combined-Consumer-Gateway - value: "{{ cluster.name }}" + value: "consumer-gateway-a" - filter: intelligent_route overlay_file: /etc/praxis/routing/routing-overlay.json model_header: X-Model diff --git a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml index 28c3720..38e8577 100644 --- a/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml +++ b/tests/e2e/topologies/grid-token-rate-limit/configs/consumer/praxis-valkey-b.yaml @@ -70,7 +70,7 @@ filter_chains: - filter: headers response_set: - name: X-Grid-Combined-Consumer-Gateway - value: "{{ cluster.name }}" + value: "consumer-gateway-b" - filter: intelligent_route overlay_file: /etc/praxis/routing/routing-overlay.json model_header: X-Model diff --git a/tests/e2e/topologies/grid-token-rate-limit/forge.yaml b/tests/e2e/topologies/grid-token-rate-limit/forge.yaml index 124780b..80a6e0c 100644 --- a/tests/e2e/topologies/grid-token-rate-limit/forge.yaml +++ b/tests/e2e/topologies/grid-token-rate-limit/forge.yaml @@ -15,7 +15,7 @@ spec: clusters: - name: west - stacks: [metallb, west-operator-base, vcr-backend, provider-boundary, west-trust-bootstrap, west-site, provider-gateway, valkey, consumer-west-a, consumer-west-b] + stacks: [metallb, west-operator-base, west-operator-seed, vcr-backend, provider-boundary, west-trust-bootstrap, west-site, site-trust-bootstrap, provider-gateway, valkey, consumer-west-a, consumer-west-b] properties: region: west role: combined @@ -24,7 +24,7 @@ spec: operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" vcrImagePullPolicy: IfNotPresent - imagePullPolicy: IfNotPresent + imagePullPolicy: Never gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" gatewayImageTag: "v0.1.3" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" @@ -33,7 +33,7 @@ spec: overlaySyncImageTag: "v0.1.3" - name: central - stacks: [metallb, central-operator-base, vcr-backend, provider-boundary, central-trust-bootstrap, central-site, provider-gateway] + stacks: [metallb, central-operator-base, central-operator-seed, vcr-backend, provider-boundary, central-trust-bootstrap, central-site, site-trust-bootstrap, provider-gateway] properties: region: central role: combined @@ -42,7 +42,7 @@ spec: operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" vcrImagePullPolicy: IfNotPresent - imagePullPolicy: IfNotPresent + imagePullPolicy: Never gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" gatewayImageTag: "v0.1.3" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" @@ -51,7 +51,7 @@ spec: overlaySyncImageTag: "v0.1.3" - name: east - stacks: [metallb, east-operator-base, vcr-backend, provider-boundary, east-trust-bootstrap, east-site, provider-gateway] + stacks: [metallb, east-operator-base, east-operator-seed, vcr-backend, provider-boundary, east-trust-bootstrap, east-site, site-trust-bootstrap, provider-gateway] properties: region: east role: combined @@ -60,7 +60,7 @@ spec: operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" vcrImagePullPolicy: IfNotPresent - imagePullPolicy: IfNotPresent + imagePullPolicy: Never gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" gatewayImageTag: "v0.1.3" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" @@ -99,7 +99,6 @@ spec: swim: siteName: "west" seeds: "" - advertiseAddress: "{{ captures.west.swim-lb-ip }}:7946" service: enabled: true type: "LoadBalancer" @@ -119,6 +118,30 @@ spec: timeout: "60s" interval: "2s" + west-operator-seed: + description: Add the captured remote SWIM seeds to the west operator + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "west" + seeds: "{{ captures.central.swim-lb-ip }}:7946,{{ captures.east.swim-lb-ip }}:7946" + advertiseAddress: "{{ captures.west.swim-lb-ip }}:7946" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" + central-operator-base: description: Central Grid operator initial deployment with SWIM LoadBalancer steps: @@ -135,7 +158,6 @@ spec: swim: siteName: "central" seeds: "" - advertiseAddress: "{{ captures.central.swim-lb-ip }}:7946" service: enabled: true type: "LoadBalancer" @@ -154,6 +176,29 @@ spec: key: swim-lb-ip timeout: "60s" interval: "2s" + central-operator-seed: + description: Add the captured remote SWIM seeds to the central operator + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "central" + seeds: "{{ captures.west.swim-lb-ip }}:7946,{{ captures.east.swim-lb-ip }}:7946" + advertiseAddress: "{{ captures.central.swim-lb-ip }}:7946" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" east-operator-base: description: East Grid operator initial deployment with SWIM LoadBalancer @@ -171,7 +216,6 @@ spec: swim: siteName: "east" seeds: "" - advertiseAddress: "{{ captures.east.swim-lb-ip }}:7946" service: enabled: true type: "LoadBalancer" @@ -190,6 +234,29 @@ spec: key: swim-lb-ip timeout: "60s" interval: "2s" + east-operator-seed: + description: Add the captured remote SWIM seeds to the east operator + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "east" + seeds: "{{ captures.west.swim-lb-ip }}:7946,{{ captures.central.swim-lb-ip }}:7946" + advertiseAddress: "{{ captures.east.swim-lb-ip }}:7946" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" vcr-backend: description: vllm-vcr inference backend for demo scenarios @@ -233,9 +300,9 @@ spec: kubectl --context "$context" apply -f -; sed -i -e "s/SITE_PLACEHOLDER/{{ cluster.name }}/g" -e "s/CANDIDATE_ID_PLACEHOLDER/$(case '{{ cluster.name }}' in west) echo f4c20a10;; central) echo c7fa24b0;; east) echo 9d876274;; esac)/g" - tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/provider/praxis.yaml; + .forge/runtime/{{ cluster.name }}/provider/praxis.yaml; kubectl --context "$context" -n grid-system create configmap provider-gateway-config - --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/provider/praxis.yaml --dry-run=client -o yaml | + --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/provider/praxis.yaml --dry-run=client -o yaml | kubectl --context "$context" apply -f - west-trust-bootstrap: @@ -250,7 +317,7 @@ spec: - -c - >- set -euo pipefail; - runtime=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/trust.yaml; + runtime=.forge/runtime/{{ cluster.name }}/trust.yaml; central=$(kubectl --context kind-grid-token-rate-limit-central -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); east=$(kubectl --context kind-grid-token-rate-limit-east -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); sed -i "s/PEER_CENTRAL_FINGERPRINT/${central}/g; s/PEER_EAST_FINGERPRINT/${east}/g" "$runtime"; @@ -268,7 +335,7 @@ spec: - -c - >- set -euo pipefail; - runtime=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/trust.yaml; + runtime=.forge/runtime/{{ cluster.name }}/trust.yaml; west=$(kubectl --context kind-grid-token-rate-limit-west -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); east=$(kubectl --context kind-grid-token-rate-limit-east -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); sed -i "s/PEER_WEST_FINGERPRINT/${west}/g; s/PEER_EAST_FINGERPRINT/${east}/g" "$runtime"; @@ -286,7 +353,7 @@ spec: - -c - >- set -euo pipefail; - runtime=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/trust.yaml; + runtime=.forge/runtime/{{ cluster.name }}/trust.yaml; west=$(kubectl --context kind-grid-token-rate-limit-west -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); central=$(kubectl --context kind-grid-token-rate-limit-central -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); sed -i "s/PEER_WEST_FINGERPRINT/${west}/g; s/PEER_CENTRAL_FINGERPRINT/${central}/g" "$runtime"; @@ -339,6 +406,11 @@ spec: region: west zone: west-1 providerSiteLabel: west + egress: + address: "{{ captures.west.provider-gateway-ip }}:8443" + tls: + mode: Mutual + serverName: west.grid.internal inferenceProviders: - name: vcr-west-provider gridNetworkRef: grid-token-rate-limit @@ -405,6 +477,11 @@ spec: region: central zone: central-1 providerSiteLabel: central + egress: + address: "{{ captures.central.provider-gateway-ip }}:8443" + tls: + mode: Mutual + serverName: central.grid.internal inferenceProviders: - name: vcr-central-provider gridNetworkRef: grid-token-rate-limit @@ -471,6 +548,11 @@ spec: region: east zone: east-1 providerSiteLabel: east + egress: + address: "{{ captures.east.provider-gateway-ip }}:8443" + tls: + mode: Mutual + serverName: east.grid.internal inferenceProviders: - name: vcr-east-provider gridNetworkRef: grid-token-rate-limit @@ -493,6 +575,20 @@ spec: interval: "30s" timeout: "5s" + site-trust-bootstrap: + description: Pin the generated local provider certificate for GridSite probing + steps: + - type: exec + command: + - bash + - -c + - >- + set -euo pipefail; + context=kind-grid-token-rate-limit-{{ cluster.name }}; + fingerprint=$(kubectl --context "$context" -n grid-system get secret provider-gateway-tls -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -outform DER | sha256sum | awk '{print $1}'); + kubectl --context "$context" -n grid-system patch gridsite "{{ cluster.name }}" --type merge + -p "{\"spec\":{\"trust\":{\"canonicalFingerprints\":[\"$fingerprint\"]}}}" + provider-gateway: description: Praxis provider gateway with mTLS and credential mounts steps: @@ -571,7 +667,7 @@ spec: - >- kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} -n grid-system create configmap consumer-gateway-config - --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f - - type: helm @@ -652,7 +748,7 @@ spec: - >- kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} -n grid-system create configmap consumer-gateway-a-config - --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/consumer-a/praxis.yaml + --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer-a/praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f - - type: helm @@ -717,7 +813,7 @@ spec: - >- kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} -n grid-system create configmap consumer-gateway-b-config - --from-file=praxis.yaml=tests/e2e/topologies/grid-token-rate-limit/.forge/runtime/{{ cluster.name }}/consumer-b/praxis.yaml + --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer-b/praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-token-rate-limit-{{ cluster.name }} apply -f - - type: helm diff --git a/xtask/src/env.rs b/xtask/src/env.rs index 39a5ee9..4dac854 100644 --- a/xtask/src/env.rs +++ b/xtask/src/env.rs @@ -5,6 +5,7 @@ pub(crate) mod combined_site_demo; pub(crate) mod config; pub(crate) mod consumer; pub(crate) mod external_provider; +pub(crate) mod forge_config; pub(crate) mod gateway; pub(crate) mod glb; pub(crate) mod glb_demo; @@ -197,6 +198,16 @@ pub(crate) fn demo_root(forge_config: &Path) -> PathBuf { /// Actions for the `env` subcommand. #[derive(Debug, Subcommand)] pub(crate) enum Action { + /// Materialize a Forge config with `GRID_XTASK_*` image overrides. + MaterializeForgeConfig { + /// Source Forge environment config. + #[arg(long)] + forge_config: PathBuf, + /// Destination for the rendered config. Defaults beside the source. + #[arg(long)] + output: Option, + }, + /// Create or update the test environment. Up { /// Path to the environment config file. @@ -1043,6 +1054,11 @@ mod llmd_pool_metrics_demo_cli_tests { )] pub(crate) fn run(action: &Action) -> Result<(), Box> { match action { + Action::MaterializeForgeConfig { forge_config, output } => { + let resolved = forge_config::materialize(forge_config, output.as_deref())?; + eprintln!("materialized Forge config: {}", resolved.display()); + Ok(()) + }, Action::Up { config } => env_up(config), Action::Down { config } => env_down(config), Action::Status { config } => env_status(config), diff --git a/xtask/src/env/forge_config.rs b/xtask/src/env/forge_config.rs new file mode 100644 index 0000000..bfcb043 --- /dev/null +++ b/xtask/src/env/forge_config.rs @@ -0,0 +1,108 @@ +//! Forge configuration materialization for local image overrides. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use super::image_overrides; + +/// Render a Forge environment with the explicitly selected demo images. +pub(crate) fn materialize(source: &Path, output: Option<&Path>) -> Result> { + let content = fs::read_to_string(source)?; + let mut config: serde_yaml::Value = serde_yaml::from_str(&content)?; + apply_image_overrides(&mut config)?; + let destination = output.map_or_else( + || { + source.with_file_name(format!( + "{}.resolved.yaml", + source.file_stem().and_then(|s| s.to_str()).unwrap_or("forge") + )) + }, + Path::to_path_buf, + ); + fs::write(&destination, serde_yaml::to_string(&config)?)?; + Ok(destination) +} + +#[expect( + clippy::too_many_lines, + reason = "The bounded image-property rewrite is easiest to audit as one operation." +)] +/// Apply the selected image references to every Forge cluster property. +fn apply_image_overrides(config: &mut serde_yaml::Value) -> Result<(), Box> { + let pull_policy = image_overrides::image_pull_policy(); + let gateway = image_overrides::gateway_image(); + let operator = image_overrides::operator_image(); + let overlay_sync = image_overrides::overlay_sync_image(); + let vcr = image_overrides::vcr_image(); + + if pull_policy == "Never" + && (std::env::var_os("GRID_XTASK_GATEWAY_IMAGE").is_none() + || std::env::var_os("GRID_XTASK_OPERATOR_IMAGE").is_none() + || std::env::var_os("GRID_XTASK_OVERLAY_SYNC_IMAGE").is_none()) + { + return Err("GRID_XTASK_GATEWAY_IMAGE, GRID_XTASK_OPERATOR_IMAGE, and GRID_XTASK_OVERLAY_SYNC_IMAGE are required when GRID_XTASK_IMAGE_PULL_POLICY=Never".into()); + } + + let (gateway_repo, gateway_tag) = parse_image_ref(&gateway); + let (operator_repo, operator_tag) = parse_image_ref(&operator); + let (overlay_repo, overlay_tag) = parse_image_ref(&overlay_sync); + + let clusters = config + .get_mut("spec") + .and_then(|spec| spec.get_mut("clusters")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or("Forge config must contain spec.clusters")?; + + for cluster in clusters { + let properties = cluster + .get_mut("properties") + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or("Forge cluster must contain properties")?; + for (key, value) in [ + ("gatewayImage", gateway.clone()), + ("gatewayImageRepo", gateway_repo.clone()), + ("gatewayImageTag", gateway_tag.clone()), + ("operatorImage", operator.clone()), + ("operatorImageRepo", operator_repo.clone()), + ("operatorImageTag", operator_tag.clone()), + ("overlaySyncImage", overlay_sync.clone()), + ("overlaySyncImageRepo", overlay_repo.clone()), + ("overlaySyncImageTag", overlay_tag.clone()), + ("vcrImage", vcr.clone()), + ("imagePullPolicy", pull_policy.clone()), + ] { + properties.insert( + serde_yaml::Value::String(key.to_owned()), + serde_yaml::Value::String(value), + ); + } + } + Ok(()) +} + +/// Split an image reference into repository and tag components. +fn parse_image_ref(image: &str) -> (String, String) { + image.rsplit_once(':').map_or_else( + || (image.to_owned(), "latest".to_owned()), + |(repo, tag)| (repo.to_owned(), tag.to_owned()), + ) +} + +#[cfg(test)] +mod tests { + use super::parse_image_ref; + + #[test] + fn parses_tagged_and_untagged_images() { + assert_eq!( + parse_image_ref("repo/image:tag"), + ("repo/image".to_owned(), "tag".to_owned()) + ); + assert_eq!( + parse_image_ref("repo/image"), + ("repo/image".to_owned(), "latest".to_owned()) + ); + } +} From 883f72c3a9090118420420f4177af9ddc359e422 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 22 Aug 2026 05:39:17 +0000 Subject: [PATCH 6/6] fix(xtask): remove redundant test imports Keep the provider traffic test compatible with the current Grid lint rules\nafter rebasing the quota topology onto Grid main. Signed-off-by: Brent Salisbury --- xtask/src/env/provider_traffic_demo.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/xtask/src/env/provider_traffic_demo.rs b/xtask/src/env/provider_traffic_demo.rs index 7bd6057..91d1b73 100644 --- a/xtask/src/env/provider_traffic_demo.rs +++ b/xtask/src/env/provider_traffic_demo.rs @@ -2878,8 +2878,6 @@ fn collect_image_evidence() -> Result, Box