diff --git a/Cargo.lock b/Cargo.lock index adf091d..11abbbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1774,6 +1774,7 @@ dependencies = [ "serde", "serde_json", "serde_json_canonicalizer", + "serde_yaml", "sha2", "swim", "thiserror 2.0.20", diff --git a/charts/grid-operator/crds/gridnetwork.yaml b/charts/grid-operator/crds/gridnetwork.yaml index 7822f0d..4007ed5 100644 --- a/charts/grid-operator/crds/gridnetwork.yaml +++ b/charts/grid-operator/crds/gridnetwork.yaml @@ -381,6 +381,24 @@ spec: items: type: string type: array + selectionPolicy: + description: |- + Local request distribution policy for the active selection group. + + This is independent of scoring. When absent, the overlay carries no + selection override and Praxis uses deterministic selection. + nullable: true + properties: + mode: + description: Local selection mode used by the data-plane gateway. + enum: + - deterministic + - roundRobin + - random + type: string + required: + - mode + type: object staleCandidateTtlSeconds: description: |- Maximum age in seconds before a stale (`fresh=false`) remote routing diff --git a/charts/grid-site/templates/gridnetwork.yaml b/charts/grid-site/templates/gridnetwork.yaml index 379f297..bd5cf48 100644 --- a/charts/grid-site/templates/gridnetwork.yaml +++ b/charts/grid-site/templates/gridnetwork.yaml @@ -1,4 +1,15 @@ {{- if .Values.gridNetwork.name }} +{{- $existing := lookup "grid.praxis-proxy.io/v1alpha1" "GridNetwork" .Release.Namespace .Values.gridNetwork.name }} +{{- $configuredSelectionPolicy := .Values.gridNetwork.selectionPolicy }} +{{- $existingSelectionPolicy := dig "spec" "selectionPolicy" nil $existing }} +{{- $selectionPolicy := $configuredSelectionPolicy }} +{{- if not $selectionPolicy }} + {{- if $existingSelectionPolicy }} + {{- $selectionPolicy = $existingSelectionPolicy }} + {{- else if not $existing }} + {{- $selectionPolicy = dict "mode" "roundRobin" }} + {{- end }} +{{- end }} apiVersion: grid.praxis-proxy.io/v1alpha1 kind: GridNetwork metadata: @@ -33,6 +44,12 @@ spec: admissionPolicy: {{- toYaml . | nindent 4 }} {{- end }} + {{- with $selectionPolicy }} + {{- if .mode }} + selectionPolicy: + mode: {{ .mode | quote }} + {{- end }} + {{- end }} {{- with .Values.gridNetwork.metricsRefreshInterval }} {{- if . }} metricsRefreshInterval: {{ . | quote }} diff --git a/charts/grid-site/tests/gridnetwork_test.yaml b/charts/grid-site/tests/gridnetwork_test.yaml index fbe5df9..7ad37ae 100644 --- a/charts/grid-site/tests/gridnetwork_test.yaml +++ b/charts/grid-site/tests/gridnetwork_test.yaml @@ -32,6 +32,15 @@ tests: path: spec.gridId value: "" + - it: defaults a new GridNetwork to roundRobin selection + set: + gridNetwork.name: prod-grid + gridSite.name: prod-site + asserts: + - equal: + path: spec.selectionPolicy.mode + value: roundRobin + - it: renders budgetPolicy.tenants verbatim when set set: gridNetwork.name: prod-grid diff --git a/charts/grid-site/values.schema.json b/charts/grid-site/values.schema.json index 19b825b..ac1b416 100644 --- a/charts/grid-site/values.schema.json +++ b/charts/grid-site/values.schema.json @@ -71,6 +71,17 @@ } } }, + "selectionPolicy": { + "type": "object", + "additionalProperties": false, + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "enum": ["deterministic", "roundRobin", "random"] + } + } + }, "metricsRefreshInterval": { "type": "string", "pattern": "^([1-9][0-9]*s|[1-9][0-9]{3,}ms)$" diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 7822f0d..4007ed5 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -381,6 +381,24 @@ spec: items: type: string type: array + selectionPolicy: + description: |- + Local request distribution policy for the active selection group. + + This is independent of scoring. When absent, the overlay carries no + selection override and Praxis uses deterministic selection. + nullable: true + properties: + mode: + description: Local selection mode used by the data-plane gateway. + enum: + - deterministic + - roundRobin + - random + type: string + required: + - mode + type: object staleCandidateTtlSeconds: description: |- Maximum age in seconds before a stale (`fresh=false`) remote routing diff --git a/docs/README.md b/docs/README.md index 79fe397..69714bd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,9 @@ - [Routing](architecture/routing.md) — versioned overlay contract, revision lifecycle, candidate ordering, `intelligent_route`, `peer_identity_trust`, and provider-side request forwarding. +- [Provider Selection and Load Balancing](architecture/provider-selection-and-load-balancing.md) — + eligibility, routing groups, scoring, selection modes, affinity, and + overlay lifecycle. - [Scoring](architecture/scoring.md) — operator-side candidate scoring, metrics input, and request-time scoring boundaries. - [Auth and Policy](architecture/auth.md) — provider authentication strategies, @@ -30,6 +33,9 @@ ## Demos +- [Provider Traffic Selection](../tests/e2e/topologies/grid-provider-traffic/README.md) — + runnable three-cluster topology for Grid selection groups and request-time + round-robin provider choice. - [Grid QuickStarts](https://github.com/praxis-proxy/demos) — deployable demonstrations with automated runtime proofs of routing, failover, security boundaries, and provider lifecycle. diff --git a/docs/architecture/provider-selection-and-load-balancing.md b/docs/architecture/provider-selection-and-load-balancing.md new file mode 100644 index 0000000..7c435ac --- /dev/null +++ b/docs/architecture/provider-selection-and-load-balancing.md @@ -0,0 +1,408 @@ +# Provider Selection and Load Balancing + +Grid and Praxis divide provider routing into two parts: + +- Grid observes provider state asynchronously, decides which providers are + eligible, orders them, and publishes a versioned routing overlay. +- Praxis consumes that overlay and makes the final request-time choice locally + in the `intelligent_route` filter. + +This separation keeps Kubernetes, Grid reconciliation, EPP metrics, and remote +coordination out of the request hot path. + +```mermaid +flowchart LR + client[Client] --> consumer[Consumer gateway] + consumer --> route[intelligent_route] + route --> active[First viable selection group] + active --> a[Provider gateway A] + active --> b[Provider gateway B] + active --> c[Provider gateway C] + route -. not used while group 0 is viable .-> fallback[Lower-priority group] + fallback --> d[Provider gateway D] +``` + +A, B, and C can share active traffic when the selected policy permits it. D is +fallback capacity and is not selected while the earlier group remains viable. +A selection group is a priority and resilience boundary; it is not a score +bucket and does not represent a traffic percentage. + +This layer balances requests across provider gateways. After a provider gateway +is selected, its local serving stack can make a separate backend-level decision. +For example, an llm-d provider gateway can delegate endpoint selection to EPP. +Grid does not use round-robin to choose individual inference replicas hidden +behind one provider gateway. + +## Choose a configuration + +Use these starting points, then adjust them to match the deployment's routing +goal: + +| Goal | Routing policy | Scoring strategy | Selection mode | +|---|---|---|---| +| Share traffic across nearby generic providers | `geographyFirst` | `noMetrics` | `roundRobin` | +| Keep nearby providers active and remote providers as fallback | `geographyFirst` | Any | `roundRobin` | +| Send new traffic to the highest-ranked provider | `geographyFirst` or `scoreFirst` | Any | `deterministic` | +| Share traffic across sites without inference metrics | `scoreFirst` | `noMetrics` | `roundRobin` | +| Randomize selection inside the preferred provider group | Either | Any | `random` | + +The three policy fields answer different questions: + +- `routingPolicy`: Which providers belong in the same active or fallback group? +- `scoringPolicy`: How should providers be ranked using available metrics? +- `selectionPolicy`: How should Praxis choose within the active group? + +## The decision sequence + +```mermaid +flowchart TD + request[Request for a capability] --> eligible[Eligibility and admission] + eligible --> groups[Order candidates and create priority groups] + groups --> affinity{Permitted affinity binding?} + affinity -->|yes| reuse[Reuse bound provider] + affinity -->|no| viable[Find first viable group] + viable --> mode{Selection mode} + mode --> deterministic[Deterministic] + mode --> roundRobin[Round-robin] + mode --> random[Random] +``` + +Scores contribute to candidate ordering. They do not create groups. The +selection mode operates only inside the first group that can serve the request. + +## Eligibility and admission + +Before ordering, Grid builds candidates for the requested capability. The +overlay already reflects capability matching, authorization and trust, +provider health, freshness, and provider availability. Admission is a hard +boundary: + +| Admission state | New requests | Existing sessions | +|---|---:|---:| +| `newAndExisting` | Allowed | Allowed | +| `existingOnly` | Not selected | Allowed when the binding is permitted | +| `excluded` | Not selected | Not selected | + +An `existingOnly` provider can finish work for a session that is already bound +to it, but it does not receive new bindings. An excluded provider cannot be +selected. Neither scoring nor a selection mode can override these states. + +## Routing policy and groups + +`spec.routingPolicy` controls candidate ordering and hard priority boundaries. +The supported values are `geographyFirst` and `scoreFirst`. + +### `geographyFirst` + +Candidates are ordered by admission, locality tier, freshness, score, and +deterministic identity tie-breakers. Groups are separated by admission, +locality tier, and freshness. Scores order candidates within a group but do +not split that group. + +```text +Closest healthy and fresh tier: A, B, C <- active selection +More distant healthy tier: D, E <- fallback +``` + +In plain language: balance within the closest healthy provider tier and use +more distant capacity as fallback. A remote provider does not join local +active traffic merely because its score is higher. + +### `scoreFirst` + +Candidates are ordered by admission, freshness, score, locality, and +deterministic identity tie-breakers. Groups are separated by admission and +freshness only. Fresh admitted providers from different sites can therefore +share one active group. Score differences affect order, not group membership. + +In plain language: allow fresh admitted providers across sites to participate +in the same active traffic group. + +## Scoring policy + +`spec.scoringPolicy.strategy` selects the provider-level signal used for score +calculation. It is independent of request-time selection: + +- `noMetrics` requires no EPP, Prometheus, or inference-specific metrics. + Dynamic score contributions are zero, while health, admission, freshness, + authorization, locality, affinity, and selection policy still apply. This is + the normal choice for generic or heterogeneous provider gateways. Use it when + llm-d EPP metrics are unavailable or are not comparable across providers. +- `queueDepth` uses asynchronously observed, normalized provider-pool queue + pressure. Lower pressure produces a higher preference score. It requires + comparable queue metrics and a meaningful queue capacity. For an llm-d + provider, Grid retrieves this signal from the configured EPP metrics endpoint. +- `kvCachePressure` uses provider-level KV-cache utilization as a capacity + pressure signal. Lower utilization produces a higher score. It is not + request-specific prefix-cache affinity; that decision belongs inside the + inference scheduler. For an llm-d provider, Grid retrieves this signal from + the configured EPP metrics endpoint. + +Grid currently uses one explicitly selected strategy rather than blending +unrelated signals into an opaque total. Missing local samples can use the +implementation's neutral fallback values, and a recent local sample can be +reused while it remains within `staleMetricsSeconds`. Deployments using a +metric strategy should provide fresh, comparable telemetry for every competing +provider. + +The important rule is: + +```text +score != traffic weight +``` + +Scores are preference and observability signals. They do not turn a score of +`0.8` versus `0.4` into a 2:1 traffic split. With `roundRobin`, candidates in +the active group receive equal turns regardless of their scores. + +For detailed metric input and normalization, see [Provider Scoring](scoring.md). + +## Selection policy + +`spec.selectionPolicy.mode` controls request-time selection inside the first +viable group. The selection mode is applied from an accepted in-memory +snapshot by Praxis; Grid is not called for each request. + +### `deterministic` + +Selects the first viable candidate in the active group. This is strict +preference behavior: Grid's ordering determines which provider receives new +unbound traffic. It is useful when locality, score, primary/standby order, or +predictability should dominate. When `selectionPolicy` is absent from an +overlay, Praxis uses `deterministic`. + +### `roundRobin` + +Takes equal turns across viable candidates in the active group. It does not +require inference metrics and does not distribute across lower-priority groups +while the active group is viable. It balances selections, not necessarily +tokens, latency, request cost, or concurrent work. Session affinity is checked +before this mode runs. + +### `random` + +Selects uniformly from viable candidates in the active group. It follows the +same admission, group, and affinity rules as round-robin. Random state is local +to the gateway process and is not a global coordinator. + +## Policy matrix + +| Routing policy | Selection policy | Effective behavior | +|---|---|---| +| `geographyFirst` | `deterministic` | Strict preference for the highest-ranked provider in the closest viable tier | +| `geographyFirst` | `roundRobin` | Equal selection in the closest viable tier; remote tiers are fallback | +| `geographyFirst` | `random` | Uniform selection in the closest viable tier | +| `scoreFirst` | `deterministic` | Strict preference for the highest-ranked fresh admitted provider across sites | +| `scoreFirst` | `roundRobin` | Equal selection across fresh admitted providers in the active group | +| `scoreFirst` | `random` | Uniform selection across fresh admitted providers in the active group | + +The scoring strategy changes ordering, not the selection mode: + +| Scoring strategy | Metrics required | Deterministic | Round-robin | +|---|---|---|---| +| `noMetrics` | No | Ordering and deterministic tie-breaks decide | Equal selection inside the active group | +| `queueDepth` | Compatible queue metrics | Highest queue-based preference is first | Scores remain visible; selection remains equal | +| `kvCachePressure` | Compatible KV metrics | Highest available-capacity preference is first | Scores remain visible; selection remains equal | + +## Configuration examples + +### Generic provider-gateway balancing + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: provider-grid +spec: + routingPolicy: geographyFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin +``` + +Providers in the nearest viable group share selections equally. No inference +metrics are required, and remote groups remain available for fallback. + +### Strict metric preference + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: inference-grid +spec: + routingPolicy: scoreFirst + scoringPolicy: + strategy: queueDepth + selectionPolicy: + mode: deterministic + metricsRefreshInterval: "10s" +``` + +Grid refreshes the selected signal asynchronously. Deterministic selection +uses the resulting ordering; it does not query EPP during a request. + +### Cross-site active/active selection + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: active-active-grid +spec: + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin +``` + +Fresh, admitted providers from multiple sites can share the active group. + +### Random selection + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: GridNetwork +metadata: + name: random-provider-grid +spec: + routingPolicy: geographyFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: random +``` + +Random selection is uniform within the active group. It is useful when an +equal probabilistic distribution is sufficient and a repeating sequence is not +required. + +## Request-time behavior and affinity + +For a new request, Praxis reads the accepted snapshot, resolves the requested +capability, checks affinity, finds the first viable group, applies the selection mode, +and records a binding when the selection succeeds. + +```text +Client + | + | request + v +Consumer gateway / intelligent_route + | 1. Read accepted in-memory overlay + | 2. Resolve capability and eligibility + | 3. Check session affinity + | 4. Find the first viable group + | 5. Apply the configured selection mode + | 6. Record the successful binding + v +Selected provider gateway + | + v +Provider backend +``` + +For an existing permitted binding, no new selection mode is applied: + +```text +Client with an existing session + | + v +intelligent_route + | + | permitted affinity binding found + v +Previously selected provider +``` + +Round-robin does not move an established session just to improve aggregate +balance. The observed request distribution can therefore differ from an exact +split when sessions generate different amounts of traffic. + +## Multiple consumer gateways + +The design supports multiple consumer gateways. Each gateway receives an +accepted overlay snapshot and keeps its own local selection state: + +```mermaid +flowchart TB + overlay[Accepted Grid overlay] + overlay --> one[Consumer gateway 1
local cursor: A to B to C] + overlay --> two[Consumer gateway 2
local cursor: A to B to C] +``` + +Counters are not coordinated globally. Each gateway can produce a balanced +local sequence, while aggregate traffic depends on request rates, affinity, +restarts, and snapshot replacement. A globally synchronized quota would need +a different coordination design and would add hot-path trade-offs. + +## Overlay lifecycle and re-ranking + +```mermaid +flowchart TD + signals[Provider health and optional EPP metrics] + reconcile[Grid operator reconciliation
eligibility, admission, ordering,
scores, groups, selection policy] + overlay[Content-addressed routing overlay] + publish[Overlay validation and publication] + snapshot[Praxis atomically loads snapshot
group index + local selection state] + request[Request-time selection from memory] + + signals --> reconcile --> overlay --> publish --> snapshot --> request +``` + +Reconciliation is triggered by watched provider, site, and network changes, +remote Grid state, and the periodic `metricsRefreshInterval`. The default +periodic cadence is 300 seconds for plaintext metrics; TLS-protected metrics +use a 60-second safety cap. A configured interval must be at least one second. +The interval controls observation and overlay publication, not request-path +latency. An operator can cause an earlier reconcile through a real watched +resource change. + +After an overlay is accepted, requests do not call Grid, Kubernetes, +ConfigMaps, EPP, Prometheus, or a remote scoring service. An unchanged semantic +revision should not continually rebuild selection state. An accepted semantic +change may create fresh snapshot-scoped state. + +## Failure and fallback + +Grid observes health, admission, freshness, and optional metrics +asynchronously. Praxis serves from the last accepted snapshot until a valid +new overlay is delivered. An unavailable or excluded provider cannot receive +new selections, and the first viable group is preferred. Later groups are +fallback capacity, not part of normal active distribution. + +Failover is therefore bounded by observation, reconciliation, overlay +distribution, and snapshot acceptance. It is not an immediate request-time +call to Grid. A request already sent upstream can fail before a newer snapshot +is accepted; do not assume automatic retry unless the gateway configuration +explicitly provides it. + +## Overlay contract and future weighting + +`selectionPolicy` is optional in both the Grid API and the overlay. An omitted +field remains omitted, and Praxis interprets it as deterministic selection. +The Helm chart explicitly renders `roundRobin` by default. Users applying a +`GridNetwork` directly can either set the selection mode explicitly or omit the policy +to select `deterministic`. + +`selection_group` and `selection_policy` are part of the semantic digest when +present. Group numbers are zero-based and contiguous per capability. Unknown +mode values and malformed policy structures are rejected. Overlays without +the optional selection fields remain valid and use deterministic selection. + +The provider ordering used to construct groups is also part of the routing +contract. In particular, the updated `geographyFirst` ordering evaluates +freshness before score after locality. Existing resources that omit +`selectionPolicy` still use deterministic selection, but an upgrade can change +which provider is first when candidates differ in freshness. Deployments that +require a fixed primary provider should set their routing and selection policy +explicitly and validate the resulting overlay during upgrade. + +Weighted selection is a future extension, not part of the current API. A +future mode such as `weightedRandom` would need an explicit overlay weight, +normalization, capacity semantics, missing-metric behavior, bounds, and +stability controls. It must not be inferred from score, rank, metric presence, +or candidate count, and it must not change admission, locality, +authorization, freshness, or group boundaries. diff --git a/docs/architecture/routing.md b/docs/architecture/routing.md index 5be5d35..438ca3b 100644 --- a/docs/architecture/routing.md +++ b/docs/architecture/routing.md @@ -32,6 +32,12 @@ llm-d / EPP / inference backend Grid does not proxy traffic. It writes the overlay used by Praxis filters. +For the complete provider-selection model, including selection groups and the +`deterministic`, `roundRobin`, and `random` modes, see [Provider Selection +and Load Balancing](provider-selection-and-load-balancing.md). Routing policy +defines candidate ordering and hard group boundaries; selection policy controls +request distribution within the first viable group. + ## Control-plane rendering path For each `GridNetwork` and gateway reference, the operator: @@ -363,8 +369,8 @@ The operator orders candidates before writing the overlay. Ordering proceeds in two phases: 1. **Scoring.** Each provider is scored by `scoring::score_backends` using - the weights selected by `GridNetwork.spec.scoringPolicy.strategy`, - optional live metrics, and optional CRDT-propagated provider metrics. + the signal selected by `GridNetwork.spec.scoringPolicy.strategy`, optional + live metrics, and optional CRDT-propagated provider metrics. Providers with no live metrics use neutral metric scores. See [Scoring](scoring.md) for the available strategies. @@ -375,7 +381,7 @@ in two phases: | Policy | Order | |---|---| - | `geographyFirst` (default) | admission, locality, score descending, freshness, deterministic tie-break | + | `geographyFirst` (default) | admission, locality, freshness, score descending, deterministic tie-break | | `scoreFirst` | admission, freshness, score descending, locality, deterministic tie-break | The deterministic tie-break is `(site, name, cluster)`. After diff --git a/docs/architecture/scoring.md b/docs/architecture/scoring.md index c9796a0..3f25bb4 100644 --- a/docs/architecture/scoring.md +++ b/docs/architecture/scoring.md @@ -1,5 +1,10 @@ # Provider Scoring +For the complete relationship between scoring, routing groups, and request-time +selection, see [Provider Selection and Load Balancing](provider-selection-and-load-balancing.md). +Scores influence candidate ordering; they are not traffic weights and do not +split selection groups. + Grid scores provider pools when the operator renders a Praxis routing overlay. Praxis reads that overlay from memory at request time; it does not call Grid, Kubernetes, the operator, or an EPP metrics endpoint on the request path. @@ -53,7 +58,7 @@ spec: Omitting `scoringPolicy` has the same effect. Every admitted candidate receives zero dynamic score. "No metrics" does not mean "no policy": health, admission, freshness, model compatibility, geography, selection tiers, session affinity, -and Praxis picker policy continue to apply. +and Praxis selection policy continue to apply. Use this strategy for heterogeneous grids, external APIs such as OpenAI, Anthropic, or Bedrock, and providers that do not expose comparable pool diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 4e8fa86..2c059e5 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -44,6 +44,7 @@ zeroize = { workspace = true } # Server-side rmcp features, only for the mcp_probe integration tests, which # spin up a real Streamable HTTP MCP server to probe against. rmcp = { workspace = true, features = ["server", "transport-streamable-http-server"] } +serde_yaml = { workspace = true } tower = { workspace = true } [lints] diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 8826651..7f7876e 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -4459,6 +4459,7 @@ mod tests { network: "net".to_owned(), local_site: "site".to_owned(), candidates: Vec::new(), + selection_policy: None, generated_at: Some("2026-07-29T01:00:00Z".to_owned()), }, }, diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index c340642..09c4570 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -60,7 +60,7 @@ pub enum ScoringStrategy { /// Do not prefer providers using dynamic metrics. /// /// All score contributions are zero. Health, admission, freshness, - /// geography, selection tiers, session affinity, and request-time picker + /// geography, selection tiers, session affinity, and request-time selection /// policy still apply. This is the generic default. #[default] NoMetrics, @@ -143,6 +143,28 @@ pub struct ScoringPolicyConfig { pub strategy: ScoringStrategy, } +/// Local request-selection mode applied by Praxis inside the active group. +#[derive(Clone, Copy, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum SelectionMode { + /// Select the first admitted candidate in the active group. + #[default] + Deterministic, + /// Distribute new requests equally in the active group. + RoundRobin, + /// Distribute new requests randomly in the active group. + Random, +} + +/// Request selection policy published in the routing overlay. +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct SelectionPolicyConfig { + /// Local selection mode used by the data-plane gateway. + pub mode: SelectionMode, +} + /// Resolve the effective [`scoring::ScoringWeights`] from a scoring policy. /// /// The public API selects one scorer. The weight adapter is internal and @@ -517,7 +539,8 @@ pub struct GridNetworkSpec { /// Admission state (`newAndExisting` before `existingOnly`) always /// outranks both geography and score in either mode. In `scoreFirst` /// mode, freshness also outranks both; in `geographyFirst` mode, - /// freshness is a tiebreaker below geography and score. + /// freshness is below locality but above score so it cannot interleave + /// candidates across freshness-based selection groups. #[serde(default, skip_serializing_if = "Option::is_none")] pub routing_policy: Option, @@ -540,6 +563,13 @@ pub struct GridNetworkSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub admission_policy: Option, + /// Local request distribution policy for the active selection group. + /// + /// This is independent of scoring. When absent, the overlay carries no + /// selection override and Praxis uses deterministic selection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selection_policy: Option, + /// Maximum time between metric refreshes and score/ranking recalculation. /// /// This controls the `GridNetwork` reconcile cadence for provider metrics; @@ -1933,6 +1963,37 @@ mod tests { assert!(result.is_err(), "unknown strategy must be rejected"); } + #[test] + fn selection_policy_round_trips_and_rejects_unknown_mode() { + let spec: GridNetworkSpec = serde_json::from_value(serde_json::json!({ + "seeds": [], + "selectionPolicy": { "mode": "roundRobin" } + })) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!( + spec.selection_policy.map(|policy| policy.mode), + Some(SelectionMode::RoundRobin) + ); + + let result = serde_json::from_value::(serde_json::json!({ + "seeds": [], + "selectionPolicy": { "mode": "weightedRandom" } + })); + let Err(error) = result else { + std::process::abort(); + }; + assert!(error.to_string().contains("unknown variant")); + } + + #[test] + fn round_robin_mode_serializes_for_explicit_policy() { + let serialized = serde_json::to_value(&SelectionPolicyConfig { + mode: SelectionMode::RoundRobin, + }) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!(serialized, serde_json::json!({"mode": "roundRobin"})); + } + #[test] fn scoring_policy_rejects_removed_weights_field() { let json = serde_json::json!({ diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index ff2456d..f02ca58 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -23,7 +23,7 @@ use std::collections::{BTreeMap, BTreeSet}; use k8s_openapi::api::core::v1::ConfigMap; use crate::{ - crd::grid_network::{ClusterEndpointConfig, TransportMode}, + crd::grid_network::{ClusterEndpointConfig, SelectionMode, TransportMode}, resources::routing_overlay::{RoutingCandidate, RoutingOverlay}, }; @@ -150,6 +150,7 @@ pub(crate) fn generate_consumer_praxis_config( } let candidates_yaml = render_candidates(&overlay.candidates); + let selection_policy_yaml = render_selection_policy(overlay.selection_policy.as_ref()); let local_site = yaml_scalar(&overlay.local_site)?; let credential_inject_section = render_credential_inject(&overlay.candidates, credential_mount_base); @@ -171,6 +172,7 @@ pub(crate) fn generate_consumer_praxis_config( \x20 - filter: intelligent_route\n\ \x20 local_site: {local_site}\n\ \x20 model_header: \"X-Model\"\n\ + {selection_policy_yaml}\ \x20 candidates:\n\ {candidates_yaml}" ); @@ -230,7 +232,24 @@ fn render_candidates(candidates: &[RoutingCandidate]) -> String { candidates.iter().map(render_candidate).collect::>().join("\n") } +/// Render the explicit Grid-owned request-selection policy. +fn render_selection_policy(policy: Option<&crate::crd::grid_network::SelectionPolicyConfig>) -> String { + let Some(policy) = policy else { + return String::new(); + }; + let mode = match policy.mode { + SelectionMode::Deterministic => "deterministic", + SelectionMode::RoundRobin => "roundRobin", + SelectionMode::Random => "random", + }; + format!(" selection_policy:\n mode: {mode}\n") +} + /// Render one `intelligent_route` candidate. +#[expect( + clippy::too_many_lines, + reason = "Candidate YAML fields are kept together to mirror the wire contract." +)] fn render_candidate(c: &RoutingCandidate) -> String { let mut lines = vec![ format!( @@ -251,6 +270,15 @@ fn render_candidate(c: &RoutingCandidate) -> String { ), format!(" fresh: {}", c.fresh), ]; + if let Some(admission) = c.admission_state { + lines.push(format!( + " admission_state: {}", + serde_json::to_string(&admission).unwrap_or_default() + )); + } + if let Some(group) = c.selection_group { + lines.push(format!(" selection_group: {group}")); + } if let Some(cred) = &c.credential { lines.extend(render_credential_reference(cred)); } @@ -525,6 +553,7 @@ mod tests { score: None, score_breakdown: None, rank: None, + selection_group: None, } } @@ -557,6 +586,7 @@ mod tests { score: None, score_breakdown: None, rank: None, + selection_group: None, } } @@ -565,6 +595,7 @@ mod tests { network: "test-net".to_owned(), local_site: "site-a".to_owned(), candidates, + selection_policy: None, generated_at: None, } } @@ -591,6 +622,22 @@ mod tests { .collect() } + fn selection_policy_mode(config: &str) -> Option { + let parsed: serde_yaml::Value = serde_yaml::from_str(config).ok()?; + parsed + .get("filter_chains")? + .as_sequence()? + .first()? + .get("filters")? + .as_sequence()? + .iter() + .find(|filter| filter.get("filter").and_then(serde_yaml::Value::as_str) == Some("intelligent_route"))? + .get("selection_policy")? + .get("mode")? + .as_str() + .map(str::to_owned) + } + // ----------------------------------------------------------------------- // Renderer: basic structure // ----------------------------------------------------------------------- @@ -626,6 +673,24 @@ mod tests { assert!(yaml.contains("gateway-site-a"), "cluster must appear in load_balancer"); } + #[test] + fn explicit_selection_policy_reaches_intelligent_route_config() { + let mut overlay = simple_overlay(vec![plain_candidate( + "inference_model", + "model", + "site-a", + "cluster-a", + true, + )]); + overlay.selection_policy = Some(crate::crd::grid_network::SelectionPolicyConfig { + mode: SelectionMode::RoundRobin, + }); + let endpoints = endpoint_coverage(&overlay); + let config = generate_consumer_praxis_config(&overlay, MOUNT_BASE, &endpoints, "/run/tls", 8080) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!(selection_policy_mode(&config).as_deref(), Some("roundRobin")); + } + #[test] fn generated_praxis_yaml_omits_operator_overlay_metadata() { let mut candidate = plain_candidate("inference_model", "model-a", "site-a", "gateway-site-a", true); @@ -633,6 +698,7 @@ mod tests { candidate.admission_state = Some(AdmissionState::NewAndExisting); candidate.selection_tier = Some(LocalityTier::SameSite); candidate.rank = Some(0); + candidate.selection_group = Some(0); let mut overlay = simple_overlay(vec![candidate]); overlay.generated_at = Some("2026-07-24T12:00:00Z".to_owned()); @@ -646,12 +712,14 @@ mod tests { ) .unwrap(); - for forbidden in ["stable_id", "admission_state", "selection_tier", "rank", "generated_at"] { + for forbidden in ["stable_id", "selection_tier", "rank", "generated_at"] { assert!( !yaml.contains(forbidden), "operator-only metadata field {forbidden} must not enter generated Praxis YAML" ); } + assert!(yaml.contains("admission_state: \"new_and_existing\"")); + assert!(yaml.contains("selection_group: 0")); } #[test] @@ -932,6 +1000,7 @@ mod tests { network: "n".to_owned(), local_site: String::new(), candidates: vec![], + selection_policy: None, generated_at: None, }; assert!( diff --git a/operator/src/resources/overlay_bridge.rs b/operator/src/resources/overlay_bridge.rs index 9007d93..1442e3c 100644 --- a/operator/src/resources/overlay_bridge.rs +++ b/operator/src/resources/overlay_bridge.rs @@ -95,8 +95,10 @@ mod tests { score: None, score_breakdown: None, rank: None, + selection_group: None, }) .collect(), + selection_policy: None, generated_at: None, } } diff --git a/operator/src/resources/overlay_envelope.rs b/operator/src/resources/overlay_envelope.rs index f1d8939..9461dbe 100644 --- a/operator/src/resources/overlay_envelope.rs +++ b/operator/src/resources/overlay_envelope.rs @@ -4,9 +4,9 @@ //! //! Wraps a routing overlay in a versioned envelope with a SHA-256 digest //! computed over the [RFC 8785][] canonical form of the semantic payload. -//! The semantic payload includes only routing-relevant fields (`network`, -//! `local_site`, `candidates`), so the revision changes only when routing -//! behavior changes — timestamp and provenance changes are no-ops. +//! The semantic payload includes `network`, `local_site`, and `candidates`, +//! plus `selection_policy` when present, so the revision changes only when +//! routing behavior changes — timestamp and provenance changes are no-ops. //! //! The envelope is published as the `routing-overlay.json` key in the overlay //! `ConfigMap`, alongside the `routing-config.json` legacy key. @@ -182,11 +182,17 @@ pub struct EnvelopeBuildResult { /// [`ProjectedCredentialRef`]: super::routing_overlay::ProjectedCredentialRef pub fn compute_semantic_digest(overlay: &RoutingOverlay) -> Result { let candidates_value = serde_json::to_value(&overlay.candidates)?; - let semantic_payload = serde_json::json!({ + let mut semantic_payload = serde_json::json!({ "candidates": candidates_value, "local_site": overlay.local_site, "network": overlay.network, }); + if let Some(policy) = &overlay.selection_policy { + let Some(object) = semantic_payload.as_object_mut() else { + return Err(serde_json::Error::custom("semantic payload must be an object")); + }; + object.insert("selection_policy".to_owned(), serde_json::to_value(policy)?); + } let canonical = serde_json_canonicalizer::to_vec(&semantic_payload).map_err(serde_json::Error::custom)?; @@ -307,7 +313,9 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }], + selection_policy: None, generated_at: Some("2026-07-29T00:00:00Z".to_owned()), } } @@ -341,6 +349,7 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }, RoutingCandidate { kind: "inference_model".to_owned(), @@ -355,8 +364,10 @@ mod tests { score: None, score_breakdown: None, rank: Some(1), + selection_group: None, }, ], + selection_policy: None, generated_at: Some("2026-07-29T01:00:00Z".to_owned()), } } diff --git a/operator/src/resources/routing_overlay.rs b/operator/src/resources/routing_overlay.rs index f922bb0..5820277 100644 --- a/operator/src/resources/routing_overlay.rs +++ b/operator/src/resources/routing_overlay.rs @@ -269,6 +269,7 @@ pub(crate) fn remote_crdt_provider_to_candidates(provider: &crdt::ProviderState) score: None, score_breakdown: None, rank: None, + selection_group: None, }) .collect() } @@ -876,6 +877,13 @@ pub struct RoutingCandidate { /// Zero-based position in the final sorted overlay. #[serde(skip_serializing_if = "Option::is_none")] pub rank: Option, + + /// Zero-based active selection group for request distribution. + /// + /// Group metadata is additive and does not change the default ordered + /// routing behavior until the data plane explicitly enables it. + #[serde(skip_serializing_if = "Option::is_none")] + pub selection_group: Option, } /// The full routing overlay for a single [`GridNetwork`]. @@ -904,10 +912,15 @@ pub struct RoutingOverlay { /// higher than remote candidates. pub local_site: String, - /// Routing candidates, ordered by admission state, locality tier, score, - /// freshness, then alphabetical tiebreak. + /// Routing candidates, ordered by admission state, locality tier, + /// freshness, score, then alphabetical tiebreak. pub candidates: Vec, + /// Optional explicit local selection policy for Praxis. An absent field is + /// intentionally backward-compatible and means deterministic selection. + #[serde(skip_serializing_if = "Option::is_none")] + pub selection_policy: Option, + /// RFC 3339 timestamp of when this overlay was rendered. #[serde(skip_serializing_if = "Option::is_none")] pub generated_at: Option, @@ -1016,6 +1029,71 @@ fn locality_sort_key(tier: Option) -> u8 { } } +/// Assign deterministic priority groups after the existing candidate sort. +/// +/// Groups are hard eligibility/priority boundaries, not score buckets. The +/// current producer contract deliberately leaves metric differences inside a +/// group so a later traffic-weight field can distribute among eligible +/// candidates without changing the active priority boundary. +/// +/// Geography-first keeps admission, locality, and freshness as boundaries; +/// Score-first keeps admission and freshness as boundaries. Scores are +/// therefore never used to split a group, and floating-point equality is not +/// part of the grouping contract. +#[derive(Clone, Copy)] +struct GroupState { + /// Admission boundary for the previous candidate. + admission: AdmissionState, + /// Freshness boundary for the previous candidate. + fresh: bool, + /// Locality boundary for the previous candidate. + tier: LocalityTier, + /// Group assigned to the previous candidate. + group: u32, +} + +/// Assign zero-based contiguous groups for each capability. +#[expect( + clippy::too_many_lines, + reason = "The two routing policies are explicit at this contract boundary." +)] +fn assign_selection_groups(candidates: &mut [RoutingCandidate], policy: crate::crd::grid_network::RoutingPolicy) { + let mut last_by_capability: HashMap<(String, String), GroupState> = HashMap::new(); + for candidate in candidates { + let key = (candidate.kind.clone(), candidate.name.clone()); + let admission = candidate.admission_state.unwrap_or(AdmissionState::NewAndExisting); + let tier = candidate.selection_tier.unwrap_or(LocalityTier::Unknown); + let next = match last_by_capability.get(&key) { + Some(last) => { + let boundary = match policy { + crate::crd::grid_network::RoutingPolicy::GeographyFirst => { + admission != last.admission || tier != last.tier || candidate.fresh != last.fresh + }, + crate::crd::grid_network::RoutingPolicy::ScoreFirst => { + admission != last.admission || candidate.fresh != last.fresh + }, + }; + if boundary { + last.group.saturating_add(1) + } else { + last.group + } + }, + None => 0, + }; + candidate.selection_group = Some(next); + last_by_capability.insert( + key, + GroupState { + admission, + fresh: candidate.fresh, + tier, + group: next, + }, + ); + } +} + // --------------------------------------------------------------------------- // Renderer // --------------------------------------------------------------------------- @@ -1048,10 +1126,14 @@ fn locality_sort_key(tier: Option) -> u8 { /// **`GeographyFirst`** (default): /// 1. admission state: `new_and_existing` before `existing_only`; /// 2. geography tier: same site, same zone, same region, cross region; -/// 3. scoring engine score, descending; -/// 4. `fresh=true` before `fresh=false`; +/// 3. freshness: `fresh=true` before `fresh=false`; +/// 4. scoring engine score, descending; /// 5. deterministic `(site, name, cluster)` tiebreak. /// +/// Freshness intentionally precedes score in this policy. Group assignment +/// treats freshness as a hard boundary, so this ordering prevents a stale +/// candidate from interleaving with fresh candidates of the same capability. +/// /// **`ScoreFirst`**: /// 1. admission state: `new_and_existing` before `existing_only`; /// 2. `fresh=true` before `fresh=false`; @@ -1220,8 +1302,8 @@ pub fn render_routing_overlay_with_admission( admission_sort_key(a.admission_state) .cmp(&admission_sort_key(b.admission_state)) .then(locality_sort_key(a.selection_tier).cmp(&locality_sort_key(b.selection_tier))) - .then_with(|| score_of(&b.cluster).total_cmp(&score_of(&a.cluster))) .then(b.fresh.cmp(&a.fresh)) + .then_with(|| score_of(&b.cluster).total_cmp(&score_of(&a.cluster))) .then(a.site.cmp(&b.site)) .then(a.name.cmp(&b.name)) .then(a.cluster.cmp(&b.cluster)) @@ -1255,10 +1337,19 @@ pub fn render_routing_overlay_with_admission( } } + assign_selection_groups(&mut candidates, policy); + + // Preserve omission for existing GridNetwork resources. New Helm + // installations resolve their round-robin default in chart values, while + // an upgraded resource without this field must retain deterministic + // Praxis compatibility behavior. + let selection_policy = network.spec.selection_policy.clone(); + Ok(RoutingOverlay { network: network_name.to_owned(), local_site: local_site.to_owned(), candidates, + selection_policy, generated_at: generated_at.map(str::to_owned), }) } @@ -1454,6 +1545,7 @@ fn candidates_from_provider( score: None, score_breakdown: None, rank: None, + selection_group: None, }); } } @@ -1669,6 +1761,156 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + #[expect( + clippy::too_many_arguments, + reason = "Test fixture mirrors all grouping dimensions explicitly." + )] + fn group_candidate( + name: &str, + site: &str, + cluster: &str, + admission_state: AdmissionState, + tier: LocalityTier, + fresh: bool, + score: f64, + ) -> RoutingCandidate { + RoutingCandidate { + kind: CANDIDATE_KIND.to_owned(), + name: name.to_owned(), + site: site.to_owned(), + cluster: cluster.to_owned(), + fresh, + credential: None, + stable_id: None, + admission_state: Some(admission_state), + selection_tier: Some(tier), + score: Some(score), + score_breakdown: None, + rank: None, + selection_group: None, + } + } + + #[test] + fn equivalent_candidates_share_group_without_identity_comparison() { + let mut candidates = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.8, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.8, + ), + ]; + assign_selection_groups(&mut candidates, crate::crd::grid_network::RoutingPolicy::GeographyFirst); + assert_eq!(candidates[0].selection_group, Some(0)); + assert_eq!(candidates[1].selection_group, Some(0)); + } + + #[test] + fn changed_routing_properties_split_groups_deterministically() { + let mut candidates = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.8, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::CrossRegion, + true, + 0.8, + ), + group_candidate( + "model-a", + "site-c", + "cluster-c", + AdmissionState::ExistingOnly, + LocalityTier::CrossRegion, + false, + 0.2, + ), + ]; + assign_selection_groups(&mut candidates, crate::crd::grid_network::RoutingPolicy::GeographyFirst); + assert_eq!( + candidates.iter().map(|c| c.selection_group).collect::>(), + vec![Some(0), Some(1), Some(2)] + ); + } + + #[test] + fn metric_differences_stay_in_same_group_for_both_policies() { + let mut geography = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.9, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.1, + ), + ]; + assign_selection_groups(&mut geography, crate::crd::grid_network::RoutingPolicy::GeographyFirst); + assert_eq!( + geography.iter().map(|c| c.selection_group).collect::>(), + vec![Some(0), Some(0)] + ); + + let mut score_first = vec![ + group_candidate( + "model-a", + "site-a", + "cluster-a", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.9, + ), + group_candidate( + "model-a", + "site-b", + "cluster-b", + AdmissionState::NewAndExisting, + LocalityTier::SameSite, + true, + 0.1, + ), + ]; + assign_selection_groups(&mut score_first, crate::crd::grid_network::RoutingPolicy::ScoreFirst); + assert_eq!( + score_first.iter().map(|c| c.selection_group).collect::>(), + vec![Some(0), Some(0)] + ); + } + fn test_site(name: &str, network: &str) -> GridSite { serde_json::from_value(serde_json::json!({ "apiVersion": "grid.praxis-proxy.io/v1alpha1", @@ -1713,6 +1955,27 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + #[test] + fn omitted_selection_policy_remains_omitted_from_overlay() { + let network = test_network_score_first("net"); + let provider = test_provider("provider", "net", &["model"]); + let overlay = render_routing_overlay( + &network, + &[], + &[provider], + &[], + "site-a", + None, + None, + &scoring::ScoringWeights::default(), + ) + .unwrap_or_else(|_| std::process::abort()); + assert!( + overlay.selection_policy.is_none(), + "an upgraded GridNetwork that omits selectionPolicy must not be migrated implicitly" + ); + } + fn test_provider_with_selector( name: &str, network: &str, @@ -2308,9 +2571,9 @@ mod tests { // provider is also unavailable, but its fresh=false signals that its // metrics are stale. // - // Default GeographyFirst sort: admission → locality → score → fresh → tiebreak. - // The degraded local (SameSite, fresh=false) outranks the fresh API - // (CrossRegion, fresh=true) because geography sorts above freshness. + // Default GeographyFirst sort: admission → locality → fresh → score → tiebreak. + // Freshness is evaluated before score after locality, so a fresh remote + // candidate can precede a degraded local candidate. let network = test_network("fallback-net"); let local_degraded = test_provider_with_backend_kind_and_phase("provider-local", "fallback-net", "local", "Degraded"); @@ -2346,8 +2609,8 @@ mod tests { assert!(api_c.fresh, "API provider with absent status must have fresh=true"); assert_eq!( overlay.candidates.first().map(|c| c.cluster.as_str()), - Some("provider-local"), - "Degraded local must rank before API (GeographyFirst: locality outranks freshness)" + Some("provider-api"), + "with equal unknown locality, GeographyFirst freshness precedes score" ); } diff --git a/overlay-sync/src/types.rs b/overlay-sync/src/types.rs index 3e7b459..57fb06f 100644 --- a/overlay-sync/src/types.rs +++ b/overlay-sync/src/types.rs @@ -116,11 +116,35 @@ pub(crate) struct RoutingOverlay { /// Routing candidates ordered by the operator's sort. pub(crate) candidates: Vec, + /// Optional local request-selection policy from the Grid operator. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) selection_policy: Option, + /// RFC 3339 timestamp of when this overlay was rendered. #[serde(skip_serializing_if = "Option::is_none")] pub(crate) generated_at: Option, } +/// Selection policy copied without interpretation by overlay-sync. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct SelectionPolicy { + /// Selection mode consumed by Praxis. + pub(crate) mode: SelectionMode, +} + +/// Wire-level mode validation. Overlay-sync does not choose or execute it. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum SelectionMode { + /// Ordered selection. + Deterministic, + /// Equal local rotation. + RoundRobin, + /// Local random selection. + Random, +} + /// A single routing candidate. #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct RoutingCandidate { @@ -166,11 +190,16 @@ pub(crate) struct RoutingCandidate { /// Zero-based position in the final sorted overlay. #[serde(skip_serializing_if = "Option::is_none")] pub(crate) rank: Option, + + /// Producer-assigned active selection group. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) selection_group: Option, } /// Credential reference projected alongside a routing candidate. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] pub(crate) struct ProjectedCredential { /// Authentication strategy. pub(crate) strategy: String, @@ -182,6 +211,7 @@ pub(crate) struct ProjectedCredential { /// A reference to a Kubernetes Secret holding a credential value. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] pub(crate) struct ProjectedCredentialRef { /// Secret name. pub(crate) name: String, diff --git a/overlay-sync/src/validation.rs b/overlay-sync/src/validation.rs index cff86cc..3b38799 100644 --- a/overlay-sync/src/validation.rs +++ b/overlay-sync/src/validation.rs @@ -8,10 +8,13 @@ use std::{fmt, fmt::Write as _}; +#[cfg(test)] use serde::ser::Error as _; use sha2::{Digest as _, Sha256}; -use crate::types::{OverlayEnvelope, OverlayScope, RoutingOverlay}; +#[cfg(test)] +use crate::types::RoutingOverlay; +use crate::types::{OverlayEnvelope, OverlayScope}; // --------------------------------------------------------------------------- // Constants @@ -126,15 +129,12 @@ pub(crate) fn validate_envelope( }); } - let envelope: OverlayEnvelope = serde_json::from_slice(raw).map_err(|e| ValidationError { - reason: RejectionReason::Malformed, - detail: format!("JSON parse failed: {e}"), - })?; + let (raw_value, envelope) = parse_envelope(raw)?; validate_schema_version(&envelope.schema_version)?; validate_scope(&envelope.scope, expected_scope)?; validate_revision_digest_agreement(&envelope)?; - validate_content_digest(&envelope)?; + validate_content_digest(&raw_value, &envelope)?; let revision = envelope.revision.value; @@ -153,6 +153,20 @@ pub(crate) fn validate_envelope( }) } +/// Parse the raw envelope while converting both parser failures into the +/// bounded validation error type. +fn parse_envelope(raw: &[u8]) -> Result<(serde_json::Value, OverlayEnvelope), ValidationError> { + let raw_value: serde_json::Value = serde_json::from_slice(raw).map_err(|e| ValidationError { + reason: RejectionReason::Malformed, + detail: format!("JSON parse failed: {e}"), + })?; + let envelope: OverlayEnvelope = serde_json::from_value(raw_value.clone()).map_err(|e| ValidationError { + reason: RejectionReason::Malformed, + detail: format!("envelope structure invalid: {e}"), + })?; + Ok((raw_value, envelope)) +} + /// Check that the schema version is supported. fn validate_schema_version(version: &str) -> Result<(), ValidationError> { if version.starts_with(SUPPORTED_MAJOR) { @@ -224,8 +238,15 @@ fn validate_revision_digest_agreement(envelope: &OverlayEnvelope) -> Result<(), } /// Recompute the semantic digest and verify it matches the envelope. -fn validate_content_digest(envelope: &OverlayEnvelope) -> Result<(), ValidationError> { - let recomputed = compute_semantic_digest(&envelope.overlay).map_err(|e| ValidationError { +fn validate_content_digest( + raw_envelope: &serde_json::Value, + envelope: &OverlayEnvelope, +) -> Result<(), ValidationError> { + let raw_overlay = raw_envelope.get("overlay").ok_or_else(|| ValidationError { + reason: RejectionReason::Malformed, + detail: "overlay payload is missing".to_owned(), + })?; + let recomputed = compute_raw_semantic_digest(raw_overlay).map_err(|e| ValidationError { reason: RejectionReason::Malformed, detail: format!("cannot canonicalize overlay: {e}"), })?; @@ -239,21 +260,64 @@ fn validate_content_digest(envelope: &OverlayEnvelope) -> Result<(), ValidationE Ok(()) } +/// Compute the semantic digest from the raw overlay JSON. +/// +/// The operator's semantic payload includes the complete candidate objects. +/// Computing from raw JSON keeps additive routing metadata digest-significant +/// even when this sidecar does not understand the new field yet. +fn compute_raw_semantic_digest(overlay: &serde_json::Value) -> Result { + let object = overlay + .as_object() + .ok_or_else(|| "overlay must be a JSON object".to_owned())?; + let network = object + .get("network") + .ok_or_else(|| "overlay.network is missing".to_owned())?; + let local_site = object + .get("local_site") + .ok_or_else(|| "overlay.local_site is missing".to_owned())?; + let candidates = object + .get("candidates") + .ok_or_else(|| "overlay.candidates is missing".to_owned())?; + let mut semantic_payload = serde_json::json!({ + "candidates": candidates, + "local_site": local_site, + "network": network, + }); + if let Some(selection_policy) = object.get("selection_policy") { + semantic_payload + .as_object_mut() + .ok_or_else(|| "semantic payload is not an object".to_owned())? + .insert("selection_policy".to_owned(), selection_policy.clone()); + } + let canonical = serde_json_canonicalizer::to_vec(&semantic_payload) + .map_err(|e| format!("RFC 8785 canonicalization failed: {e}"))?; + let digest: [u8; 32] = Sha256::digest(&canonical).into(); + Ok(hex_encode(&digest)) +} + // --------------------------------------------------------------------------- // Digest computation (same algorithm as the operator) // --------------------------------------------------------------------------- /// Compute the SHA-256 digest of the RFC 8785 canonical semantic payload. /// -/// The semantic payload includes only `network`, `local_site`, and -/// `candidates` — the same fields used by the operator. +/// The semantic payload includes `network`, `local_site`, and `candidates`, +/// plus `selection_policy` when present — the same fields used by the +/// operator. +#[cfg(test)] fn compute_semantic_digest(overlay: &RoutingOverlay) -> Result { let candidates_value = serde_json::to_value(&overlay.candidates)?; - let semantic_payload = serde_json::json!({ + let mut semantic_payload = serde_json::json!({ "candidates": candidates_value, "local_site": overlay.local_site, "network": overlay.network, }); + if let Some(policy) = &overlay.selection_policy { + let Some(object) = semantic_payload.as_object_mut() else { + return Err(serde_json::Error::custom("semantic payload must be an object")); + }; + object.insert("selection_policy".to_owned(), serde_json::to_value(policy)?); + } let canonical = serde_json_canonicalizer::to_vec(&semantic_payload).map_err(serde_json::Error::custom)?; @@ -308,7 +372,9 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }], + selection_policy: None, generated_at: Some("2026-07-29T00:00:00Z".to_owned()), } } @@ -354,6 +420,50 @@ mod tests { result.unwrap(); } + #[test] + fn additive_candidate_field_is_digest_significant_and_accepted() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["candidates"][0]["future_field"] = serde_json::json!({"mode": "x"}); + let digest = compute_raw_semantic_digest(&value["overlay"]).unwrap(); + value["revision"]["value"] = serde_json::Value::String(digest.clone()); + value["content_digest"]["value"] = serde_json::Value::String(digest); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(result.is_ok(), "additive candidate metadata should survive validation"); + } + + #[test] + fn additive_candidate_field_mutation_without_digest_update_is_rejected() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["candidates"][0]["future_field"] = serde_json::json!({"mode": "x"}); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(matches!(result.unwrap_err().reason, RejectionReason::DigestMismatch)); + } + + #[test] + fn credential_value_fields_remain_rejected() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["candidates"][0]["credential"] = serde_json::json!({ + "strategy": "bearer_token", + "secretRef": { + "name": "credential", + "namespace": "ns", + "key": "token" + }, + "token": "must-not-enter-overlay" + }); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(matches!(result.unwrap_err().reason, RejectionReason::Malformed)); + } + #[test] fn oversized_payload_rejected() { let env = valid_envelope(); @@ -437,6 +547,37 @@ mod tests { ); } + #[test] + fn selection_policy_fixture_digest_matches_operator_contract() { + let fixture = include_str!("../../tests/fixtures/overlay-contract/v1/valid-selection-policy.json"); + let envelope: OverlayEnvelope = serde_json::from_str(fixture).unwrap(); + let recomputed = compute_semantic_digest(&envelope.overlay).unwrap(); + assert_eq!( + recomputed, envelope.revision.value, + "selection policy fixture must match the producer digest; computed={recomputed}" + ); + let scope = test_scope(); + validate_envelope(fixture.as_bytes(), &scope, 1_048_576, None) + .expect("selection policy fixture must pass transparent validation"); + } + + #[test] + fn unknown_selection_policy_field_is_rejected() { + let env = valid_envelope(); + let mut value = serde_json::to_value(env).unwrap(); + value["overlay"]["selection_policy"] = serde_json::json!({ + "mode": "roundRobin", + "unexpected": true + }); + let digest = compute_raw_semantic_digest(&value["overlay"]).unwrap(); + value["revision"]["value"] = serde_json::Value::String(digest.clone()); + value["content_digest"]["value"] = serde_json::Value::String(digest); + + let raw = serde_json::to_vec(&value).unwrap(); + let result = validate_envelope(&raw, &test_scope(), 1_048_576, None); + assert!(matches!(result.unwrap_err().reason, RejectionReason::Malformed)); + } + #[test] fn hex_encode_lowercase() { let bytes = [0xAB, 0xCD, 0x01, 0x23]; diff --git a/overlay-sync/src/watcher.rs b/overlay-sync/src/watcher.rs index 9a41e83..1316321 100644 --- a/overlay-sync/src/watcher.rs +++ b/overlay-sync/src/watcher.rs @@ -481,22 +481,47 @@ mod tests { score: None, score_breakdown: None, rank: Some(0), + selection_group: None, }], + selection_policy: None, generated_at: Some("2026-07-29T00:00:00Z".to_owned()), } } fn overlay_digest(overlay: &crate::types::RoutingOverlay) -> String { - let canonical = serde_json::json!({ + let mut semantic_payload = serde_json::json!({ "candidates": serde_json::to_value(&overlay.candidates).unwrap(), "local_site": overlay.local_site, "network": overlay.network, }); - let canonical_bytes = serde_json_canonicalizer::to_vec(&canonical).unwrap(); + if let Some(policy) = &overlay.selection_policy { + semantic_payload + .as_object_mut() + .unwrap() + .insert("selection_policy".to_owned(), serde_json::to_value(policy).unwrap()); + } + let canonical_bytes = serde_json_canonicalizer::to_vec(&semantic_payload).unwrap(); let digest: [u8; 32] = sha2::Sha256::new().chain_update(&canonical_bytes).finalize().into(); digest.iter().map(|b| format!("{b:02x}")).collect() } + #[test] + fn overlay_digest_includes_selection_policy() { + let scope = ExpectedScope { + network: "test-net".to_owned(), + gateway: "gw".to_owned(), + namespace: "ns".to_owned(), + local_site: "site-a".to_owned(), + }; + let without_policy = test_overlay(&scope); + let mut with_policy = without_policy.clone(); + with_policy.selection_policy = Some(crate::types::SelectionPolicy { + mode: crate::types::SelectionMode::RoundRobin, + }); + + assert_ne!(overlay_digest(&without_policy), overlay_digest(&with_policy)); + } + fn test_provenance() -> crate::types::OverlayProvenance { crate::types::OverlayProvenance { producer: "grid-operator".to_owned(), diff --git a/tests/e2e/topologies/grid-combined-site/forge.yaml b/tests/e2e/topologies/grid-combined-site/forge.yaml index 7686b30..a3bc1cb 100644 --- a/tests/e2e/topologies/grid-combined-site/forge.yaml +++ b/tests/e2e/topologies/grid-combined-site/forge.yaml @@ -210,6 +210,11 @@ spec: gridId: grid-combined-site-v1 region: west zone: west-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin swim: probeInterval: "5s" suspicionTimeout: "15s" @@ -268,6 +273,11 @@ spec: gridId: grid-combined-site-v1 region: central zone: central-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin swim: probeInterval: "5s" suspicionTimeout: "15s" @@ -326,6 +336,11 @@ spec: gridId: grid-combined-site-v1 region: east zone: east-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin swim: probeInterval: "5s" suspicionTimeout: "15s" diff --git a/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml b/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml index debf2d2..1a68c7f 100644 --- a/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml +++ b/tests/e2e/topologies/grid-combined-site/resources/common/vcr-provider-workload.yaml @@ -36,6 +36,8 @@ spec: - ALL readOnlyRootFilesystem: false runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 env: - name: MODEL value: "Qwen/Qwen3-0.6B" diff --git a/tests/e2e/topologies/grid-provider-traffic/README.md b/tests/e2e/topologies/grid-provider-traffic/README.md new file mode 100644 index 0000000..5cea41d --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/README.md @@ -0,0 +1,232 @@ +# Provider Traffic Selection + +This topology demonstrates Grid publishing a provider-selection contract and Praxis applying that contract locally for each request. It creates three Kind clusters, three independently attributable provider gateways, and one consumer gateway. + +The focused proof sends 60 requests without session affinity. With +`selectionPolicy.mode: roundRobin`, the expected result is a repeating +three-provider cycle and exactly 20 responses from each provider. The first +measured request can use any provider because readiness probes may already have +advanced the gateway-local cursor. + +This scenario tests provider selection across Grid sites. It does not test load balancing among replicas hidden behind one provider gateway, distributed token quotas, or cloud bursting. + +## Topology + +```mermaid +flowchart TB + client[Client requests] + + subgraph a[Kind cluster: provider-a] + consumer[Consumer gateway
intelligent_route] + operatorA[Grid operator] + gatewayA[Provider gateway A] + simulatorA[VCR simulator A] + operatorA -->|accepted overlay| consumer + gatewayA --> simulatorA + end + + subgraph b[Kind cluster: provider-b] + operatorB[Grid operator] + gatewayB[Provider gateway B] + simulatorB[VCR simulator B] + gatewayB --> simulatorB + end + + subgraph c[Kind cluster: provider-c] + operatorC[Grid operator] + gatewayC[Provider gateway C] + simulatorC[VCR simulator C] + gatewayC --> simulatorC + end + + client --> consumer + consumer -->|candidate A| gatewayA + consumer -->|candidate B over mTLS| gatewayB + consumer -->|candidate C over mTLS| gatewayC + + operatorA <-->|SWIM state| operatorB + operatorB <-->|SWIM state| operatorC + operatorC <-->|SWIM state| operatorA +``` + +Only `provider-a` runs a consumer gateway. Every cluster runs a Grid operator, a provider gateway, and a VCR inference simulator. SWIM distributes provider state between the operators. Each operator reconciles that state into a versioned overlay; the consumer uses its local accepted copy. + +## Who makes each decision? + +```mermaid +flowchart LR + state[Provider health, trust,
admission, and site state] + grid[Grid reconciliation] + overlay[Versioned routing overlay
groups + selection policy] + praxis[Praxis intelligent_route
accepted in-memory snapshot] + provider[Selected provider gateway] + backend[Provider-local simulator] + + state --> grid --> overlay --> praxis --> provider --> backend +``` + +Grid decides which candidates are eligible, their priority group, and the selection policy published in the overlay. Praxis chooses a candidate at request time from that already-accepted snapshot. Requests do not call Grid, Kubernetes, SWIM, or a metrics service. + +The provider gateway then resolves its configured local backend. That is a separate routing boundary: Grid selects provider gateways, not individual inference replicas hidden behind them. + +## Request decision flow + +```mermaid +flowchart TD + request[Request arrives at consumer] + snapshot{Accepted overlay available?} + model{Matching model candidates?} + affinity{Permitted session binding?} + group[Find first viable selection group] + mode{Selection mode} + deterministic[Choose first ranked candidate] + roundRobin[Choose next candidate in local cycle] + random[Choose uniformly from the group] + forward[Forward to selected provider gateway] + reject[Return routing error] + + request --> snapshot + snapshot -->|no| reject + snapshot -->|yes| model + model -->|no| reject + model -->|yes| affinity + affinity -->|yes| forward + affinity -->|no| group --> mode + mode -->|deterministic| deterministic --> forward + mode -->|roundRobin| roundRobin --> forward + mode -->|random| random --> forward +``` + +The proof uses unbound requests so session affinity cannot pin the sequence to one provider. All three candidates are fresh, admitted, and assigned to `selection_group: 0`. + +## Configuration + +The topology configures the Grid network with an explicit policy: + +```yaml +gridNetwork: + name: grid-provider-traffic + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin +``` + +`scoreFirst` with `noMetrics` puts the three fresh, admitted providers in the same active group. `roundRobin` then gives each candidate an equal turn. Scores are not weights, and lower-priority groups do not participate while this group remains viable. + +The complete environment is in [`forge.yaml`](./forge.yaml). Supporting files are intentionally local to this topology: + +| Path | Purpose | +|---|---| +| [`configs/consumer/praxis.yaml`](./configs/consumer/praxis.yaml) | Consumer filter chain, accepted overlay, provider-hop clusters, and mTLS endpoints | +| [`configs/provider/praxis.yaml`](./configs/provider/praxis.yaml) | Provider identity validation, exact candidate routing, credential injection, and response attribution | +| [`resources/common/vcr-provider-workload.yaml`](./resources/common/vcr-provider-workload.yaml) | One attributed VCR backend per provider cluster | +| [`resources/common/backend-network-policy.yaml`](./resources/common/backend-network-policy.yaml) | Restricts backend access to the provider gateway | + +## Run the proof + +Prerequisites include Docker, Kind, `kubectl`, Helm, OpenSSL, and `praxis-forge`. Build the Grid operator and Praxis AI gateway images before using the default `Never` pull policy. + +Build `praxis-forge` and the two source images from clean Grid and AI checkouts: + +```console +# From the Grid repository. +cargo build -p forge +docker build -f deploy/operator/Containerfile \ + -t grid-operator:provider-traffic-demo . + +# From the Praxis AI repository. +docker build -f Containerfile \ + -t praxis-ai:provider-traffic-demo . +``` + +The Forge binary is written to `target/debug/praxis-forge`. Add that directory +to `PATH` or invoke the binary by its full path. Verify the environment before +creating clusters: + +```console +target/debug/praxis-forge config validate \ + --config tests/e2e/topologies/grid-provider-traffic/forge.yaml +cargo test -p xtask provider_traffic --locked +``` + +```console +export GRID_XTASK_OPERATOR_IMAGE=grid-operator:provider-traffic-demo +export GRID_XTASK_GATEWAY_IMAGE=praxis-ai:provider-traffic-demo +export GRID_XTASK_VCR_IMAGE=ghcr.io/neuralmagic/vllm-vcr:vllm0.23 +export GRID_XTASK_IMAGE_PULL_POLICY=Never + +cargo xtask env run-grid-provider-traffic-demo \ + --forge-config tests/e2e/topologies/grid-provider-traffic/forge.yaml \ + --quick \ + --teardown +``` + +For registry-hosted images, use immutable tags or digests and set +`GRID_XTASK_IMAGE_PULL_POLICY=IfNotPresent`. Do not reuse evidence from a run +that used different source commits or image digests. + +## Expected result + +A qualifying run demonstrates: + +- three Kind clusters become healthy; +- SWIM discovers and authorizes the remote sites; +- the accepted overlay contains three stable provider candidates; +- every candidate is in selection group `0`; +- the overlay publishes `selection_policy.mode: roundRobin`; +- all 60 requests return successfully; +- attribution follows a rotation-equivalent three-provider cycle, such as + `provider-a`, `provider-b`, `provider-c` or `provider-b`, `provider-c`, + `provider-a`; +- each provider serves exactly 20 requests; +- the semantic overlay revision remains stable during traffic; +- teardown removes the clusters and shared network. + +Provider identity comes from request-scoped HTTP response attribution, not only +from logs or expected configuration. A balanced count without a repeating +ordered cycle is insufficient for the strict round-robin proof. The cycle's +starting provider is not significant. + +## Example sequence + +```mermaid +sequenceDiagram + participant C as Client + participant E as Consumer gateway + participant A as Provider A + participant B as Provider B + participant P as Provider C + + Note over E: Earlier readiness probes may have advanced the local cursor + C->>E: First measured request + E->>A: Next candidate in group 0 + A-->>C: 200 + provider-a attribution + C->>E: Request 2 + E->>B: Next candidate in group 0 + B-->>C: 200 + provider-b attribution + C->>E: Request 3 + E->>P: Next candidate in group 0 + P-->>C: 200 + provider-c attribution + Note over E: Local cursor returns to provider-a +``` + +## Troubleshooting + +| Symptom | Check | +|---|---| +| Forge stops before cluster creation | Validate `forge.yaml` and confirm `praxis-forge` is installed | +| A local image is missing | Build the configured tag or use registry images with a pull-enabled policy | +| Only one provider receives traffic | Confirm requests have no reusable session ID and all candidates are in group `0` | +| Remote providers do not appear | Check SWIM addresses, discovered `GridSite` resources, fingerprints, and trust authorization | +| A provider rejects the request | Check candidate stable ID, model/path validation, mTLS identity, and credential mounting | +| Counts are balanced but not cyclic | Confirm `roundRobin`, a stable overlay, and one consumer process for the measured sequence; any rotation of A/B/C is valid | +| A request hangs | Preserve the failure; do not discard or silently retry it | + +## Related documentation + +- [Provider Selection and Load Balancing](../../../../docs/architecture/provider-selection-and-load-balancing.md) +- [Routing](../../../../docs/architecture/routing.md) +- [Provider Scoring](../../../../docs/architecture/scoring.md) +- [Consumer Config](../../../../docs/architecture/consumer-config.md) diff --git a/tests/e2e/topologies/grid-provider-traffic/configs/consumer/praxis.yaml b/tests/e2e/topologies/grid-provider-traffic/configs/consumer/praxis.yaml new file mode 100644 index 0000000..bc77426 --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/configs/consumer/praxis.yaml @@ -0,0 +1,75 @@ +listeners: + - name: proxy + address: "0.0.0.0:8080" + filter_chains: + - main + +filter_chains: + - name: main + filters: + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Provider-Traffic-Consumer-Gateway + value: "{{ cluster.name }}" + - filter: intelligent_route + overlay_file: /etc/praxis/routing/routing-overlay.json + model_header: X-Model + provider_hop_clusters: + - vcr-provider-a-provider + - vcr-provider-b-provider + - vcr-provider-c-provider + expected_overlay_scope: + network: grid-provider-traffic + gateway: consumer-gateway + namespace: grid-system + local_site: "{{ cluster.name }}" + reload: + enabled: true + debounce_ms: 500 + session_affinity: + enabled: true + header: X-Session-Id + ttl_secs: 3600 + - filter: load_balancer + clusters: + - name: vcr-provider-a-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-a.grid.internal + verify: true + endpoints: + - "{{ captures.provider-a.provider-gateway-ip }}:8443" + - name: vcr-provider-b-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-b.grid.internal + verify: true + endpoints: + - "{{ captures.provider-b.provider-gateway-ip }}:8443" + - name: vcr-provider-c-provider + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-c.grid.internal + verify: true + endpoints: + - "{{ captures.provider-c.provider-gateway-ip }}:8443" + +admin: + address: "127.0.0.1:9901" + +shutdown_timeout_secs: 5 diff --git a/tests/e2e/topologies/grid-provider-traffic/configs/provider/praxis.yaml b/tests/e2e/topologies/grid-provider-traffic/configs/provider/praxis.yaml new file mode 100644 index 0000000..aa41394 --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/configs/provider/praxis.yaml @@ -0,0 +1,60 @@ +# Provider gateway fixture used by the provider-traffic xtask tests. +listeners: + - name: provider + address: "0.0.0.0:8443" + filter_chains: + - provider-inference + tls: + certificates: + - cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + client_ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert_mode: require + +filter_chains: + - name: provider-inference + filters: + - filter: peer_identity_trust + trusted_peers: + - organization: ai-grid + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Provider-Traffic-Provider-Gateway + value: "SITE_PLACEHOLDER" + - filter: provider_route + provider_id: "SITE_PLACEHOLDER" + model_header: X-Model + emit_demo_attribution: true + routes: + - candidate_id: "CANDIDATE_ID_PLACEHOLDER" + model: Qwen/Qwen3-0.6B + paths: + - /v1/chat/completions + - /v1/responses + cluster: vcr-backend + credential: + strategy: bearer_token + secretRef: + name: vcr-inference-credential + namespace: grid-system + key: token + - filter: credential_inject + credentials: + - strategy: bearer_token + name: vcr-inference-credential + namespace: grid-system + key: token + file: /etc/praxis/credentials/vcr-inference/token + - filter: load_balancer + clusters: + - name: vcr-backend + endpoints: + - "vcr-inference-SITE_PLACEHOLDER.grid-system.svc.cluster.local:8000" + +admin: + address: "127.0.0.1:9901" +shutdown_timeout_secs: 5 diff --git a/tests/e2e/topologies/grid-provider-traffic/forge.yaml b/tests/e2e/topologies/grid-provider-traffic/forge.yaml new file mode 100644 index 0000000..972f6c2 --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/forge.yaml @@ -0,0 +1,511 @@ +apiVersion: forge.praxis.dev/v1alpha1 +kind: Environment + +metadata: + name: grid-provider-traffic + +spec: + runtime: + provider: docker + clusterPrefix: grid-provider-traffic + + network: + crossCluster: true + dnsZone: grid-provider-traffic.test + + clusters: + - name: provider-a + stacks: [metallb, provider-a-operator-base, vcr-backend, provider-a-site, provider-gateway, consumer-gateway] + properties: + region: provider-a + role: combined + siteName: provider-a + gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" + imagePullPolicy: IfNotPresent + gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" + gatewayImageTag: "v0.1.3" + operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" + operatorImageTag: "v0.1.3" + + - name: provider-b + stacks: [metallb, provider-b-operator-base, vcr-backend, provider-b-site, provider-gateway] + properties: + region: provider-b + role: combined + siteName: provider-b + gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" + imagePullPolicy: IfNotPresent + gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" + gatewayImageTag: "v0.1.3" + operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" + operatorImageTag: "v0.1.3" + + - name: provider-c + stacks: [metallb, provider-c-operator-base, vcr-backend, provider-c-site, provider-gateway] + properties: + region: provider-c + role: combined + siteName: provider-c + gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" + imagePullPolicy: IfNotPresent + gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" + gatewayImageTag: "v0.1.3" + operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" + operatorImageTag: "v0.1.3" + + stacks: + metallb: + description: MetalLB load balancer with auto-configured address pool + steps: + - type: url + url: https://raw.githubusercontent.com/metallb/metallb/v0.14.9/config/manifests/metallb-native.yaml + sha256: 951065e85692aa106f1bb5d5a487d9306154923a794ab1d82122881cbaf588e4 + - type: wait + resource: deployment/controller + namespace: metallb-system + condition: available + timeout: "120s" + - type: metallb-auto-pool + name: forge-pool + + provider-a-operator-base: + description: West Grid operator initial deployment with SWIM LoadBalancer + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "provider-a" + seeds: "" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/grid-operator-swim + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: swim-lb-ip + timeout: "60s" + interval: "2s" + + provider-b-operator-base: + description: Central Grid operator initial deployment with SWIM LoadBalancer + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "provider-b" + seeds: "" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/grid-operator-swim + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: swim-lb-ip + timeout: "60s" + interval: "2s" + + provider-c-operator-base: + description: East Grid operator initial deployment with SWIM LoadBalancer + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: "0.1.0" + namespace: grid-system + values: + image: + repository: "{{ cluster.properties.operatorImageRepo }}" + tag: "{{ cluster.properties.operatorImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + swim: + siteName: "provider-c" + seeds: "" + service: + enabled: true + type: "LoadBalancer" + gateway: + serviceName: "provider-gateway" + port: "8443" + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/grid-operator-swim + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: swim-lb-ip + timeout: "60s" + interval: "2s" + + vcr-backend: + description: vllm-vcr inference backend for demo scenarios + steps: + - type: manifest + path: resources/common/grid-system-namespace.yaml + - type: template-manifest + path: resources/common/vcr-provider-workload.yaml + - type: manifest + path: resources/common/backend-network-policy.yaml + - type: wait + resource: deployment/vcr-inference-{{ cluster.name }} + namespace: grid-system + condition: available + timeout: "120s" + + provider-a-site: + description: West combined site with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site + chart: charts/grid-site + version: "0.1.0" + namespace: grid-system + values: + commonLabels: + grid.praxis-proxy.io/auto-discover-sites: "true" + gridNetwork: + name: grid-provider-traffic + gridId: grid-provider-traffic-v1 + region: provider-a + zone: provider-a-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: consumer-gateway + namespace: grid-system + localSiteName: provider-a + gridSite: + name: provider-a + region: provider-a + zone: provider-a-1 + providerSiteLabel: provider-a + inferenceProviders: + - name: vcr-provider-a-provider + gridNetworkRef: grid-provider-traffic + providerKind: vllm-vcr + backendKind: local + endpoint: "http://vcr-inference-provider-a.grid-system.svc.cluster.local:8000" + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: provider-a + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: "30s" + timeout: "5s" + + provider-b-site: + description: Central combined site with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site + chart: charts/grid-site + version: "0.1.0" + namespace: grid-system + values: + commonLabels: + grid.praxis-proxy.io/auto-discover-sites: "true" + gridNetwork: + name: grid-provider-traffic + gridId: grid-provider-traffic-v1 + region: provider-b + zone: provider-b-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: consumer-gateway + namespace: grid-system + localSiteName: provider-b + gridSite: + name: provider-b + region: provider-b + zone: provider-b-1 + providerSiteLabel: provider-b + inferenceProviders: + - name: vcr-provider-b-provider + gridNetworkRef: grid-provider-traffic + providerKind: vllm-vcr + backendKind: local + endpoint: "http://vcr-inference-provider-b.grid-system.svc.cluster.local:8000" + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: provider-b + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: "30s" + timeout: "5s" + + provider-c-site: + description: East combined site with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site + chart: charts/grid-site + version: "0.1.0" + namespace: grid-system + values: + commonLabels: + grid.praxis-proxy.io/auto-discover-sites: "true" + gridNetwork: + name: grid-provider-traffic + gridId: grid-provider-traffic-v1 + region: provider-c + zone: provider-c-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + swim: + probeInterval: "5s" + suspicionTimeout: "15s" + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: consumer-gateway + namespace: grid-system + localSiteName: provider-c + gridSite: + name: provider-c + region: provider-c + zone: provider-c-1 + providerSiteLabel: provider-c + inferenceProviders: + - name: vcr-provider-c-provider + gridNetworkRef: grid-provider-traffic + providerKind: vllm-vcr + backendKind: local + endpoint: "http://vcr-inference-provider-c.grid-system.svc.cluster.local:8000" + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: provider-c + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: "30s" + timeout: "5s" + + provider-gateway: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: helm + release: provider-gateway + chart: charts/praxis-gateway + version: "0.1.0" + namespace: grid-system + values: + fullnameOverride: "provider-gateway" + image: + repository: "{{ cluster.properties.gatewayImageRepo }}" + tag: "{{ cluster.properties.gatewayImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: "provider-gateway-config" + port: + containerPort: 8443 + name: "https-mtls" + service: + type: "LoadBalancer" + port: 8443 + health: + readiness: + tcpSocket: + port: "https-mtls" + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: "https-mtls" + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: "provider-gateway-tls" + credentials: + - name: "vcr-inference-credential" + mountPath: "/etc/praxis/credentials/vcr-inference" + podLabels: + grid.praxis-proxy.io/backend-access: "provider-gateway" + grid.praxis-proxy.io/provider-site: "{{ cluster.name }}" + - type: wait + resource: deployment/provider-gateway + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/provider-gateway + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: provider-gateway-ip + timeout: "60s" + interval: "2s" + + consumer-gateway: + description: Praxis consumer gateway with operator-managed overlay + steps: + - type: template-file + source: configs/consumer/praxis.yaml + target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + - type: exec + command: + - bash + - -c + - >- + kubectl --context kind-grid-provider-traffic-{{ cluster.name }} -n grid-system + create configmap consumer-gateway-config + --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + --dry-run=client -o yaml | + kubectl --context kind-grid-provider-traffic-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway + chart: charts/praxis-gateway + version: "0.1.0" + namespace: grid-system + values: + fullnameOverride: "consumer-gateway" + image: + repository: "{{ cluster.properties.gatewayImageRepo }}" + tag: "{{ cluster.properties.gatewayImageTag }}" + pullPolicy: "{{ cluster.properties.imagePullPolicy }}" + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: "consumer-gateway-config" + service: + type: "LoadBalancer" + overlay: + enabled: true + existingConfigMap: "grid-overlay-grid-provider-traffic-consumer-gateway" + tls: + enabled: true + existingSecret: "consumer-gateway-tls" + podLabels: + grid.praxis-proxy.io/consumer-site: "{{ cluster.name }}" + - type: wait + resource: deployment/consumer-gateway + namespace: grid-system + condition: available + timeout: "120s" + - type: capture + resource: svc/consumer-gateway + namespace: grid-system + jsonpath: "{.status.loadBalancer.ingress[0].ip}" + key: consumer-gateway-ip + timeout: "60s" + interval: "2s" diff --git a/tests/e2e/topologies/grid-provider-traffic/resources/common/backend-network-policy.yaml b/tests/e2e/topologies/grid-provider-traffic/resources/common/backend-network-policy.yaml new file mode 100644 index 0000000..f0cecbd --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/resources/common/backend-network-policy.yaml @@ -0,0 +1,43 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: vcr-inference-allow-provider-gateway-only + namespace: grid-system + labels: + app.kubernetes.io/part-of: grid-provider-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: vllm-vcr + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + grid.praxis-proxy.io/backend-access: "provider-gateway" + ports: + - protocol: TCP + port: 8000 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: vcr-inference-allow-grid-operator-health + namespace: grid-system + labels: + app.kubernetes.io/part-of: grid-provider-traffic +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: vllm-vcr + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: grid-operator + ports: + - protocol: TCP + port: 8000 diff --git a/tests/e2e/topologies/grid-provider-traffic/resources/common/grid-system-namespace.yaml b/tests/e2e/topologies/grid-provider-traffic/resources/common/grid-system-namespace.yaml new file mode 100644 index 0000000..b816c84 --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/resources/common/grid-system-namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: grid-system + labels: + name: grid-system + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted diff --git a/tests/e2e/topologies/grid-provider-traffic/resources/common/vcr-provider-workload.yaml b/tests/e2e/topologies/grid-provider-traffic/resources/common/vcr-provider-workload.yaml new file mode 100644 index 0000000..6194e9b --- /dev/null +++ b/tests/e2e/topologies/grid-provider-traffic/resources/common/vcr-provider-workload.yaml @@ -0,0 +1,117 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vcr-inference-{{ cluster.name }} + namespace: grid-system + labels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/part-of: grid-provider-traffic +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/instance: "{{ cluster.name }}" + template: + metadata: + labels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/instance: "{{ cluster.name }}" + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: vcr + image: "{{ cluster.properties.vcrImage }}" + imagePullPolicy: "{{ cluster.properties.imagePullPolicy }}" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + env: + - name: MODEL + value: "Qwen/Qwen3-0.6B" + - name: MOCK_PD_ROLE + value: "both" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: VLLM_PORT + value: "8000" + - name: MOCK_MAX_NUM_SEQS + value: "4" + - name: MOCK_KV_CACHE_SIZE + value: "64" + - name: MOCK_TOKENS_PER_BLOCK + value: "16" + - name: MOCK_MAX_MODEL_LEN + value: "512" + - name: MOCK_TTFT_MS + value: "50" + - name: MOCK_ITL_MS + value: "20" + - name: MOCK_TIME_FACTOR_UNDER_LOAD + value: "2.0" + ports: + - name: http + containerPort: 8000 + startupProbe: + httpGet: + path: /v1/models + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /health + port: 8000 + periodSeconds: 5 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /health + port: 8000 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + memory: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: vcr-inference-{{ cluster.name }} + namespace: grid-system + labels: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/part-of: grid-provider-traffic +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: vllm-vcr + app.kubernetes.io/instance: "{{ cluster.name }}" + ports: + - name: http + port: 8000 + targetPort: 8000 diff --git a/tests/fixtures/overlay-contract/v1/manifest.json b/tests/fixtures/overlay-contract/v1/manifest.json index f30813a..873b013 100644 --- a/tests/fixtures/overlay-contract/v1/manifest.json +++ b/tests/fixtures/overlay-contract/v1/manifest.json @@ -42,6 +42,11 @@ "expected": "accept", "revision": "75b057d750d9db77030ecd5a073c235c56b2b0460d3d517340b3e44020e83056" }, + "valid-selection-policy.json": { + "content_digest": "d3321586b5e414b8bd80c07a9d16ac10dce38d6810e3591a8526a6bdd0639007", + "expected": "accept", + "revision": "d3321586b5e414b8bd80c07a9d16ac10dce38d6810e3591a8526a6bdd0639007" + }, "malformed-envelope.json": { "expected": "reject", "reason": "malformed_structure" diff --git a/tests/fixtures/overlay-contract/v1/valid-selection-policy.json b/tests/fixtures/overlay-contract/v1/valid-selection-policy.json new file mode 100644 index 0000000..1bd8f46 --- /dev/null +++ b/tests/fixtures/overlay-contract/v1/valid-selection-policy.json @@ -0,0 +1,56 @@ +{ + "schema_version": "1.0.0", + "revision": { + "kind": "content_addressed", + "algorithm": "sha256", + "value": "d3321586b5e414b8bd80c07a9d16ac10dce38d6810e3591a8526a6bdd0639007" + }, + "content_digest": { + "algorithm": "sha256", + "value": "d3321586b5e414b8bd80c07a9d16ac10dce38d6810e3591a8526a6bdd0639007" + }, + "scope": { + "network": "test-net", + "gateway": "gw", + "namespace": "ns", + "local_site": "site-a" + }, + "provenance": { + "producer": "grid-operator", + "producer_version": "0.1.3", + "source_name": "test-net", + "source_uid": "uid-selection-policy", + "source_generation": 1, + "rendered_at": "2026-08-12T00:00:00Z" + }, + "overlay": { + "network": "test-net", + "local_site": "site-a", + "selection_policy": { + "mode": "roundRobin" + }, + "candidates": [ + { + "kind": "inference_model", + "name": "model-a", + "site": "site-a", + "cluster": "cluster-a", + "fresh": true, + "stable_id": "abcd1234", + "rank": 0, + "selection_group": 0 + }, + { + "kind": "inference_model", + "name": "model-b", + "site": "site-b", + "cluster": "cluster-b", + "fresh": true, + "stable_id": "efgh5678", + "rank": 1, + "selection_group": 0 + } + ], + "generated_at": "2026-08-12T00:00:00Z" + } +} diff --git a/xtask/src/env.rs b/xtask/src/env.rs index 947eb4a..ec11ebf 100644 --- a/xtask/src/env.rs +++ b/xtask/src/env.rs @@ -16,6 +16,7 @@ pub(crate) mod kubectl; pub(crate) mod llmd_pool_metrics_demo; pub(crate) mod operator; pub(crate) mod operator_overlay; +pub(crate) mod provider_traffic_demo; pub(crate) mod providers; pub(crate) mod trust; pub(crate) mod verify; @@ -997,6 +998,17 @@ pub(crate) enum Action { #[arg(long)] kv_cache: bool, }, + + /// Create the focused provider-gateway traffic demo, then prove equal + /// selection across its active provider group. + RunGridProviderTrafficDemo { + /// Path to the Forge environment config file. + #[arg(long, default_value = "tests/e2e/topologies/grid-provider-traffic/forge.yaml")] + forge_config: PathBuf, + /// Demo mode and teardown options. Only `--quick` is supported. + #[command(flatten)] + options: GlbDemoOptions, + }, } #[cfg(test)] @@ -1122,6 +1134,9 @@ pub(crate) fn run(action: &Action) -> Result<(), Box> { metrics_mtls, kv_cache, } => llmd_pool_metrics_demo::run(forge_config, options, *metrics_mtls, *kv_cache), + Action::RunGridProviderTrafficDemo { forge_config, options } => { + provider_traffic_demo::run(forge_config, options) + }, } } diff --git a/xtask/src/env/combined_site_demo.rs b/xtask/src/env/combined_site_demo.rs index 9b15b45..d98199f 100644 --- a/xtask/src/env/combined_site_demo.rs +++ b/xtask/src/env/combined_site_demo.rs @@ -3407,34 +3407,6 @@ fn require_local_image(image: &str) -> Result<(), Box> { .into()) } -/// Verify that `image` has a numeric USER (or UID:GID) so Kubernetes can -/// enforce `runAsNonRoot` without `runAsUser` in the pod spec. -fn require_numeric_image_user(image: &str) -> Result<(), Box> { - let output = Command::new("docker") - .args(["inspect", "--format", "{{.Config.User}}", image]) - .output()?; - if !output.status.success() { - return Err(format!( - "cannot inspect image {image:?}: {}", - String::from_utf8_lossy(&output.stderr).trim() - ) - .into()); - } - let user = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - if user.is_empty() { - return Err(format!("image {image:?} has no USER set; runAsNonRoot requires a numeric user").into()); - } - let valid = user.split(':').all(|part| part.parse::().is_ok()); - if !valid { - return Err(format!( - "image {image:?} has non-numeric USER {user:?}; \ - Kubernetes cannot verify runAsNonRoot with a non-numeric user" - ) - .into()); - } - Ok(()) -} - /// Load local container images into all Kind clusters. /// /// Reads image references from the `GRID_XTASK_*_IMAGE` environment variables @@ -3458,9 +3430,6 @@ fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result eprintln!(" verified local image: {image}"); } - require_numeric_image_user(&vcr)?; - eprintln!(" verified numeric USER for {vcr}"); - for cluster in CLUSTERS { for image in [&gateway, &operator, &vcr] { eprintln!(" loading {image} into {cluster}..."); diff --git a/xtask/src/env/provider_traffic_demo.rs b/xtask/src/env/provider_traffic_demo.rs new file mode 100644 index 0000000..65fcda0 --- /dev/null +++ b/xtask/src/env/provider_traffic_demo.rs @@ -0,0 +1,3039 @@ +//! Narrated, evidence-backed provider-traffic demo scenarios. +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::{Path, PathBuf}, + process::Command, + sync::atomic::{AtomicU64, Ordering}, + time::{Duration, Instant}, +}; + +use serde::Serialize; + +use super::{DemoMode, GlbDemoOptions, certs, glb, kubectl, operator, safe_truncate_str}; + +// ----------------------------------------------------------------------------- +// Constants +// ----------------------------------------------------------------------------- + +/// Directory where generated TLS certificates are stored. +const CERTS_DIR: &str = "tests/env/certs"; + +/// Ordered provider-site cluster names in the provider-traffic scenario. +/// +/// The consumer entrypoint is deployed only in `CONSUMER_SITE`; the other +/// clusters contain provider gateways and backends only. +const CLUSTERS: &[&str] = &["provider-a", "provider-b", "provider-c"]; + +/// The single consumer gateway used by the focused request-routing proof. +const CONSUMER_SITE: &str = "provider-a"; + +/// Consumer gateway TLS secret name (matches Helm `existingSecret` reference). +const CONSUMER_TLS_SECRET: &str = "consumer-gateway-tls"; + +/// Evidence JSON schema version. +const EVIDENCE_SCHEMA_VERSION: &str = "1"; + +/// Kubernetes namespace for all Grid components. +const GRID_SYSTEM_NS: &str = "grid-system"; + +/// Overlay `ConfigMap` name created by the Grid operator for consumer gateways. +const OVERLAY_CONFIGMAP: &str = "grid-overlay-grid-provider-traffic-consumer-gateway"; + +/// Provider credential secret name (matches Helm `credentials[0].name`). +const VCR_INFERENCE_CREDENTIAL: &str = "vcr-inference-credential"; + +/// Stable terminal separator that also remains readable in captured logs. +const OUTPUT_RULE: &str = "==============================================================================="; + +/// Provider gateway service name advertised via SWIM for cross-site discovery. +const PROVIDER_GATEWAY_SERVICE: &str = "provider-gateway"; + +/// Provider gateway port advertised via SWIM for cross-site discovery. +const PROVIDER_GATEWAY_PORT: &str = "8443"; + +/// Provider gateway TLS secret name (matches Helm `existingSecret` reference). +const PROVIDER_TLS_SECRET: &str = "provider-gateway-tls"; + +/// Same-CA client identity with an organization rejected by `peer_identity_trust`. +const WRONG_ORG_TLS_SECRET: &str = "wrong-org-client-tls"; + +/// Number of environment setup phases shown to the user. +const SETUP_PHASES: usize = 14; + +/// Makes retry probe names unique while retaining a recognizable prefix. +static PROBE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +// ----------------------------------------------------------------------------- +// Context +// ----------------------------------------------------------------------------- + +/// Provider-traffic demo execution context. +struct ProviderTrafficContext { + /// Canonical demo root directory for resolving configs, resources, and + /// other demo-relative assets. + demo_root: PathBuf, + /// Path to the resolved Forge config. + resolved_config: PathBuf, + /// Path to the forge binary. + forge_bin: PathBuf, +} + +// ----------------------------------------------------------------------------- +// Overlay State +// ----------------------------------------------------------------------------- + +/// Per-site overlay snapshot captured from the `ConfigMap`. +#[derive(Clone, Debug, serde::Deserialize, Serialize)] +struct OverlayData { + /// Kubernetes `ConfigMap` `resourceVersion` (per-cluster, never compared + /// across clusters). + resource_version: String, + /// Content-addressed semantic revision from the + /// `grid.praxis-proxy.io/overlay-revision` annotation. Changes only when + /// routing-relevant fields change; safe to compare across clusters. + semantic_revision: String, + /// Candidate name to stable ID mapping. + stable_ids: BTreeMap, + /// Full candidate details for evidence (kind, name, site, cluster, model). + candidates: Vec, +} + +/// Subset of `RoutingCandidate` fields kept for evidence and validation. +#[derive(Clone, Debug, serde::Deserialize, Serialize)] +struct OverlayCandidate { + /// Candidate kind (e.g. `inference_model`). + kind: String, + /// Model name. + name: String, + /// Site name. + site: String, + /// Upstream cluster identifier (used as the candidate key). + cluster: String, + /// Deterministic stable ID for session binding. + stable_id: String, + /// Whether the candidate's metrics snapshot is fresh. + fresh: Option, + /// Admission state emitted by the operator, when present. + admission_state: Option, + /// Explicit priority group emitted by the operator, when present. + selection_group: Option, +} + +/// Pre- and post-SWIM overlay snapshots for evidence. +#[derive(Clone, Debug, Default, serde::Deserialize, Serialize)] +struct OverlayState { + /// Per-site overlays before SWIM seeding (local candidates only). + pre_swim: BTreeMap, + /// Per-site overlays after global convergence. + post_swim: BTreeMap, +} + +// ----------------------------------------------------------------------------- +// Evidence +// ----------------------------------------------------------------------------- + +/// Evidence written to `results.json`. +#[derive(Debug, serde::Deserialize, Serialize)] +struct Evidence { + /// Evidence schema version. + schema_version: String, + /// Demo mode that was executed. + mode: String, + /// Topology name. + topology: String, + /// List of cluster names. + clusters: Vec, + /// Proof results for each assertion. + proof_results: BTreeMap, + /// Exact image references used. + images: BTreeMap, + /// Pre- and post-SWIM overlay snapshots. + overlay_state: OverlayState, + /// Cluster health status. + cluster_health: Vec, + /// Component deployment status. + components: Vec, + /// SWIM membership views. + swim_membership: Vec, + /// Provider response samples. + provider_responses: Vec, + /// Security assertion results. + security_results: Vec, + /// Teardown success. + teardown_success: bool, +} + +/// Evidence for one proof assertion. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ProofResult { + /// Whether the proof passed. + success: bool, + /// Human-readable reason. + reason: String, + /// Observed facts that support this result. + observed_facts: BTreeMap, + /// Duration of the assertion in milliseconds. + duration_ms: u64, +} + +/// Cluster health status. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ClusterHealth { + /// Cluster name. + name: String, + /// Whether the cluster is healthy. + healthy: bool, + /// API server response time in milliseconds. + api_response_ms: Option, + /// Number of ready nodes. + ready_nodes: u32, +} + +/// Component deployment status. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ComponentStatus { + /// Component name. + name: String, + /// Deployment namespace. + namespace: String, + /// Ready replicas. + ready_replicas: u32, + /// Desired replicas. + desired_replicas: u32, + /// Whether the component is ready. + ready: bool, +} + +/// SWIM membership view. +#[derive(Debug, serde::Deserialize, Serialize)] +struct SwimMembership { + /// Site name. + site: String, + /// Local node ID. + local_node: String, + /// List of known peers. + peers: Vec, + /// Membership convergence status. + converged: bool, +} + +/// Provider response metadata. +#[derive(Debug, serde::Deserialize, Serialize)] +struct ProviderResponse { + /// Consumer site that made the request. + consumer_site: String, + /// Provider site that served the request. + provider_site: String, + /// Provider instance ID. + provider_instance: String, + /// Session ID. + session_id: String, + /// Serving revision. + serving_revision: String, + /// Response time in milliseconds. + response_time_ms: u64, + /// Whether the response was successful. + success: bool, +} + +/// Security assertion result. +#[derive(Debug, serde::Deserialize, Serialize)] +struct SecurityResult { + /// Type of security test. + test_type: String, + /// Expected result (allow/deny). + expected: String, + /// Actual result. + actual: String, + /// Whether the test passed. + passed: bool, + /// Additional context. + context: String, +} + +// ----------------------------------------------------------------------------- +// Utility Functions +// ----------------------------------------------------------------------------- + +/// Format current UTC timestamp for run IDs. +fn format_utc_timestamp() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or_else(|_| "unknown".to_owned(), |duration| format!("{}", duration.as_secs())) +} + +/// Format current UTC timestamp in ISO format. +fn format_utc_iso() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now().duration_since(UNIX_EPOCH).map_or_else( + |_| "unknown-utc".to_owned(), + |duration| format!("{}-utc", duration.as_secs()), + ) +} + +/// Resolve the evidence directory path. +fn resolve_evidence_dir( + forge_config: &Path, + options: &GlbDemoOptions, + run_id: &str, +) -> Result> { + if let Some(dir) = &options.evidence_dir { + Ok(dir.clone()) + } else { + let config_dir = forge_config + .parent() + .ok_or("forge config should have parent directory")?; + Ok(config_dir.join(format!("evidence-{run_id}"))) + } +} + +// ----------------------------------------------------------------------------- +// Assertion Framework +// ----------------------------------------------------------------------------- + +/// Result of a runtime assertion. +type AssertionResult = Result>; + +/// Create a successful proof result with observed facts. +fn proof_success(reason: &str, observed_facts: BTreeMap, duration: Duration) -> ProofResult { + ProofResult { + success: true, + reason: reason.to_owned(), + observed_facts, + duration_ms: u64::try_from(duration.as_millis().min(u128::from(u64::MAX))).unwrap_or(u64::MAX), + } +} + +/// Create a failed proof result with observed facts. +fn proof_failure(reason: &str, observed_facts: BTreeMap, duration: Duration) -> ProofResult { + ProofResult { + success: false, + reason: reason.to_owned(), + observed_facts, + duration_ms: u64::try_from(duration.as_millis().min(u128::from(u64::MAX))).unwrap_or(u64::MAX), + } +} + +/// Execute an assertion with timing and error handling. +fn run_assertion(name: &str, assertion_fn: F) -> AssertionResult +where + F: FnOnce() -> AssertionResult, +{ + let start = Instant::now(); + eprintln!(" [ASSERT] {name}"); + + let result = assertion_fn(); + let _duration = start.elapsed(); + + match &result { + Ok(proof) => { + if proof.success { + eprintln!(" [OK] {name}: {}", proof.reason); + } else { + eprintln!(" [FAIL] {name}: {}", proof.reason); + } + }, + Err(e) => { + eprintln!(" [ERROR] {name}: {e}"); + }, + } + + result.map_err(|e| { + // Convert assertion errors to proof failures + format!("Assertion {name} failed: {e}").into() + }) +} + +/// Poll for a condition with bounded retries. +fn poll_until(condition: F, timeout: Duration, interval: Duration) -> Result> +where + F: Fn() -> Result, Box>, +{ + let start = Instant::now(); + + loop { + match condition() { + Ok(Some(result)) => return Ok(result), + Ok(None) => { + if start.elapsed() >= timeout { + return Err("Timeout waiting for condition".into()); + } + std::thread::park_timeout(interval); + }, + Err(e) => return Err(e), + } + } +} + +/// Wait for a deployment to be ready. +fn wait_for_deployment(deployment: &str, namespace: &str, context: &str) -> Result<(), Box> { + kubectl::wait_for_rollout_ns(context, deployment, namespace, "deployment")?; + Ok(()) +} + +/// Pod-security overrides for ephemeral curl pods in restricted namespaces. +/// +/// `kubectl run` names the container after the pod, so the container name must +/// match. The curlimages/curl image runs as UID 100. +fn curl_pod_overrides(pod_name: &str, curl_args: &[&str]) -> String { + let args = curl_args.strip_prefix(&["curl"]).unwrap_or(curl_args); + serde_json::json!({ + "spec": { + "automountServiceAccountToken": false, + "securityContext": { + "runAsNonRoot": true, + "seccompProfile": { "type": "RuntimeDefault" } + }, + "containers": [{ + "name": pod_name, + "image": "curlimages/curl:8.12.1", + "command": ["curl"], + "args": args, + "securityContext": { + "runAsUser": 100, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { "drop": ["ALL"] } + } + }] + } + }) + .to_string() +} + +/// Run an ephemeral curl pod with restricted `PodSecurity` context. +fn run_curl_probe(context: &str, pod_name: &str, curl_args: &[&str]) -> Result { + let sequence = PROBE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let prefix = pod_name.get(..pod_name.len().min(40)).unwrap_or(pod_name); + let unique_pod_name = format!("{prefix}-{sequence}"); + let overrides = curl_pod_overrides(&unique_pod_name, curl_args); + Command::new("kubectl") + .args([ + "run", + &unique_pod_name, + "--image=curlimages/curl:8.12.1", + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "--rm", + "-i", + "--restart=Never", + "--overrides", + &overrides, + ]) + .output() +} + +/// Run an ephemeral curl pod with additional kubectl flags (e.g. `--labels`). +fn response_header(output: &[u8], name: &str) -> Option { + let expected = name.to_ascii_lowercase(); + String::from_utf8_lossy(output).lines().find_map(|line| { + let (header, value) = line.split_once(':')?; + header.eq_ignore_ascii_case(&expected).then(|| value.trim().to_owned()) + }) +} + +/// Wait for the demo environment to be ready. +fn wait_for_environment_ready() -> Result> { + // Wait for Grid operators to converge + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + eprintln!(" [WAIT] {cluster}: Grid operator convergence"); + + // Wait for deployment to be ready + wait_for_deployment("grid-operator", "grid-system", &context)?; + + // Every site has a provider gateway; only the designated ingress site + // has the consumer gateway used by the request proof. + if *cluster == CONSUMER_SITE { + wait_for_deployment("consumer-gateway", "grid-system", &context)?; + } + wait_for_deployment("provider-gateway", "grid-system", &context)?; + + eprintln!(" [OK] {cluster}: Gateways ready"); + } + + Ok("Three provider sites converged; one consumer entrypoint and all provider gateways are ready".to_owned()) +} + +// ----------------------------------------------------------------------------- +// Runtime Assertions +// ----------------------------------------------------------------------------- + +/// Assert exactly three provider clusters exist and are healthy. +#[expect( + clippy::too_many_lines, + reason = "The proof reports one bounded fact set per cluster." +)] +fn assert_cluster_health() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let mut cluster_health = Vec::new(); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + // Check API server responsiveness + let api_start = Instant::now(); + let output = Command::new("kubectl") + .args(["cluster-info", "--context", &context]) + .output()?; + + let api_response_ms = + u64::try_from(api_start.elapsed().as_millis().min(u128::from(u64::MAX))).unwrap_or(u64::MAX); + let healthy = output.status.success(); + + // Get node count + let nodes_output = Command::new("kubectl") + .args([ + "get", + "nodes", + "--context", + &context, + "-o", + "jsonpath={.items[*].status.conditions[?(@.type=='Ready')].status}", + ]) + .output()?; + + let ready_nodes = if nodes_output.status.success() { + u32::try_from( + String::from_utf8_lossy(&nodes_output.stdout) + .split_whitespace() + .filter(|s| s == &"True") + .count() + .min(u32::MAX as usize), + ) + .unwrap_or(u32::MAX) + } else { + 0 + }; + + cluster_health.push(ClusterHealth { + name: cluster.to_string(), + healthy, + api_response_ms: Some(api_response_ms), + ready_nodes, + }); + + observed_facts.insert(format!("{cluster}_healthy"), serde_json::Value::Bool(healthy)); + observed_facts.insert( + format!("{cluster}_ready_nodes"), + serde_json::Value::Number(ready_nodes.into()), + ); + } + + let all_healthy = cluster_health.iter().all(|c| c.healthy && c.ready_nodes > 0); + observed_facts.insert( + "total_clusters".to_owned(), + serde_json::Value::Number(CLUSTERS.len().into()), + ); + + if all_healthy { + Ok(proof_success( + &format!("All {} clusters are healthy with ready nodes", CLUSTERS.len()), + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more clusters are unhealthy", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert one provider stack per site and one consumer entrypoint. +#[expect( + clippy::too_many_lines, + reason = "The proof checks the required deployed components together." +)] +fn assert_component_deployment() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let mut components = Vec::new(); + + let static_components = ["grid-operator", "provider-gateway"]; + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + let mock_name = format!("vcr-inference-{cluster}"); + let cluster_components: Vec<&str> = static_components + .iter() + .copied() + .chain(std::iter::once(mock_name.as_str())) + .collect(); + let cluster_components = if *cluster == CONSUMER_SITE { + cluster_components + .into_iter() + .chain(std::iter::once("consumer-gateway")) + .collect::>() + } else { + cluster_components + }; + + for component in &cluster_components { + let output = Command::new("kubectl") + .args([ + "get", + "deployment", + component, + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "-o", + "jsonpath={.status.readyReplicas},{.status.replicas}", + ]) + .output()?; + + if output.status.success() { + let status_str = String::from_utf8_lossy(&output.stdout); + let parts: Vec<&str> = status_str.trim().split(',').collect(); + + let ready_replicas = parts.first().and_then(|s| s.parse::().ok()).unwrap_or(0); + let desired_replicas = parts.get(1).and_then(|s| s.parse::().ok()).unwrap_or(0); + let ready = ready_replicas > 0 && ready_replicas == desired_replicas; + + components.push(ComponentStatus { + name: format!("{cluster}-{component}"), + namespace: GRID_SYSTEM_NS.to_owned(), + ready_replicas, + desired_replicas, + ready, + }); + + observed_facts.insert(format!("{cluster}_{component}_ready"), serde_json::Value::Bool(ready)); + observed_facts.insert( + format!("{cluster}_{component}_replicas"), + serde_json::Value::Number(ready_replicas.into()), + ); + } else { + components.push(ComponentStatus { + name: format!("{cluster}-{component}"), + namespace: GRID_SYSTEM_NS.to_owned(), + ready_replicas: 0, + desired_replicas: 1, + ready: false, + }); + observed_facts.insert(format!("{cluster}_{component}_ready"), serde_json::Value::Bool(false)); + } + } + } + + let expected_count = components.len(); + let all_ready = components.len() == expected_count && components.iter().all(|c| c.ready); + observed_facts.insert( + "total_components".to_owned(), + serde_json::Value::Number(expected_count.into()), + ); + + if all_ready { + Ok(proof_success( + &format!( + "{} components are ready across {} provider sites with one consumer entrypoint", + expected_count, + CLUSTERS.len() + ), + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more components are not ready", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert every site's `GridNetwork` reports all three remote sites connected. +#[expect( + clippy::too_many_lines, + clippy::unnecessary_wraps, + reason = "The assertion framework requires a fallible, named proof boundary." +)] +fn assert_swim_convergence() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let expected_remote_sites = CLUSTERS.len() - 1; + let mut all_converged = true; + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + let status = poll_until( + || { + let output = Command::new("kubectl") + .args([ + "get", + "gridnetwork/grid-provider-traffic", + "--context", + &context, + "-o", + "jsonpath={.status.phase},{.status.connectedSites}", + ]) + .output()?; + if !output.status.success() { + return Ok(None); + } + let value = String::from_utf8_lossy(&output.stdout); + let Some((phase, connected)) = value.trim().split_once(',') else { + return Ok(None); + }; + let connected = connected.parse::().unwrap_or_default(); + Ok((phase == "Active" && connected == expected_remote_sites).then_some((phase.to_owned(), connected))) + }, + Duration::from_secs(90), + Duration::from_secs(3), + ); + + match status { + Ok((phase, connected)) => { + observed_facts.insert(format!("{cluster}_phase"), serde_json::Value::String(phase)); + observed_facts.insert( + format!("{cluster}_connected_sites"), + serde_json::Value::Number(connected.into()), + ); + }, + Err(error) => { + all_converged = false; + observed_facts.insert(format!("{cluster}_error"), serde_json::Value::String(error.to_string())); + }, + } + } + + observed_facts.insert( + "expected_remote_sites".to_owned(), + serde_json::Value::Number(expected_remote_sites.into()), + ); + + if all_converged { + Ok(proof_success( + "Every GridNetwork is Active with all three remote sites connected", + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more GridNetworks did not report all three remote sites connected", + observed_facts, + start.elapsed(), + )) + } +} + +/// Verify that both remote sites are discovered and routing-eligible. +/// +/// The locally declared placement site is not a remote SWIM discovery result +/// and is excluded from this assertion. +#[expect( + clippy::too_many_lines, + clippy::unnecessary_wraps, + reason = "The assertion framework requires a fallible, named proof boundary." +)] +fn assert_site_auto_discovery() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let expected_remote_count = CLUSTERS.len() - 1; + let mut all_ok = true; + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let result = poll_until( + || { + let output = Command::new("kubectl") + .args([ + "get", "gridsite", + "-l", "grid.praxis-proxy.io/auto-discovered=true", + "--context", &context, + "-n", GRID_SYSTEM_NS, + "-o", "jsonpath={range .items[*]}{.metadata.name}\t{.status.phase}\t{.status.reason}\t{.spec.egress.address}\t{.spec.egress.tls.serverName}\t{.spec.trust.canonicalFingerprints}\n{end}", + ]) + .output()?; + if !output.status.success() { + return Ok(None); + } + let body = String::from_utf8_lossy(&output.stdout); + let mut verified = Vec::new(); + let port_suffix = format!(":{PROVIDER_GATEWAY_PORT}"); + for line in body.lines() { + let fields: Vec<&str> = line.trim().split('\t').collect(); + let [name, phase, reason, addr, server_name, fingerprints, ..] = fields.as_slice() else { + continue; + }; + if *phase == "Active" + && *reason == "TlsVerified" + && addr.ends_with(&port_suffix) + && !server_name.is_empty() + && !fingerprints.is_empty() + { + verified.push(serde_json::json!({ + "name": name, + "phase": phase, + "reason": reason, + "egressAddress": addr, + "serverName": server_name, + "hasFingerprints": true, + })); + } + } + if verified.len() == expected_remote_count { + Ok(Some(verified)) + } else { + Ok(None) + } + }, + Duration::from_secs(120), + Duration::from_secs(5), + ); + + match result { + Ok(remotes) => { + observed_facts.insert(format!("{cluster}_remote_sites"), serde_json::Value::Array(remotes)); + }, + Err(error) => { + all_ok = false; + observed_facts.insert(format!("{cluster}_error"), serde_json::Value::String(error.to_string())); + }, + } + } + + observed_facts.insert( + "expected_remote_count".to_owned(), + serde_json::Value::Number(expected_remote_count.into()), + ); + + if all_ok { + Ok(proof_success( + "Every cluster has two Active/TlsVerified remote `GridSites` with :8443 addresses and trust configuration", + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more clusters lack Active auto-discovered remote GridSites", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert each consumer receives and accepts a versioned overlay. +/// +/// Validates per-site: `ConfigMap` exists with non-empty data, gateway deployment +/// is ready, and a request is routed through the accepted overlay. +#[expect( + clippy::too_many_lines, + reason = "The proof checks overlay, gateway, and routing acceptance together." +)] +fn assert_overlay_acceptance() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let mut all_accepted = true; + + { + let cluster = CONSUMER_SITE; + let context = format!("kind-grid-provider-traffic-{cluster}"); + + // Step 1: Overlay ConfigMap exists + let overlay_output = Command::new("kubectl") + .args([ + "get", + "configmap", + "grid-overlay-grid-provider-traffic-consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.metadata.resourceVersion}", + ]) + .output()?; + + let resource_version = String::from_utf8_lossy(&overlay_output.stdout).trim().to_owned(); + let overlay_exists = overlay_output.status.success() && !resource_version.is_empty(); + + // Step 2: Overlay ConfigMap has non-empty data + let overlay_data_output = Command::new("kubectl") + .args([ + "get", + "configmap", + "grid-overlay-grid-provider-traffic-consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.data}", + ]) + .output()?; + + let data_str = String::from_utf8_lossy(&overlay_data_output.stdout).trim().to_owned(); + let has_data = overlay_data_output.status.success() && !data_str.is_empty() && data_str != "{}"; + + // Step 3: Consumer gateway deployment is ready + let deploy_output = Command::new("kubectl") + .args([ + "get", + "deployment/consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.status.readyReplicas}", + ]) + .output()?; + + let gateway_ready = deploy_output.status.success() + && String::from_utf8_lossy(&deploy_output.stdout) + .trim() + .parse::() + .is_ok_and(|n| n > 0); + + // Step 4: A valid request proves that the accepted overlay is serving. + // Praxis health endpoints are exposed only on the pod-local admin listener. + let routing_output = run_curl_probe( + &context, + &format!("overlay-routing-{cluster}"), + &[ + "curl", + "--fail-with-body", + "--silent", + "--show-error", + "--max-time", + "10", + "--header", + "Content-Type: application/json", + "--header", + "Authorization: Bearer consumer-token", + "--data", + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"overlay probe"}],"max_tokens":16}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ], + )?; + + let routing_ok = routing_output.status.success(); + + let overlay_accepted = overlay_exists && has_data && gateway_ready && routing_ok; + if !overlay_accepted { + all_accepted = false; + } + + observed_facts.insert( + format!("{cluster}_overlay_configmap_exists"), + serde_json::Value::Bool(overlay_exists), + ); + observed_facts.insert(format!("{cluster}_overlay_has_data"), serde_json::Value::Bool(has_data)); + observed_facts.insert( + format!("{cluster}_gateway_ready"), + serde_json::Value::Bool(gateway_ready), + ); + observed_facts.insert( + format!("{cluster}_overlay_routing_ok"), + serde_json::Value::Bool(routing_ok), + ); + observed_facts.insert( + format!("{cluster}_resource_version"), + serde_json::Value::String(resource_version), + ); + } + + observed_facts.insert( + "all_overlays_accepted".to_owned(), + serde_json::Value::Bool(all_accepted), + ); + + if all_accepted { + Ok(proof_success( + "Consumer entrypoint overlay ConfigMap exists with data, gateway ready, and routing passes", + observed_facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + "One or more sites failed overlay acceptance (ConfigMap/data/ready/routing)", + observed_facts, + start.elapsed(), + )) + } +} + +/// Assert consumer and operator cannot access provider credentials. +fn require_local_image(image: &str) -> Result<(), Box> { + let status = Command::new("docker") + .args(["image", "inspect", image]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status()?; + if status.success() { + return Ok(()); + } + Err(format!( + "required local image {image:?} is absent; build it or set \ + GRID_XTASK_IMAGE_PULL_POLICY=IfNotPresent with registry image overrides" + ) + .into()) +} + +/// Load local container images into all Kind clusters. +/// +/// Reads image references from the `GRID_XTASK_*_IMAGE` environment variables +/// (same source as `apply_image_overrides`). When `imagePullPolicy` is not +/// `Never`, this is a no-op. +#[expect( + clippy::too_many_lines, + reason = "Image loading is one bounded setup operation across the three clusters." +)] +fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result<(), Box> { + let pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); + if pull_policy != "Never" { + eprintln!(" skipping Kind image loading (pull policy is {pull_policy})"); + return Ok(()); + } + + let gateway = + std::env::var("GRID_XTASK_GATEWAY_IMAGE").unwrap_or_else(|_| "praxis-ai:provider-traffic-demo".to_owned()); + let operator = + std::env::var("GRID_XTASK_OPERATOR_IMAGE").unwrap_or_else(|_| "grid-operator:provider-traffic-demo".to_owned()); + let vcr = crate::env::image_overrides::vcr_image(); + + for image in [&gateway, &operator, &vcr] { + require_local_image(image)?; + eprintln!(" verified local image: {image}"); + } + + for cluster in CLUSTERS { + for image in [&gateway, &operator, &vcr] { + eprintln!(" loading {image} into {cluster}..."); + let output = Command::new(forge_bin.as_os_str()) + .arg("--config") + .arg(resolved_config) + .args(["--non-interactive", "cluster", "load-image", cluster, image]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to load {image} into {cluster}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + } + eprintln!(" [OK] {cluster}: all images loaded"); + } + Ok(()) +} + +/// Generate TLS certificates for all provider-traffic identities. +/// +/// Must be called BEFORE `forge up` so the certificates exist on the host +/// when `install_provider_boundary` creates the Kubernetes Secrets. +fn stage_provider_boundary() -> Result<(), Box> { + let identities: Vec = CLUSTERS.iter().map(|c| (*c).to_owned()).collect(); + certs::generate_all(&identities)?; + + let wrong_ca = ::certs::generate_ca("Combined Site untrusted test CA")?; + fs::write(Path::new(CERTS_DIR).join("untrusted-ca.pem"), wrong_ca.cert_pem)?; + + eprintln!(" [OK] TLS certificates generated for provider-a, provider-b, provider-c"); + Ok(()) +} + +/// Create TLS and credential Secrets in every provider-traffic cluster. +/// +/// Must be called AFTER `forge up` since the clusters must exist. Gateway +/// deployments are restarted so pods pick up the new volume mounts. +fn install_provider_boundary() -> Result<(), Box> { + let certs_dir = Path::new(CERTS_DIR); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + apply_tls_secret(&context, cluster, CONSUMER_TLS_SECRET, certs_dir)?; + apply_tls_secret(&context, cluster, PROVIDER_TLS_SECRET, certs_dir)?; + apply_tls_secret(&context, "wrong-org-client", WRONG_ORG_TLS_SECRET, certs_dir)?; + + eprintln!(" [OK] {cluster}: TLS secrets installed"); + } + + Ok(()) +} + +/// Create a TLS secret from the generated cert, key, and CA files. +#[expect( + clippy::too_many_lines, + reason = "TLS secret creation is one bounded setup operation." +)] +fn apply_tls_secret( + context: &str, + identity: &str, + secret_name: &str, + certs_dir: &Path, +) -> Result<(), Box> { + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "create", + "secret", + "generic", + secret_name, + &format!( + "--from-file=tls.crt={}", + certs_dir.join(format!("{identity}-cert.pem")).display() + ), + &format!( + "--from-file=tls.key={}", + certs_dir.join(format!("{identity}-key.pem")).display() + ), + &format!("--from-file=ca.crt={}", certs_dir.join("ca.pem").display()), + "--dry-run=client", + "-o", + "yaml", + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to render {identity} Secret/{secret_name}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + kubectl::apply_manifest(context, &String::from_utf8(output.stdout)?) +} + +/// Generate a random 32-byte hex provider credential. +fn generate_provider_credential() -> Result> { + let output = Command::new("openssl").args(["rand", "-hex", "32"]).output()?; + if !output.status.success() { + return Err("openssl failed to generate provider credential".into()); + } + let token = String::from_utf8(output.stdout)?.trim().to_owned(); + if token.len() != 64 || !token.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err("openssl returned an invalid provider credential".into()); + } + Ok(token) +} + +/// Create an Opaque Secret with a `token` key. +fn apply_credential_secret(context: &str, secret_name: &str, token: &str) -> Result<(), Box> { + let manifest = format!( + r#"{{"apiVersion":"v1","kind":"Secret","metadata":{{"name":"{secret_name}","namespace":"{GRID_SYSTEM_NS}"}},"type":"Opaque","stringData":{{"token":"{token}"}}}}"#, + ); + kubectl::apply_manifest(context, &manifest) +} + +/// Data extracted from the operator-created overlay `ConfigMap`. +/// Read a single cluster's overlay `ConfigMap` and return structured data. +/// +/// Captures both the Kubernetes `resourceVersion` (per-cluster) and the +/// semantic revision from the `grid.praxis-proxy.io/overlay-revision` +/// annotation (content-addressed, safe to compare across clusters). +#[expect( + clippy::too_many_lines, + reason = "The reader validates and parses one Kubernetes ConfigMap response." +)] +fn read_cluster_overlay(cluster: &str) -> Result> { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + OVERLAY_CONFIGMAP, + "-o", + "json", + ]) + .output()?; + + if !output.status.success() { + return Err(format!("{cluster}: overlay ConfigMap not found").into()); + } + + let cm: serde_json::Value = + serde_json::from_slice(&output.stdout).map_err(|e| format!("{cluster}: overlay ConfigMap invalid: {e}"))?; + + let resource_version = cm + .pointer("/metadata/resourceVersion") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_owned(); + + let semantic_revision = cm + .pointer("/metadata/annotations/grid.praxis-proxy.io~1overlay-revision") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_owned(); + + let routing_json = cm + .pointer("/data/routing-config.json") + .and_then(|v| v.as_str()) + .ok_or_else(|| format!("{cluster}: overlay missing routing-config.json"))?; + + let parsed: serde_json::Value = + serde_json::from_str(routing_json).map_err(|e| format!("{cluster}: routing-config.json invalid: {e}"))?; + + let raw_candidates = parsed + .get("candidates") + .and_then(|v| v.as_array()) + .ok_or_else(|| format!("{cluster}: overlay missing candidates array"))?; + + let mut stable_ids = BTreeMap::new(); + let mut candidates = Vec::new(); + + for c in raw_candidates { + let cluster_field = c.get("cluster").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let stable_id = c.get("stable_id").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let kind = c.get("kind").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let name = c.get("name").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let site = c.get("site").and_then(|v| v.as_str()).unwrap_or("").to_owned(); + let fresh = c.get("fresh").and_then(serde_json::Value::as_bool); + let admission_state = c + .get("admission_state") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let selection_group = c + .get("selection_group") + .and_then(serde_json::Value::as_u64) + .and_then(|value| u32::try_from(value).ok()); + + if !cluster_field.is_empty() && !stable_id.is_empty() { + stable_ids.insert(cluster_field.clone(), stable_id.clone()); + } + + candidates.push(OverlayCandidate { + kind, + name, + site, + cluster: cluster_field, + stable_id, + fresh, + admission_state, + selection_group, + }); + } + + Ok(OverlayData { + resource_version, + semantic_revision, + stable_ids, + candidates, + }) +} + +/// Establish the non-traffic preconditions for the measured picker proof. +/// +/// This gate deliberately performs no request. It verifies one consumer +/// replica, the explicit round-robin policy, three fresh `NewAndExisting` +/// candidates in group zero, and three consecutive identical semantic +/// revisions. The measured request window starts only after this gate passes. +#[expect( + clippy::too_many_lines, + reason = "The readiness barrier checks all non-traffic serving invariants." +)] +fn wait_for_round_robin_readiness() -> Result, Box> { + let context = "kind-grid-provider-traffic-provider-a"; + let replicas = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "deployment/consumer-gateway", + "-o", + "jsonpath={.spec.replicas},{.status.replicas},{.status.readyReplicas}", + ]) + .output()?; + let replica_text = String::from_utf8_lossy(&replicas.stdout).trim().to_owned(); + if !replicas.status.success() || replica_text != "1,1,1" { + return Err(format!("consumer gateway is not exactly one ready replica: {replica_text}").into()); + } + + let mut stable_revision: Option = None; + let mut stable_signature: Option = None; + for observation in 1..=3 { + let overlay = read_cluster_overlay("provider-a")?; + let configmap = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + OVERLAY_CONFIGMAP, + "-o", + "json", + ]) + .output()?; + if !configmap.status.success() { + return Err("consumer overlay ConfigMap could not be read".into()); + } + let configmap_json: serde_json::Value = serde_json::from_slice(&configmap.stdout)?; + let routing_text = configmap_json + .pointer("/data/routing-config.json") + .and_then(serde_json::Value::as_str) + .ok_or("consumer overlay ConfigMap has no routing-config.json data")?; + let routing: serde_json::Value = serde_json::from_str(routing_text)?; + let mode = routing + .pointer("/selection_policy/mode") + .and_then(serde_json::Value::as_str) + .ok_or("overlay does not publish selection_policy.mode")?; + if mode != "roundRobin" { + return Err(format!("overlay selection mode is {mode}, expected roundRobin").into()); + } + + let mut candidate_signature = Vec::new(); + for candidate in &overlay.candidates { + if candidate.selection_group != Some(0) { + return Err(format!( + "candidate {} is not in group 0: {:?}", + candidate.cluster, candidate.selection_group + ) + .into()); + } + if candidate.fresh == Some(false) { + return Err(format!("candidate {} is stale", candidate.cluster).into()); + } + if candidate + .admission_state + .as_deref() + .is_some_and(|state| state != "new_and_existing") + { + return Err(format!( + "candidate {} is not NewAndExisting: {:?}", + candidate.cluster, candidate.admission_state + ) + .into()); + } + candidate_signature.push(format!( + "{}:{}:{:?}:{:?}", + candidate.cluster, candidate.stable_id, candidate.selection_group, candidate.admission_state + )); + } + candidate_signature.sort(); + let signature = candidate_signature.join("|"); + if overlay.candidates.len() != 3 { + return Err(format!("expected three candidates, found {}", overlay.candidates.len()).into()); + } + if stable_revision + .as_ref() + .is_some_and(|revision| revision != &overlay.semantic_revision) + || stable_signature.as_ref().is_some_and(|previous| previous != &signature) + { + return Err("overlay changed while establishing the measured proof precondition".into()); + } + stable_revision = Some(overlay.semantic_revision); + stable_signature = Some(signature); + if observation < 3 { + std::thread::park_timeout(Duration::from_secs(2)); + } + } + + let serving_revision = stable_revision + .as_deref() + .ok_or("round-robin readiness did not establish a semantic revision")?; + wait_for_consumer_gateway_revision(serving_revision)?; + + let mut facts = BTreeMap::new(); + facts.insert("consumer_gateway_replicas".to_owned(), serde_json::json!(replica_text)); + facts.insert("semantic_revision".to_owned(), serde_json::json!(stable_revision)); + facts.insert("candidate_signature".to_owned(), serde_json::json!(stable_signature)); + facts.insert("selection_policy".to_owned(), serde_json::json!("roundRobin")); + facts.insert("candidate_count".to_owned(), serde_json::json!(3)); + facts.insert( + "praxis_serving_revision".to_owned(), + serde_json::json!(serving_revision), + ); + Ok(facts) +} + +/// Read bounded logs from the single consumer gateway used by the measured +/// provider-traffic proof. +fn consumer_gateway_logs() -> Result> { + let output = Command::new("kubectl") + .args([ + "--context", + "kind-grid-provider-traffic-provider-a", + "-n", + GRID_SYSTEM_NS, + "logs", + "deployment/consumer-gateway", + "-c", + "praxis", + "--tail=300", + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to read consumer gateway logs: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 160) + ) + .into()); + } + Ok(strip_csi_sgr(&String::from_utf8_lossy(&output.stdout))) +} + +/// Strip the ANSI SGR sequences emitted by the gateway's tracing subscriber. +fn strip_csi_sgr(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(character) = chars.next() { + if character == '\x1b' { + if chars.next() == Some('[') { + for final_byte in chars.by_ref() { + if final_byte.is_ascii_alphabetic() { + break; + } + } + } + } else { + output.push(character); + } + } + output +} + +/// Wait until Praxis reports that the exact final overlay revision is serving. +/// +/// The `ConfigMap` and projected file can be ready before the gateway watcher +/// has accepted the file. Starting the measured window earlier would count +/// requests against the initial local-only snapshot and invalidate the +/// round-robin proof. +fn wait_for_consumer_gateway_revision(revision: &str) -> Result<(), Box> { + const TIMEOUT: Duration = Duration::from_secs(90); + const POLL: Duration = Duration::from_secs(2); + let deadline = Instant::now() + TIMEOUT; + loop { + let logs = consumer_gateway_logs()?; + let accepted = latest_log_field(&logs, "accepted_revision"); + let serving = latest_log_field(&logs, "serving_revision"); + if accepted.as_deref() == Some(revision) && serving.as_deref() == Some(revision) { + return Ok(()); + } + if Instant::now() >= deadline { + let accepted_summary = accepted + .as_deref() + .map_or_else(|| "none".to_owned(), |value| safe_truncate_str(value, 16)); + let serving_summary = serving + .as_deref() + .map_or_else(|| "none".to_owned(), |value| safe_truncate_str(value, 16)); + return Err(format!( + "consumer gateway did not report accepted/serving overlay revision {} within {TIMEOUT:?} (accepted={}, serving={})", + safe_truncate_str(revision, 16), + accepted_summary, + serving_summary + ) + .into()); + } + std::thread::park_timeout(POLL); + } +} + +/// Return the latest exact tracing field value from bounded gateway logs. +fn latest_log_field(logs: &str, field: &str) -> Option { + logs.lines() + .rev() + .find_map(|line| { + let prefix = format!("{field}="); + line.match_indices(&prefix).find_map(|(index, _)| { + let at_boundary = index == 0 + || line + .get(..index) + .and_then(|value| value.chars().next_back()) + .is_some_and(char::is_whitespace); + if !at_boundary { + return None; + } + let value = line.get(index + prefix.len()..)?; + Some( + value + .strip_prefix('"') + .map_or_else( + || value.split_whitespace().next().unwrap_or(""), + |value| value.split('"').next().unwrap_or(""), + ) + .to_owned(), + ) + }) + }) + .filter(|value| !value.is_empty()) +} + +/// Build the expected provider candidate name set. +fn expected_candidates() -> BTreeSet { + let mut expected = BTreeSet::new(); + for cluster in CLUSTERS { + expected.insert(format!("vcr-{cluster}-provider")); + } + + expected +} + +/// Wait for each cluster's operator to produce its expected local candidate. +/// +/// Pre-SWIM, each operator only knows its local `InferenceProvider` resources. +/// Global convergence (all candidates on every cluster) happens after SWIM +/// seeding in a separate phase. +#[expect( + clippy::too_many_lines, + reason = "The readiness poll checks each local cluster's overlay." +)] +fn wait_for_local_overlays() -> Result, Box> { + let timeout = Duration::from_secs(180); + let interval = Duration::from_secs(5); + let start = Instant::now(); + + eprintln!(" Polling for local overlay ConfigMaps (timeout {timeout:?})..."); + + while start.elapsed() < timeout { + let mut overlays = BTreeMap::new(); + let mut all_ready = true; + + for cluster in CLUSTERS { + let expected_local = format!("vcr-{cluster}-provider"); + match read_cluster_overlay(cluster) { + Ok(data) if data.stable_ids.contains_key(&expected_local) => { + eprintln!( + " {cluster}: {expected_local} present (semantic_rev={}, stable_id={})", + data.semantic_revision, + data.stable_ids.get(&expected_local).map_or("?", String::as_str) + ); + overlays.insert((*cluster).to_owned(), data); + }, + Ok(data) => { + eprintln!( + " {cluster}: overlay present but missing {expected_local} (has: {:?})", + data.stable_ids.keys().collect::>() + ); + all_ready = false; + }, + Err(_) => { + all_ready = false; + }, + } + } + + if all_ready && overlays.len() == CLUSTERS.len() { + return Ok(overlays); + } + + std::thread::park_timeout(interval); + } + + collect_overlay_diagnostics(); + Err(format!("Local overlay ConfigMaps not ready after {timeout:?}").into()) +} + +/// Wait until every provider-traffic cluster serves the same candidate set. +#[expect( + clippy::too_many_lines, + reason = "The convergence poll compares the bounded three-cluster overlay set." +)] +fn wait_for_global_overlay_convergence( + expected: &BTreeSet, +) -> Result, Box> { + let timeout = Duration::from_secs(180); + let interval = Duration::from_secs(5); + let start = Instant::now(); + + eprintln!( + " Waiting for global overlay convergence \ + ({} candidates on all clusters, timeout {timeout:?})...", + expected.len() + ); + + while start.elapsed() < timeout { + let mut overlays = BTreeMap::new(); + let mut all_converged = true; + + for cluster in CLUSTERS { + match read_cluster_overlay(cluster) { + Ok(data) => { + let missing: Vec<&String> = expected.iter().filter(|c| !data.stable_ids.contains_key(*c)).collect(); + if missing.is_empty() { + overlays.insert((*cluster).to_owned(), data); + } else { + eprintln!(" {cluster}: missing candidates {missing:?}"); + all_converged = false; + } + }, + Err(_) => { + all_converged = false; + }, + } + } + + if all_converged && overlays.len() == CLUSTERS.len() { + let reference_site = CLUSTERS.first().ok_or("CLUSTERS is empty")?; + let reference = overlays + .get(*reference_site) + .ok_or("reference site missing from overlays")?; + + for cluster in CLUSTERS.iter().skip(1) { + let site_data = overlays + .get(*cluster) + .ok_or_else(|| format!("{cluster} missing from overlays"))?; + for candidate in expected { + let ref_id = reference + .stable_ids + .get(candidate) + .ok_or_else(|| format!("{reference_site}: missing {candidate}"))?; + let site_id = site_data + .stable_ids + .get(candidate) + .ok_or_else(|| format!("{cluster}: missing {candidate}"))?; + if site_id != ref_id { + return Err(format!( + "stable_id mismatch for {candidate}: \ + {reference_site}={ref_id} vs {cluster}={site_id}" + ) + .into()); + } + } + } + + eprintln!( + " Global overlay converged: {} candidates on all {} clusters, stable_ids agree", + expected.len(), + CLUSTERS.len() + ); + return Ok(overlays); + } + + std::thread::park_timeout(interval); + } + + collect_overlay_diagnostics(); + Err(format!("Global overlay convergence failed after {timeout:?}").into()) +} + +/// Collect diagnostic information when the overlay `ConfigMap` fails to converge. +#[expect( + clippy::too_many_lines, + reason = "Diagnostics intentionally report each bounded control-plane boundary." +)] +fn collect_overlay_diagnostics() { + eprintln!(" [DIAG] Collecting overlay failure diagnostics...\n"); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + eprintln!(" ======== {cluster} ========"); + + eprintln!(" [DIAG] {cluster}: 1. Operator deployment SWIM env vars"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "deployment", + "grid-operator", + "-o", + "jsonpath={range .spec.template.spec.containers[0].env[*]}{.name}={.value}{'\\n'}{end}", + ]) + .status(), + ); + + eprintln!("\n [DIAG] {cluster}: 2. SWIM service details"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "svc", + "grid-operator-swim", + "-o", + "wide", + ]) + .status(), + ); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "endpoints", + "grid-operator-swim", + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 3. Operator pod status"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "pods", + "-l", + "app.kubernetes.io/name=grid-operator", + "-o", + "wide", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 4. Operator logs (last 50 lines, unfiltered)"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "logs", + "deployment/grid-operator", + "--tail=50", + ]) + .status(), + ); + + eprintln!("\n [DIAG] {cluster}: 5. GridNetwork CRD status"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 6. Overlay ConfigMap content"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + OVERLAY_CONFIGMAP, + "-o", + "json", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 7. InferenceProvider CRs"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "inferenceprovider", + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 8. Helm values for grid-operator"); + drop( + Command::new("helm") + .args([ + "get", + "values", + "grid-operator", + "--namespace", + GRID_SYSTEM_NS, + "--kube-context", + &context, + "-o", + "yaml", + ]) + .status(), + ); + + eprintln!(" [DIAG] {cluster}: 9. NetworkPolicy in grid-system"); + drop( + Command::new("kubectl") + .args(["--context", &context, "-n", GRID_SYSTEM_NS, "get", "networkpolicy"]) + .status(), + ); + + eprintln!(); + } + + eprintln!(" [DIAG] 10. Cross-cluster SWIM connectivity check"); + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + for target in CLUSTERS { + if *target == *cluster { + continue; + } + let target_context = format!("kind-grid-provider-traffic-{target}"); + if let Ok(output) = Command::new("kubectl") + .args([ + "--context", + &target_context, + "-n", + GRID_SYSTEM_NS, + "get", + "svc", + "grid-operator-swim", + "-o", + "jsonpath={.status.loadBalancer.ingress[0].ip}", + ]) + .output() + { + let ip = String::from_utf8_lossy(&output.stdout); + eprintln!(" [DIAG] {cluster} -> {target} (SWIM LB {ip}): testing TCP 7946"); + drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "exec", + "deployment/grid-operator", + "--", + "sh", + "-c", + &format!( + "timeout 3 sh -c 'echo | nc -w 2 {ip} 7946' && echo REACHABLE || echo UNREACHABLE" + ), + ]) + .status(), + ); + } + } + } +} + +/// Materialize provider gateway configuration from the pre-SWIM overlay map. +/// +/// For each cluster, extracts the `stable_id` for the local +/// `vcr-{cluster}-provider` candidate and renders the provider praxis.yaml +/// template with: +/// - `SITE_PLACEHOLDER` → cluster name +/// - `CANDIDATE_ID_PLACEHOLDER` → stable ID from the overlay +/// +/// Creates the `provider-gateway-config` `ConfigMap` in each cluster. +#[expect( + clippy::too_many_lines, + reason = "Provider config materialization is one bounded setup phase." +)] +fn materialize_provider_config( + overlays: &BTreeMap, + demo_root: &Path, +) -> Result<(), Box> { + let template_path = demo_root.join("configs/provider/praxis.yaml"); + let template = + fs::read_to_string(template_path).map_err(|e| format!("failed to read provider config template: {e}"))?; + + for cluster in CLUSTERS { + let provider_name = format!("vcr-{cluster}-provider"); + let overlay = overlays + .get(*cluster) + .ok_or_else(|| format!("no overlay data for cluster {cluster}"))?; + let stable_id = overlay.stable_ids.get(&provider_name).ok_or_else(|| { + format!( + "{cluster}: no candidate {provider_name} in overlay (has: {:?})", + overlay.stable_ids.keys().collect::>() + ) + })?; + + eprintln!(" {cluster}: {provider_name} -> {stable_id}"); + + let rendered = template + .replace("SITE_PLACEHOLDER", cluster) + .replace("CANDIDATE_ID_PLACEHOLDER", stable_id); + + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let create = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "create", + "configmap", + "provider-gateway-config", + &format!("--from-literal=praxis.yaml={rendered}"), + "--dry-run=client", + "-o", + "yaml", + ]) + .output()?; + + if !create.status.success() { + return Err(format!( + "failed to render provider-gateway-config for {cluster}: {}", + String::from_utf8_lossy(&create.stderr).trim() + ) + .into()); + } + + kubectl::apply_manifest(&context, &String::from_utf8(create.stdout)?)?; + eprintln!(" [OK] {cluster}: provider-gateway-config created (stable_id={stable_id})"); + } + + Ok(()) +} + +/// Materialize the Forge configuration with image overrides. +fn materialize_config(source: &Path) -> Result> { + let content = fs::read_to_string(source)?; + let mut config: serde_yaml::Value = serde_yaml::from_str(&content)?; + apply_image_overrides(&mut config); + let rendered = serde_yaml::to_string(&config)?; + let parent = source.parent().ok_or("source config must have parent directory")?; + let output = parent.join(".forge.resolved.yaml"); + fs::write(&output, rendered)?; + Ok(output) +} + +/// Apply image overrides from environment variables to the Forge configuration. +#[expect( + clippy::too_many_lines, + clippy::collapsible_if, + reason = "Image override application with structured YAML manipulation; nested ifs follow YAML structure hierarchy" +)] +fn apply_image_overrides(config: &mut serde_yaml::Value) { + let gateway_image = + std::env::var("GRID_XTASK_GATEWAY_IMAGE").unwrap_or_else(|_| "praxis-ai:provider-traffic-demo".to_owned()); + let operator_image = + std::env::var("GRID_XTASK_OPERATOR_IMAGE").unwrap_or_else(|_| "grid-operator:provider-traffic-demo".to_owned()); + let vcr_image = crate::env::image_overrides::vcr_image(); + let image_pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); + + let (gateway_repo, gateway_tag) = parse_image_ref(&gateway_image); + let (operator_repo, operator_tag) = parse_image_ref(&operator_image); + + if let Some(spec) = config.get_mut("spec") { + if let Some(clusters) = spec.get_mut("clusters") { + if let Some(clusters_array) = clusters.as_sequence_mut() { + for cluster in clusters_array { + if let Some(properties) = cluster.get_mut("properties") { + if let Some(props_map) = properties.as_mapping_mut() { + let pairs = [ + ("gatewayImage", &gateway_image), + ("operatorImage", &operator_image), + ("vcrImage", &vcr_image), + ("imagePullPolicy", &image_pull_policy), + ("gatewayImageRepo", &gateway_repo), + ("gatewayImageTag", &gateway_tag), + ("operatorImageRepo", &operator_repo), + ("operatorImageTag", &operator_tag), + ]; + for (key, val) in pairs { + props_map.insert( + serde_yaml::Value::String(key.to_owned()), + serde_yaml::Value::String(val.clone()), + ); + } + } + } + } + } + } + } +} + +/// Parse image reference into (repo, tag) components. +fn parse_image_ref(image: &str) -> (String, String) { + if let Some(colon_pos) = image.rfind(':') { + let (repo, tag) = image.split_at(colon_pos); + // Skip the ':' character + let tag = tag.strip_prefix(':').unwrap_or(tag); + (repo.to_owned(), tag.to_owned()) + } else { + (image.to_owned(), "latest".to_owned()) + } +} + +/// Prepare setup context from configuration. +fn prepare_setup(forge_config: &Path) -> Result> { + let root = super::demo_root(forge_config); + eprintln!("Forge config: {}", forge_config.display()); + eprintln!("Demo root: {}", root.display()); + let resolved_config = materialize_config(forge_config)?; + let forge_bin = glb::resolve_forge_binary() + .ok_or("praxis-forge binary not found")? + .into(); + + Ok(ProviderTrafficContext { + demo_root: root, + resolved_config, + forge_bin, + }) +} + +/// Authorize auto-discovered remote `GridSites` with identity trust material. +/// +/// For each local cluster, waits for the two remote auto-discovered `GridSites`, +/// verifies the SWIM-advertised certificate matches the staged identity, then +/// patches `spec.egress.tls.serverName` and `spec.trust.canonicalFingerprints`. +/// The controller transitions the site to Active naturally after the patch. +fn authorize_discovered_sites() -> Result<(), Box> { + const TRUST_TIMEOUT: Duration = Duration::from_secs(120); + const GRID_NETWORK: &str = "grid-provider-traffic"; + + for local in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{local}"); + eprintln!(); + eprintln!(" {local}: authorizing remote provider sites"); + for remote in CLUSTERS { + if *remote == *local { + continue; + } + let site_name = format!("{GRID_NETWORK}-{remote}"); + operator::wait_for_auto_gridsite(&context, &site_name, GRID_NETWORK, TRUST_TIMEOUT)?; + let canonical_fp = certs::site_certificate_fingerprint(remote)?; + operator::wait_for_expected_site_certificate(&context, &site_name, &canonical_fp, TRUST_TIMEOUT)?; + let server_name = format!("{remote}.grid.internal"); + operator::patch_gridsite_identity_trust(&context, &site_name, &canonical_fp, &server_name)?; + operator::wait_for_gridsite_phase(&context, &site_name, "Active", TRUST_TIMEOUT)?; + } + } + eprintln!(" [OK] All auto-discovered remote GridSites authorized and Active"); + Ok(()) +} + +/// Deploy the provider-traffic environment. +#[expect( + clippy::too_many_lines, + reason = "sequential setup steps: each step depends on the previous; splitting obscures the setup flow" +)] +fn deploy_setup(context: &ProviderTrafficContext) -> Result> { + let total_phases = SETUP_PHASES; + let mut phase = 0; + let mut next = || { + phase += 1; + phase + }; + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Resolving Forge config and building images", + next(), + total_phases + ); + + // Validate the resolved forge configuration + let output = Command::new(&context.forge_bin) + .args(["config", "validate", "--config"]) + .arg(&context.resolved_config) + .output()?; + + if !output.status.success() { + return Err(format!( + "Forge config validation failed: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + eprintln!(" [OK] Forge config resolved to {}", context.resolved_config.display()); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Generating TLS certificates for all sites", + next(), + total_phases + ); + + stage_provider_boundary()?; + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Creating three provider Kind clusters: provider-a, provider-b, provider-c", + next(), + total_phases + ); + + let config_status = Command::new(&context.forge_bin) + .args(["up", "--config"]) + .arg(&context.resolved_config) + .status()?; + + if !config_status.success() { + return Err("Failed to create provider-traffic clusters".into()); + } + + eprintln!(" [OK] All three provider clusters created"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Loading container images into Kind clusters", + next(), + total_phases + ); + + load_images_into_clusters(&context.forge_bin, &context.resolved_config)?; + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying infrastructure stacks", next(), total_phases); + + let apply_stack = |forge_bin: &Path, + resolved_config: &Path, + cluster: &str, + stack: &str| + -> Result<(), Box> { + eprintln!(" applying {stack} to {cluster}..."); + let stack_status = Command::new(forge_bin) + .arg("--config") + .arg(resolved_config) + .args(["--non-interactive", "stack", "apply", cluster, stack]) + .status()?; + if !stack_status.success() { + return Err(format!("Failed to apply {stack} to {cluster}").into()); + } + Ok(()) + }; + + for cluster in CLUSTERS { + apply_stack(&context.forge_bin, &context.resolved_config, cluster, "metallb")?; + } + for cluster in CLUSTERS { + let op_stack = format!("{cluster}-operator-base"); + apply_stack(&context.forge_bin, &context.resolved_config, cluster, &op_stack)?; + } + eprintln!(" [OK] Infrastructure stacks applied"); + + eprintln!(); + eprintln!("[SETUP {}/{}] Verifying Grid operators are ready", next(), total_phases); + + for cluster in CLUSTERS { + let ctx = format!("kind-grid-provider-traffic-{cluster}"); + wait_for_deployment("grid-operator", GRID_SYSTEM_NS, &ctx)?; + eprintln!(" [OK] {cluster}: Grid operator ready"); + } + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying VCR backends", next(), total_phases); + + for cluster in CLUSTERS { + let ctx = format!("kind-grid-provider-traffic-{cluster}"); + let credential = generate_provider_credential()?; + apply_credential_secret(&ctx, VCR_INFERENCE_CREDENTIAL, &credential)?; + eprintln!(" [OK] {cluster}: vcr-inference-credential created"); + apply_stack(&context.forge_bin, &context.resolved_config, cluster, "vcr-backend")?; + } + eprintln!(" [OK] VCR backends deployed"); + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying grid-site resources", next(), total_phases); + + for cluster in CLUSTERS { + let site_stack = format!("{cluster}-site"); + apply_stack(&context.forge_bin, &context.resolved_config, cluster, &site_stack)?; + } + eprintln!(" [OK] Grid site resources deployed"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Waiting for local overlay ConfigMaps", + next(), + total_phases + ); + + let pre_swim_overlays = wait_for_local_overlays()?; + eprintln!(" [OK] Local overlay ConfigMaps ready"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Materializing provider config and installing trust", + next(), + total_phases + ); + + install_provider_boundary()?; + + materialize_provider_config(&pre_swim_overlays, &context.demo_root)?; + eprintln!(" [OK] Provider config materialized, trust installed"); + + eprintln!(); + eprintln!("[SETUP {}/{}] Deploying provider gateways", next(), total_phases); + + for cluster in CLUSTERS { + apply_stack( + &context.forge_bin, + &context.resolved_config, + cluster, + "provider-gateway", + )?; + } + eprintln!(" [OK] Provider gateways deployed"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Deploying the single consumer gateway", + next(), + total_phases + ); + + apply_stack( + &context.forge_bin, + &context.resolved_config, + CONSUMER_SITE, + "consumer-gateway", + )?; + eprintln!(" [OK] Consumer gateway deployed in {CONSUMER_SITE}"); + + eprintln!(); + eprintln!( + "[SETUP {}/{}] SWIM discovery and trust authorization", + next(), + total_phases + ); + + configure_swim_peers(&context.forge_bin, &context.resolved_config)?; + authorize_discovered_sites()?; + + eprintln!(); + eprintln!( + "[SETUP {}/{}] Waiting for global overlay convergence", + next(), + total_phases + ); + + let expected = expected_candidates(); + let post_swim_overlays = wait_for_global_overlay_convergence(&expected)?; + let environment_status = wait_for_environment_ready()?; + eprintln!(" [OK] {environment_status}"); + + Ok(OverlayState { + pre_swim: pre_swim_overlays, + post_swim: post_swim_overlays, + }) +} + +// ----------------------------------------------------------------------------- +// Demo Scenarios +// ----------------------------------------------------------------------------- + +/// Run the quick-mode proof scenarios using the assertion framework. +/// +/// Run the six focused provider-traffic proof scenarios. +#[expect( + clippy::too_many_lines, + reason = "The focused demo presents its six proof phases in order." +)] +fn run_quick_scenarios() -> BTreeMap { + let mut results = BTreeMap::new(); + let mut scenario_num: usize = 0; + let mut scenario = || { + scenario_num += 1; + scenario_num + }; + + eprintln!(); + eprintln!("=== QUICK MODE SCENARIOS ==="); + eprintln!(); + + eprintln!("[SCENARIO {}] Verify three provider clusters are healthy", scenario()); + run_and_insert(&mut results, "cluster_health", assert_cluster_health); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify component deployment", scenario()); + run_and_insert(&mut results, "component_deployment", assert_component_deployment); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify SWIM convergence", scenario()); + run_and_insert(&mut results, "swim_convergence", assert_swim_convergence); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify site auto-discovery", scenario()); + run_and_insert(&mut results, "site_auto_discovery", assert_site_auto_discovery); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify overlay acceptance", scenario()); + run_and_insert(&mut results, "overlay_acceptance", assert_overlay_acceptance); + + eprintln!(); + eprintln!(); + eprintln!( + "[SCENARIO {}] Prove equal round-robin across distinct provider gateways", + scenario() + ); + run_and_insert( + &mut results, + "provider_gateway_round_robin", + assert_provider_gateway_round_robin, + ); + + results +} + +/// Send serial, unbound requests through one consumer gateway and verify that +/// the active no-metrics round-robin picker distributes them across the +/// distinct provider gateways. This is intentionally a request-path proof: +/// the request itself is the only source of the attribution counts. +#[expect( + clippy::too_many_lines, + clippy::unnecessary_wraps, + reason = "The assertion framework requires a fallible, named traffic proof boundary." +)] +fn assert_provider_gateway_round_robin() -> AssertionResult { + let start = Instant::now(); + let context = "kind-grid-provider-traffic-provider-a"; + let mut counts: BTreeMap = BTreeMap::new(); + let mut sequence = Vec::new(); + let mut failures = Vec::new(); + let mut overlay_changes = Vec::new(); + + let readiness = match wait_for_round_robin_readiness() { + Ok(facts) => facts, + Err(error) => { + return Ok(proof_failure( + &format!("round-robin readiness gate failed: {error}"), + BTreeMap::from([(String::from("readiness_error"), serde_json::json!(error.to_string()))]), + start.elapsed(), + )); + }, + }; + let baseline_overlay = match read_cluster_overlay("provider-a") { + Ok(overlay) => overlay, + Err(error) => { + return Ok(proof_failure( + &format!("could not capture baseline overlay after readiness: {error}"), + BTreeMap::new(), + start.elapsed(), + )); + }, + }; + + for request_number in 1..=60 { + if let Ok(current_overlay) = read_cluster_overlay("provider-a") + && (current_overlay.resource_version != baseline_overlay.resource_version + || current_overlay.semantic_revision != baseline_overlay.semantic_revision) + { + overlay_changes.push(serde_json::json!({ + "request": request_number, + "resource_version": current_overlay.resource_version, + "semantic_revision": current_overlay.semantic_revision, + })); + } + let request_label = format!("provider-traffic-rr-{request_number:03}"); + let output = run_curl_probe( + context, + &format!("rr-{request_number}"), + &[ + "curl", + "--fail-with-body", + "--include", + "--silent", + "--show-error", + "--header", + "Content-Type: application/json", + "--header", + "Authorization: Bearer consumer-token", + "--data", + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"provider traffic proof"}],"max_tokens":8}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ], + ); + + match output { + Ok(output) if output.status.success() => { + let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway") + .or_else(|| response_header(&output.stdout, "x-grid-provider-traffic-provider-gateway")) + .or_else(|| response_header(&output.stdout, "x-grid-provider-gateway")); + if let Some(provider) = provider { + *counts.entry(provider.clone()).or_default() += 1; + sequence.push(provider); + } else { + failures.push(format!("{request_label}: provider attribution header missing")); + } + }, + Ok(output) => failures.push(format!( + "{request_label}: HTTP probe failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )), + Err(error) => failures.push(format!("{request_label}: probe execution failed: {error}")), + } + } + + let canonical = ["provider-a", "provider-b", "provider-c"]; + let exact_counts = canonical + .iter() + .all(|name| counts.get(*name).copied().unwrap_or(0) == 20); + let cycle_start = sequence + .first() + .and_then(|first| canonical.iter().position(|name| *name == first)); + let repeating_cycle = sequence.len() == 60 + && cycle_start.is_some_and(|start_index| { + sequence.iter().enumerate().all(|(index, provider)| { + canonical + .get((start_index + index) % canonical.len()) + .is_some_and(|expected| *expected == provider.as_str()) + }) + }); + let balanced = exact_counts && repeating_cycle && failures.is_empty(); + + let mut facts = BTreeMap::new(); + facts.insert("request_count".to_owned(), serde_json::json!(sequence.len())); + facts.insert("provider_counts".to_owned(), serde_json::json!(counts)); + facts.insert("ordered_provider_sequence".to_owned(), serde_json::json!(sequence)); + facts.insert("cycle_start_provider".to_owned(), serde_json::json!(sequence.first())); + facts.insert("exact_20_each".to_owned(), serde_json::json!(exact_counts)); + facts.insert( + "repeating_three_provider_cycle".to_owned(), + serde_json::json!(repeating_cycle), + ); + facts.insert("failures".to_owned(), serde_json::json!(failures)); + facts.insert("selection_policy".to_owned(), serde_json::json!("roundRobin")); + facts.insert("scoring_strategy".to_owned(), serde_json::json!("noMetrics")); + facts.insert("readiness".to_owned(), serde_json::json!(readiness)); + facts.insert( + "baseline_resource_version".to_owned(), + serde_json::json!(baseline_overlay.resource_version), + ); + facts.insert( + "baseline_semantic_revision".to_owned(), + serde_json::json!(baseline_overlay.semantic_revision), + ); + let overlay_stable = overlay_changes.is_empty(); + facts.insert( + "overlay_changes_during_requests".to_owned(), + serde_json::json!(overlay_changes), + ); + + if balanced && overlay_stable { + Ok(proof_success( + "60 serial requests distributed evenly across three distinct provider gateways", + facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + &format!( + "provider gateway distribution or overlay stability failed: counts={counts:?}, overlay_changes={overlay_changes:?}" + ), + facts, + start.elapsed(), + )) + } +} + +/// Run one proof assertion and retain failures as structured evidence. +fn run_and_insert(results: &mut BTreeMap, name: &str, assertion_fn: fn() -> AssertionResult) { + match run_assertion(name, assertion_fn) { + Ok(proof) => { + results.insert(name.to_owned(), proof); + }, + Err(error) => { + eprintln!(" [X] {name} failed: {error}"); + results.insert( + name.to_owned(), + proof_failure(&format!("{name} failed: {error}"), BTreeMap::new(), Duration::ZERO), + ); + }, + } +} + +/// Configure SWIM peer discovery by updating each operator with peer seed addresses. +fn configure_swim_peers(_forge_bin: &Path, _resolved_config: &Path) -> Result<(), Box> { + let mut swim_ips = BTreeMap::new(); + + for cluster in CLUSTERS { + let ip = read_swim_lb_ip(cluster)?; + eprintln!(" {cluster}: SWIM LB IP = {ip}"); + swim_ips.insert(*cluster, ip); + } + + for cluster in CLUSTERS { + let this_ip = swim_ips + .get(cluster) + .ok_or_else(|| format!("{cluster}: missing SWIM IP"))?; + let mut peer_parts = Vec::new(); + for c in CLUSTERS { + if *c != *cluster { + let ip = swim_ips.get(c).ok_or_else(|| format!("{c}: missing SWIM IP"))?; + peer_parts.push(format!("{ip}:7946")); + } + } + let peer_seeds = peer_parts.join(","); + update_operator_swim_config(cluster, this_ip, &peer_seeds)?; + } + + for cluster in CLUSTERS { + let ctx = format!("kind-grid-provider-traffic-{cluster}"); + wait_for_deployment("grid-operator", GRID_SYSTEM_NS, &ctx)?; + eprintln!(" [OK] {cluster}: operator restarted with SWIM config"); + } + + Ok(()) +} + +/// Read the SWIM `LoadBalancer` IP for a cluster directly from Kubernetes. +fn read_swim_lb_ip(cluster: &str) -> Result> { + let context = format!("kind-grid-provider-traffic-{cluster}"); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "svc", + "grid-operator-swim", + "-o", + "jsonpath={.status.loadBalancer.ingress[0].ip}", + ]) + .output()?; + + if !output.status.success() { + return Err(format!( + "{cluster}: cannot read SWIM service: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + + let ip = String::from_utf8(output.stdout)?.trim().to_owned(); + if ip.is_empty() { + return Err(format!("{cluster}: SWIM LoadBalancer has no ingress IP").into()); + } + + Ok(ip) +} + +/// Update a single operator's SWIM configuration with peer addresses. +#[expect( + clippy::too_many_lines, + reason = "The Helm upgrade carries the bounded SWIM configuration contract." +)] +fn update_operator_swim_config( + cluster: &str, + advertise_ip: &str, + seeds: &str, +) -> Result<(), Box> { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + let operator_image = + std::env::var("GRID_XTASK_OPERATOR_IMAGE").unwrap_or_else(|_| "grid-operator:provider-traffic-demo".to_owned()); + let image_pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); + let (operator_repo, operator_tag) = parse_image_ref(&operator_image); + + let seeds_escaped = seeds.replace(',', "\\,"); + + let upgrade_output = Command::new("helm") + .args([ + "upgrade", + "grid-operator", + "charts/grid-operator", + "--version", + "0.1.0", + "--namespace", + "grid-system", + "--kube-context", + &context, + "--reuse-values", + "--set", + &format!("image.repository={operator_repo}"), + "--set", + &format!("image.tag={operator_tag}"), + "--set", + &format!("image.pullPolicy={image_pull_policy}"), + "--set", + &format!("swim.siteName={cluster}"), + "--set", + &format!("swim.advertiseAddress={advertise_ip}:7946"), + "--set", + &format!("swim.seeds={seeds_escaped}"), + "--set", + "swim.service.enabled=true", + "--set", + "swim.service.type=LoadBalancer", + "--set", + &format!("gateway.serviceName={PROVIDER_GATEWAY_SERVICE}"), + "--set-string", + &format!("gateway.port={PROVIDER_GATEWAY_PORT}"), + ]) + .output()?; + + if !upgrade_output.status.success() { + return Err(format!( + "Failed to update SWIM config for {cluster}: {}", + String::from_utf8_lossy(&upgrade_output.stderr) + ) + .into()); + } + + eprintln!(" [OK] {cluster}: SWIM peers configured"); + + assert_operator_gateway_env(&context, cluster)?; + + Ok(()) +} + +/// Post-upgrade assertion: the operator deployment must reflect the expected +/// gateway discovery contract after every Helm upgrade. +fn assert_operator_gateway_env(context: &str, cluster: &str) -> Result<(), Box> { + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "deployment/grid-operator", + "-o", + "jsonpath={.spec.template.spec.containers[0].env}", + ]) + .output()?; + + let env_json = String::from_utf8_lossy(&output.stdout); + + let has_service = env_json.contains(&format!( + "\"name\":\"GRID_GATEWAY_SERVICE_NAME\",\"value\":\"{PROVIDER_GATEWAY_SERVICE}\"" + )); + let has_port = env_json.contains(&format!( + "\"name\":\"GRID_GATEWAY_PORT\",\"value\":\"{PROVIDER_GATEWAY_PORT}\"" + )); + + if !has_service || !has_port { + return Err(format!( + "{cluster}: operator gateway env mismatch after helm upgrade \ + (expected GRID_GATEWAY_SERVICE_NAME={PROVIDER_GATEWAY_SERVICE}, \ + GRID_GATEWAY_PORT={PROVIDER_GATEWAY_PORT}); got: {env_json}" + ) + .into()); + } + + Ok(()) +} + +/// Tear down only the provider-traffic Forge environment. +fn teardown_environment(context: &ProviderTrafficContext) -> Result<(), Box> { + eprintln!(); + eprintln!("=== TEARDOWN ==="); + + let status = Command::new(&context.forge_bin) + .args(["down", "--config"]) + .arg(&context.resolved_config) + .status()?; + + if !status.success() { + return Err("failed to tear down provider-traffic environment".into()); + } + + eprintln!(" [OK] Environment torn down successfully"); + Ok(()) +} + +/// Run the focused provider-traffic demo. +#[expect( + clippy::too_many_lines, + reason = "The public demo entrypoint keeps setup, proof, evidence, and teardown visible." +)] +pub(crate) fn run(forge_config: &Path, options: &GlbDemoOptions) -> Result<(), Box> { + if options.mode() != DemoMode::Quick { + return Err("provider-traffic supports only the focused quick proof".into()); + } + let mode = DemoMode::Quick; + let run_id = format_utc_timestamp(); + let wall_start = Instant::now(); + let _started_at = format_utc_iso(); + + let evidence_dir = resolve_evidence_dir(forge_config, options, &run_id)?; + fs::create_dir_all(&evidence_dir)?; + + let setup_ctx = prepare_setup(forge_config); + let mut teardown_success = false; + let mut run_error = None; + let mut overlay_state = OverlayState::default(); + let mut image_evidence = BTreeMap::new(); + + let proof_results = match &setup_ctx { + Ok(context) => { + eprintln!("{OUTPUT_RULE}"); + eprintln!("Grid Provider Traffic Demo"); + eprintln!("Mode: {}", if mode == DemoMode::Quick { "quick" } else { "full" }); + eprintln!("Config: {}", forge_config.display()); + eprintln!("{OUTPUT_RULE}"); + + match deploy_setup(context) { + Ok(state) => { + overlay_state = state; + image_evidence = match collect_image_evidence() { + Ok(collected_images) => collected_images, + Err(error) => { + run_error = Some(format!("image evidence collection failed: {error}")); + BTreeMap::new() + }, + }; + eprintln!(); + eprintln!("{OUTPUT_RULE}"); + eprintln!("ENVIRONMENT READY - Starting proof scenarios"); + eprintln!("{OUTPUT_RULE}"); + + let scenario_results = run_quick_scenarios(); + + let failed_proofs: Vec<&str> = scenario_results + .iter() + .filter_map(|(name, proof)| (!proof.success).then_some(name.as_str())) + .collect(); + if !failed_proofs.is_empty() { + run_error = Some(format!("runtime proofs failed: {}", failed_proofs.join(", "))); + } + + // Teardown if requested + if options.teardown && (run_error.is_none() || !options.keep_on_failure) { + match teardown_environment(context) { + Ok(()) => teardown_success = true, + Err(error) => { + eprintln!("[WARN] Teardown failed: {error}"); + run_error = Some(match run_error { + Some(previous) => format!("{previous}; teardown failed: {error}"), + None => format!("teardown failed: {error}"), + }); + }, + } + } + + scenario_results + }, + Err(e) => { + eprintln!("[FAIL] Environment setup failed: {e}"); + run_error = Some(format!("environment setup failed: {e}")); + + if options.teardown && !options.keep_on_failure { + if let Err(cleanup_err) = teardown_environment(context) { + eprintln!("[WARN] Cleanup after setup failure also failed: {cleanup_err}"); + run_error = Some(format!("environment setup failed: {e}; cleanup failed: {cleanup_err}")); + } else { + teardown_success = true; + } + } + + BTreeMap::new() + }, + } + }, + Err(e) => { + eprintln!("[FAIL] Setup preparation failed: {e}"); + run_error = Some(format!("setup preparation failed: {e}")); + BTreeMap::new() + }, + }; + + let evidence = Evidence { + schema_version: EVIDENCE_SCHEMA_VERSION.to_owned(), + mode: if mode == DemoMode::Quick { "quick" } else { "full" }.to_owned(), + topology: "provider-traffic".to_owned(), + clusters: CLUSTERS.iter().map(|&s| s.to_owned()).collect(), + proof_results, + images: image_evidence, + overlay_state, + cluster_health: Vec::new(), // Will be populated during runtime assertions + components: Vec::new(), // Will be populated during runtime assertions + swim_membership: Vec::new(), // Will be populated during runtime assertions + provider_responses: Vec::new(), // Will be populated during runtime assertions + security_results: Vec::new(), // Will be populated during runtime assertions + teardown_success, + }; + + // Write evidence + let evidence_file = evidence_dir.join("results.json"); + let evidence_json = serde_json::to_string_pretty(&evidence)?; + fs::write(&evidence_file, evidence_json)?; + + eprintln!(); + eprintln!("{OUTPUT_RULE}"); + eprintln!("Demo completed in {:.1}s", wall_start.elapsed().as_secs_f64()); + eprintln!("Evidence: {}", evidence_file.display()); + eprintln!("{OUTPUT_RULE}"); + + match run_error { + Some(error) => Err(error.into()), + None => Ok(()), + } +} + +/// Collect actual image evidence from the deployed clusters. +#[expect( + clippy::too_many_lines, + reason = "Evidence collection queries the bounded set of deployed component images." +)] +fn collect_image_evidence() -> Result, Box> { + let mut image_evidence = BTreeMap::new(); + + for cluster in CLUSTERS { + let context = format!("kind-grid-provider-traffic-{cluster}"); + + // Get grid operator image + let operator_output = Command::new("kubectl") + .args([ + "get", + "deployment/grid-operator", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if operator_output.status.success() { + let operator_image = String::from_utf8_lossy(&operator_output.stdout).trim().to_owned(); + image_evidence.insert(format!("{cluster}_operator"), operator_image); + } + + // Get consumer gateway image + let consumer_output = Command::new("kubectl") + .args([ + "get", + "deployment/consumer-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if consumer_output.status.success() { + let consumer_image = String::from_utf8_lossy(&consumer_output.stdout).trim().to_owned(); + image_evidence.insert(format!("{cluster}_consumer_gateway"), consumer_image); + } + + // Get provider gateway image + let provider_output = Command::new("kubectl") + .args([ + "get", + "deployment/provider-gateway", + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if provider_output.status.success() { + let provider_image = String::from_utf8_lossy(&provider_output.stdout).trim().to_owned(); + image_evidence.insert(format!("{cluster}_provider_gateway"), provider_image); + } + + // Get VCR inference image + let vcr_output = Command::new("kubectl") + .args([ + "get", + &format!("deployment/vcr-inference-{cluster}"), + "--context", + &context, + "-n", + "grid-system", + "-o", + "jsonpath={.spec.template.spec.containers[0].image}", + ]) + .output()?; + + if vcr_output.status.success() { + let mock_image = String::from_utf8_lossy(&vcr_output.stdout).trim().to_owned(); + image_evidence.insert(format!("{cluster}_vcr_inference"), mock_image); + } + } + + Ok(image_evidence) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn proof_success_creation() { + let mut facts = BTreeMap::new(); + facts.insert("cluster_count".to_owned(), serde_json::Value::Number(3.into())); + facts.insert("all_healthy".to_owned(), serde_json::Value::Bool(true)); + + let proof = proof_success("Test success", facts.clone(), Duration::from_millis(100)); + + assert!(proof.success); + assert_eq!(proof.reason, "Test success"); + assert_eq!(proof.duration_ms, 100); + assert_eq!(proof.observed_facts.len(), 2); + assert_eq!( + proof.observed_facts.get("all_healthy"), + Some(&serde_json::Value::Bool(true)) + ); + } + + #[test] + fn proof_failure_creation() { + let mut facts = BTreeMap::new(); + facts.insert("error_code".to_owned(), serde_json::Value::Number(500.into())); + + let proof = proof_failure("Test failure", facts.clone(), Duration::from_millis(50)); + + assert!(!proof.success); + assert_eq!(proof.reason, "Test failure"); + assert_eq!(proof.duration_ms, 50); + assert_eq!(proof.observed_facts.len(), 1); + assert_eq!( + proof.observed_facts.get("error_code"), + Some(&serde_json::Value::Number(500.into())) + ); + } + + #[test] + fn assertion_result_error_handling() { + let assertion_fn = || -> AssertionResult { Err("Simulated assertion failure".into()) }; + + let result = run_assertion("test_assertion", assertion_fn); + assert!(result.is_err()); + + let Err(error) = result else { + std::process::abort(); + }; + let error_msg = error.to_string(); + assert!(error_msg.contains("Assertion test_assertion failed")); + assert!(error_msg.contains("Simulated assertion failure")); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "This test exercises the complete evidence schema.")] + fn evidence_serialization() { + let evidence = Evidence { + schema_version: "test".to_owned(), + mode: "quick".to_owned(), + topology: "provider-traffic".to_owned(), + clusters: vec![ + "provider-a".to_owned(), + "provider-b".to_owned(), + "provider-c".to_owned(), + ], + proof_results: BTreeMap::new(), + images: BTreeMap::new(), + overlay_state: OverlayState::default(), + cluster_health: vec![ClusterHealth { + name: "provider-a".to_owned(), + healthy: true, + api_response_ms: Some(100), + ready_nodes: 1, + }], + components: vec![ComponentStatus { + name: "provider-a-grid-operator".to_owned(), + namespace: "grid-system".to_owned(), + ready_replicas: 1, + desired_replicas: 1, + ready: true, + }], + swim_membership: vec![SwimMembership { + site: "provider-a".to_owned(), + local_node: "provider-a-operator".to_owned(), + peers: vec!["provider-b-operator".to_owned(), "provider-c-operator".to_owned()], + converged: true, + }], + provider_responses: vec![], + security_results: vec![], + teardown_success: true, + }; + + let Ok(json) = serde_json::to_string(&evidence) else { + std::process::abort(); + }; + assert!(json.contains("\"schema_version\":\"test\"")); + assert!(json.contains("\"topology\":\"provider-traffic\"")); + assert!(json.contains("\"healthy\":true")); + assert!(json.contains("\"ready_replicas\":1")); + assert!(json.contains("\"converged\":true")); + + // Verify deserialization + let Ok(_deserialized) = serde_json::from_str::(&json) else { + std::process::abort(); + }; + } + + #[test] + fn proof_count_validation() { + let names = [ + "cluster_health", + "component_deployment", + "swim_convergence", + "site_auto_discovery", + "overlay_acceptance", + "provider_gateway_round_robin", + ]; + assert_eq!(names.len(), 6); + assert_eq!(names[0], "cluster_health"); + assert_eq!(names[5], "provider_gateway_round_robin"); + assert_eq!(CLUSTERS.len(), 3); + assert_eq!(CLUSTERS, &["provider-a", "provider-b", "provider-c"]); + } + + #[test] + fn evidence_schema_version() { + assert_eq!(EVIDENCE_SCHEMA_VERSION, "1"); + } + + #[test] + fn curl_pod_overrides_meets_restricted_pod_security() { + let json = curl_pod_overrides("test-probe", &["curl", "--fail", "http://example.test"]); + let actual: serde_json::Value = serde_json::from_str(&json).unwrap_or_else(|_| std::process::abort()); + assert_eq!( + actual, + serde_json::json!({ + "spec": { + "automountServiceAccountToken": false, + "securityContext": { + "runAsNonRoot": true, + "seccompProfile": { "type": "RuntimeDefault" } + }, + "containers": [{ + "name": "test-probe", + "image": "curlimages/curl:8.12.1", + "command": ["curl"], + "args": ["--fail", "http://example.test"], + "securityContext": { + "runAsUser": 100, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { "drop": ["ALL"] } + } + }] + } + }) + ); + } + + #[test] + fn provider_traffic_constants_describe_focused_topology() { + assert_eq!(CLUSTERS, &["provider-a", "provider-b", "provider-c"]); + assert_eq!(CONSUMER_SITE, "provider-a"); + assert_eq!(EVIDENCE_SCHEMA_VERSION, "1"); + } +}