From 1118a1cc3032b21c443292ead07ad9137e0f6cc7 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Mon, 31 Aug 2026 06:31:34 +0000 Subject: [PATCH] docs(demos): expand cloud burst resilience example Signed-off-by: Brent Salisbury --- demos/grid-cloud-burst/README.md | 1476 +++++++++-------- .../08-regional-local-pools.yaml | 202 +++ .../09-regional-inference-providers.yaml | 65 + .../10-observability-jaeger.yaml | 67 + .../10-openai-overflow-providers.yaml | 53 + .../11-azure-overflow-providers.yaml | 63 + .../12-azure-provider-gateways.yaml | 195 +++ .../13-consumer-east-praxis.yaml | 246 +++ .../14-consumer-west-praxis.yaml | 246 +++ .../example-manifests/README.md | 31 + .../configure-azure-consumer-routes.sh | 46 + .../scripts/configure-otel.sh | 28 + .../scripts/configure-ui-traces.sh | 12 + .../scripts/diagnose-east-withdrawal.sh | 91 + .../scripts/diagnose-request-stall.sh | 79 + .../scripts/latency-one-provider-cycles.sh | 134 ++ .../scripts/qualify-cloud-burst.sh | 307 ++++ .../scripts/validate-azure-provider.sh | 79 + 18 files changed, 2699 insertions(+), 721 deletions(-) create mode 100644 demos/grid-cloud-burst/example-manifests/08-regional-local-pools.yaml create mode 100644 demos/grid-cloud-burst/example-manifests/09-regional-inference-providers.yaml create mode 100644 demos/grid-cloud-burst/example-manifests/10-observability-jaeger.yaml create mode 100644 demos/grid-cloud-burst/example-manifests/10-openai-overflow-providers.yaml create mode 100644 demos/grid-cloud-burst/example-manifests/11-azure-overflow-providers.yaml create mode 100644 demos/grid-cloud-burst/example-manifests/12-azure-provider-gateways.yaml create mode 100644 demos/grid-cloud-burst/example-manifests/13-consumer-east-praxis.yaml create mode 100644 demos/grid-cloud-burst/example-manifests/14-consumer-west-praxis.yaml create mode 100755 demos/grid-cloud-burst/scripts/configure-azure-consumer-routes.sh create mode 100755 demos/grid-cloud-burst/scripts/configure-otel.sh create mode 100755 demos/grid-cloud-burst/scripts/configure-ui-traces.sh create mode 100755 demos/grid-cloud-burst/scripts/diagnose-east-withdrawal.sh create mode 100755 demos/grid-cloud-burst/scripts/diagnose-request-stall.sh create mode 100755 demos/grid-cloud-burst/scripts/latency-one-provider-cycles.sh create mode 100755 demos/grid-cloud-burst/scripts/qualify-cloud-burst.sh create mode 100755 demos/grid-cloud-burst/scripts/validate-azure-provider.sh diff --git a/demos/grid-cloud-burst/README.md b/demos/grid-cloud-burst/README.md index ec9c0b8..b127d39 100644 --- a/demos/grid-cloud-burst/README.md +++ b/demos/grid-cloud-burst/README.md @@ -1,790 +1,824 @@ - + -# Praxis Grid Adaptive Cloud Bursting +# Grid Regional Failover and Cloud Burst -https://github.com/user-attachments/assets/6cb33a69-288d-4195-8f80-c6bc537f2d41 +https://github.com/user-attachments/assets/ab6b3166-7ddc-4b4f-999e-b4fbeede8c7c -This demo presents **Praxis Grid adaptive inference routing and cloud -bursting for Kubernetes**. - -It demonstrates how Grid can continuously react to changing provider health, -locality, queue pressure, token consumption, and available overflow capacity -while keeping the Grid control plane **out of the synchronous inference request -path**. - -```text -Grid observes and computes policy asynchronously - | - v - versioned routing snapshot - | - v - Praxis gateway - | - local selection - | - v - selected provider -``` - -Praxis does not call Grid, Kubernetes, Prometheus, llm-d/EPP, or another remote -scoring service for each request. It executes the latest accepted policy -snapshot locally. - ---- - -## Architecture document - -The broader policy architecture used by this demo is summarized in the -architecture and policy sections below. The full architecture document is -maintained with the implementation planning materials. - -That document covers: - -- admission and provider eligibility; -- same-site, same-zone, and same-region active provider grouping; -- weighted and pressure-aware placement; -- queue-depth and KV-cache pressure policies; -- local rebalancing before cloud overflow; -- independent burst and overflow-provider policies; -- token governance; -- multi-gateway and multi-provider scaling; -- provider security boundaries; -- observability and policy feedback loops; and -- the path toward future cost-aware placement. - ---- - -# What the demo proves - -The recording composes several Grid and Praxis capabilities into one end-to-end -scenario: - -- **local-first inference routing** across Kubernetes-hosted providers; -- **active provider grouping** without falsifying site or region identity; -- **weighted request placement** inside an active provider group; -- **queue-aware provider pressure** using llm-d/EPP metrics; -- **reactive overflow routing** when preferred capacity becomes constrained; -- **independent admission, grouping, placement, burst, and overflow policies**; -- **soft token governance** backed by shared distributed token state; -- **token continuity across multiple Praxis gateways and provider changes**; -- **OpenTelemetry route and provider attribution**; -- **versioned routing state** so requests can be correlated with the policy - snapshot that served them; and -- **recovery back toward preferred Kubernetes inference capacity** as pressure - falls. - -The overflow tier is provider-neutral. Depending on deployment policy, it may -contain providers such as: - -- AWS Bedrock; -- Azure-hosted model services; -- Anthropic; -- OpenAI; -- Google Vertex AI / Gemini; and -- other compatible model-provider APIs. - -The recording does not claim that every provider above is exercised in this -specific run. They represent the external-provider class supported by the -architecture. - ---- - -# User story: keep Grid out of the request hot path - -> **As a platform operator, I want Grid to make sophisticated multi-cluster -> routing decisions without making every inference request wait on the control -> plane.** - -Grid is the asynchronous policy plane. Praxis is the fast local execution -plane. +This experimental demo shows inference traffic moving through four local +providers in two provider sites and four external overflow routes. It covers +healthy site-local balancing, backend failure, cross-site fallback, pressure-driven +admission, Azure/OpenAI overflow, recovery, shared sliding-window soft quotas, +and request-level provider attribution. + +The implementation is early work in progress. Required changes are still +carried on development branches while their upstream contracts are reviewed. +Configuration and deployment may change as those proposals settle. Recreating +the environment currently requires manual assembly and familiarity with +Kubernetes, Grid, and Praxis. + +## User stories + +- **As an inference-platform operator**, I want traffic to remain on healthy + site-local capacity, fail over to another Grid site when necessary, and use + external providers only when the local fleet cannot accept new work. +- **As a reliability engineer**, I want backend failure and queue pressure to + produce different, explainable admission states while requests continue from + the last accepted routing snapshot. +- **As an application owner**, I want one authenticated token allocation to + follow my workload across consumer gateways and provider changes so failover + cannot create a second allowance. +- **As an operator investigating a request**, I want the observed path to + distinguish the entry gateway, provider boundary, concrete inference backend + or cloud route, quota decision, and upstream HTTP result. + +## What this demo proves + +| Capability | Demonstrated behavior | +|---|---| +| Regional balancing | East traffic uses `llm-d-east-1/2`; west uses `llm-d-west-1/2`. | +| Single-backend failure | Traffic stays site-local and uses the surviving backend. | +| Provider-site failure | Traffic uses healthy Grid providers in the other site. | +| Full local-fleet failure | Traffic uses the Azure OpenAI and OpenAI external provider group. | +| Multi-cloud overflow | Azure OpenAI and OpenAI routes share the external provider group. | +| Pressure admission | Queue pressure moves locals to `existing_only`; new traffic uses overflow. | +| Recovery | Recovered locals rejoin after Grid publishes and Praxis accepts new state. | +| Exact attribution | Request paths distinguish the provider gateway from its selected backend. | +| Hot-path isolation | Grid publishes state asynchronously; Praxis selects from a local snapshot. | +| Shared soft quota | Three application-scoped sliding windows remain continuous across consumers and provider changes. | + +The demo does not claim gradual percentage-based cloud bursting, cost-aware +placement, or billing. Its cloud transition is group +fallback: external capacity becomes active when no preferred Grid provider can +accept a new request. + +The recording includes an Azure-attributed HTTP 429 returned by Azure's upstream +service. That is distinct from a Praxis quota denial: a Praxis denial occurs +before routing and therefore has no provider hop. + +## Topology + +Two consumer gateways can reach two provider sites. Each provider +site has a provider gateway and two local inference simulators. Azure OpenAI and +OpenAI routes in each site form the final overflow tier. ```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% flowchart TB - subgraph CP["Grid control / policy plane"] - D["Discover
providers"] - O["Observe
health + metrics"] - A["Apply
admission"] - G["Build active
provider groups"] - P["Compute traffic
placement"] - B["Compute burst /
overflow policy"] - V["Publish versioned
routing snapshot"] - - D --> O --> A --> G --> P --> B --> V - end + Client["Inference client"] - subgraph DP["Praxis fast request path"] - R["Request"] - I["Authenticate"] - T["Apply token
policy"] - S["Read local
snapshot"] - F["Resolve
affinity"] - C["Choose first
viable group"] - W["Select provider
locally"] - U["Forward"] - - R --> I --> T --> S --> F --> C --> W --> U + subgraph Entry["Consumer gateways"] + direction LR + EastConsumer["Consumer East
quota + local snapshot"] + WestConsumer["Consumer West
quota + local snapshot"] end - V -. "asynchronous
snapshot delivery" .-> S -``` - -Grid may reconcile changing distributed state at its own cadence. - -Praxis continues serving from the last accepted routing snapshot and atomically -moves to a newer snapshot after validation. - ---- - -# User story: understand policy as independent decisions + subgraph Fleet["Preferred Grid-hosted capacity"] + direction LR + subgraph East["East provider site"] + direction TB + EastGateway["East provider gateway"] + East1["llm-d-east-1"] + East2["llm-d-east-2"] + EastGateway --> East1 + EastGateway --> East2 + end + subgraph West["West provider site"] + direction TB + WestGateway["West provider gateway"] + West1["llm-d-west-1"] + West2["llm-d-west-2"] + WestGateway --> West1 + WestGateway --> West2 + end + end -> **As an operator, I want each routing policy to answer one clear question so -> I can change one behavior without accidentally changing several others.** + OverflowGroup["Final eligible group
multi-cloud overflow"] + subgraph Overflow["External providers"] + direction LR + Azure["Azure OpenAI
east + west routes"] + OpenAI["OpenAI
east + west routes"] + end -The demo treats routing as a composition of independent policies: - -| Policy | Question | -|---|---| -| **Admission** | May this provider receive new or existing traffic? | -| **Grouping** | Which eligible providers actively compete together? | -| **Placement** | How is traffic divided inside the active group? | -| **Burst policy** | How much traffic should leave preferred capacity? | -| **Overflow policy** | Where should overflow traffic go? | -| **Affinity** | Should an existing session remain on its current provider? | -| **Token policy** | How is a user or workload's token consumption governed? | + Client --> EastConsumer + Client --> WestConsumer + EastConsumer -->|"preferred"| EastGateway + WestConsumer -->|"preferred"| WestGateway + EastConsumer -. "regional fallback" .-> WestGateway + WestConsumer -. "regional fallback" .-> EastGateway + EastConsumer -. "no eligible local group" .-> OverflowGroup + WestConsumer -. "no eligible local group" .-> OverflowGroup + OverflowGroup --> Azure + OverflowGroup --> OpenAI +``` + +The local providers use +[`llm-d-inference-sim`](https://github.com/llm-d/llm-d-inference-sim) to expose +an OpenAI-compatible endpoint and Prometheus-compatible pressure signals. It +stands in for an inference stack. Grid does not manage model serving, +scheduling, or inference-engine internals. + +## Component ownership + +| Component | Responsibility | Not its request-time responsibility | +|---|---|---| +| Grid operator | Observe health and llm-d-compatible metrics, compute admission and locality groups, publish versioned overlays. | Proxy inference requests or select a destination per request. | +| Overlay sync | Validate and deliver accepted routing state to Praxis. | Score providers or run inference. | +| Praxis consumer | Parse the request, read the accepted snapshot, select a viable provider, forward the request. | Query Kubernetes, Grid, EPP, or Prometheus per request. | +| Praxis provider | Validate candidate/model/path, resolve a provider-local backend, apply final-hop credentials. | Recompute Grid placement. | +| llm-d stack | Manage provider-local inference scheduling and expose operational signals. | Compute Grid-wide locality or overflow policy. | +| External provider | Serve eligible overflow requests. | Participate in Grid's synchronous control path. | ```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - H["Hard constraints
health, trust, capability,
freshness, policy"] - A["Admission
may provider receive traffic?"] - G["Grouping
who competes together?"] - L["Placement
how is local share divided?"] - B["Burst
how much leaves preferred tier?"] - O["Overflow
where does burst traffic go?"] - X["Final groups +
traffic weights"] - P["Praxis executes
locally"] - - H --> A --> G --> L --> B --> O --> X --> P +flowchart LR + subgraph Control["Asynchronous control plane"] + Metrics["Provider health and metrics"] --> Grid["Grid policy"] + Grid --> Overlay["Versioned overlay"] + Overlay --> Sync["Validation and sync"] + end + subgraph Request["Synchronous request path"] + Client["Request"] --> Consumer["Praxis consumer"] + Consumer --> Provider["Praxis provider"] + Provider --> Backend["Selected backend"] + end + Sync -. "atomic snapshot update" .-> Consumer ``` -The key rule is: - -> **Do not collapse locality, queue pressure, provider cost, and traffic -> percentage into one opaque score.** +Grid reconciles at its own cadence. Praxis continues serving from the last +accepted snapshot and switches only after a newer snapshot has been validated +and loaded. ---- +## Resilience model -# User story: keep provider topology truthful +Resilience is layered rather than expressed as a single undifferentiated +fallback. The accepted overlay preserves locality, separates health withdrawal +from pressure admission, and activates the external group only after preferred +Grid capacity is no longer eligible for a new request. -> **As a regional platform owner, I want multiple providers in different sites -> to actively share traffic without pretending they are the same physical -> site.** +### Routing order -Grid keeps site and region identity intact. +For a request entering the east consumer: -For example, two providers can remain distinct: +| Priority | Candidate tier | Used when | +|---:|---|---| +| 1 | Healthy east providers | Normal local operation. | +| 2 | Healthy west providers | East has no provider eligible for new traffic. | +| 3 | External overflow | No Grid provider is eligible, or admission restricts all locals. | -```text -Qwen East - site: east - region: region-a +The west consumer uses the same order with west and east reversed. Disabling +both east backends should not immediately use an external API while healthy +west capacity remains eligible. Cloud is the final viable group. -Qwen West - site: west - region: region-a -``` - -while still participating in the same active regional group. +### Healthy request ```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - E["Qwen East
site: east
region: region-a"] - W["Qwen West
site: west
region: region-a"] - G["Active group
scope: sameRegion"] - P["Praxis weighted
selection"] - - E --> G - W --> G - G --> P -``` - -Grouping answers: +sequenceDiagram + participant C as Client + participant E as East consumer + participant P as East provider gateway + participant B as East backend pair -```text -Which providers may actively compete together? + C->>E: New unbound request + E->>E: Select from accepted snapshot + E->>P: Candidate and routing context + P->>P: Validate candidate, model, and path + P->>B: Resolve east backend + B-->>P: Inference response + P-->>E: Gateway and backend attribution + E-->>C: HTTP 200 ``` -Traffic weight answers: - -```text -How much new traffic should each eligible member receive? -``` - -Those remain separate policies. - ---- - -# User story: rebalance locally before paying for overflow - -> **As a GPU fleet owner, I want Grid to use available Kubernetes-hosted -> inference capacity before introducing external-provider cost.** - -One hot provider should not make the entire preferred tier look saturated. +### Availability fallback ```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - M["Per-provider
pressure + capacity"] - R["Rebalance traffic
inside preferred tier"] - H{"Enough preferred-tier
headroom remains?"} - L["Remain on
preferred providers"] - B["Allocate bounded
overflow share"] - - M --> R --> H - H -->|yes| L - H -->|no| B +flowchart LR + Healthy["East pair healthy"] --> East["Select east"] + OneDown["One east down"] --> Survivor["Select surviving east"] + RegionDown["Both east down"] --> West["Select healthy west"] + AllDown["All four locals down"] --> Cloud["Select overflow"] ``` -### Example +### Failure and recovery sequence shown in the recording -Suppose: +The failure demonstration is progressive and symmetric across the two sites: -```text -East queue pressure: 0.95 -West queue pressure: 0.35 -``` - -The first response should be to reduce East's new-traffic share and move more -traffic toward West. - -```text -Before pressure: - -East 50% -West 50% -Cloud 0% - -East becomes hot: +| Failure state | Expected route | Recovery behavior | +|---|---|---| +| One local backend unavailable | The surviving backend in the same site | The withdrawn backend rejoins after health and overlay convergence. | +| Both backends in one site unavailable | Healthy Grid capacity in the other site | Restoring the site returns new traffic according to locality and the accepted overlay. | +| All four local backends unavailable | The external provider group, including Azure OpenAI and OpenAI | Restoring local capacity withdraws cloud preference after the recovered overlay is accepted. | -East 15% -West 85% -Cloud 0% -``` - -Only when the preferred tier as a whole loses enough usable headroom should -Grid begin allocating traffic to overflow providers. - ---- - -# User story: use soft burst before hard fallback +These are availability transitions, not queue-pressure transitions. An unavailable +endpoint is withdrawn from new traffic by health reconciliation. A healthy but +pressured endpoint remains available for existing work while admission changes to +`existing_only` for new requests. In both cases, the gateways continue using their +last accepted snapshot until Grid publishes and Praxis accepts the replacement +revision; the request path does not synchronously call Grid or Kubernetes. -> **As an application owner, I want the platform to add external capacity -> gradually instead of waiting for local inference to fail completely.** - -The target behavior has three stages: - -1. **Local rebalance** — move traffic among healthy preferred providers. -2. **Soft burst** — move only the necessary share to overflow capacity. -3. **Hard fallback** — if preferred providers cannot accept new work, overflow - handles eligible new traffic. +The complete recovery path is: ```mermaid -%%{init: {"htmlLabels": false}}%% stateDiagram-v2 - [*] --> LocalOnly - - LocalOnly --> SoftBurst: preferred headroom falls - SoftBurst --> LocalOnly: sustained recovery - - SoftBurst --> HardFallback: no preferred new capacity - HardFallback --> SoftBurst: preferred capacity returns - HardFallback --> LocalOnly: full sustained recovery -``` - -### Example progression - -| State | Preferred providers | Overflow | -|---|---:|---:| -| Healthy | 100% | 0% | -| Moderate sustained pressure | 90% | 10% | -| Higher pressure | 70% | 30% | -| No preferred new capacity | 0% new traffic | 100% new traffic | -| Recovery | ramps upward | ramps downward | - -The exact percentages are policy results, not hard-coded product constants. - ---- - -# User story: pressure changes placement without overriding admission - -> **As an inference operator, I want queue or KV-cache pressure to shift -> traffic gradually while admission remains the hard safety boundary.** - -Admission answers: - -```text -Can this provider receive traffic? -``` - -Placement answers: - -```text -How much new traffic should this eligible provider receive? -``` + [*] --> LocalBalanced: all local backends healthy + LocalBalanced --> EastSurvivor: one east backend fails + EastSurvivor --> LocalBalanced: east backend restored and overlay accepted + LocalBalanced --> WestFallback: both east backends fail + WestFallback --> LocalBalanced: east site restored and overlay accepted + WestFallback --> CloudOverflow: remaining local backends fail + CloudOverflow --> LocalBalanced: local backends restored and overlay accepted + LocalBalanced --> WestSurvivor: one west backend fails + WestSurvivor --> LocalBalanced: west backend restored and overlay accepted +``` + +Each transition is visible in the request history as a changed provider gateway +and exact inference-provider attribution. The recording shows the local survivor +first, then cross-site Grid capacity, then the external group only when no local +provider is viable. Recovery is likewise observable: health returns, Grid +publishes a new versioned overlay, Praxis loads it, and subsequent new requests +return to local providers. Any request error during convergence remains visible; +it is not converted into a successful result by the demo. + +For a failure test, scale the simulator Deployment rather than the provider +gateway or Grid components. Grid observes the missing Service endpoint and +withdraws that provider through normal health reconciliation: + +```bash +kubectl -n grid-system scale deployment/llm-d-east-1 --replicas=0 +# restore it after the test +kubectl -n grid-system scale deployment/llm-d-east-1 --replicas=1 +``` + +The failover controls use this same Deployment operation. A provider absent +from the accepted overlay remains visible as configured-but-withdrawn until +health and the replacement overlay converge. + +### Pressure fallback + +Queue depth is collected from each configured llm-d-compatible metrics +endpoint. With capacity 10, the validated scenario set the queue to 9. That +crossed the admission boundary and moved all local providers to +`existing_only` for new requests. ```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - P["Provider
metrics"] - A{"Admission
state"} - W["Placement
weight"] - S["Praxis
selection"] - E["No new selection
existing affinity
may continue"] - X["No traffic"] - - P --> A - A -->|new + existing| W - A -->|existing only| E - A -->|excluded| X - W --> S -``` - -Example: - -```text -Provider A - admission: new_and_existing - traffic weight: 20 - -Provider B - admission: new_and_existing - traffic weight: 80 -``` - -Both remain eligible, but B receives a larger share of new unbound traffic. - -If A moves to `existing_only`, its prior weight no longer allows it to receive -new selections. - ---- - -# User story: avoid routing flaps - -> **As an operator, I want routing to react to sustained pressure without -> generating a new policy snapshot for every small metric movement.** - -Pressure-aware placement should be damped. - -```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - M["Raw queue /
KV pressure"] - C["Convert to
available capacity"] - S["Smooth across
observations"] - W["Compute candidate
weights"] - D{"Material distribution
change?"} - K["Keep current
published weights"] - P["Publish new
traffic weights"] - - M --> C --> S --> W --> D - D -->|no| K - D -->|yes| P -``` - -Useful anti-flap controls include: - -- smoothing across observations; -- separate pressure and recovery behavior; -- bounded minimum and maximum weights; -- material-change thresholds; -- admission hysteresis; -- minimum dwell time; and -- recovery hold-down. - -Small oscillations should not automatically generate continuous routing churn. - ---- - -# User story: make burst amount independent from burst destination - -> **As a cloud economics owner, I want to decide how much traffic needs -> external capacity separately from which external provider receives it.** - -There are two independent decisions: - -```text -HOW MUCH traffic should overflow? -``` - -and: - -```text -WHERE should that overflow traffic go? -``` - -```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - T["Total new
traffic"] - B["Burst policy
70% preferred
30% overflow"] - L["Preferred tier
70%"] - O["Overflow tier
30%"] - D["Overflow distribution
policy"] - - T --> B - B --> L - B --> O - O --> D - - D --> C1["AWS
Bedrock"] - D --> C2["Azure"] - D --> C3["Anthropic"] - D --> C4["OpenAI"] - D --> C5["Vertex AI /
Gemini"] - D --> CX["Other
providers"] -``` - -### Example - -Grid decides: - -```text -overflow share = 30% -``` - -The overflow policy independently decides: - -```text -Bedrock 50% -Azure 25% -Anthropic 15% -OpenAI 10% -``` - -The approximate total distribution becomes: - -```text -Preferred providers 70.0% -Bedrock 15.0% -Azure 7.5% -Anthropic 4.5% -OpenAI 3.0% -``` - -Changing the cloud-provider ratio should not change the burst percentage. -Changing the burst percentage should not change the relative overflow-provider -ratio. - ---- - -# User story: use different signals for different policy decisions - -> **As a policy author, I want queue depth, KV-cache pressure, locality, and -> future cost signals to affect only the decisions they are configured to -> influence.** - -One deployment might eventually use: - -```text -Burst policy - input: aggregate preferred-tier queue / headroom - result: 20% overflow - -Preferred-tier placement - input: KV-cache pressure - result: East 70% / West 30% of retained local share - -Overflow placement - input: future cost policy - result: Provider A 75% / Provider B 25% of overflow share -``` - -```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - Q["Aggregate queue /
capacity pressure"] - KV["KV-cache
pressure"] - Cost["Future cost /
provider policy"] - - Burst["Burst policy
preferred vs overflow"] - Local["Preferred-tier
placement"] - Cloud["Overflow-tier
placement"] - - Final["Compose final
traffic weights"] - Overlay["Grid routing
snapshot"] - Praxis["Praxis local
selection"] - - Q --> Burst - KV --> Local - Cost --> Cloud - - Burst --> Final - Local --> Final - Cloud --> Final - - Final --> Overlay --> Praxis -``` - -Praxis does not need to understand which signals produced the final weights. - ---- - -# User story: scale to many Praxis gateways - -> **As a networking operator, I want many Praxis gateways to execute the same -> routing policy without coordinating every weighted selection.** - -Grid publishes routing state by routing perspective. - -```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - O["Versioned routing
snapshot"] - - O --> G1["Praxis
gateway 1"] - O --> G2["Praxis
gateway 2"] - O --> G3["Praxis
gateway 3"] - O --> GN["Praxis
gateway N"] - - G1 --> F["Provider fleet"] - G2 --> F - G3 --> F - GN --> F -``` - -Round-robin counters and weighted draws are local. There is no distributed lock -for each provider selection. - -Over sufficient request volume, aggregate traffic approaches the policy -published by Grid. - -The shared state that does require coordination is token usage, which is -separate from routing selection state. - ---- - -# User story: keep soft token governance independent from routing - -> **As a platform owner, I want a user's token usage to remain continuous even -> while traffic moves between gateways, sites, clusters, and providers.** - -The demo uses shared token state separately from routing. - -```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - R["Request"] - I["Trusted user /
workload identity"] - L["Shared token
ledger"] - S["Soft allocation
status"] - G["Grid / Praxis
routing"] - P["Selected local or
overflow provider"] - U["Actual usage
reconciliation"] - - R --> I --> L --> S --> G --> P --> U - U --> L -``` - -A soft token allocation can report: - -```text -within allocation -approaching allocation -over allocation -``` - -without necessarily denying the request. - -### Example +sequenceDiagram + participant M as llm-d metrics + participant G as Grid + participant O as Routing overlay + participant P as Praxis + participant X as External provider + + M-->>G: Queue crosses admission threshold + G->>G: Mark locals existing_only + G->>O: Publish semantic revision + O-->>P: Validate and load revision + P->>P: Skip local groups for new request + P->>X: Select overflow provider +``` + +`existing_only` can preserve an established binding where policy permits it, +but prevents a new unbound selection. It differs from a fully excluded or +unhealthy provider. + +The pressure action changes only the simulator metric. It does not call Grid, +edit the overlay, pause traffic, or wait for reconciliation: + +```yaml +fake-metrics: + waiting-requests: 9 +``` + +With `queueCapacity: 10`, the observed queue signal crosses the configured +admission boundary. Grid publishes the resulting admission change and Praxis +continues serving its previous immutable snapshot until the replacement is +accepted. + +## Exact request attribution + +| Field | Example | Meaning | +|---|---|---| +| Provider gateway | `east` | Provider security and routing boundary. | +| Inference provider | `llm-d-east-1` | Provider-local backend resolved by `provider_route`. | + +With optional response attribution enabled, the gateway returns both values: + +```http +x-ai-demo-provider-gateway: east +x-ai-inference-provider: llm-d-east-1 +``` + +The inference-provider value comes only from trusted `provider_route.cluster` +metadata after candidate, model, and path validation. A client-provided header +with that name is removed and cannot override the selected backend. + +The corresponding OpenTelemetry span is `provider.route`, with bounded +provider, backend, model, candidate, and optional overlay-revision attributes. +Praxis core still owns propagation, sampling, request spans, and OTLP export. + +## Representative configuration + +These fragments explain the contract. Use the complete resources in +[`example-manifests`](./example-manifests/) as the source for these examples. + +### Grid policy + +```yaml +gridNetwork: + name: grid-cloud-burst-rhoai + routingPolicy: geographyFirst + scoringPolicy: + strategy: queueDepth + selectionPolicy: + mode: roundRobin + grouping: + localityScope: sameSite +``` + +- `geographyFirst` keeps the closest viable providers first. +- `queueDepth` uses the configured llm-d-compatible signal for scoring and + admission. +- `roundRobin` distributes new unbound requests inside the active group. +- `sameSite` keeps each local pair in its site-local group. + +### Local provider + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: llm-d-east-1 + namespace: grid-system +spec: + gridNetworkRef: grid-cloud-burst-rhoai + providerKind: llm-d-inference-sim + backendKind: local + endpoint: http://llm-d-east-1.grid-system.svc.cluster.local:8000 + routingClusterRef: llm-d-east-1 + models: + - name: gpt-4o-mini + capabilities: [text_generation] + contextWindow: 4096 + siteSelector: + matchLabels: + grid.praxis-proxy.io/provider-site: east-local + metricsConfig: + metricsEndpoint: http://llm-d-east-1.grid-system.svc.cluster.local:8000 + path: /metrics + queueCapacity: 10 + signalNames: + queueDepth: vllm:num_requests_waiting + timeout: 2s + healthCheck: + path: /health + interval: 5s + timeout: 2s +``` + +The provider name, `routingClusterRef`, provider-gateway route, Service, and +Deployment must agree so attribution names the actual selected backend. + +The local simulator exposes the queue signal that Grid observes. The two +simulators in each site use the same shape; only the provider and Service names +change: + +```yaml +data: + config.yaml: | + model: "gpt-4o-mini" + mode: "echo" + fake-metrics: + running-requests: 0 + waiting-requests: 0 + kv-cache-usage: 0.1 +``` + +The matching provider points its metrics configuration at that Service: + +```yaml +metricsConfig: + metricsEndpoint: http://llm-d-east-1.grid-system.svc.cluster.local:8000 + path: /metrics + queueCapacity: 10 + signalNames: + queueDepth: vllm:num_requests_waiting +``` + +### External overflow provider + +```yaml +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: openai-east + namespace: grid-system +spec: + gridNetworkRef: grid-cloud-burst-rhoai + providerKind: openai + backendKind: api_provider + endpoint: https://api.openai.com + routingClusterRef: openai-east + models: + - name: gpt-4o-mini + capabilities: [text_generation] + contextWindow: 4096 + auth: + strategy: bearer_token + manual: true + secretRef: + name: openai-api-key + namespace: grid-system + key: token +``` + +The credential belongs at the provider boundary. It must not appear in Grid +routing metadata, response attribution, logs, or trace attributes. + +### External cluster + +Praxis must set both TLS SNI and HTTP authority: + +```yaml +- name: openai-east + http: + authority: api.openai.com + endpoints: + - api.openai.com:443 + tls: + sni: api.openai.com + verify: true + idle_timeout_ms: 5000 +``` + +`authority` is nested under `http`. SNI alone can produce HTTP 421 even when +TLS succeeds. The idle timeout avoids reusing a connection after the upstream +has closed it. Do not use unauthenticated `/v1/models` as a health probe for a +credentialed route; a 401 would incorrectly mark a usable provider unhealthy. + +### Shared sliding-window quota + +Both consumer gateways use the same identity/model key contract and Valkey +namespace. The demo applications each reserve 20 tokens against an independent +60-token rolling minute. `soft` enforcement reports over-allocation without +turning it into a pre-provider rejection: + +```yaml +- filter: token_rate_limit + enforcement: soft + key: + principal: + source: metadata + name: identity.user_id + onMissing: reject + model: + source: header + name: x-model + onMissing: reject + backend: + kind: valkey + url: ${TOKEN_RATE_LIMIT_VALKEY_URL} + namespace: praxis:token-rate-limit:v1 + rules: + - name: app1 + match: + metadata: + identity.user_id: app1 + estimation: + strategy: fixed + tokens: 20 + token_budgets: + - window: 1m + capacity: 60 +``` + +Authentication establishes `identity.user_id` before the limiter runs. The +limiter reserves before provider selection and reconciles actual response usage +afterward. Provider and overlay changes therefore do not change the quota key or +reset the rolling window. The complete app1/app2/app3 rules are in the two +consumer manifests. + +## Example resources + +This directory contains two complementary reference sets: + +| Set | Files | Purpose | +|---|---|---| +| Core adaptive-burst reference | `01`-`06` and [`example-manifests/README.md`](./example-manifests/README.md) | The original compact policy composition, including weighted placement, soft quota, provider-boundary security, OpenAI, and optional Bedrock. | +| Recorded regional-resilience topology | `08`-`14` and `scripts/` | The v2.1 two-site environment shown in the recording, including four local simulators, round-robin selection, Azure/OpenAI overflow, diagnostics, and optional tracing. | + +Treat these as separate topology examples. Do not apply both GridNetwork/provider +sets blindly into the same namespace. Start from the compact `01`-`06` reference +when studying policy composition; use `08`-`14` when reproducing the recorded +regional failover and multi-cloud path. + +| File | Purpose | +|---|---| +| [`08-regional-local-pools.yaml`](./example-manifests/08-regional-local-pools.yaml) | Four simulator ConfigMaps, Deployments, and Services. | +| [`09-regional-inference-providers.yaml`](./example-manifests/09-regional-inference-providers.yaml) | Four local providers with health and queue metrics. | +| [`10-openai-overflow-providers.yaml`](./example-manifests/10-openai-overflow-providers.yaml) | Two external overflow providers. | +| [`11-azure-overflow-providers.yaml`](./example-manifests/11-azure-overflow-providers.yaml) | Secret-backed Azure OpenAI overflow provider templates. | +| [`12-azure-provider-gateways.yaml`](./example-manifests/12-azure-provider-gateways.yaml) | Azure provider gateway templates with mTLS, Entra token injection, sealed routing, and upstream authority/SNI. | +| [`13-consumer-east-praxis.yaml`](./example-manifests/13-consumer-east-praxis.yaml) | Declarative east consumer ConfigMap with local, OpenAI, and dedicated Azure provider hops. | +| [`14-consumer-west-praxis.yaml`](./example-manifests/14-consumer-west-praxis.yaml) | Declarative west consumer ConfigMap with local, OpenAI, and dedicated Azure provider hops. | +| [`10-observability-jaeger.yaml`](./example-manifests/10-observability-jaeger.yaml) | Optional OTLP-compatible Jaeger deployment. | +| [`scripts/validate-azure-provider.sh`](./scripts/validate-azure-provider.sh) | One-token provider-boundary smoke test that derives the live candidate and sealed revision. | +| [`configure-otel.sh`](./scripts/configure-otel.sh) | Configures gateway OTLP export. | +| [`configure-ui-traces.sh`](./scripts/configure-ui-traces.sh) | Connects the UI to trace queries. | + +### Azure OpenAI overflow (optional) + +Azure is an additional external provider class, not another local Grid site. +The Azure gateway obtains an Entra bearer token server-side with the `azure_ad` +filter, then sends the OpenAI-compatible request to +`/openai/v1/chat/completions`. The Azure resource hostname is used for both +HTTP authority and TLS SNI. Grid only publishes the provider identity, +capability, health, and overflow-group membership; it does not acquire Azure +tokens or call Azure on the request path. + +The `11-azure-overflow-providers.yaml` file contains only placeholders. Create +the `azure-openai-client-secret` Secret from a protected secret manager and +configure the gateway with `AZURE_CLIENT_SECRET` through a Secret-backed +environment reference. Never put the client secret in a ConfigMap, overlay, +README, evidence, or browser-visible data. + +The Azure candidates share the same accepted overflow selection group as the +OpenAI candidates and use `roundRobin` within that group. Before applying the +provider resources, validate the direct Azure deployment independently; then +validate the provider gateway path and only afterward add the candidates to the +Grid overlay. The v2.1 recording exercises Azure through the consumer path as +well as OpenAI; the external services retain their own independent quotas and +availability. + +For a reusable provider-boundary smoke test after deployment, run +`scripts/validate-azure-provider.sh` from this directory. It reads the live +candidate ID and optional sealed overlay revision from the selected Azure +ConfigMap, mounts the existing `gateway-tls` Secret, and sends one one-token +request. Azure credentials remain server-side in the gateway Secret. The script +requires HTTP 200 plus `x-ai-demo-provider-gateway: azure-east|azure-west` and +`x-ai-inference-provider: azure-upstream`; it does not replace the qualifying +consumer-to-Grid overflow test. + +Before enabling Azure candidates in the consumer overlay, run +`scripts/configure-azure-consumer-routes.sh` to validate the generated change. +It adds `azure-east` and `azure-west` to `provider_hop_clusters` and maps them +to the dedicated Azure provider-gateway Services. The declarative source for +the cold deployment is `13-consumer-east-praxis.yaml` and +`14-consumer-west-praxis.yaml`. The endpoints are +`azure-east.grid-system.svc.cluster.local:8443` and +`azure-west.grid-system.svc.cluster.local:8443`; the SNI values use the +certificate-covered east/west identities. The script is dry-run by default and +checks that local/OpenAI routes are unchanged and that a second render is +identical. Set `APPLY_AZURE_CONSUMER_ROUTES=1` only after reviewing that +validation, then roll the consumers and run the provider-boundary probe and the +consumer-to-Grid test. Azure has been validated through the consumer path; +provider-attributed Azure 429 responses remain subject to the external Azure +resource's own quota and rate limits. + +These extend an existing Grid and Praxis environment. They do not create the +complete cluster, trust material, gateways, overlay sync, routes, or external +credentials by themselves. + +## Deployment outline + +### Prerequisites + +- Kubernetes or OpenShift. +- Grid operator and overlay sync from the development branch below. +- Praxis gateways built with the corresponding AI routing changes. +- `kubectl` or `oc`, Helm, and permission to create Grid resources. +- An external API credential if real overflow is exercised. +- Optional OTLP, Jaeger, and the routing UI. + +Install the base Grid and Praxis components, establish consumer-to-provider +trust, and confirm gateway readiness. Create `grid-system/openai-api-key` with +a `token` entry through normal secret management; never commit it here. + +Apply the reusable provider resources: + +```bash +kubectl apply -f example-manifests/08-regional-local-pools.yaml +kubectl apply -f example-manifests/09-regional-inference-providers.yaml +kubectl apply -f example-manifests/10-openai-overflow-providers.yaml +``` + +Configure provider gateways with route and cluster names matching the eight +`routingClusterRef` values. Wait until Grid health and admission converge, then +confirm the published overlay revision matches the revision served by Praxis. + +The provider gateway keeps one route entry per selected candidate. The route +key is the Grid-rendered stable ID; the cluster name is the concrete upstream +target: + +```yaml +- filter: provider_route + routes: + - candidate_id: + model: gpt-4o-mini + provider_model: gpt-4o-mini + paths: [/v1/chat/completions, /v1/responses] + cluster: llm-d-east-1 +``` + +The consumer declares provider-hop clusters separately and maps them to the +provider gateway Services. This keeps gateway identity distinct from the +selected inference provider: + +```yaml +- filter: intelligent_route + provider_hop_clusters: + - llm-d-east-1 + - llm-d-east-2 + - llm-d-west-1 + - llm-d-west-2 + - openai-east + - openai-west + - azure-east + - azure-west +``` + +Optional tracing: + +```bash +kubectl apply -f example-manifests/10-observability-jaeger.yaml +./scripts/configure-otel.sh +./scripts/configure-ui-traces.sh +``` + +The tracing UI needs two distinct Jaeger addresses: ```text -Alice allocation: 10,000 tokens - -Usage: 8,500 -Status: within allocation - -Usage: 10,500 -Status: over allocation -Action: report / observe -Request: may continue under soft policy +JAEGER_URL=http://jaeger-query.grid-system.svc.cluster.local:16686 +JAEGER_UI_URL=https:// +TRACING_UI_TOKEN_TRACE_LOOKUP=true ``` -A separate policy can choose hard enforcement where required. +`JAEGER_URL` is the in-cluster query endpoint used by the UI server. +`JAEGER_UI_URL` is the browser-reachable Route used for trace links. Do not set +the browser URL to the cluster-local Service address, and do not allow the UI +server to fall back to `http://localhost:16686` in a Kubernetes deployment. -Routing changes do not reset the token ledger. Token-policy changes do not reset -routing state. +For this OpenShift environment, obtain the public URL rather than hard-coding a +cluster-specific hostname: + +```bash +export JAEGER_URL=http://jaeger-query.grid-system.svc.cluster.local:16686 +export JAEGER_UI_URL="https://$(kubectl -n grid-system get route jaeger-query-public -o jsonpath='{.spec.host}')" +./scripts/configure-ui-traces.sh +``` ---- +The checked-in UI Deployment should declare both variables so a fresh +declarative deployment does not depend on running the helper afterward. When +updating them, preserve the rest of the container definition. After rollout, +verify the UI status API reports `jaeger_reachable: true`, its server-side +query URL is the cluster-local Service, and generated trace links use the +public Route. -# User story: preserve security at the provider boundary +Token trace lookup injects a unique W3C trace context into each request created +by the UI and queries Jaeger only for that exact trace ID. The lookup is +asynchronous and bounded; a request row remains explicitly unindexed when the +trace does not arrive. Do not correlate a row with the latest trace or by +timestamp alone. -> **As a security engineer, I want external-provider credentials to stay at the -> final provider hop and never become part of Grid traffic-selection metadata.** +If the routing UI login is enabled, create its credentials as a Secret before +applying the UI Deployment. Supply values interactively so the password is not +stored in the README, a manifest, or shell history: -```mermaid -sequenceDiagram - participant C as Client - participant CG as Consumer gateway - participant PG as Provider gateway - participant CP as External provider - - C->>CG: authenticated request - CG->>CG: token policy + local selection - CG->>PG: selected provider hop - PG->>PG: authorize route / model / path - PG->>PG: replace caller credential - PG->>PG: provider-local model mapping - PG->>CP: TLS + provider credential - CP-->>PG: inference response - PG-->>CG: response - CG-->>C: response +```bash +read -r -p "UI username: " TRACING_UI_AUTH_USERNAME +read -r -s -p "UI password: " TRACING_UI_AUTH_PASSWORD +printf '\n' +kubectl -n praxis-tracing-cloud-burst create secret generic praxis-tracing-cloud-burst-ui-auth --from-literal=username="$TRACING_UI_AUTH_USERNAME" --from-literal=password="$TRACING_UI_AUTH_PASSWORD" --dry-run=client -o yaml | kubectl apply -f - +unset TRACING_UI_AUTH_USERNAME TRACING_UI_AUTH_PASSWORD ``` -Grid selects a provider candidate. The provider gateway owns the final serving -and credential boundary. +The UI Deployment must read `TRACING_UI_AUTH_USERNAME` and +`TRACING_UI_AUTH_PASSWORD` from the `username` and `password` keys in this +Secret using `valueFrom.secretKeyRef`. Do not place either value directly in +the Deployment. This Secret is separate from application quota credentials, +OpenAI credentials, and Azure credentials. -That keeps the routing architecture generic across model-provider APIs with -different authentication mechanisms. +The diagnostic and qualification scripts intentionally have no default UI +credentials. Export `UI_USER` and `UI_PASSWORD` from an approved local secret +source for the duration of a run, then unset them. Do not add fallback passwords +to the scripts or manifests. ---- +Review the scripts first; they assume names from this development environment. -# User story: make policy changes observable +## Validation matrix -> **As an operator, I want to know why traffic moved and which routing snapshot -> made the decision.** +Use unique request/session IDs and clear UI history before each phase. Wait for +Kubernetes state, provider health, overlay publication, and Praxis serving +revision convergence before measuring traffic. -The demo records route/provider attribution through OpenTelemetry and associates -requests with the serving routing revision. +| Scenario | Action | Expected result | +|---|---|---| +| Healthy east | 20 unbound east requests. | Both east providers appear. | +| Healthy west | 20 unbound west requests. | Both west providers appear. | +| One east down | Scale `llm-d-east-1` to zero. | All new traffic uses `llm-d-east-2`. | +| East down | Scale both east simulators to zero. | Traffic uses west, not cloud. | +| One west down | Scale `llm-d-west-1` to zero. | All new traffic uses `llm-d-west-2`. | +| West down | Scale both west simulators to zero. | Traffic uses east, not cloud. | +| All locals down | Scale all four simulators to zero. | Traffic uses the Azure OpenAI/OpenAI external group. | +| Queue pressure | Put all local queues over admission. | New traffic uses overflow. | +| Recovery | Restore replicas/queues and wait. | Traffic returns to its preferred site-local group. | -Useful evidence includes: +Example failure injection: -```text -request ID -gateway -logical model/service -serving routing revision -selection group -selection mode -selected provider -provider site / region -admission state -traffic weight -queue / pressure evidence -token allocation state +```bash +kubectl -n grid-system scale deployment llm-d-east-1 --replicas=0 ``` -```mermaid -%%{init: {"flowchart": {"htmlLabels": false}}}%% -flowchart TB - T["Provider + Praxis
telemetry"] - O["Observability
store"] - A["Analyze routing
behavior"] - R["Human or future
policy recommendation"] - V["Validate + version
policy change"] - G["Grid computes next
routing snapshot"] - - T --> O --> A --> R --> V --> G -``` - -This creates a path toward future closed-loop optimization without putting an -AI model or dashboard into the synchronous request path. - ---- - -# Presentation flow - -The narrated recording follows this sequence: - -1. **Architecture** — Grid computes policy asynchronously; Praxis executes it - locally. -2. **Reactive burst** — use preferred capacity first, then introduce bounded - overflow. -3. **Independent policies** — admission, grouping, placement, burst, and - overflow remain separate. -4. **Soft token limits** — distributed token usage remains continuous and - over-allocation can be reported without forced denial. -5. **Live demo** — baseline traffic, pressure, route movement, overflow, and - recovery. -6. **Outro** — Grid remains off the request hot path. - ---- - -# Live demo narrative - -A representative run is: - -```text -1. BASELINE - - preferred Kubernetes providers are healthy - token usage is visible - Grid publishes the baseline routing snapshot - Praxis routes locally - -2. PRESSURE +Always restore the environment: - provider queue pressure rises - llm-d / EPP exposes the signal - Grid observes the pressure - -3. LOCAL REBALANCE - - Grid changes provider placement - less new traffic goes to the pressured pool - more goes to preferred providers with headroom - -4. OVERFLOW - - preferred-tier headroom becomes insufficient - Grid publishes a new policy revision - some eligible new traffic moves to overflow capacity - -5. TOKEN CONTINUITY - - the same user/workload token state is preserved - routing changes do not create a new token allowance - -6. RECOVERY - - local pressure falls - Grid reduces the overflow share - preferred capacity resumes more traffic - -7. STEADY STATE - - Grid remains asynchronous - Praxis continues selecting locally +```bash +kubectl -n grid-system scale deployment llm-d-east-1 llm-d-east-2 llm-d-west-1 llm-d-west-2 --replicas=1 ``` ---- +Do not treat retries alone as convergence proof. State, overlay revision, +Praxis serving revision, HTTP status, response headers, and UI path must agree. -# Build inputs +## Validation evidence -The burst-routing work used for the current demo is composed from the following -development branches/checkpoints: +The original repeatable failure matrix established the local and regional +fallback behavior before Azure was added. Those preserved counts remain useful +because they isolate Grid health withdrawal from external-provider behavior: -- **Praxis** — - [`nerdalert/praxis`](https://github.com/nerdalert/praxis/tree/burst-routing-v1), - branch `burst-routing-v1`, composed checkpoint `cf2a7bb6`. -- **Praxis AI** — - [`nerdalert/ai`](https://github.com/nerdalert/ai/tree/burst-routing-v1), - branch `burst-routing-v1`, checkpoint `557cd37`. -- **Grid** — - [`nerdalert/grid`](https://github.com/nerdalert/grid/tree/burst-routing-v1), - branch `burst-routing-v1`, checkpoint `8c8635c`. -- **Tracing UI** — - [`nerdalert/praxis-tracing`](https://github.com/nerdalert/praxis-tracing/tree/burst-routing-v1), - branch `burst-routing-v1`. +| Scenario | Result | Selected inference providers | +|---|---:|---| +| Healthy east | 20/20 HTTP 200 | `llm-d-east-1`: 8, `llm-d-east-2`: 12 | +| Healthy west | 20/20 HTTP 200 | `llm-d-west-1`: 7, `llm-d-west-2`: 13 | +| One east disabled | 10/10 HTTP 200 | `llm-d-east-2`: 10 | +| Both east disabled | 10/10 HTTP 200 | `llm-d-west-1`: 3, `llm-d-west-2`: 7 | +| One west disabled | 4/4 HTTP 200 | `llm-d-west-2`: 4 | +| Both west disabled | 4/4 HTTP 200 | `llm-d-east-1`: 3, `llm-d-east-2`: 1 | +| All four disabled | 6/6 HTTP 200 | openai-west: 5, openai-east: 1 | +| Queue at 9/10 | HTTP 200 via overflow | openai-east and openai-west | +| Pressure recovery | HTTP 200 via local | `llm-d-east-1` and `llm-d-east-2` | +| Final 2 requests/sec | 10/10 HTTP 200 | zero 502 responses | -These are development checkpoints used for the composed demo. They should not -be interpreted as statements that every behavior described in the broader -architecture document has already merged upstream. - ---- - - +The full names shown by the UI were `llm-d-east-1`, `llm-d-east-2`, +`llm-d-west-1`, `llm-d-west-2`, `openai-east`, and `openai-west`. A direct +header check separately reported gateway `east` and backend `llm-d-east-1`. +All simulator Deployments were restored to `1/1`, queues returned to zero, and +no recent 403, 421, 502, or 503 gateway errors remained. + +The counts prove participation and fallback, not a statistical service-level +guarantee. Round-robin state is local to each Praxis process; aggregate traffic +converges with sufficient unbound volume. + +The v2.1 recording extends that matrix rather than rewriting its historical +counts. It visibly demonstrates: + +| Recorded phase | Evidence shown | +|---|---| +| Healthy local fleet | Requests enter through both consumers and identify concrete east/west local backends. | +| Progressive withdrawal | A site-local survivor is used first, followed by healthy capacity in the other Grid site. | +| Multi-cloud overflow | Azure OpenAI and OpenAI routes appear after local providers are no longer eligible. | +| Azure upstream limit | An Azure-attributed HTTP 429 remains distinguishable from a Praxis quota denial with no provider hop. | +| Queue pressure | Runtime queue metrics cross admission and local providers move to `existing_only` without restarting the serving simulators. | +| Recovery | Local providers become eligible again after health or pressure clears and a replacement overlay is accepted. | +| Quota continuity | The same app/model sliding windows remain visible across local, regional, and cloud provider changes. | + +## Security and limitations + +- Provider credentials remain in Secrets and are injected only at the final + provider hop. +- Client backend-attribution headers are removed before forwarding. +- Prompts, responses, credentials, authorization headers, and raw session IDs + are not default trace attributes. +- External API traffic can incur cost. Bound traffic and rotate temporary + credentials. +- This is not a supported release bundle or one-command installer. +- Simulator pressure is not production qualification of every llm-d/EPP or + inference-engine metric contract. +- The demo proves hard group fallback, not gradual weighted spillover. +- A `provider.route` span records a routing decision, not downstream completion. + +## Development inputs + +| Component | Repository and branch | Role | +|---|---|---| +| Praxis core | [`nerdalert/praxis:burst-routing-v1`](https://github.com/nerdalert/praxis/tree/burst-routing-v1) | Proxy/runtime and trusted identity. | +| Praxis AI | [`nerdalert/ai:burst-routing-v1-running`](https://github.com/nerdalert/ai/tree/burst-routing-v1-running) | Selection, provider routing, model rewrite, credentials, attribution. | +| Grid | [`nerdalert/grid:burst-routing-v1-running`](https://github.com/nerdalert/grid/tree/burst-routing-v1-running) | Health, admission, grouping, and overlay publication. | +| Experimental | [`nerdalert/experimental:feat/cloud-burst-resilience-demo`](https://github.com/nerdalert/experimental/tree/feat/cloud-burst-resilience-demo/demos/grid-cloud-burst) | Documentation and example resources. | +| Tracing UI | [`nerdalert/praxis-tracing:burst-routing-v1`](https://github.com/nerdalert/praxis-tracing/tree/burst-routing-v1/grid-cloud-burst/routing-observability-ui) | Interactive request history, provider attribution, controls, and trace inspection. | + +Related upstream work: + +- [AI provider selection foundation](https://github.com/praxis-proxy/ai/pull/731) +- [Grid provider selection groups](https://github.com/praxis-proxy/grid/pull/65) +- [AI provider-backend attribution](https://github.com/praxis-proxy/ai/pull/834) + +## Summary + +Healthy traffic stays with the site-local pair. Losing one backend leaves the +survivor in service; losing the site moves new requests to healthy Grid capacity +in the other site. Only after the local fleet is unavailable or restricted for +new admission does the Azure OpenAI/OpenAI group become active. Recovery returns +new traffic locally without resetting the shared token windows. + +Grid computes and publishes these policy transitions asynchronously. Praxis +executes the accepted snapshot locally, enforces the shared token policy, and +reports the exact gateway and inference backend or cloud route selected for each +request. + + diff --git a/demos/grid-cloud-burst/example-manifests/08-regional-local-pools.yaml b/demos/grid-cloud-burst/example-manifests/08-regional-local-pools.yaml new file mode 100644 index 0000000..70302fd --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/08-regional-local-pools.yaml @@ -0,0 +1,202 @@ +# Two local llm-d-inference-sim providers per regional gateway. +# Names are intentionally shared across Deployment, Service, InferenceProvider, +# and provider-gateway load_balancer clusters. +apiVersion: v1 +kind: ConfigMap +metadata: + name: llm-d-east-1-config + namespace: grid-system +data: + config.yaml: | + port: 8000 + model: "gpt-4o-mini" + served-model-name: ["gpt-4o-mini"] + mode: "echo" + force-dummy-tokenizer: true + max-num-seqs: 4 + max-waiting-queue-length: 128 + fake-metrics: + running-requests: 0 + waiting-requests: 0 + kv-cache-usage: 0.1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: llm-d-east-2-config + namespace: grid-system +data: + config.yaml: | + port: 8000 + model: "gpt-4o-mini" + served-model-name: ["gpt-4o-mini"] + mode: "echo" + force-dummy-tokenizer: true + max-num-seqs: 4 + max-waiting-queue-length: 128 + fake-metrics: + running-requests: 0 + waiting-requests: 0 + kv-cache-usage: 0.1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: llm-d-west-1-config + namespace: grid-system +data: + config.yaml: | + port: 8000 + model: "gpt-4o-mini" + served-model-name: ["gpt-4o-mini"] + mode: "echo" + force-dummy-tokenizer: true + max-num-seqs: 4 + max-waiting-queue-length: 128 + fake-metrics: + running-requests: 0 + waiting-requests: 0 + kv-cache-usage: 0.1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: llm-d-west-2-config + namespace: grid-system +data: + config.yaml: | + port: 8000 + model: "gpt-4o-mini" + served-model-name: ["gpt-4o-mini"] + mode: "echo" + force-dummy-tokenizer: true + max-num-seqs: 4 + max-waiting-queue-length: 128 + fake-metrics: + running-requests: 0 + waiting-requests: 0 + kv-cache-usage: 0.1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-d-east-1 + namespace: grid-system + labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-east-1} +spec: + replicas: 1 + selector: {matchLabels: {grid.praxis-proxy.io/pool: llm-d-east-1}} + template: + metadata: {labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-east-1}} + spec: + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: simulator + image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2 + imagePullPolicy: IfNotPresent + args: ["--config", "/config/config.yaml"] + ports: [{name: http, containerPort: 8000}] + volumeMounts: [{name: config, mountPath: /config, readOnly: true}] + readinessProbe: {httpGet: {path: /health, port: http}, periodSeconds: 5} + securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}, readOnlyRootFilesystem: true} + resources: {requests: {cpu: 100m, memory: 128Mi}, limits: {cpu: "1", memory: 512Mi}} + volumes: [{name: config, configMap: {name: llm-d-east-1-config}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-d-east-2 + namespace: grid-system + labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-east-2} +spec: + replicas: 1 + selector: {matchLabels: {grid.praxis-proxy.io/pool: llm-d-east-2}} + template: + metadata: {labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-east-2}} + spec: + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: simulator + image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2 + imagePullPolicy: IfNotPresent + args: ["--config", "/config/config.yaml"] + ports: [{name: http, containerPort: 8000}] + volumeMounts: [{name: config, mountPath: /config, readOnly: true}] + readinessProbe: {httpGet: {path: /health, port: http}, periodSeconds: 5} + securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}, readOnlyRootFilesystem: true} + resources: {requests: {cpu: 100m, memory: 128Mi}, limits: {cpu: "1", memory: 512Mi}} + volumes: [{name: config, configMap: {name: llm-d-east-2-config}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-d-west-1 + namespace: grid-system + labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-west-1} +spec: + replicas: 1 + selector: {matchLabels: {grid.praxis-proxy.io/pool: llm-d-west-1}} + template: + metadata: {labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-west-1}} + spec: + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: simulator + image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2 + imagePullPolicy: IfNotPresent + args: ["--config", "/config/config.yaml"] + ports: [{name: http, containerPort: 8000}] + volumeMounts: [{name: config, mountPath: /config, readOnly: true}] + readinessProbe: {httpGet: {path: /health, port: http}, periodSeconds: 5} + securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}, readOnlyRootFilesystem: true} + resources: {requests: {cpu: 100m, memory: 128Mi}, limits: {cpu: "1", memory: 512Mi}} + volumes: [{name: config, configMap: {name: llm-d-west-1-config}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-d-west-2 + namespace: grid-system + labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-west-2} +spec: + replicas: 1 + selector: {matchLabels: {grid.praxis-proxy.io/pool: llm-d-west-2}} + template: + metadata: {labels: {app.kubernetes.io/name: llm-d-inference-sim, grid.praxis-proxy.io/pool: llm-d-west-2}} + spec: + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: simulator + image: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2 + imagePullPolicy: IfNotPresent + args: ["--config", "/config/config.yaml"] + ports: [{name: http, containerPort: 8000}] + volumeMounts: [{name: config, mountPath: /config, readOnly: true}] + readinessProbe: {httpGet: {path: /health, port: http}, periodSeconds: 5} + securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}, readOnlyRootFilesystem: true} + resources: {requests: {cpu: 100m, memory: 128Mi}, limits: {cpu: "1", memory: 512Mi}} + volumes: [{name: config, configMap: {name: llm-d-west-2-config}}] +--- +apiVersion: v1 +kind: Service +metadata: {name: llm-d-east-1, namespace: grid-system} +spec: {selector: {grid.praxis-proxy.io/pool: llm-d-east-1}, ports: [{name: http, port: 8000, targetPort: http}]} +--- +apiVersion: v1 +kind: Service +metadata: {name: llm-d-east-2, namespace: grid-system} +spec: {selector: {grid.praxis-proxy.io/pool: llm-d-east-2}, ports: [{name: http, port: 8000, targetPort: http}]} +--- +apiVersion: v1 +kind: Service +metadata: {name: llm-d-west-1, namespace: grid-system} +spec: {selector: {grid.praxis-proxy.io/pool: llm-d-west-1}, ports: [{name: http, port: 8000, targetPort: http}]} +--- +apiVersion: v1 +kind: Service +metadata: {name: llm-d-west-2, namespace: grid-system} +spec: {selector: {grid.praxis-proxy.io/pool: llm-d-west-2}, ports: [{name: http, port: 8000, targetPort: http}]} diff --git a/demos/grid-cloud-burst/example-manifests/09-regional-inference-providers.yaml b/demos/grid-cloud-burst/example-manifests/09-regional-inference-providers.yaml new file mode 100644 index 0000000..30e9062 --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/09-regional-inference-providers.yaml @@ -0,0 +1,65 @@ +# Two local InferenceProviders per site. Use localityScope: sameSite on the +# GridNetwork so east and west remain isolated active groups. +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: {name: llm-d-east-1, namespace: grid-system} +spec: + gridNetworkRef: grid-cloud-burst-rhoai + providerKind: llm-d-inference-sim + backendKind: local + endpoint: http://llm-d-east-1.grid-system.svc.cluster.local:8000 + routingClusterRef: llm-d-east-1 + capacityWeight: 50 + models: [{name: gpt-4o-mini, capabilities: [text_generation], contextWindow: 4096}] + siteSelector: {matchLabels: {grid.praxis-proxy.io/provider-site: east-local}} + accessPolicy: {siteSelector: {matchLabels: {}}} + metricsConfig: {metricsEndpoint: http://llm-d-east-1.grid-system.svc.cluster.local:8000, path: /metrics, queueCapacity: 10, signalNames: {queueDepth: vllm:num_requests_waiting}, timeout: 2s} + healthCheck: {path: /health, interval: 5s, timeout: 2s} +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: {name: llm-d-east-2, namespace: grid-system} +spec: + gridNetworkRef: grid-cloud-burst-rhoai + providerKind: llm-d-inference-sim + backendKind: local + endpoint: http://llm-d-east-2.grid-system.svc.cluster.local:8000 + routingClusterRef: llm-d-east-2 + capacityWeight: 50 + models: [{name: gpt-4o-mini, capabilities: [text_generation], contextWindow: 4096}] + siteSelector: {matchLabels: {grid.praxis-proxy.io/provider-site: east-local}} + accessPolicy: {siteSelector: {matchLabels: {}}} + metricsConfig: {metricsEndpoint: http://llm-d-east-2.grid-system.svc.cluster.local:8000, path: /metrics, queueCapacity: 10, signalNames: {queueDepth: vllm:num_requests_waiting}, timeout: 2s} + healthCheck: {path: /health, interval: 5s, timeout: 2s} +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: {name: llm-d-west-1, namespace: grid-system} +spec: + gridNetworkRef: grid-cloud-burst-rhoai + providerKind: llm-d-inference-sim + backendKind: local + endpoint: http://llm-d-west-1.grid-system.svc.cluster.local:8000 + routingClusterRef: llm-d-west-1 + capacityWeight: 50 + models: [{name: gpt-4o-mini, capabilities: [text_generation], contextWindow: 4096}] + siteSelector: {matchLabels: {grid.praxis-proxy.io/provider-site: west-local}} + accessPolicy: {siteSelector: {matchLabels: {}}} + metricsConfig: {metricsEndpoint: http://llm-d-west-1.grid-system.svc.cluster.local:8000, path: /metrics, queueCapacity: 10, signalNames: {queueDepth: vllm:num_requests_waiting}, timeout: 2s} + healthCheck: {path: /health, interval: 5s, timeout: 2s} +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: {name: llm-d-west-2, namespace: grid-system} +spec: + gridNetworkRef: grid-cloud-burst-rhoai + providerKind: llm-d-inference-sim + backendKind: local + endpoint: http://llm-d-west-2.grid-system.svc.cluster.local:8000 + routingClusterRef: llm-d-west-2 + capacityWeight: 50 + models: [{name: gpt-4o-mini, capabilities: [text_generation], contextWindow: 4096}] + siteSelector: {matchLabels: {grid.praxis-proxy.io/provider-site: west-local}} + accessPolicy: {siteSelector: {matchLabels: {}}} + metricsConfig: {metricsEndpoint: http://llm-d-west-2.grid-system.svc.cluster.local:8000, path: /metrics, queueCapacity: 10, signalNames: {queueDepth: vllm:num_requests_waiting}, timeout: 2s} + healthCheck: {path: /health, interval: 5s, timeout: 2s} diff --git a/demos/grid-cloud-burst/example-manifests/10-observability-jaeger.yaml b/demos/grid-cloud-burst/example-manifests/10-observability-jaeger.yaml new file mode 100644 index 0000000..b7dc7c8 --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/10-observability-jaeger.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jaeger + namespace: grid-system + labels: + app.kubernetes.io/name: jaeger + app.kubernetes.io/part-of: grid-cloud-burst +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: jaeger + template: + metadata: + labels: + app.kubernetes.io/name: jaeger + app.kubernetes.io/part-of: grid-cloud-burst + spec: + containers: + - name: jaeger + image: jaegertracing/all-in-one:1.57 + imagePullPolicy: IfNotPresent + env: + - name: COLLECTOR_OTLP_ENABLED + value: "true" + ports: + - name: otlp-grpc + containerPort: 4317 + - name: query + containerPort: 16686 + readinessProbe: + httpGet: + path: / + port: query + initialDelaySeconds: 5 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: jaeger-collector + namespace: grid-system + labels: + app.kubernetes.io/name: jaeger +spec: + selector: + app.kubernetes.io/name: jaeger + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc +--- +apiVersion: v1 +kind: Service +metadata: + name: jaeger-query + namespace: grid-system + labels: + app.kubernetes.io/name: jaeger +spec: + selector: + app.kubernetes.io/name: jaeger + ports: + - name: query + port: 16686 + targetPort: query diff --git a/demos/grid-cloud-burst/example-manifests/10-openai-overflow-providers.yaml b/demos/grid-cloud-burst/example-manifests/10-openai-overflow-providers.yaml new file mode 100644 index 0000000..b57df64 --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/10-openai-overflow-providers.yaml @@ -0,0 +1,53 @@ +# OpenAI is an overflow provider. Its InferenceProvider endpoint must describe +# the real external backend; the provider-gateway route separately owns the +# final-hop TLS/authority and credential injection configuration. The paired +# provider-gateway ConfigMaps must set both tls.sni and http.authority to +# api.openai.com; SNI alone can produce HTTP 421 from the upstream. In Praxis +# configuration authority is nested under the cluster's http field. +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: openai-east + namespace: grid-system +spec: + gridNetworkRef: grid-cloud-burst-rhoai + backendKind: api_provider + providerKind: openai + endpoint: https://api.openai.com + routingClusterRef: openai-east + capacityWeight: 50 + models: + - name: gpt-4o-mini + capabilities: [text_generation] + contextWindow: 4096 + auth: + strategy: bearer_token + manual: true + secretRef: {name: openai-api-key, namespace: grid-system, key: token} + siteSelector: + matchLabels: {grid.praxis-proxy.io/provider-site: east-cloud} + accessPolicy: {siteSelector: {matchLabels: {}}} +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: openai-west + namespace: grid-system +spec: + gridNetworkRef: grid-cloud-burst-rhoai + backendKind: api_provider + providerKind: openai + endpoint: https://api.openai.com + routingClusterRef: openai-west + capacityWeight: 50 + models: + - name: gpt-4o-mini + capabilities: [text_generation] + contextWindow: 4096 + auth: + strategy: bearer_token + manual: true + secretRef: {name: openai-api-key, namespace: grid-system, key: token} + siteSelector: + matchLabels: {grid.praxis-proxy.io/provider-site: west-cloud} + accessPolicy: {siteSelector: {matchLabels: {}}} diff --git a/demos/grid-cloud-burst/example-manifests/11-azure-overflow-providers.yaml b/demos/grid-cloud-burst/example-manifests/11-azure-overflow-providers.yaml new file mode 100644 index 0000000..6e33aa6 --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/11-azure-overflow-providers.yaml @@ -0,0 +1,63 @@ +# Azure OpenAI overflow providers. +# +# This file is intentionally a template: never commit a client secret. Render +# the Secret value from a protected environment/secret manager, then apply it. +# The paired provider-gateway configuration must use the azure_ad filter, +# authority/SNI equal to the Azure resource hostname, and the Azure v1 path. +apiVersion: v1 +kind: Secret +metadata: + name: azure-openai-client-secret + namespace: grid-system +type: Opaque +stringData: + client-secret: REPLACE_FROM_SECRET_MANAGER +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: azure-east + namespace: grid-system +spec: + gridNetworkRef: grid-cloud-burst-rhoai + backendKind: api_provider + providerKind: azure_openai + # Replace the hostname with the assigned Azure OpenAI resource. + endpoint: https://cloudburst-aoai-nxtdv.openai.azure.com + routingClusterRef: azure-east + capacityWeight: 50 + models: + - name: gpt-4o-mini + capabilities: [text_generation] + contextWindow: 4096 + auth: + strategy: bearer_token + manual: true + secretRef: {name: azure-openai-client-secret, namespace: grid-system, key: client-secret} + siteSelector: + matchLabels: {grid.praxis-proxy.io/provider-site: east-cloud} + accessPolicy: {siteSelector: {matchLabels: {}}} +--- +apiVersion: grid.praxis-proxy.io/v1alpha1 +kind: InferenceProvider +metadata: + name: azure-west + namespace: grid-system +spec: + gridNetworkRef: grid-cloud-burst-rhoai + backendKind: api_provider + providerKind: azure_openai + endpoint: https://cloudburst-aoai-nxtdv.openai.azure.com + routingClusterRef: azure-west + capacityWeight: 50 + models: + - name: gpt-4o-mini + capabilities: [text_generation] + contextWindow: 4096 + auth: + strategy: bearer_token + manual: true + secretRef: {name: azure-openai-client-secret, namespace: grid-system, key: client-secret} + siteSelector: + matchLabels: {grid.praxis-proxy.io/provider-site: west-cloud} + accessPolicy: {siteSelector: {matchLabels: {}}} diff --git a/demos/grid-cloud-burst/example-manifests/12-azure-provider-gateways.yaml b/demos/grid-cloud-burst/example-manifests/12-azure-provider-gateways.yaml new file mode 100644 index 0000000..3fdea36 --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/12-azure-provider-gateways.yaml @@ -0,0 +1,195 @@ +# Template for the dedicated Azure provider gateways. Render this file with +# the deployment script; placeholders are never valid credential values. +# The gateway uses the same mTLS server secret as the demo gateways, while the +# Entra client secret is injected only as AZURE_CLIENT_SECRET from a Secret. +apiVersion: v1 +kind: ConfigMap +metadata: + name: azure-east-config + namespace: grid-system +data: + praxis.yaml: | + listeners: + - name: provider + address: 0.0.0.0:8443 + filter_chains: [provider-inference] + tls: + certificates: + - cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + client_ca: {ca_path: /etc/praxis/tls/ca.crt} + client_cert_mode: require + filter_chains: + - name: provider-inference + filters: + - filter: peer_identity_trust + trusted_peers: [{organization: ai-grid}] + - filter: provider_route + provider_id: azure-east + model_header: X-Model + emit_demo_attribution: true + routes: + - candidate_id: ${AZURE_EAST_CANDIDATE_ID} + model: gpt-4o-mini + provider_model: gpt-4o-mini + paths: [/v1/chat/completions, /v1/responses] + cluster: azure-upstream + - filter: path_rewrite + replace: + pattern: "^/v1/chat/completions/?$" + replacement: /openai/v1/chat/completions + conditions: + - when: {path: /v1/chat/completions, methods: [POST]} + - filter: azure_ad + tenant_id: ${AZURE_TENANT_ID} + client_id: ${AZURE_CLIENT_ID} + scope: https://cognitiveservices.azure.com/.default + client_secret_env_var: AZURE_CLIENT_SECRET + - filter: load_balancer + clusters: + - name: azure-upstream + http: {authority: ${AZURE_OPENAI_HOST}} + endpoints: [${AZURE_OPENAI_HOST}:443] + tls: {sni: ${AZURE_OPENAI_HOST}, verify: true} + idle_timeout_ms: 5000 + insecure_options: {allow_private_endpoints: false} + admin: {address: 127.0.0.1:9901} + shutdown_timeout_secs: 5 + telemetry: + otlp_endpoint: http://jaeger-collector.grid-system.svc.cluster.local:4317 + sampling_rate: 1.0 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: azure-west-config + namespace: grid-system +data: + praxis.yaml: | + listeners: + - name: provider + address: 0.0.0.0:8443 + filter_chains: [provider-inference] + tls: + certificates: + - cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + client_ca: {ca_path: /etc/praxis/tls/ca.crt} + client_cert_mode: require + filter_chains: + - name: provider-inference + filters: + - filter: peer_identity_trust + trusted_peers: [{organization: ai-grid}] + - filter: provider_route + provider_id: azure-west + model_header: X-Model + emit_demo_attribution: true + routes: + - candidate_id: ${AZURE_WEST_CANDIDATE_ID} + model: gpt-4o-mini + provider_model: gpt-4o-mini + paths: [/v1/chat/completions, /v1/responses] + cluster: azure-upstream + - filter: path_rewrite + replace: + pattern: "^/v1/chat/completions/?$" + replacement: /openai/v1/chat/completions + conditions: + - when: {path: /v1/chat/completions, methods: [POST]} + - filter: azure_ad + tenant_id: ${AZURE_TENANT_ID} + client_id: ${AZURE_CLIENT_ID} + scope: https://cognitiveservices.azure.com/.default + client_secret_env_var: AZURE_CLIENT_SECRET + - filter: load_balancer + clusters: + - name: azure-upstream + http: {authority: ${AZURE_OPENAI_HOST}} + endpoints: [${AZURE_OPENAI_HOST}:443] + tls: {sni: ${AZURE_OPENAI_HOST}, verify: true} + idle_timeout_ms: 5000 + insecure_options: {allow_private_endpoints: false} + admin: {address: 127.0.0.1:9901} + shutdown_timeout_secs: 5 + telemetry: + otlp_endpoint: http://jaeger-collector.grid-system.svc.cluster.local:4317 + sampling_rate: 1.0 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: azure-east + namespace: grid-system + labels: {app.kubernetes.io/instance: azure-east, app.kubernetes.io/name: praxis-gateway} +spec: + replicas: 1 + selector: {matchLabels: {app.kubernetes.io/instance: azure-east, app.kubernetes.io/name: praxis-gateway}} + template: + metadata: {labels: {app.kubernetes.io/instance: azure-east, app.kubernetes.io/name: praxis-gateway}} + spec: + automountServiceAccountToken: false + imagePullSecrets: [{name: burst-registry-pull-current}, {name: burst-registry-pull}] + containers: + - name: praxis + image: ${AZURE_IMAGE} + args: [--config, /etc/praxis/praxis.yaml] + ports: [{containerPort: 8443, name: https-mtls}] + env: + - name: AZURE_CLIENT_SECRET + valueFrom: {secretKeyRef: {name: azure-openai-client-secret, key: client-secret}} + readinessProbe: {tcpSocket: {port: 8443}, initialDelaySeconds: 3, periodSeconds: 5} + livenessProbe: {tcpSocket: {port: 8443}, initialDelaySeconds: 5, periodSeconds: 10} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: {drop: [ALL]}} + volumeMounts: + - {name: config, mountPath: /etc/praxis, readOnly: true} + - {name: tls, mountPath: /etc/praxis/tls, readOnly: true} + volumes: + - {name: config, configMap: {name: azure-east-config}} + - {name: tls, secret: {secretName: gateway-tls}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: azure-west + namespace: grid-system + labels: {app.kubernetes.io/instance: azure-west, app.kubernetes.io/name: praxis-gateway} +spec: + replicas: 1 + selector: {matchLabels: {app.kubernetes.io/instance: azure-west, app.kubernetes.io/name: praxis-gateway}} + template: + metadata: {labels: {app.kubernetes.io/instance: azure-west, app.kubernetes.io/name: praxis-gateway}} + spec: + automountServiceAccountToken: false + imagePullSecrets: [{name: burst-registry-pull-current}, {name: burst-registry-pull}] + containers: + - name: praxis + image: ${AZURE_IMAGE} + args: [--config, /etc/praxis/praxis.yaml] + ports: [{containerPort: 8443, name: https-mtls}] + env: + - name: AZURE_CLIENT_SECRET + valueFrom: {secretKeyRef: {name: azure-openai-client-secret, key: client-secret}} + readinessProbe: {tcpSocket: {port: 8443}, initialDelaySeconds: 3, periodSeconds: 5} + livenessProbe: {tcpSocket: {port: 8443}, initialDelaySeconds: 5, periodSeconds: 10} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: {drop: [ALL]}} + volumeMounts: + - {name: config, mountPath: /etc/praxis, readOnly: true} + - {name: tls, mountPath: /etc/praxis/tls, readOnly: true} + volumes: + - {name: config, configMap: {name: azure-west-config}} + - {name: tls, secret: {secretName: gateway-tls}} +--- +apiVersion: v1 +kind: Service +metadata: {name: azure-east, namespace: grid-system} +spec: + selector: {app.kubernetes.io/instance: azure-east, app.kubernetes.io/name: praxis-gateway} + ports: [{name: https-mtls, port: 8443, targetPort: https-mtls}] +--- +apiVersion: v1 +kind: Service +metadata: {name: azure-west, namespace: grid-system} +spec: + selector: {app.kubernetes.io/instance: azure-west, app.kubernetes.io/name: praxis-gateway} + ports: [{name: https-mtls, port: 8443, targetPort: https-mtls}] diff --git a/demos/grid-cloud-burst/example-manifests/13-consumer-east-praxis.yaml b/demos/grid-cloud-burst/example-manifests/13-consumer-east-praxis.yaml new file mode 100644 index 0000000..d5a27f5 --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/13-consumer-east-praxis.yaml @@ -0,0 +1,246 @@ +# Declarative consumer gateway configuration for the cloud-burst demo. +# Includes the Azure provider-hop clusters and preserves local/OpenAI routes. +# Demo-only basic-auth values are not production credentials; external +# provider credentials remain Secret-backed and are never stored here. +apiVersion: v1 +kind: ConfigMap +metadata: + name: consumer-east-config + namespace: grid-system +data: + praxis.yaml: | + listeners: + - name: proxy + address: 0.0.0.0:8080 + filter_chains: + - main + filter_chains: + - name: main + filters: + - filter: basic_auth + realm: Grid cloud-burst demo + strip_authorization: true + credentials: + - username: alice + password: alice-secret + - username: bob + password: bob-secret + - username: app1 + password: app1-secret + - username: app2 + password: app2-secret + - username: app3 + password: app3-secret + # Join a valid incoming W3C trace and propagate it to the provider. + - filter: trace_context + - filter: token_rate_limit + enforcement: soft + key: + principal: + source: metadata + name: identity.user_id + onMissing: reject + model: + source: header + name: x-model + onMissing: reject + allowedModels: + - gpt-4o-mini + reservationTimeout: 2m + backend: + kind: valkey + url: ${TOKEN_RATE_LIMIT_VALKEY_URL} + namespace: praxis:token-rate-limit:v1 + limits: + maxKeys: 32 + maxKeyLength: 256 + maxActiveReservations: 16 + rules: + - name: alice + match: + metadata: + identity.user_id: alice + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 1m + capacity: 6000 + - name: bob + match: + metadata: + identity.user_id: bob + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 50000 + - name: app1 + match: + metadata: + identity.user_id: app1 + estimation: + strategy: fixed + tokens: 20 + token_budgets: + - window: 1m + capacity: 60 + - name: app2 + match: + metadata: + identity.user_id: app2 + estimation: + strategy: fixed + tokens: 20 + token_budgets: + - window: 1m + capacity: 60 + - name: app3 + match: + metadata: + identity.user_id: app3 + estimation: + strategy: fixed + tokens: 20 + token_budgets: + - window: 1m + capacity: 60 + - name: default + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 50000 + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Combined-Consumer-Gateway + value: east consumer gateway + - filter: intelligent_route + overlay_file: /etc/praxis/routing/routing-overlay.json + model_header: X-Model + provider_hop_clusters: + - openai-east + - openai-west + - llm-d-east-1 + - llm-d-east-2 + - llm-d-west-1 + - llm-d-west-2 + - azure-east + - azure-west + expected_overlay_scope: + network: grid-cloud-burst-rhoai + gateway: consumer-east + namespace: grid-system + local_site: east-local + reload: + enabled: true + debounce_ms: 500 + session_affinity: + enabled: true + header: X-Session-Id + ttl_secs: 3600 + - filter: token_count + provider: openai + - filter: load_balancer + clusters: + - name: llm-d-east-1 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - provider-east.grid-system.svc.cluster.local:8443 + - name: llm-d-east-2 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - provider-east.grid-system.svc.cluster.local:8443 + - name: llm-d-west-1 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - provider-west.grid-system.svc.cluster.local:8443 + - name: llm-d-west-2 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - provider-west.grid-system.svc.cluster.local:8443 + - name: openai-east + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - provider-east.grid-system.svc.cluster.local:8443 + - name: openai-west + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - provider-west.grid-system.svc.cluster.local:8443 + - name: azure-east + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - azure-east.grid-system.svc.cluster.local:8443 + - name: azure-west + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - azure-west.grid-system.svc.cluster.local:8443 + insecure_options: + allow_private_endpoints: true + admin: + address: 127.0.0.1:9901 + shutdown_timeout_secs: 5 + telemetry: + otlp_endpoint: http://jaeger-collector.grid-system.svc.cluster.local:4317 + sampling_rate: 1.0 diff --git a/demos/grid-cloud-burst/example-manifests/14-consumer-west-praxis.yaml b/demos/grid-cloud-burst/example-manifests/14-consumer-west-praxis.yaml new file mode 100644 index 0000000..62d715f --- /dev/null +++ b/demos/grid-cloud-burst/example-manifests/14-consumer-west-praxis.yaml @@ -0,0 +1,246 @@ +# Declarative consumer gateway configuration for the cloud-burst demo. +# Includes the Azure provider-hop clusters and preserves local/OpenAI routes. +# Demo-only basic-auth values are not production credentials; external +# provider credentials remain Secret-backed and are never stored here. +apiVersion: v1 +kind: ConfigMap +metadata: + name: consumer-west-config + namespace: grid-system +data: + praxis.yaml: | + listeners: + - name: proxy + address: 0.0.0.0:8080 + filter_chains: + - main + filter_chains: + - name: main + filters: + - filter: basic_auth + realm: Grid cloud-burst demo + strip_authorization: true + credentials: + - username: alice + password: alice-secret + - username: bob + password: bob-secret + - username: app1 + password: app1-secret + - username: app2 + password: app2-secret + - username: app3 + password: app3-secret + # Join a valid incoming W3C trace and propagate it to the provider. + - filter: trace_context + - filter: token_rate_limit + enforcement: soft + key: + principal: + source: metadata + name: identity.user_id + onMissing: reject + model: + source: header + name: x-model + onMissing: reject + allowedModels: + - gpt-4o-mini + reservationTimeout: 2m + backend: + kind: valkey + url: ${TOKEN_RATE_LIMIT_VALKEY_URL} + namespace: praxis:token-rate-limit:v1 + limits: + maxKeys: 32 + maxKeyLength: 256 + maxActiveReservations: 16 + rules: + - name: alice + match: + metadata: + identity.user_id: alice + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 1m + capacity: 6000 + - name: bob + match: + metadata: + identity.user_id: bob + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 50000 + - name: app1 + match: + metadata: + identity.user_id: app1 + estimation: + strategy: fixed + tokens: 20 + token_budgets: + - window: 1m + capacity: 60 + - name: app2 + match: + metadata: + identity.user_id: app2 + estimation: + strategy: fixed + tokens: 20 + token_budgets: + - window: 1m + capacity: 60 + - name: app3 + match: + metadata: + identity.user_id: app3 + estimation: + strategy: fixed + tokens: 20 + token_budgets: + - window: 1m + capacity: 60 + - name: default + estimation: + strategy: fixed + tokens: 5 + token_budgets: + - window: 10m + capacity: 50000 + - filter: json_body_field + field: model + header: X-Model + - filter: headers + response_set: + - name: X-Grid-Combined-Consumer-Gateway + value: west consumer gateway + - filter: intelligent_route + overlay_file: /etc/praxis/routing/routing-overlay.json + model_header: X-Model + provider_hop_clusters: + - openai-east + - openai-west + - llm-d-east-1 + - llm-d-east-2 + - llm-d-west-1 + - llm-d-west-2 + - azure-east + - azure-west + expected_overlay_scope: + network: grid-cloud-burst-rhoai + gateway: consumer-west + namespace: grid-system + local_site: west-local + reload: + enabled: true + debounce_ms: 500 + session_affinity: + enabled: true + header: X-Session-Id + ttl_secs: 3600 + - filter: token_count + provider: openai + - filter: load_balancer + clusters: + - name: llm-d-west-1 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - provider-west.grid-system.svc.cluster.local:8443 + - name: llm-d-west-2 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - provider-west.grid-system.svc.cluster.local:8443 + - name: llm-d-east-1 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - provider-east.grid-system.svc.cluster.local:8443 + - name: llm-d-east-2 + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - provider-east.grid-system.svc.cluster.local:8443 + - name: openai-east + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - provider-east.grid-system.svc.cluster.local:8443 + - name: openai-west + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - provider-west.grid-system.svc.cluster.local:8443 + - name: azure-east + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-east.grid-system.svc + verify: true + endpoints: + - azure-east.grid-system.svc.cluster.local:8443 + - name: azure-west + tls: + ca: + ca_path: /etc/praxis/tls/ca.crt + client_cert: + cert_path: /etc/praxis/tls/tls.crt + key_path: /etc/praxis/tls/tls.key + sni: provider-west.grid-system.svc + verify: true + endpoints: + - azure-west.grid-system.svc.cluster.local:8443 + insecure_options: + allow_private_endpoints: true + admin: + address: 127.0.0.1:9901 + shutdown_timeout_secs: 5 + telemetry: + otlp_endpoint: http://jaeger-collector.grid-system.svc.cluster.local:4317 + sampling_rate: 1.0 diff --git a/demos/grid-cloud-burst/example-manifests/README.md b/demos/grid-cloud-burst/example-manifests/README.md index 6e50d8c..80ba8da 100644 --- a/demos/grid-cloud-burst/example-manifests/README.md +++ b/demos/grid-cloud-burst/example-manifests/README.md @@ -72,3 +72,34 @@ deploy time: Replace the placeholders with real values (or use the demo's render scripts) before applying. + +## Regional resilience topology used by the v2.1 recording + +Files `08` through `14` are a separate, larger topology. They model two +consumer gateways, east and west provider sites, four local inference +simulators, and an Azure OpenAI/OpenAI external provider group. + +| File | Purpose | +| --- | --- | +| `08-regional-local-pools.yaml` | Four independently controllable local simulator Deployments and Services. | +| `09-regional-inference-providers.yaml` | Local providers with health and queue-depth admission signals. | +| `10-openai-overflow-providers.yaml` | East and west OpenAI overflow candidates. | +| `11-azure-overflow-providers.yaml` | Azure OpenAI provider templates with Secret references and endpoint placeholders. | +| `12-azure-provider-gateways.yaml` | Dedicated Azure gateway Services and Deployments with mTLS and Entra token injection. | +| `13-consumer-east-praxis.yaml` | East consumer pipeline with shared soft quota and all provider-hop clusters. | +| `14-consumer-west-praxis.yaml` | West consumer equivalent of the same identity, quota, and routing contract. | +| `10-observability-jaeger.yaml` | Optional Jaeger deployment for request/route inspection. | + +The numeric overlap at `10` is intentional: observability is optional and does +not participate in provider ordering. + +This set extends an already installed Grid and Praxis environment. It does not +create trust material, external-provider Secrets, Grid operators, overlay sync, +or the base consumer/provider gateway deployments. Apply the local simulator +and provider resources first, configure the gateway routes, then add Azure only +after its direct and provider-boundary validation passes. + +Do not apply the `01`-`06` and `08`-`14` GridNetwork/provider sets together +without reconciling their network names, sites, and provider policies. The +former is the compact adaptive-burst reference; the latter reproduces the +regional resilience recording. diff --git a/demos/grid-cloud-burst/scripts/configure-azure-consumer-routes.sh b/demos/grid-cloud-burst/scripts/configure-azure-consumer-routes.sh new file mode 100755 index 0000000..adec79d --- /dev/null +++ b/demos/grid-cloud-burst/scripts/configure-azure-consumer-routes.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Add the optional Azure provider gateways to the consumer's authenticated +# provider-hop contract. It is a validation bridge for the current installation; +# the normal install should render the same entries declaratively from Helm or +# the demo's gateway ConfigMap templates. It preserves local/OpenAI routes. + +: "${KUBECONFIG:?set KUBECONFIG to the target cluster}" +kube_bin="${KUBE_BIN:-kubectl}" +namespace="${GRID_NAMESPACE:-grid-system}" +apply="${APPLY_AZURE_CONSUMER_ROUTES:-0}" + +for configmap in consumer-east-config consumer-west-config; do + current="$($kube_bin -n "$namespace" get configmap "$configmap" -o jsonpath='{.data.praxis\.yaml}')" + updated="$(printf '%s\n' "$current" | yq -oy ' + (.filter_chains[].filters[] | select(.filter == "intelligent_route").provider_hop_clusters) |= + (((. // []) + ["azure-east", "azure-west"]) | unique) + | (.filter_chains[].filters[] | select(.filter == "load_balancer").clusters) |= + (map(select(.name != "azure-east" and .name != "azure-west")) + [ + {"name": "azure-east", "tls": {"ca": {"ca_path": "/etc/praxis/tls/ca.crt"}, "client_cert": {"cert_path": "/etc/praxis/tls/tls.crt", "key_path": "/etc/praxis/tls/tls.key"}, "sni": "provider-east.grid-system.svc", "verify": true}, "endpoints": ["azure-east.grid-system.svc.cluster.local:8443"]}, + {"name": "azure-west", "tls": {"ca": {"ca_path": "/etc/praxis/tls/ca.crt"}, "client_cert": {"cert_path": "/etc/praxis/tls/tls.crt", "key_path": "/etc/praxis/tls/tls.key"}, "sni": "provider-west.grid-system.svc", "verify": true}, "endpoints": ["azure-west.grid-system.svc.cluster.local:8443"]} + ])')" + + # Validate the generated configuration before touching the cluster. + normalized_after="$(printf '%s\n' "$updated" | yq -o=json | jq -S '.')" + local_openai_before="$(printf '%s\n' "$current" | yq -o=json '[.filter_chains[].filters[] | select(.filter == "load_balancer").clusters[] | select(.name | test("^(llm-d|openai)-"))]' | jq -S '.')" + local_openai_after="$(printf '%s\n' "$updated" | yq -o=json '[.filter_chains[].filters[] | select(.filter == "load_balancer").clusters[] | select(.name | test("^(llm-d|openai)-"))]' | jq -S '.')" + [[ "$local_openai_before" == "$local_openai_after" ]] || { echo "$configmap: existing local/OpenAI routes changed" >&2; exit 1; } + printf '%s\n' "$updated" | yq -o=json '.filter_chains[].filters[] | select(.filter == "load_balancer").clusters[] | select(.name == "azure-east") | (.endpoints[0] == "azure-east.grid-system.svc.cluster.local:8443" and .tls.sni == "provider-east.grid-system.svc")' | grep -qx true || { echo "$configmap: azure-east endpoint/SNI validation failed" >&2; exit 1; } + printf '%s\n' "$updated" | yq -o=json '.filter_chains[].filters[] | select(.filter == "load_balancer").clusters[] | select(.name == "azure-west") | (.endpoints[0] == "azure-west.grid-system.svc.cluster.local:8443" and .tls.sni == "provider-west.grid-system.svc")' | grep -qx true || { echo "$configmap: azure-west endpoint/SNI validation failed" >&2; exit 1; } + repeated="$(printf '%s\n' "$updated" | yq -oy ' + (.filter_chains[].filters[] | select(.filter == "intelligent_route").provider_hop_clusters) |= (((. // []) + ["azure-east", "azure-west"]) | unique) + | (.filter_chains[].filters[] | select(.filter == "load_balancer").clusters) |= (map(select(.name != "azure-east" and .name != "azure-west")) + [ + {"name": "azure-east", "tls": {"ca": {"ca_path": "/etc/praxis/tls/ca.crt"}, "client_cert": {"cert_path": "/etc/praxis/tls/tls.crt", "key_path": "/etc/praxis/tls/tls.key"}, "sni": "provider-east.grid-system.svc", "verify": true}, "endpoints": ["azure-east.grid-system.svc.cluster.local:8443"]}, + {"name": "azure-west", "tls": {"ca": {"ca_path": "/etc/praxis/tls/ca.crt"}, "client_cert": {"cert_path": "/etc/praxis/tls/tls.crt", "key_path": "/etc/praxis/tls/tls.key"}, "sni": "provider-west.grid-system.svc", "verify": true}, "endpoints": ["azure-west.grid-system.svc.cluster.local:8443"]} + ])' | yq -o=json | jq -S '.')" + [[ "$normalized_after" == "$repeated" ]] || { echo "$configmap: transformation is not idempotent" >&2; exit 1; } + patch="$(jq -n --arg value "$updated" '{data:{"praxis.yaml":$value}}')" + if [[ "$apply" == 1 ]]; then + "$kube_bin" -n "$namespace" patch configmap "$configmap" --type merge -p "$patch" >/dev/null + echo "configured $configmap for Azure provider hops" + else + echo "validated $configmap; dry-run only (set APPLY_AZURE_CONSUMER_ROUTES=1 to apply)" + fi +done diff --git a/demos/grid-cloud-burst/scripts/configure-otel.sh b/demos/grid-cloud-burst/scripts/configure-otel.sh new file mode 100755 index 0000000..e8b2e27 --- /dev/null +++ b/demos/grid-cloud-burst/scripts/configure-otel.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Configure the existing gateway ConfigMaps to export real gateway spans. +# +# SAFETY: require an explicit opt-in because this changes all four gateway +# ConfigMaps and causes a rollout. The current composed image includes the +# Praxis runtime fix and retains its tracing guard for the server lifetime. + +: "${KUBECONFIG:?set KUBECONFIG to the target cluster}" +if [[ "${ALLOW_GATEWAY_OTEL_UNTIL_CORE_FIX:-}" != "1" ]]; then + echo "gateway OTLP export is blocked: rerun with ALLOW_GATEWAY_OTEL_UNTIL_CORE_FIX=1 after verifying the fixed image" >&2 + exit 2 +fi +namespace="${GRID_NAMESPACE:-grid-system}" +collector_endpoint="${OTLP_ENDPOINT:-http://jaeger-collector.grid-system.svc.cluster.local:4317}" + +for configmap in consumer-east-config consumer-west-config provider-east-config provider-west-config; do + current=$(kubectl -n "$namespace" get configmap "$configmap" -o jsonpath='{.data.praxis\.yaml}') + if printf '%s\n' "$current" | grep -q '^telemetry:'; then + echo "$configmap already has telemetry configuration" + continue + fi + updated=$(printf '%s\ntelemetry:\n otlp_endpoint: %s\n sampling_rate: 1.0\n' "$current" "$collector_endpoint") + patch=$(jq -n --arg value "$updated" '[{"op":"replace","path":"/data/praxis.yaml","value":$value}]') + kubectl -n "$namespace" patch configmap "$configmap" --type=json -p "$patch" >/dev/null + echo "configured $configmap" +done diff --git a/demos/grid-cloud-burst/scripts/configure-ui-traces.sh b/demos/grid-cloud-burst/scripts/configure-ui-traces.sh new file mode 100755 index 0000000..f57224d --- /dev/null +++ b/demos/grid-cloud-burst/scripts/configure-ui-traces.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${KUBECONFIG:?set KUBECONFIG to the target cluster}" +ui_namespace="${UI_NAMESPACE:-praxis-tracing-cloud-burst}" +deployment="${UI_DEPLOYMENT:-praxis-tracing-cloud-burst-ui}" +jaeger_url="${JAEGER_URL:-http://jaeger-query.grid-system.svc.cluster.local:16686}" + +kubectl -n "$ui_namespace" set env "deployment/$deployment" \ + JAEGER_URL="$jaeger_url" \ + JAEGER_UI_URL="${JAEGER_UI_URL:-$jaeger_url}" +kubectl -n "$ui_namespace" rollout status "deployment/$deployment" --timeout=180s diff --git a/demos/grid-cloud-burst/scripts/diagnose-east-withdrawal.sh b/demos/grid-cloud-burst/scripts/diagnose-east-withdrawal.sh new file mode 100755 index 0000000..5fbc4d4 --- /dev/null +++ b/demos/grid-cloud-burst/scripts/diagnose-east-withdrawal.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +K="${KUBECONFIG:-/tmp/rhoai-sim-validation.kubeconfig}" +NS="${GRID_NAMESPACE:-grid-system}" +UI="${UI_URL:?set UI_URL to the deployed cloud-burst UI URL}" +USER="${UI_USER:?set UI_USER to the deployed UI username}" +PASS="${UI_PASSWORD:?set UI_PASSWORD to the deployed UI password}" +EVIDENCE_ROOT="${EVIDENCE_ROOT:-$ROOT/evidence}" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +E="$EVIDENCE_ROOT/east-withdrawal-$STAMP" +mkdir -p "$E/raw" +SCENARIO=focused-east-withdrawal +PROVIDER=llm-d-east-1 +TRAFFIC_PID="" + +log() { printf '[%s] %s\n' "$(date -u +%FT%T.%3NZ)" "$*" | tee -a "$E/runner.log" >&2; } +kc() { kubectl --kubeconfig "$K" -n "$NS" "$@"; } +ui() { curl -ksS --connect-timeout 5 --max-time 20 -u "$USER:$PASS" "$@"; } + +cleanup() { + set +e + if [[ -n "$TRAFFIC_PID" ]]; then kill "$TRAFFIC_PID" 2>/dev/null || true; fi + ui -X POST -H 'content-type: application/json' -d "{\"provider\":\"$PROVIDER\",\"disabled\":false}" "$UI/api/v1/cloud-burst/provider" >"$E/raw/restore-provider.json" 2>&1 || true + for _ in {1..90}; do + ready="$(kc get deploy "$PROVIDER" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" + [[ "$ready" == 1 ]] && break + sleep 1 + done + capture_state cleanup-final || true + kc logs deploy/consumer-east --since=20m 2>/dev/null | sed -E 's/(Authorization:|authorization:|api[_-]?key[=:])[^[:space:]]+/\1 [REDACTED]/Ig' >"$E/raw/consumer-east.log" || true + log "cleanup complete; evidence=$E" +} +trap cleanup EXIT INT TERM + +capture_state() { + local label="$1" now desired ready endpoints overlay phase + now="$(date -u +%FT%T.%3NZ)" + desired="$(kc get deploy "$PROVIDER" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" + ready="$(kc get deploy "$PROVIDER" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" + endpoints="$(kc get endpoints "$PROVIDER" -o json 2>/dev/null | jq -c '[.subsets[]?.addresses[]?.ip] // []' || echo '[]')" + overlay="$(kc get cm grid-overlay-grid-cloud-burst-rhoai-consumer-east -o json 2>/dev/null | jq -c '.data["routing-overlay.json"] | fromjson | {revision:.revision, candidates:(.overlay.candidates|map({cluster,stable_id,selection_group,admission_state}))}' 2>/dev/null || echo '{"revision":null,"candidates":[]}' )" + phase="$(kc get gridnetwork grid-cloud-burst-rhoai -o jsonpath='{.status.phase}' 2>/dev/null || true)" + jq -nc --arg at "$now" --arg scenario "$SCENARIO" --arg label "$label" --arg desired "$desired" --arg ready "$ready" --arg phase "$phase" --argjson endpoints "$endpoints" --argjson overlay "$overlay" \ + '{at:$at,scenario:$scenario,label:$label,deployment:{desired:($desired|tonumber?),ready:($ready|tonumber?)},endpoints:$endpoints,overlay:$overlay,gridnetwork_phase:$phase}' >>"$E/state-samples.jsonl" +} + +traffic_loop() { + local seq=0 body status rid at + while :; do + seq=$((seq+1)); rid="east-withdraw-${STAMP}-${seq}" + at="$(date -u +%FT%T.%3NZ)" + body="$(ui -X POST -H 'content-type: application/json' -H "X-Request-ID: $rid" -d '{"consumer":"a","app":"app1"}' "$UI/api/v1/token-rate-limit/requests" 2>"$E/raw/request-${seq}.err" || true)" + status="$(jq -r '.record.http.status // .record.status // 0' <<<"$body" 2>/dev/null || echo 0)" + jq -nc --arg at "$at" --arg id "$rid" --argjson status "${status:-0}" --argjson body "$(jq -c '.record // null' <<<"$body" 2>/dev/null || echo null)" \ + '{at:$at,request_id:$id,http_status:$status,record:$body}' >>"$E/requests.jsonl" + sleep 0.5 + done +} + +main() { + printf '{"started_at":"%s","scenario":"%s","rate_per_second":2,"provider":"%s","no_retries":true}\n' "$(date -u +%FT%TZ)" "$SCENARIO" "$PROVIDER" >"$E/metadata.json" + capture_state before + [[ "$(kc get deploy "$PROVIDER" -o jsonpath='{.spec.replicas}')" == 1 ]] || { log "precondition failed: desired replicas"; exit 1; } + [[ "$(kc get deploy "$PROVIDER" -o jsonpath='{.status.readyReplicas}')" == 1 ]] || { log "precondition failed: ready replicas"; exit 1; } + traffic_loop & TRAFFIC_PID=$! + sleep 5 + log "disabling $PROVIDER while traffic continues" + ui -X POST -H 'content-type: application/json' -d "{\"provider\":\"$PROVIDER\",\"disabled\":true}" "$UI/api/v1/cloud-burst/provider" >"$E/raw/disable-provider.json" + old_revision="$(jq -r '.overlay.revision.value // empty' "$E/state-samples.jsonl" | head -1)" + withdrawal_revision="" + stable=0 + while :; do + capture_state polling + latest="$(tail -1 "$E/state-samples.jsonl")" + current_revision="$(jq -r '.overlay.revision.value // empty' <<<"$latest")" + candidate_present="$(jq -r --arg p "$PROVIDER" '[.overlay.candidates[]|select(.cluster==$p)]|length' <<<"$latest")" + if [[ -n "$current_revision" && "$current_revision" != "$old_revision" && "$candidate_present" == 0 ]]; then + [[ -n "$withdrawal_revision" ]] || { withdrawal_revision="$current_revision"; log "withdrawal revision observed: $withdrawal_revision"; } + stable=$((stable+1)) + else + stable=0 + fi + [[ "$stable" -ge 30 ]] && break + sleep 1 + done + jq --arg revision "$withdrawal_revision" '. + {withdrawal_revision:$revision}' "$E/metadata.json" >"$E/metadata.tmp" && mv "$E/metadata.tmp" "$E/metadata.json" + log "withdrawal revision served for 30 consecutive state samples" +} +main "$@" diff --git a/demos/grid-cloud-burst/scripts/diagnose-request-stall.sh b/demos/grid-cloud-burst/scripts/diagnose-request-stall.sh new file mode 100755 index 0000000..0a7546b --- /dev/null +++ b/demos/grid-cloud-burst/scripts/diagnose-request-stall.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +K="${KUBECONFIG:-/tmp/rhoai-sim-validation.kubeconfig}" +NS=grid-system +UI="${UI_URL:?set UI_URL to the deployed cloud-burst UI URL}" +E="$ROOT/evidence/request-stall-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$E/raw" +PROVIDER=llm-d-east-1 +TRAFFIC_PID="" +STARTED="$(date -u +%FT%TZ)" + +kc() { kubectl --kubeconfig "$K" -n "$NS" "$@"; } +: "${UI_USER:?set UI_USER to the tracing UI username}" +: "${UI_PASSWORD:?set UI_PASSWORD to the tracing UI password}" +ui() { curl -ksS --connect-timeout 5 --max-time 35 -u "$UI_USER:$UI_PASSWORD" "$@"; } +log() { printf '[%s] %s\n' "$(date -u +%FT%T.%3NZ)" "$*" | tee -a "$E/runner.log" >&2; } + +snapshot() { + local label="$1" now overlay phase endpoint_json + now="$(date -u +%FT%T.%3NZ)" + endpoint_json="$(kc get endpointslice -l kubernetes.io/service-name="$PROVIDER" -o json 2>/dev/null | jq -c '[.items[].endpoints[]?|{addresses,conditions}]' || echo '[]')" + overlay="$(kc get cm grid-overlay-grid-cloud-burst-rhoai-consumer-east -o json 2>/dev/null | jq -c '.data["routing-overlay.json"]|fromjson|{revision:.revision,candidates:(.overlay.candidates|map({cluster,stable_id,selection_group,admission_state}))}' 2>/dev/null || echo '{"revision":null,"candidates":[]}' )" + phase="$(kc get gridnetwork grid-cloud-burst-rhoai -o jsonpath='{.status.phase}' 2>/dev/null || true)" + jq -nc --arg at "$now" --arg label "$label" --arg phase "$phase" --argjson endpoints "$endpoint_json" --argjson overlay "$overlay" \ + '{at:$at,label:$label,endpoints:$endpoints,gridnetwork_phase:$phase,overlay:$overlay}' >>"$E/state.jsonl" +} + +traffic() { + local n=0 id start end body_file timing_file status + while :; do + n=$((n+1)); id="stall-${STARTED}-${n}"; start="$(date -u +%FT%T.%3NZ)" + body_file="$E/raw/response-$n.body"; timing_file="$E/raw/response-$n.timing.json" + set +e + ui -o "$body_file" -D "$E/raw/response-$n.headers" -w '%{json}' \ + -X POST -H 'content-type: application/json' -H "X-Request-ID: $id" \ + -d '{"consumer":"a","app":"app2"}' "$UI/api/v1/token-rate-limit/requests" >"$timing_file" 2>"$E/raw/response-$n.curlerr" + local rc=$? + set -e + end="$(date -u +%FT%T.%3NZ)" + status="$(jq -r '.record.http.status // .record.status // 0' "$body_file" 2>/dev/null || echo 0)" + jq -nc --arg id "$id" --arg started "$start" --arg ended "$end" --argjson rc "$rc" \ + --argjson timing "$(jq -c '. // {}' "$timing_file" 2>/dev/null || echo '{}')" \ + --argjson status "${status:-0}" --argjson record "$(jq -c '.record // null' "$body_file" 2>/dev/null || echo null)" \ + '{request_id:$id,started_at:$started,ended_at:$ended,curl_exit:$rc,http_status:$status,curl_timing:$timing,record:$record}' >>"$E/requests.jsonl" + sleep 1 + done +} + +cleanup() { + set +e + if [[ -n "$TRAFFIC_PID" ]]; then kill "$TRAFFIC_PID" 2>/dev/null || true; fi + ui -X POST -H 'content-type: application/json' -d "{\"provider\":\"$PROVIDER\",\"disabled\":false}" "$UI/api/v1/cloud-burst/provider" >"$E/raw/restore.json" 2>&1 || true + for _ in {1..90}; do + [[ "$(kc get deploy "$PROVIDER" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" == 1 ]] && break + sleep 1 + done + snapshot cleanup-final || true + for workload in consumer-east consumer-west provider-east provider-west; do + kc logs "deploy/$workload" --since-time="$STARTED" 2>/dev/null | sed -E 's/(Authorization:|authorization:|api[_-]?key[=:])[^[:space:]]+/\1 [REDACTED]/Ig' >"$E/raw/$workload.log" || true + done + log "cleanup complete; evidence=$E" +} +trap cleanup EXIT INT TERM + +main() { + printf '{"started_at":"%s","provider":"%s","rate_per_second":1,"retries":false}\n' "$STARTED" "$PROVIDER" >"$E/metadata.json" + for p in llm-d-east-1 llm-d-east-2 llm-d-west-1 llm-d-west-2; do + [[ "$(kc get deploy "$p" -o jsonpath='{.spec.replicas}')" == 1 && "$(kc get deploy "$p" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" == 1 ]] || { log "preflight failed: $p"; exit 1; } + done + snapshot preflight + traffic & TRAFFIC_PID=$! + sleep 5 + log "disabling $PROVIDER while one request per second continues" + ui -X POST -H 'content-type: application/json' -d "{\"provider\":\"$PROVIDER\",\"disabled\":true}" "$UI/api/v1/cloud-burst/provider" >"$E/raw/disable.json" + for _ in {1..45}; do snapshot polling; sleep 1; done +} +main "$@" diff --git a/demos/grid-cloud-burst/scripts/latency-one-provider-cycles.sh b/demos/grid-cloud-burst/scripts/latency-one-provider-cycles.sh new file mode 100755 index 0000000..ae51b8c --- /dev/null +++ b/demos/grid-cloud-burst/scripts/latency-one-provider-cycles.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +K="${KUBECONFIG:-/tmp/rhoai-sim-validation.kubeconfig}" +NS="${GRID_NAMESPACE:-grid-system}" +UI="${UI_URL:?set UI_URL to the deployed cloud-burst UI URL}" +USER="${UI_USER:?set UI_USER to the deployed UI username}" +PASS="${UI_PASSWORD:?set UI_PASSWORD to the deployed UI password}" +EVIDENCE_ROOT="${EVIDENCE_ROOT:-$ROOT/evidence}" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +E="$EVIDENCE_ROOT/latency-one-provider-$STAMP" +mkdir -p "$E/raw" +TRAFFIC_PID="" +SCENARIO="preflight" +LOCAL=(llm-d-east-1 llm-d-east-2 llm-d-west-1 llm-d-west-2) + +log() { printf '[%s] %s\n' "$(date -u +%FT%T.%3NZ)" "$*" | tee -a "$E/runner.log" >&2; } +kc() { kubectl --kubeconfig "$K" -n "$NS" "$@"; } +ui() { curl -ksS --connect-timeout 5 --max-time 20 -u "$USER:$PASS" "$@"; } + +snapshot() { + local label="$1" now overlay phase + now="$(date -u +%FT%T.%3NZ)" + overlay="$(kc get cm grid-overlay-grid-cloud-burst-rhoai-consumer-east -o json 2>/dev/null | jq -c '.data["routing-overlay.json"]|fromjson|{revision:.revision,candidates:(.overlay.candidates|map({cluster,stable_id,selection_group,admission_state}))}' 2>/dev/null || echo '{"revision":null,"candidates":[]}' )" + phase="$(kc get gridnetwork grid-cloud-burst-rhoai -o jsonpath='{.status.phase}' 2>/dev/null || true)" + jq -nc --arg at "$now" --arg scenario "$SCENARIO" --arg label "$label" --arg phase "$phase" --argjson overlay "$overlay" \ + '{at:$at,scenario:$scenario,label:$label,gridnetwork_phase:$phase,overlay:$overlay}' >>"$E/state-samples.jsonl" +} + +set_disabled() { + local provider="$1" disabled="$2" + ui -X POST -H 'content-type: application/json' -d "{\"provider\":\"$provider\",\"disabled\":$disabled}" \ + "$UI/api/v1/cloud-burst/provider" >"$E/raw/${SCENARIO}-${provider}-${disabled}.json" +} + +restore_all() { + local provider + ui -X POST -H 'content-type: application/json' -d '{"on":false,"mode":"sim"}' "$UI/api/v1/cloud-burst/load" >"$E/raw/${SCENARIO}-pressure-off.json" 2>&1 || true + for provider in "${LOCAL[@]}"; do set_disabled "$provider" false; done + for _ in {1..90}; do + local ready=0 + for provider in "${LOCAL[@]}"; do + [[ "$(kc get deploy "$provider" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" == 1 ]] && ready=$((ready+1)) + done + [[ "$ready" == 4 ]] && return 0 + sleep 1 + done + return 1 +} + +request_one() { + local consumer="$1" app="app1" id start end body status latency provider gateway + id="lat-${STAMP}-${SCENARIO}-$$-$(date -u +%s%N)" + start="$(date -u +%FT%T.%3NZ)"; start_ns="$(date +%s%N)" + body="$(ui -X POST -H 'content-type: application/json' -H "X-Request-ID: $id" -d "{\"consumer\":\"$consumer\",\"app\":\"$app\"}" "$UI/api/v1/token-rate-limit/requests" 2>/dev/null || true)" + end="$(date -u +%FT%T.%3NZ)"; end_ns="$(date +%s%N)" + latency="$(awk -v s="$start_ns" -v e="$end_ns" 'BEGIN{printf "%.3f",(e-s)/1000000}')" + status="$(jq -r '.record.http.status // .record.status // 0' <<<"$body" 2>/dev/null || echo 0)" + provider="$(jq -r '.record.inference_provider // ""' <<<"$body" 2>/dev/null || true)" + gateway="$(jq -r '.record.route.provider_gateway // ""' <<<"$body" 2>/dev/null || true)" + jq -nc --arg scenario "$SCENARIO" --arg consumer "$consumer" --arg id "$id" --arg started "$start" --arg ended "$end" \ + --arg latency "$latency" --argjson status "${status:-0}" --arg provider "$provider" --arg gateway "$gateway" \ + '{scenario:$scenario,consumer:$consumer,request_id:$id,started_at:$started,ended_at:$ended,latency_ms:($latency|tonumber),http_status:$status,inference_provider:$provider,provider_gateway:$gateway,record:($body.record//null)}' \ + --argjson body "$(jq -c '. // {}' <<<"$body" 2>/dev/null || echo '{}')" >>"$E/requests.jsonl" +} + +traffic_loop() { + local n=0 + while :; do + n=$((n+1)); request_one "$([[ $((n%2)) == 0 ]] && echo b || echo a)" & + sleep 0.5 + wait || true + done +} + +sequential_20() { + local n + for n in {1..20}; do request_one "$([[ $((n%2)) == 0 ]] && echo b || echo a)"; done +} + +cycle() { + local keep="$1" provider + SCENARIO="cycle-${keep}" + log "starting $SCENARIO; survivor=$keep" + restore_all + snapshot cycle-start + traffic_loop & TRAFFIC_PID=$! + sleep 5 + for provider in "${LOCAL[@]}"; do + [[ "$provider" == "$keep" ]] && continue + log "disabling $provider; traffic remains at 2 requests/second" + set_disabled "$provider" true + snapshot "disabled-$provider" + sleep 30 + done + log "collecting 20 sequential requests with only $keep enabled" + snapshot one-provider-start + sequential_20 + snapshot one-provider-end + for provider in "${LOCAL[@]}"; do + [[ "$provider" == "$keep" ]] && continue + log "restoring $provider while traffic continues" + set_disabled "$provider" false + snapshot "restored-$provider" + sleep 10 + done + kill "$TRAFFIC_PID" 2>/dev/null || true; wait "$TRAFFIC_PID" 2>/dev/null || true; TRAFFIC_PID="" + snapshot cycle-end +} + +cleanup() { + set +e + if [[ -n "$TRAFFIC_PID" ]]; then kill "$TRAFFIC_PID" 2>/dev/null || true; fi + restore_all >/dev/null 2>&1 || true + snapshot cleanup-final || true + kc logs deploy/consumer-east --since=30m 2>/dev/null >"$E/raw/consumer-east.log" || true + kc logs deploy/consumer-west --since=30m 2>/dev/null >"$E/raw/consumer-west.log" || true + log "cleanup complete; evidence=$E" +} +trap cleanup EXIT INT TERM + +main() { + printf '{"started_at":"%s","rate_per_second":2,"cycles":["llm-d-east-1","llm-d-west-1","llm-d-east-2"],"retries":false}\n' "$(date -u +%FT%TZ)" >"$E/metadata.json" + restore_all + for provider in "${LOCAL[@]}"; do + [[ "$(kc get deploy "$provider" -o jsonpath='{.spec.replicas}')" == 1 && "$(kc get deploy "$provider" -o jsonpath='{.status.readyReplicas}')" == 1 ]] || { log "preflight failed: $provider"; exit 1; } + done + snapshot preflight + cycle llm-d-east-1 + cycle llm-d-west-1 + cycle llm-d-east-2 +} +main "$@" diff --git a/demos/grid-cloud-burst/scripts/qualify-cloud-burst.sh b/demos/grid-cloud-burst/scripts/qualify-cloud-burst.sh new file mode 100755 index 0000000..2e5948d --- /dev/null +++ b/demos/grid-cloud-burst/scripts/qualify-cloud-burst.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Live cloud-burst qualification runner. This is intentionally a diagnostic +# script: it changes only the simulator replicas/metrics through the existing +# demo controls and preserves every observed result. It never calls Grid +# reconciliation and never retries a request. + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KUBECONFIG_PATH="${KUBECONFIG:-/tmp/rhoai-sim-validation.kubeconfig}" +NS="${GRID_NAMESPACE:-grid-system}" +UI_NS="${UI_NAMESPACE:-praxis-tracing-cloud-burst}" +UI_URL="${UI_URL:?set UI_URL to the deployed cloud-burst UI URL}" +UI_USER="${UI_USER:?set UI_USER to the deployed UI username}" +UI_PASSWORD="${UI_PASSWORD:?set UI_PASSWORD to the deployed UI password}" +NETWORK="${GRID_NETWORK:-grid-cloud-burst-rhoai}" +OVERLAY_CM="${GRID_OVERLAY_CONFIGMAP:-grid-overlay-grid-cloud-burst-rhoai-consumer-east}" +RATE="${RATE_PER_SECOND:-2}" +WINDOW="${SCENARIO_SECONDS:-10}" +REQUEST_TIMEOUT="${REQUEST_TIMEOUT_SECONDS:-20}" +EVIDENCE_ROOT="${EVIDENCE_ROOT:-$ROOT/evidence}" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +EVIDENCE="$EVIDENCE_ROOT/qualification-$STAMP" +RAW="$EVIDENCE/raw" +mkdir -p "$RAW" + +LOCAL_PROVIDERS=(llm-d-east-1 llm-d-east-2 llm-d-west-1 llm-d-west-2) +declare -A PROVIDER_TO_SIM=( + [llm-d-east-1]=llm-d-east-1 + [llm-d-east-2]=llm-d-east-2 + [llm-d-west-1]=llm-d-west-1 + [llm-d-west-2]=llm-d-west-2 +) +SCENARIO="preflight" +SAMPLER_PID="" +TRAFFIC_PID="" + +log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*" | tee -a "$EVIDENCE/runner.log" >&2; } +kc() { kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NS" "$@"; } +ui_curl() { curl -ksS --connect-timeout 5 --max-time "$REQUEST_TIMEOUT" -u "$UI_USER:$UI_PASSWORD" "$@"; } + +sanitize() { + sed -E 's/(Authorization:|authorization:|X-Api-Key:|x-api-key:|api[_-]?key[=:])[^[:space:]]+/\1 [REDACTED]/Ig; s/(password|secret|token)[=:][^,;[:space:]]+/\1=[REDACTED]/Ig' +} + +cleanup() { + set +e + for child in $(jobs -pr); do kill "$child" 2>/dev/null || true; done + if [[ -n "$TRAFFIC_PID" ]]; then kill "$TRAFFIC_PID" 2>/dev/null || true; fi + if [[ -n "$SAMPLER_PID" ]]; then kill "$SAMPLER_PID" 2>/dev/null || true; fi + log "cleanup: restoring simulator metrics and replicas" + ui_curl -X POST -H 'content-type: application/json' \ + -d '{"on":false,"mode":"sim"}' "$UI_URL/api/v1/cloud-burst/load" \ + >"$RAW/cleanup-pressure.json" 2>&1 || true + for provider in "${LOCAL_PROVIDERS[@]}"; do + ui_curl -X POST -H 'content-type: application/json' \ + -d "{\"provider\":\"$provider\",\"disabled\":false}" \ + "$UI_URL/api/v1/cloud-burst/provider" >"$RAW/cleanup-$provider.json" 2>&1 || true + done + log "cleanup: waiting for simulator deployments to report one ready replica" + for _ in {1..60}; do + ready=0 + for provider in "${LOCAL_PROVIDERS[@]}"; do + sim="${PROVIDER_TO_SIM[$provider]}" + replicas="$(kc get deploy "$sim" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" + [[ "$replicas" == 1 ]] && ready=$((ready + 1)) + done + [[ "$ready" == 4 ]] && break + sleep 1 + done + capture_logs + rmdir "$EVIDENCE/state-samples.lock.d" "$EVIDENCE/requests.lock.d" 2>/dev/null || true + sample_state "cleanup-final" || true + log "cleanup complete; evidence=$EVIDENCE" +} +trap cleanup EXIT INT TERM + +sample_state() { + local label="$1" now overlay_json grid_json + local lockdir="$EVIDENCE/state-samples.lock.d" + while ! mkdir "$lockdir" 2>/dev/null; do sleep 0.01; done + now="$(date -u +%FT%TZ)" + overlay_json="$(kc get configmap "$OVERLAY_CM" -o json 2>/dev/null | jq -c ' + .data["routing-overlay.json"] // .data["routing-config.json"] // "" | fromjson? | + {revision:(.revision // null), candidates:((.overlay.candidates // .candidates // []) | + map({name:(.name // .model // null),site:(.site // null),cluster:(.cluster // null),group:(.selection_group // .group // null),admission:(.admission_state // .admission // null),tier:(.selection_tier // .tier // null),weight:(.traffic_weight // .weight // null),backend_kind:(.backend_kind // null)}))}' 2>/dev/null || echo '{"revision":null,"candidates":[]}' )" + grid_json="$(kc get gridnetwork "$NETWORK" -o json 2>/dev/null | jq -c '{phase:(.status.phase // null),conditions:(.status.conditions // [] | map({type,reason,status,message})),generation:.metadata.generation}' 2>/dev/null || echo '{"phase":null,"conditions":[]}' )" + { + printf '{"at":"%s","scenario":"%s","label":"%s","deployments":{' "$now" "$SCENARIO" "$label" + local first=1 provider sim desired ready + for provider in "${LOCAL_PROVIDERS[@]}"; do + sim="${PROVIDER_TO_SIM[$provider]}" + desired="$(kc get deploy "$sim" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" + ready="$(kc get deploy "$sim" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" + [[ $first == 0 ]] && printf ','; first=0 + printf '%q' "$provider" >/dev/null + printf '"%s":{"desired":%s,"ready":%s}' "$provider" "${desired:-null}" "${ready:-null}" + done + printf '},"services":{' + first=1 + for provider in "${LOCAL_PROVIDERS[@]}"; do + sim="${PROVIDER_TO_SIM[$provider]}" + local endpoints + endpoints="$(kc get endpoints "$sim" -o json 2>/dev/null | jq -c '[.subsets[]?.addresses[]?.ip] // []' 2>/dev/null || echo '[]')" + [[ $first == 0 ]] && printf ','; first=0 + printf '"%s":%s' "$provider" "$endpoints" + done + printf '},"overlay":%s,"gridnetwork":%s}\n' "$overlay_json" "$grid_json" + } >>"$EVIDENCE/state-samples.jsonl" + rmdir "$lockdir" +} + +sample_loop() { + while :; do sample_state "periodic"; sleep 1; done +} + +start_sampling() { + sample_loop & SAMPLER_PID=$! +} + +capture_provenance() { + git -C "$ROOT/../.." rev-parse HEAD >"$RAW/experimental-head.txt" 2>/dev/null || true + git -C "$ROOT/../.." status --short >"$RAW/experimental-status.txt" 2>/dev/null || true + kc get deploy -o json | jq '[.items[] | {name:.metadata.name, images:[.spec.template.spec.containers[]?.image]}]' \ + >"$RAW/deployment-images.json" 2>/dev/null || true + kc get gridnetwork "$NETWORK" -o json | jq '{metadata:{name:.metadata.name,generation:.metadata.generation},status:.status}' \ + >"$RAW/gridnetwork-provenance.json" 2>/dev/null || true + kc get configmap "$OVERLAY_CM" -o json | jq 'del(.metadata.managedFields,.metadata.annotations)' \ + >"$RAW/overlay-configmap.json" 2>/dev/null || true +} + +capture_logs() { + local workload + for workload in consumer-east consumer-west provider-east provider-west azure-east azure-west; do + kc logs "deploy/$workload" --since=15m 2>/dev/null | sanitize >"$RAW/logs-$workload.txt" || true + done + kc get events --sort-by='.lastTimestamp' -o json 2>/dev/null | jq '[.items[] | {lastTimestamp:.lastTimestamp,reason:.reason,type:.type,involvedObject:.involvedObject.name,message:.message}]' \ + >"$RAW/events.json" || true +} + +record_response() { + local scenario="$1" consumer="$2" app="$3" request_id="$4" header_file="$5" body_file="$6" start="$7" end="$8" http="$9" + local lockdir="$EVIDENCE/requests.lock.d" + while ! mkdir "$lockdir" 2>/dev/null; do sleep 0.01; done + local headers_json='{}' response_json='null' record_json='null' actual_status='null' + if [[ -f "$header_file" ]]; then + headers_json="$(awk 'BEGIN{IGNORECASE=1} /^[[:space:]]*[^:]+:[[:space:]]/ { sub(/^[^:]+:[[:space:]]*/, ""); key=$0; sub(/:.*/, "", key); value=$0; sub(/^[^:]+:[[:space:]]*/, "", value); printf "%s\t%s\n", tolower(key), value }' "$header_file" | jq -Rn '[inputs | split("\t") | { (.[0]): .[1] }] | add // {}' 2>/dev/null || echo '{}')" + fi + response_json="$(jq -c . "$body_file" 2>/dev/null || echo 'null')" + record_json="$(jq -c '.record // null' <<<"$response_json" 2>/dev/null || echo 'null')" + actual_status="$(jq -r '.record.http.status // .record.status // .record.http_status // empty' <<<"$response_json" 2>/dev/null || true)" + [[ "$actual_status" =~ ^[0-9]+$ ]] || actual_status=null + jq -nc --arg scenario "$scenario" --arg consumer "$consumer" --arg app "$app" \ + --arg request_id "$request_id" --arg started "$start" --arg ended "$end" \ + --arg wrapper_status "$http" --argjson actual_status "$actual_status" \ + --argjson headers "$headers_json" --argjson record "$record_json" \ + --arg body "$(sanitize <"$body_file" | head -c 4000)" \ + '{scenario:$scenario,consumer:$consumer,application:$app,request_id:$request_id,started_at:$started,ended_at:$ended,wrapper_http_status:($wrapper_status|tonumber?),http_status:$actual_status,headers:$headers,record:$record,body:$body}' \ + | { cat >>"$EVIDENCE/requests.jsonl"; rmdir "$lockdir"; } +} + +send_request() { + local consumer="$1" app="$2" scenario="${3:-$SCENARIO}" request_id start end header_file body_file http unique + unique="${BASHPID}-$(date -u +%s%N)" + request_id="cbq-${STAMP}-${unique}-${consumer}-${app}" + header_file="$RAW/response-${unique}.headers" + body_file="$RAW/response-${unique}.body" + start="$(date -u +%FT%T.%3NZ)" + set +e + http="$(ui_curl -D "$header_file" -o "$body_file" -w '%{http_code}' \ + -X POST -H 'content-type: application/json' -H "X-Request-ID: $request_id" \ + -d "{\"consumer\":\"$consumer\",\"app\":\"$app\"}" \ + "$UI_URL/api/v1/token-rate-limit/requests" 2>"$RAW/response-${unique}.curlerr")" + local curl_rc=$? + set -e + [[ "$curl_rc" == 0 && "$http" =~ ^[0-9]{3}$ ]] || http=000 + end="$(date -u +%FT%T.%3NZ)" + record_response "$scenario" "$consumer" "$app" "$request_id" "$header_file" "$body_file" "$start" "$end" "$http" +} + +traffic_for() { + local consumer="$1" seconds="${2:-$WINDOW}" deadline pid + local -a request_pids=() + deadline=$((SECONDS + seconds)) + while (( SECONDS < deadline )); do + send_request "$consumer" "app1" "$SCENARIO" & + request_pids+=("$!") + sleep "$(awk "BEGIN {print 1/$RATE}")" + done + for pid in "${request_pids[@]}"; do wait "$pid" || true; done +} + +set_provider_disabled() { + local provider="$1" disabled="$2" + ui_curl -X POST -H 'content-type: application/json' \ + -d "{\"provider\":\"$provider\",\"disabled\":$disabled}" \ + "$UI_URL/api/v1/cloud-burst/provider" | sanitize >"$RAW/${SCENARIO}-${provider}-${disabled}.json" +} + +set_pressure() { + local on="$1" + ui_curl -X POST -H 'content-type: application/json' \ + -d "{\"on\":$on,\"mode\":\"sim\"}" "$UI_URL/api/v1/cloud-burst/load" \ + | sanitize >"$RAW/${SCENARIO}-pressure-${on}.json" +} + +wait_local_ready() { + local expected=4 ready sim provider + for _ in {1..90}; do + ready=0 + for provider in "${LOCAL_PROVIDERS[@]}"; do + sim="${PROVIDER_TO_SIM[$provider]}" + [[ "$(kc get deploy "$sim" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" == 1 ]] && ready=$((ready + 1)) + done + [[ "$ready" == "$expected" ]] && return 0 + sleep 1 + done + log "WARN local readiness timeout: ready=$ready expected=$expected" + return 1 +} + +restore_all_local() { + local provider + for provider in "${LOCAL_PROVIDERS[@]}"; do + set_provider_disabled "$provider" false + done + wait_local_ready || true +} + +preflight() { + log "preflight: checking UI, four simulators, overlay, consumers, and baseline" + local status + status="$(ui_curl -o "$RAW/preflight-status.json" -w '%{http_code}' "$UI_URL/api/status")" + [[ "$status" == 200 ]] || { log "FAIL preflight UI status HTTP $status"; return 1; } + ui_curl "$UI_URL/api/v1/cloud-burst" >"$RAW/preflight-cloud-burst.json" + ui_curl "$UI_URL/api/v1/token-rate-limit" >"$RAW/preflight-token-status.json" + for provider in "${LOCAL_PROVIDERS[@]}"; do + sim="${PROVIDER_TO_SIM[$provider]}" + [[ "$(kc get deploy "$sim" -o jsonpath='{.spec.replicas}' | tr -d '\n')" == 1 ]] || { log "FAIL $sim desired replicas"; return 1; } + [[ "$(kc get deploy "$sim" -o jsonpath='{.status.readyReplicas}' | tr -d '\n')" == 1 ]] || { log "FAIL $sim ready replicas"; return 1; } + done + sample_state "preflight" + traffic_for a 10 + traffic_for b 10 + local failures + failures="$(jq -s '[.[] | select(.scenario=="preflight" and ((.http_status // 0) < 200 or (.http_status // 0) >= 300))] | length' "$EVIDENCE/requests.jsonl")" + [[ "$failures" == 0 ]] || { log "FAIL preflight baseline request failures=$failures"; return 1; } + log "PASS preflight" +} + +write_summary() { + jq -s 'group_by(.scenario) | map({scenario:.[0].scenario,requests:length,wrapper_statuses:(map(.wrapper_http_status)|group_by(.)|map({status:.[0],count:length})),http_statuses:(map(.http_status)|group_by(.)|map({status:.[0],count:length})),admissions:(map(.record.admission // null)|group_by(.)|map({value:.[0],count:length})),providers:(map(.record.inference_provider // null)|group_by(.)|map({value:.[0],count:length})),provider_gateways:(map(.record.route.provider_gateway // null)|group_by(.)|map({value:.[0],count:length})),five_xx:(map(select((.http_status // 0)>=500 and (.http_status // 0)<600))|length),transition_502s:(map(select(.http_status==502))|length)})' \ + "$EVIDENCE/requests.jsonl" >"$EVIDENCE/scenario-summary.json" + jq -s '[.[] | select((.http_status // 0) >= 500 and (.http_status // 0) < 600)]' \ + "$EVIDENCE/requests.jsonl" >"$EVIDENCE/failures.json" + jq -s 'map(select(.http_status==502)) | length' "$EVIDENCE/requests.jsonl" >"$EVIDENCE/transition-502-count.txt" + cp "$EVIDENCE/state-samples.jsonl" "$EVIDENCE/state-timeline.jsonl" + jq -R -s 'split("\n") | map(select(length>0) | fromjson) | map({at,scenario,label,gridnetwork_phase:.gridnetwork.phase,overlay_revision:.overlay.revision})' \ + "$EVIDENCE/state-samples.jsonl" >"$EVIDENCE/revision-timeline.json" 2>/dev/null || true +} + +run_scenario() { + local name="$1" action="$2" + SCENARIO="$name" + log "scenario start: $name" + sample_state "scenario-start" + eval "$action" + sample_state "scenario-end" + log "scenario complete: $name" +} + +baseline() { traffic_for a "$WINDOW"; traffic_for b "$WINDOW"; } +one_east_down() { set_provider_disabled llm-d-east-1 true; traffic_for a "$WINDOW"; } +restore_east() { set_provider_disabled llm-d-east-1 false; traffic_for a "$WINDOW"; } +both_east_down() { set_provider_disabled llm-d-east-1 true; set_provider_disabled llm-d-east-2 true; traffic_for a "$WINDOW"; } +one_west_down() { set_provider_disabled llm-d-west-1 true; traffic_for b "$WINDOW"; } +restore_west() { set_provider_disabled llm-d-west-1 false; traffic_for b "$WINDOW"; } +both_west_down() { set_provider_disabled llm-d-west-1 true; set_provider_disabled llm-d-west-2 true; traffic_for b "$WINDOW"; } +all_local_down() { for p in "${LOCAL_PROVIDERS[@]}"; do set_provider_disabled "$p" true; done; traffic_for a "$WINDOW"; traffic_for b "$WINDOW"; } +pressure_burst() { restore_all_local; set_pressure true; traffic_for a "$WINDOW"; traffic_for b "$WINDOW"; } +pressure_recovery() { set_pressure false; traffic_for a "$WINDOW"; traffic_for b "$WINDOW"; } +combined_degradation() { restore_all_local; set_provider_disabled llm-d-east-1 true; set_provider_disabled llm-d-west-1 true; set_pressure true; traffic_for a "$WINDOW"; traffic_for b "$WINDOW"; } + +main() { + cat >"$EVIDENCE/metadata.json" <&2; exit 2 ;; esac + +config="$($kube_bin -n "$namespace" get configmap "$configmap" -o jsonpath='{.data.praxis\.yaml}')" +candidate_id="$(printf '%s\n' "$config" | sed -nE 's/^[[:space:]]*-[[:space:]]*candidate_id:[[:space:]]*([^[:space:]]+).*$/\1/p' | head -n 1)" +route_model="$(printf '%s\n' "$config" | sed -nE 's/^[[:space:]]*model:[[:space:]]*([^[:space:]]+).*$/\1/p' | head -n 1)" +route_revision="$(printf '%s\n' "$config" | sed -nE 's/^[[:space:]]*overlay_revision:[[:space:]]*([^[:space:]]+).*$/\1/p' | head -n 1)" +if [[ -z "$candidate_id" || -z "$route_model" ]]; then + echo "provider route in $configmap has no candidate_id/model" >&2; exit 1 +fi + +request_id="azure-boundary-$(date -u +%Y%m%dT%H%M%SZ)-$$" +revision_header='' +if [[ -n "$route_revision" ]]; then + revision_header="x-ai-routing-revision: $route_revision" +fi + +read -r -d '' probe_command <<'PROBE' || true +set -eu +headers=$(mktemp); body=$(mktemp) +trap 'rm "$headers" "$body"' EXIT +set -- +if [ -n "${REVISION_HEADER}" ]; then set -- -H "${REVISION_HEADER}"; fi +status=$(curl --fail-with-body -sS --connect-timeout 10 --max-time 45 \ + --cacert /tls/ca.crt --cert /tls/tls.crt --key /tls/tls.key \ + --connect-to "${SERVER_NAME}:8443:${PROVIDER_SERVICE}:8443" \ + -H "Content-Type: application/json" -H "X-Model: ${ROUTE_MODEL}" \ + -H "x-ai-routing-candidate: ${CANDIDATE_ID}" \ + -H "x-ai-routing-request-id: ${REQUEST_ID}" "$@" \ + -X POST "https://${SERVER_NAME}:8443/v1/chat/completions" \ + --data "{\"model\":\"${ROUTE_MODEL}\",\"messages\":[{\"role\":\"user\",\"content\":\"Say hi\"}],\"max_tokens\":1}" \ + -D "$headers" -o "$body" -w "%{http_code}") +printf "http_status=%s\\n" "$status" +gateway=$(awk "tolower(\$1)==\"x-ai-demo-provider-gateway:\" {print \$2}" "$headers" | tr -d "\\r") +backend=$(awk "tolower(\$1)==\"x-ai-inference-provider:\" {print \$2}" "$headers" | tr -d "\\r") +printf "x-ai-demo-provider-gateway=%s\\n" "$gateway" +printf "x-ai-inference-provider=%s\\n" "$backend" +if [ "$status" != 200 ] || [ "$gateway" != "$EXPECTED_GATEWAY" ] || [ "$backend" != "azure-upstream" ]; then + echo "unexpected Azure provider-boundary result" >&2; exit 1 +fi +PROBE + +overrides="$(jq -n --arg image "$curl_image" --arg command "$probe_command" \ + '{spec:{restartPolicy:"Never",containers:[{name:"probe",image:$image,command:["sh","-ceu",$command],env:[ + {name:"CANDIDATE_ID",value:""},{name:"ROUTE_MODEL",value:""},{name:"ROUTE_REVISION",value:""}, + {name:"REQUEST_ID",value:""},{name:"SERVER_NAME",value:""},{name:"PROVIDER_SERVICE",value:""}, + {name:"EXPECTED_GATEWAY",value:""},{name:"REVISION_HEADER",value:""}], + volumeMounts:[{name:"gateway-tls",mountPath:"/tls",readOnly:true}]}], + volumes:[{name:"gateway-tls",secret:{secretName:"gateway-tls"}}]}}' | jq \ + --arg candidate "$candidate_id" --arg model "$route_model" --arg revision "$route_revision" \ + --arg request "$request_id" --arg server "$server_name" --arg service "$provider_service" \ + --arg expected "azure-${region}" --arg revision_header "$revision_header" \ + '.spec.containers[0].env[0].value=$candidate | .spec.containers[0].env[1].value=$model | + .spec.containers[0].env[2].value=$revision | .spec.containers[0].env[3].value=$request | + .spec.containers[0].env[4].value=$server | .spec.containers[0].env[5].value=$service | + .spec.containers[0].env[6].value=$expected | .spec.containers[0].env[7].value=$revision_header')" + +echo "validating Azure provider boundary: region=$region configmap=$configmap candidate=$candidate_id" +if [[ -n "$route_revision" ]]; then echo "sealed route revision will be forwarded"; else echo "no sealed route revision configured"; fi +cleanup() { "$kube_bin" -n "$namespace" delete pod "$probe_pod" --ignore-not-found >/dev/null 2>&1 || true; } +trap cleanup EXIT +"$kube_bin" -n "$namespace" delete pod "$probe_pod" --ignore-not-found >/dev/null 2>&1 || true +"$kube_bin" -n "$namespace" run "$probe_pod" --image="$curl_image" --restart=Never --rm -i --quiet --overrides="$overrides"