From 3c9826f1041aae662b7330316c170017d97eea58 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Thu, 3 Sep 2026 03:45:12 +0000 Subject: [PATCH] chore(release): prepare Grid v0.1.4 Signed-off-by: Brent Salisbury --- .github/workflows/release.yaml | 75 +- .../workflows/triggered-integration-test.yaml | 8 +- Cargo.lock | 18 +- Cargo.toml | 2 +- charts/grid-mock-providers/Chart.yaml | 4 +- charts/grid-operator/Chart.yaml | 4 +- charts/grid-site/Chart.yaml | 4 +- charts/praxis-gateway/Chart.yaml | 4 +- charts/praxis-gateway/README.md | 15 +- charts/praxis-gateway/values.yaml | 4 +- docs/README.md | 2 + docs/architecture/operations.md | 2 +- docs/installation/existing-clusters.md | 2 +- docs/release.md | 202 ++ .../values/east-a-consumer-gateway.yaml | 4 +- .../values/east-a-grid-mock-providers.yaml | 2 +- .../combined-site/values/east-a-operator.yaml | 2 +- .../values/east-a-provider-gateway.yaml | 4 +- .../values/east-b-consumer-gateway.yaml | 4 +- .../values/east-b-grid-mock-providers.yaml | 2 +- .../combined-site/values/east-b-operator.yaml | 2 +- .../values/east-b-provider-gateway.yaml | 4 +- .../values/west-a-consumer-gateway.yaml | 4 +- .../values/west-a-grid-mock-providers.yaml | 2 +- .../combined-site/values/west-a-operator.yaml | 2 +- .../values/west-a-provider-gateway.yaml | 4 +- .../values/west-b-consumer-gateway.yaml | 4 +- .../values/west-b-grid-mock-providers.yaml | 2 +- .../combined-site/values/west-b-operator.yaml | 2 +- .../values/west-b-provider-gateway.yaml | 4 +- .../values/east-consumer-gateway.yaml | 4 +- .../values/east-consumer-operator.yaml | 2 +- .../values/east-provider-gateway.yaml | 4 +- .../values/east-provider-operator.yaml | 2 +- .../values/west-consumer-gateway.yaml | 4 +- .../values/west-consumer-operator.yaml | 2 +- .../values/west-provider-gateway.yaml | 4 +- .../values/west-provider-operator.yaml | 2 +- .../existing-clusters/inventory.example.yaml | 6 +- forge/src/command/down.rs | 75 +- forge/src/command/up.rs | 2 + forge/src/networking.rs | 29 +- forge/src/state.rs | 5 + scripts/verify-helm-chart.sh | 6 +- .../topologies/grid-combined-site/README.md | 2 +- .../topologies/grid-combined-site/forge.yaml | 30 +- tests/e2e/topologies/grid-glb-demo/README.md | 2 +- tests/e2e/topologies/grid-glb-demo/forge.yaml | 50 +- .../grid-llmd-pool-metrics/README.md | 2 +- .../grid-llmd-pool-metrics/forge.yaml | 22 +- .../grid-provider-traffic/forge.yaml | 30 +- .../README.md | 24 +- .../grid-token-rate-limit/README.md | 8 +- .../grid-token-rate-limit/forge.yaml | 36 +- .../grid-workload-inference/README.md | 2 +- xtask/src/env/combined_site_demo.rs | 1645 ++++++++++++++--- xtask/src/env/forge_config.rs | 65 +- xtask/src/env/glb.rs | 105 +- xtask/src/env/glb_demo.rs | 4 +- xtask/src/env/image_overrides.rs | 19 +- xtask/src/env/kubectl.rs | 19 +- xtask/src/env/llmd_pool_metrics_demo.rs | 167 +- ...gle_cluster_multi_gateway_qualification.rs | 291 ++- 63 files changed, 2494 insertions(+), 570 deletions(-) create mode 100644 docs/release.md diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d8f9247..da32cd5a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,8 +24,10 @@ permissions: {} env: CARGO_TERM_COLOR: always NIGHTLY_RUSTFMT: nightly-2026-03-28 - AI_ROLLUP_IMAGE_NAME: ghcr.io/praxis-proxy/grid-ai-rollup - AI_ROLLUP_SOURCE_SHA: c68149502d991d51daaff8d6bf96c18704aeb38d + OFFICIAL_AI_IMAGE_NAME: ghcr.io/praxis-proxy/ai + OFFICIAL_AI_IMAGE_TAG: 0.3.0 + OFFICIAL_AI_IMAGE_DIGEST: sha256:ccd46f8772eebcbde2f41ad35c3234d23463b8314a5865083e32baf31eddd1a8 + OFFICIAL_AI_SOURCE_SHA: b44360afb4100c5543b1180ac8df16e482978fc8 OPERATOR_IMAGE_NAME: ghcr.io/praxis-proxy/grid-operator MOCK_PROVIDERS_IMAGE_NAME: ghcr.io/praxis-proxy/grid-mock-providers OVERLAY_SYNC_IMAGE_NAME: ghcr.io/praxis-proxy/grid-overlay-sync @@ -138,7 +140,7 @@ jobs: operator_digest: ${{ steps.operator.outputs.digest }} mock_providers_digest: ${{ steps.mock_providers.outputs.digest }} overlay_sync_digest: ${{ steps.overlay_sync.outputs.digest }} - ai_rollup_digest: ${{ steps.ai_rollup.outputs.digest }} + official_ai_digest: ${{ steps.official_ai.outputs.digest }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -155,32 +157,33 @@ jobs: username: ${{ github.actor }} password: ${{ github.token }} - - name: Verify official Grid AI rollup - id: ai_rollup - env: - TAG: ${{ needs.validate.outputs.tag }} + - name: Verify official Praxis AI image + id: official_ai run: | set -euo pipefail - RELEASE_REF="${AI_ROLLUP_IMAGE_NAME}:${TAG}" - SOURCE_REF="${AI_ROLLUP_IMAGE_NAME}:sha-${AI_ROLLUP_SOURCE_SHA}" - RELEASE_DIGEST=$(docker buildx imagetools inspect "${RELEASE_REF}" \ - --format '{{.Manifest.Digest}}') - SOURCE_DIGEST=$(docker buildx imagetools inspect "${SOURCE_REF}" \ - --format '{{.Manifest.Digest}}') - if [[ ! "${RELEASE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ || \ - ! "${SOURCE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "::error::Could not resolve official AI rollup digests" + IMAGE_REF="${OFFICIAL_AI_IMAGE_NAME}:${OFFICIAL_AI_IMAGE_TAG}" + DIGEST=$(docker buildx imagetools inspect "${IMAGE_REF}" | awk '$1 == "Digest:" { print $2; exit }') + if [[ ! "${DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::Could not resolve official AI image digest" + exit 1 + fi + if [[ "${DIGEST}" != "${OFFICIAL_AI_IMAGE_DIGEST}" ]]; then + echo "::error::Official AI tag no longer resolves to the pinned digest" exit 1 fi - if [[ "${RELEASE_DIGEST}" != "${SOURCE_DIGEST}" ]]; then - echo "::error::Official AI rollup tags resolve to different digests" - echo "::error::${RELEASE_REF}: ${RELEASE_DIGEST}" - echo "::error::${SOURCE_REF}: ${SOURCE_DIGEST}" + IMMUTABLE_REF="${OFFICIAL_AI_IMAGE_NAME}@${DIGEST}" + docker pull "${IMMUTABLE_REF}" + SOURCE=$(docker image inspect "${IMMUTABLE_REF}" --format '{{index .Config.Labels "org.opencontainers.image.source"}}') + REVISION=$(docker image inspect "${IMMUTABLE_REF}" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}') + VERSION=$(docker image inspect "${IMMUTABLE_REF}" --format '{{index .Config.Labels "org.opencontainers.image.version"}}') + if [[ "${SOURCE}" != "https://github.com/praxis-proxy/ai" || \ + "${REVISION}" != "${OFFICIAL_AI_SOURCE_SHA}" || \ + "${VERSION}" != "${OFFICIAL_AI_IMAGE_TAG}" ]]; then + echo "::error::Official AI image provenance does not match the pinned release" exit 1 fi - echo "digest=${RELEASE_DIGEST}" >> "${GITHUB_OUTPUT}" - echo "Verified ${RELEASE_REF}@${RELEASE_DIGEST}" - echo "Verified ${SOURCE_REF}@${SOURCE_DIGEST}" + echo "digest=${DIGEST}" >> "${GITHUB_OUTPUT}" + echo "Verified ${IMAGE_REF}@${DIGEST} (${REVISION})" - name: Build and push operator id: operator @@ -275,8 +278,12 @@ jobs: echo "::error::${chart} version ($CHART_VERSION) != tag version ($VERSION)" exit 1 fi - if [ "$CHART_APP_VERSION" != "$TAG" ]; then - echo "::error::${chart} appVersion ($CHART_APP_VERSION) != release tag ($TAG)" + EXPECTED_APP_VERSION="$TAG" + if [ "$chart" = praxis-gateway ]; then + EXPECTED_APP_VERSION="$OFFICIAL_AI_IMAGE_TAG" + fi + if [ "$CHART_APP_VERSION" != "$EXPECTED_APP_VERSION" ]; then + echo "::error::${chart} appVersion ($CHART_APP_VERSION) != expected application version ($EXPECTED_APP_VERSION)" exit 1 fi done @@ -345,7 +352,7 @@ jobs: OPERATOR_DIGEST: ${{ needs.container.outputs.operator_digest }} MOCK_PROVIDERS_DIGEST: ${{ needs.container.outputs.mock_providers_digest }} OVERLAY_SYNC_DIGEST: ${{ needs.container.outputs.overlay_sync_digest }} - AI_ROLLUP_DIGEST: ${{ needs.container.outputs.ai_rollup_digest }} + OFFICIAL_AI_DIGEST: ${{ needs.container.outputs.official_ai_digest }} CHART_DIGEST: ${{ needs.chart.outputs.chart_digest }} SOURCE_SHA: ${{ needs.validate.outputs.source_sha }} VERSION: ${{ needs.validate.outputs.version }} @@ -354,7 +361,7 @@ jobs: ## Container Images \`\`\`text - ${AI_ROLLUP_IMAGE_NAME}:${VERSION}@${AI_ROLLUP_DIGEST} + ${OFFICIAL_AI_IMAGE_NAME}:${OFFICIAL_AI_IMAGE_TAG}@${OFFICIAL_AI_DIGEST} ${OPERATOR_IMAGE_NAME}@${OPERATOR_DIGEST} ${MOCK_PROVIDERS_IMAGE_NAME}@${MOCK_PROVIDERS_DIGEST} ${OVERLAY_SYNC_IMAGE_NAME}@${OVERLAY_SYNC_DIGEST} @@ -380,10 +387,18 @@ jobs: \`\`\` The Grid images were built from \`${SOURCE_SHA}\`. The Praxis AI - rollup was built from - \`praxis-proxy/ai@${AI_ROLLUP_SOURCE_SHA}\` and - \`praxis-proxy/praxis@0d3347c634ab4e09d882c02bf01767a1667c82db\`. + image was verified from the organization-owned image above at + \`praxis-proxy/ai@${OFFICIAL_AI_SOURCE_SHA}\`. Every reference above is immutable. + + ## Distributed token quota image requirement + + Grid publishes the routing topology and qualification for distributed + token quotas, but the standard Praxis AI ${OFFICIAL_AI_IMAGE_TAG} + container does not include the optional \`token-rate-limit-filter\` + and \`praxis-filter/basic-auth-filter\` features. Deployments using the + quota example must supply a Praxis AI image built with both features. + Grid does not publish a replacement AI rollup. EOF - name: Create GitHub release diff --git a/.github/workflows/triggered-integration-test.yaml b/.github/workflows/triggered-integration-test.yaml index a6ef3549..b185277e 100644 --- a/.github/workflows/triggered-integration-test.yaml +++ b/.github/workflows/triggered-integration-test.yaml @@ -38,10 +38,10 @@ env: KUBECTL_VERSION: v1.32.3 # The gateway image MUST be pinned by digest for reproducible evidence. # To obtain the digest for a new tag: - # docker pull ghcr.io/praxis-proxy/grid-ai-rollup: - # docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/praxis-proxy/grid-ai-rollup: - GATEWAY_IMAGE_REF: ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3 - GATEWAY_IMAGE_DIGEST: sha256:e5bdea9071f76533ab61714eeef6067c7449d0e9d811c221aa940cf3cdf9719f + # docker pull ghcr.io/praxis-proxy/ai: + # docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/praxis-proxy/ai: + GATEWAY_IMAGE_REF: ghcr.io/praxis-proxy/ai:0.3.0 + GATEWAY_IMAGE_DIGEST: sha256:ccd46f8772eebcbde2f41ad35c3234d23463b8314a5865083e32baf31eddd1a8 # Pinned identity/date/message-suffix for the local PR-merge reconstruction # in resolve-source's "Resolve source" step and glb-e2e's "Reconstruct PR # merge" step (see grid#66). Both jobs build a merge commit from the same diff --git a/Cargo.lock b/Cargo.lock index 11abbbfe..0acd7f19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -359,7 +359,7 @@ dependencies = [ [[package]] name = "certs" -version = "0.1.3" +version = "0.1.4" dependencies = [ "pem 4.0.0", "rcgen", @@ -498,7 +498,7 @@ dependencies = [ [[package]] name = "crdt" -version = "0.1.3" +version = "0.1.4" dependencies = [ "bincode", "serde", @@ -777,7 +777,7 @@ checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "forge" -version = "0.1.3" +version = "0.1.4" dependencies = [ "clap", "fs2", @@ -966,7 +966,7 @@ dependencies = [ [[package]] name = "grid-overlay-sync" -version = "0.1.3" +version = "0.1.4" dependencies = [ "axum", "clap", @@ -1648,7 +1648,7 @@ dependencies = [ [[package]] name = "mock-providers" -version = "0.1.3" +version = "0.1.4" dependencies = [ "axum", "clap", @@ -1751,7 +1751,7 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "operator" -version = "0.1.3" +version = "0.1.4" dependencies = [ "axum", "bytes", @@ -2429,7 +2429,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scoring" -version = "0.1.3" +version = "0.1.4" dependencies = [ "serde", "serde_json", @@ -2701,7 +2701,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "swim" -version = "0.1.3" +version = "0.1.4" dependencies = [ "aes-gcm", "bincode", @@ -3520,7 +3520,7 @@ dependencies = [ [[package]] name = "xtask" -version = "0.1.3" +version = "0.1.4" dependencies = [ "certs", "clap", diff --git a/Cargo.toml b/Cargo.toml index aa199488..9177071b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ ] [workspace.package] -version = "0.1.3" +version = "0.1.4" edition = "2024" rust-version = "1.96" license = "MIT" diff --git a/charts/grid-mock-providers/Chart.yaml b/charts/grid-mock-providers/Chart.yaml index 89b6971a..e72c0373 100644 --- a/charts/grid-mock-providers/Chart.yaml +++ b/charts/grid-mock-providers/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: grid-mock-providers description: Mock inference providers for Grid development and testing type: application -version: 0.1.3 -appVersion: "v0.1.3" +version: 0.1.4 +appVersion: "v0.1.4" kubeVersion: ">=1.26.0-0" home: https://github.com/praxis-proxy/grid sources: diff --git a/charts/grid-operator/Chart.yaml b/charts/grid-operator/Chart.yaml index 8d4a4da7..b90c88d8 100644 --- a/charts/grid-operator/Chart.yaml +++ b/charts/grid-operator/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: grid-operator description: Grid operator for multi-site AI inference routing with Praxis type: application -version: 0.1.3 -appVersion: "v0.1.3" +version: 0.1.4 +appVersion: "v0.1.4" kubeVersion: ">=1.26.0-0" home: https://github.com/praxis-proxy/grid sources: diff --git a/charts/grid-site/Chart.yaml b/charts/grid-site/Chart.yaml index 536f3828..3941a6dd 100644 --- a/charts/grid-site/Chart.yaml +++ b/charts/grid-site/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: grid-site description: Grid site topology resources (GridNetwork, GridSite, InferenceProvider) type: application -version: 0.1.3 -appVersion: "v0.1.3" +version: 0.1.4 +appVersion: "v0.1.4" kubeVersion: ">=1.26.0-0" home: https://github.com/praxis-proxy/grid sources: diff --git a/charts/praxis-gateway/Chart.yaml b/charts/praxis-gateway/Chart.yaml index 761c41c6..e664f5da 100644 --- a/charts/praxis-gateway/Chart.yaml +++ b/charts/praxis-gateway/Chart.yaml @@ -5,8 +5,8 @@ description: >- Deploys the Praxis process directly — not an operator. Long-term ownership moves to the future Praxis/Gateway Operator repository. type: application -version: 0.1.3 -appVersion: "v0.1.3" +version: 0.1.4 +appVersion: "0.3.0" kubeVersion: ">=1.26.0-0" home: https://github.com/praxis-proxy/grid sources: diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index 3b818a3b..f6e5d885 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -18,7 +18,7 @@ release ownership exist. Do not treat this as a permanent Grid responsibility. - Kubernetes >= 1.26 - Helm >= 3.12 - A Praxis configuration ConfigMap already created in the target namespace -- A compatible Grid AI gateway image (default: Grid v0.1.3 rollup) +- A compatible Praxis AI gateway image (default: Praxis AI 0.3.0) ## Install @@ -34,10 +34,17 @@ helm install edge-gateway charts/praxis-gateway \ --set config.existingConfigMap=edge-gateway-config ``` -The default image is the Grid v0.1.3 AI gateway rollup. Override +The default image is the official Praxis AI 0.3.0 gateway. Override `image.repository`, `image.tag`, or `image.digest` to install another compatible gateway image. Prefer a digest when reproducing a validated deployment. +The standard Praxis AI 0.3.0 image supports Grid provider selection and load +balancing. It does not include the optional `token-rate-limit-filter` and +`praxis-filter/basic-auth-filter` features required by the distributed token +quota qualification. That qualification requires an explicitly supplied Praxis +AI image built with both features; Grid does not publish a replacement AI +rollup. + The chart uses [Semantic Versioning](https://semver.org/). Its `version` identifies the chart package, while `appVersion` identifies the default Praxis AI image; these values may advance independently. @@ -47,8 +54,8 @@ AI image; these values may advance independently. | Key | Type | Default | Description | |-----|------|---------|-------------| | `replicaCount` | int | `1` | Gateway replicas. | -| `image.repository` | string | `ghcr.io/praxis-proxy/grid-ai-rollup` | Image repository. | -| `image.tag` | string | `""` | Image tag. Defaults to chart `appVersion` (`v0.1.3`). | +| `image.repository` | string | `ghcr.io/praxis-proxy/ai` | Image repository. | +| `image.tag` | string | `0.3.0` | Image tag. | | `image.digest` | string | `""` | Immutable digest (sha256:…). When set, tag is ignored. | | `image.pullPolicy` | string | `IfNotPresent` | Image pull policy. | | `imagePullSecrets` | list | `[]` | Pull secrets for private registries. | diff --git a/charts/praxis-gateway/values.yaml b/charts/praxis-gateway/values.yaml index aca6f9ef..66e57ac9 100644 --- a/charts/praxis-gateway/values.yaml +++ b/charts/praxis-gateway/values.yaml @@ -4,9 +4,9 @@ replicaCount: 1 # -- Gateway container image settings. image: # -- Image repository. - repository: ghcr.io/praxis-proxy/grid-ai-rollup + repository: ghcr.io/praxis-proxy/ai # -- Image tag. Used when digest is empty. - tag: "v0.1.3" + tag: "0.3.0" # -- Immutable image digest (sha256:<64 hex>). When set, tag is ignored. digest: "" # -- Image pull policy. diff --git a/docs/README.md b/docs/README.md index 95fa753a..edb0574a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -67,6 +67,8 @@ image preparation, execution, evidence, and cleanup. ## Development +- [Release Process](release.md) - versioning, validation, artifact publication, + and release workflow. - [Development](development.md) — build, test, format, lint, and coverage. - [Conventions](conventions.md) — coding style, testing requirements, documentation rules, and commit attribution. diff --git a/docs/architecture/operations.md b/docs/architecture/operations.md index 7c045369..d96c308a 100644 --- a/docs/architecture/operations.md +++ b/docs/architecture/operations.md @@ -1316,7 +1316,7 @@ The named demos (`grid-glb-demo`, `grid-combined-site`, `grid-llmd-pool-metrics`) do **not** need them — they override `GRID_XTASK_GATEWAY_IMAGE`/`GRID_XTASK_MOCK_EPP_IMAGE` (see `xtask/src/env/image_overrides.rs`) with published -`ghcr.io/praxis-proxy/grid-ai-rollup` images and never build +`ghcr.io/praxis-proxy/ai` images and never build from an AI repository checkout. As of this writing, neither `Containerfile.composed` nor a mock diff --git a/docs/installation/existing-clusters.md b/docs/installation/existing-clusters.md index 823629aa..46a921d7 100644 --- a/docs/installation/existing-clusters.md +++ b/docs/installation/existing-clusters.md @@ -396,7 +396,7 @@ The `grid-mock-providers` image requires startup arguments: ```yaml containers: - name: mock-inference - image: ghcr.io/praxis-proxy/grid-mock-providers:v0.1.3 + image: ghcr.io/praxis-proxy/grid-mock-providers:v0.1.4 args: ["--provider", "openai", "--port", "8080"] env: - name: MOCK_EXPECTED_BEARER_TOKEN diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 00000000..94ed14dc --- /dev/null +++ b/docs/release.md @@ -0,0 +1,202 @@ +# Release Process + +## Versioning + +Grid uses [Semantic Versioning][semver]. The workspace version is defined in +`workspace.package.version` in the root `Cargo.toml`. Workspace crates inherit +that version. + +Each Helm chart has its own `version`, which must match the Grid release. The +`appVersion` for Grid-owned workloads also matches the Grid tag. The +`praxis-gateway` chart is different: its `appVersion` identifies the default +Praxis AI image and may advance independently of Grid. + +[semver]: https://semver.org/ + +## Release Artifacts + +A Grid release publishes: + +- `ghcr.io/praxis-proxy/grid-operator`; +- `ghcr.io/praxis-proxy/grid-mock-providers`; +- `ghcr.io/praxis-proxy/grid-overlay-sync`; +- the `grid-operator`, `grid-site`, `praxis-gateway`, and + `grid-mock-providers` Helm charts; and +- a GitHub Release containing generated notes and immutable artifact digests. + +Grid does not publish a Praxis AI image or an AI rollup. The release workflow +verifies the pinned official Praxis AI image used by the `praxis-gateway` chart, +including its digest and OCI provenance. + +Optional Praxis AI filters are an explicit deployment dependency. Examples or +qualifications that require optional filters must document the required Cargo +features and require the caller to provide a compatible image. They must not +silently substitute a Grid-owned AI build. + +## Pre-release Checklist + +Before opening a release preparation pull request: + +- [ ] Update the workspace version in `Cargo.toml` and regenerate `Cargo.lock`. +- [ ] Update every Helm chart `version`. +- [ ] Update Grid workload chart `appVersion` values to the Grid tag. +- [ ] Verify the `praxis-gateway` `appVersion` and default image match the + intended official Praxis AI release. +- [ ] Update the release workflow's pinned AI tag, digest, and source revision + when the default Praxis AI image changes. +- [ ] Run `cargo +nightly-2026-03-28 fmt --all -- --check`. +- [ ] Run `cargo clippy --workspace --all-targets -- -D warnings`. +- [ ] Run `make test`, `make doc`, and `make lint`. +- [ ] Run `git diff --check` and validate the release workflow with + `actionlint`. +- [ ] Lint and render all Helm charts with their required values. +- [ ] Validate affected Forge topologies. +- [ ] Run the relevant integration qualifications when routing, overlay, or + gateway compatibility changes. +- [ ] Confirm generated evidence and local build artifacts are not committed. +- [ ] Review pull request labels so generated release notes are useful. + +The first-class integration qualifications are documented in +[the documentation index](README.md#integration-qualifications). They build +isolated Kind environments, exercise runtime behavior, record evidence, and +clean up their resources. + +## Integration qualification checklist + +Run the individual qualification commands below against fresh source builds +and fresh, uniquely tagged images. There is deliberately no aggregate release +qualification command. Run the commands sequentially because several use Kind +clusters and Docker networks whose names can otherwise collide. + +| Area | Command | Topology/config path | Classification | Required when | +|---|---|---|---|---| +| Provider traffic selection and round-robin | `cargo xtask env run-grid-provider-traffic-qualification --forge-config tests/e2e/topologies/grid-provider-traffic/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR"` | `tests/e2e/topologies/grid-provider-traffic/forge.yaml` | Release qualification | Routing, provider candidates, overlay selection policy, provider attribution, or gateway compatibility changes | +| Distributed token quota | `cargo xtask env run-grid-token-rate-limit-qualification --forge-config tests/e2e/topologies/grid-token-rate-limit/forge.yaml --image-tag "$IMAGE_TAG" --evidence-dir "$EVIDENCE_DIR"` | `tests/e2e/topologies/grid-token-rate-limit/forge.yaml` | Experimental integration qualification | Quota, identity, Basic Auth, Valkey, gateway image, or shared-consumer changes | +| Single-cluster multi-gateway | `cargo xtask env run-grid-single-cluster-multi-gateway-qualification --forge-config tests/e2e/topologies/grid-single-cluster-multi-gateway/forge.yaml --evidence-dir "$EVIDENCE_DIR"` | `tests/e2e/topologies/grid-single-cluster-multi-gateway/forge.yaml` | Integration qualification | Same-site gateway lifecycle, overlay delivery, provider selection, concurrency, or NetworkPolicy behavior changes | +| Combined-site lifecycle | `cargo xtask env run-grid-combined-site-demo --forge-config tests/e2e/topologies/grid-combined-site/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR"` | `tests/e2e/topologies/grid-combined-site/forge.yaml` | Experimental lifecycle qualification | Combined-site routing, provider add/remove/re-add, session fallback, rollout, trust, or lifecycle changes | +| GLB | `cargo xtask env run-grid-glb-demo --forge-config tests/e2e/topologies/grid-glb-demo/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR"` | `tests/e2e/topologies/grid-glb-demo/forge.yaml` | Experimental integration test | GLB, ingress, edge/provider boundary, mTLS, or network-boundary changes | +| Workload inference / no ingress | `cargo xtask env run-grid-glb-demo --forge-config tests/e2e/topologies/grid-glb-demo/forge.yaml --no-ingress --full --teardown --evidence-dir "$EVIDENCE_DIR"` | Reuses `tests/e2e/topologies/grid-glb-demo/forge.yaml`; documented by `tests/e2e/topologies/grid-workload-inference/README.md` | Experimental integration test | Workload inference, no-ingress routing, or cluster-local workload entry changes | +| llm-d pool metrics pressure and recovery | `cargo xtask env run-grid-llmd-pool-metrics-demo --forge-config tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR"` | `tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml` | Experimental integration qualification | EPP metrics, queue/KV-cache scoring, pressure transitions, recovery, or metrics mTLS changes | + +Use the following one-line command stack after setting `IMAGE_TAG` and +`EVIDENCE_DIR` for each run. The quota qualification additionally requires a +Praxis AI image built with the optional features shown below; the other +topologies require the filters documented in their READMEs and may use the +official compatible AI image where applicable. + +```console +cargo xtask env run-grid-provider-traffic-qualification --forge-config tests/e2e/topologies/grid-provider-traffic/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR" +cargo xtask env run-grid-token-rate-limit-qualification --forge-config tests/e2e/topologies/grid-token-rate-limit/forge.yaml --image-tag "$IMAGE_TAG" --evidence-dir "$EVIDENCE_DIR" +cargo xtask env run-grid-single-cluster-multi-gateway-qualification --forge-config tests/e2e/topologies/grid-single-cluster-multi-gateway/forge.yaml --evidence-dir "$EVIDENCE_DIR" +cargo xtask env run-grid-combined-site-demo --forge-config tests/e2e/topologies/grid-combined-site/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR" +cargo xtask env run-grid-glb-demo --forge-config tests/e2e/topologies/grid-glb-demo/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR" +cargo xtask env run-grid-glb-demo --forge-config tests/e2e/topologies/grid-glb-demo/forge.yaml --no-ingress --full --teardown --evidence-dir "$EVIDENCE_DIR" +cargo xtask env run-grid-llmd-pool-metrics-demo --forge-config tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml --full --teardown --evidence-dir "$EVIDENCE_DIR" +``` + +### Images and optional features + +Each run must use exact image references that are materialized into the +resolved Forge configuration and loaded into Kind when `imagePullPolicy: +Never` is used. Record the image IDs or immutable digests in the evidence. +Typical local overrides are: + +```console +export GRID_XTASK_GATEWAY_IMAGE=praxis-ai:$IMAGE_TAG +export GRID_XTASK_OPERATOR_IMAGE=grid-operator:$IMAGE_TAG +export GRID_XTASK_OVERLAY_SYNC_IMAGE=grid-overlay-sync:$IMAGE_TAG +export GRID_XTASK_MOCK_PROVIDER_IMAGE=grid-mock-providers:$IMAGE_TAG +export GRID_XTASK_VCR_IMAGE=ghcr.io/neuralmagic/vllm-vcr:vllm0.23 +export GRID_XTASK_IMAGE_PULL_POLICY=Never +``` + +The distributed token-quota qualification requires Praxis AI built with: + +```text +token-rate-limit-filter,praxis-filter/basic-auth-filter +``` + +The image label and runtime registrations must match the compiled feature +set. The standard official AI image may be suitable for provider selection and +other gateway paths, but a quota run must be **BLOCKED**, never marked PASS, if +the required filters are absent or its feature label is unavailable or +inaccurate. See the [provider traffic README](../tests/e2e/topologies/grid-provider-traffic/README.md), [quota README](../tests/e2e/topologies/grid-token-rate-limit/README.md), and the other topology READMEs for exact image preparation. + +### Evidence, cleanup, and release gates + +Create a new UTC-stamped evidence directory for every run, normally beneath +`evidence/`. Evidence from an older commit, source tree, image tag, or image +digest is not valid for the current release. Generated evidence must not be +committed. + +Every command must be run with its teardown option and must clean only the +clusters, processes, port-forwards, pods, and Docker networks it owns. A +missing prerequisite, unavailable image, missing Forge binary, or failed +readiness/convergence check is **BLOCKED** or **FAIL**, never PASS. Preserve +non-2xx responses and first failures in the evidence; do not hide them with +retries or reinterpret them as successful routing. + +`cargo xtask env validate-all` is a separate legacy/local validation suite. It +does not run the release qualification matrix above and must not be presented +as a substitute for these topology-specific commands. + +Experimental classification means that a topology is outside the default +supported release surface; it does not permit weakening assertions or ignoring +failed results. When an experimental path is changed or claimed in release +notes, run it and report its evidence independently. + +### Qualification details + +| Area | Behavior proved | Images and overrides | Evidence, runtime, and cleanup | +|---|---|---|---| +| Provider traffic selection and round-robin | Grid publishes stable provider candidates and groups; Praxis AI accepts the overlay and returns trusted attribution while eligible providers receive round-robin traffic. | Official compatible Praxis AI gateway image, plus the locally built Grid operator, overlay-sync, mock-provider, and VCR images. Use the `GRID_XTASK_*_IMAGE` overrides above when validating unreleased Grid code. | Write to the run’s UTC-stamped `EVIDENCE_DIR`; the full run is typically several minutes. `--teardown` removes run-owned resources. | +| Distributed token quota | Basic Auth precedes admission; Alice’s sliding-window budget is shared across consumers; routing spans sites; concurrency, expiry, restart persistence, Valkey fail-closed behavior, recovery, and NetworkPolicy are exercised. | Praxis AI must be built with `token-rate-limit-filter,praxis-filter/basic-auth-filter`; use `--image-tag` and the exact feature-enabled local AI image, with local Grid operator/overlay-sync/VCR images as required by the README. | Record structured quota and routing evidence under `EVIDENCE_DIR`; runtime is variable and materially longer than a smoke test. The command cleans up on completion or failure; do not use `--keep` for release evidence. | +| Single-cluster multi-gateway | Two consumer gateways independently accept and serve the same three-provider overlay inside one Kubernetes cluster and one GridSite; attributed round-robin selection, provider withdrawal/restoration, consumer failure/recovery, concurrent traffic, and NetworkPolicy boundaries are exercised. | The fixed `grid-operator:single-cluster-qualification`, `grid-overlay-sync:single-cluster-qualification`, and `praxis-ai:single-cluster-qualification` references are local-development defaults. Release validation should set `GRID_XTASK_OPERATOR_IMAGE`, `GRID_XTASK_OVERLAY_SYNC_IMAGE`, and `GRID_XTASK_GATEWAY_IMAGE` to unique references; set `GRID_XTASK_VCR_IMAGE` as needed. All resolved references are loaded into Kind under `imagePullPolicy: Never`. | The command records timestamped structured evidence and performs automatic cleanup unless `--keep` is explicitly supplied. It does not claim multi-site SWIM, WAN behavior, or a globally shared round-robin cursor. | +| Combined-site lifecycle | Combined-site bootstrap, trusted round-robin, provider drain and restoration, secondary add/remove/re-add, session fallback, revision convergence, and lifecycle cleanup. | Official compatible Praxis AI image plus local Grid operator, overlay-sync, mock-provider, and VCR images through the overrides above. | Save lifecycle timelines and request attribution under `EVIDENCE_DIR`; a full run is typically on the order of tens of minutes. `--teardown` performs bounded cleanup of owned clusters, pods, processes, and networks. | +| GLB | Global load-balancing and network-boundary behavior, including provider attribution and the configured ingress path. | Official compatible Praxis AI image plus the topology’s required local Grid/operator/mock-provider/VCR images; use the listed overrides and `Never` pull policy for local images. | Save results under `EVIDENCE_DIR`; use `--quick` for bounded diagnostics or `--full` for qualification. `--teardown` removes only run-owned resources. | +| Workload inference / no ingress | Workload inference through the no-ingress path and its provider/network behavior. This reuses the GLB command with `--no-ingress`; it is not a separate invented CLI command. | Same image set and overrides as GLB, with any optional AI features required by that topology’s README. | Save no-ingress evidence under `EVIDENCE_DIR`; use `--quick` for diagnostics or `--full` for qualification. `--teardown` removes only run-owned resources. | +| llm-d pool metrics pressure and recovery | Pool-metrics observation, pressure-aware placement, availability during transitions, and recovery after metrics return below threshold. | Official compatible Praxis AI image plus local Grid operator/overlay-sync and the llm-d/EPP images required by its README; `--metrics-mtls` and `--kv-cache` are optional command flags when the topology enables them. | Save metric, overlay, reload, request, and recovery timelines under `EVIDENCE_DIR`; runtime is variable and may be long. `--teardown` performs bounded owned-resource cleanup. | + +## Tagging A Release + +Tags use `v..`, for example `v0.1.4`. Create the tag only +from the reviewed commit after its pull request and CI checks pass. Sign the tag +using the project's normal Git signing configuration: + +```console +git tag -s v0.1.4 -m "Grid v0.1.4" +git push origin v0.1.4 +``` + +Do not move or recreate a published release tag. Prepare a new patch release +for corrections. + +## Automated Publication + +Pushing a valid release tag triggers the **Release** workflow. The workflow: + +1. verifies that the tag matches the workspace version; +2. reruns lint, tests, and documentation validation from the tagged source; +3. verifies the pinned official Praxis AI image and provenance; +4. builds and publishes immutable Grid container images with SBOM and + provenance attestations; +5. validates, packages, and publishes all Helm charts; and +6. creates the GitHub Release with generated notes and immutable digests. + +The workflow can also be dispatched for an existing immutable release tag. A +manual dispatch does not replace the requirement for a reviewed, signed tag. + +## Release Notes + +Grid uses [GitHub Releases][releases] for release notes. There is no committed +version-specific changelog. Add the user-facing summary, compatibility notes, +upgrade considerations, qualification results, and demonstration links to the +GitHub Release page. + +[releases]: https://github.com/praxis-proxy/grid/releases + +## Release Branches + +Release branches are optional. Create one from a release tag only when a +supported line needs a backport. Use `release/v..x`, cherry-pick +the focused fix, and publish a new patch tag through the normal workflow. diff --git a/examples/helm/existing-clusters/combined-site/values/east-a-consumer-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/east-a-consumer-gateway.yaml index 93066373..9878676b 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-a-consumer-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-a-consumer-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: consumer-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/combined-site/values/east-a-grid-mock-providers.yaml b/examples/helm/existing-clusters/combined-site/values/east-a-grid-mock-providers.yaml index 475e55f1..6e4fb0b8 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-a-grid-mock-providers.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-a-grid-mock-providers.yaml @@ -9,7 +9,7 @@ image: repository: ghcr.io/praxis-proxy/grid-mock-providers - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent providers: diff --git a/examples/helm/existing-clusters/combined-site/values/east-a-operator.yaml b/examples/helm/existing-clusters/combined-site/values/east-a-operator.yaml index 0d8af5d6..641c8550 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-a-operator.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-a-operator.yaml @@ -10,7 +10,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/combined-site/values/east-a-provider-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/east-a-provider-gateway.yaml index 76c010db..c6d89450 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-a-provider-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-a-provider-gateway.yaml @@ -11,8 +11,8 @@ fullnameOverride: provider-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/combined-site/values/east-b-consumer-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/east-b-consumer-gateway.yaml index 2819575d..cc76b751 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-b-consumer-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-b-consumer-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: consumer-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/combined-site/values/east-b-grid-mock-providers.yaml b/examples/helm/existing-clusters/combined-site/values/east-b-grid-mock-providers.yaml index aee83a90..2eb521bb 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-b-grid-mock-providers.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-b-grid-mock-providers.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-mock-providers - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent providers: diff --git a/examples/helm/existing-clusters/combined-site/values/east-b-operator.yaml b/examples/helm/existing-clusters/combined-site/values/east-b-operator.yaml index 1ad5484b..7fbb80b6 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-b-operator.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-b-operator.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/combined-site/values/east-b-provider-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/east-b-provider-gateway.yaml index efa88c89..f6521cb9 100644 --- a/examples/helm/existing-clusters/combined-site/values/east-b-provider-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/east-b-provider-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: provider-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/combined-site/values/west-a-consumer-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/west-a-consumer-gateway.yaml index 3cb4a685..6b1ec7dc 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-a-consumer-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-a-consumer-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: consumer-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/combined-site/values/west-a-grid-mock-providers.yaml b/examples/helm/existing-clusters/combined-site/values/west-a-grid-mock-providers.yaml index 38e8050d..2e049573 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-a-grid-mock-providers.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-a-grid-mock-providers.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-mock-providers - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent providers: diff --git a/examples/helm/existing-clusters/combined-site/values/west-a-operator.yaml b/examples/helm/existing-clusters/combined-site/values/west-a-operator.yaml index 36bd407c..ba602b37 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-a-operator.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-a-operator.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/combined-site/values/west-a-provider-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/west-a-provider-gateway.yaml index e548516b..5f1d89bd 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-a-provider-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-a-provider-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: provider-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/combined-site/values/west-b-consumer-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/west-b-consumer-gateway.yaml index 3701b1ba..3628b052 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-b-consumer-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-b-consumer-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: consumer-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/combined-site/values/west-b-grid-mock-providers.yaml b/examples/helm/existing-clusters/combined-site/values/west-b-grid-mock-providers.yaml index d946eb59..f8dd41c8 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-b-grid-mock-providers.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-b-grid-mock-providers.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-mock-providers - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent providers: diff --git a/examples/helm/existing-clusters/combined-site/values/west-b-operator.yaml b/examples/helm/existing-clusters/combined-site/values/west-b-operator.yaml index 848d6edf..b868e55b 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-b-operator.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-b-operator.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/combined-site/values/west-b-provider-gateway.yaml b/examples/helm/existing-clusters/combined-site/values/west-b-provider-gateway.yaml index ee75462c..811bcf03 100644 --- a/examples/helm/existing-clusters/combined-site/values/west-b-provider-gateway.yaml +++ b/examples/helm/existing-clusters/combined-site/values/west-b-provider-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: provider-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-gateway.yaml b/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-gateway.yaml index ea6da441..14f9d65f 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-gateway.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: consumer-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-operator.yaml b/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-operator.yaml index bc1be5b1..260a5ad7 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-operator.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/east-consumer-operator.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/east-provider-gateway.yaml b/examples/helm/existing-clusters/dedicated-edge/values/east-provider-gateway.yaml index 7b6ff5a6..c7a117a0 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/east-provider-gateway.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/east-provider-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: provider-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/east-provider-operator.yaml b/examples/helm/existing-clusters/dedicated-edge/values/east-provider-operator.yaml index 8ed50b32..d0e00199 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/east-provider-operator.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/east-provider-operator.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-gateway.yaml b/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-gateway.yaml index faf89e1c..f7b6c02b 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-gateway.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: consumer-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-operator.yaml b/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-operator.yaml index d0189f40..e8b5465a 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-operator.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/west-consumer-operator.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/west-provider-gateway.yaml b/examples/helm/existing-clusters/dedicated-edge/values/west-provider-gateway.yaml index be25d9a6..95a48799 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/west-provider-gateway.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/west-provider-gateway.yaml @@ -9,8 +9,8 @@ fullnameOverride: provider-gateway image: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 pullPolicy: IfNotPresent config: diff --git a/examples/helm/existing-clusters/dedicated-edge/values/west-provider-operator.yaml b/examples/helm/existing-clusters/dedicated-edge/values/west-provider-operator.yaml index 14586024..216cc283 100644 --- a/examples/helm/existing-clusters/dedicated-edge/values/west-provider-operator.yaml +++ b/examples/helm/existing-clusters/dedicated-edge/values/west-provider-operator.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 pullPolicy: IfNotPresent swim: diff --git a/examples/helm/existing-clusters/inventory.example.yaml b/examples/helm/existing-clusters/inventory.example.yaml index 98ab5650..c3398cdd 100644 --- a/examples/helm/existing-clusters/inventory.example.yaml +++ b/examples/helm/existing-clusters/inventory.example.yaml @@ -41,7 +41,7 @@ sites: images: operator: repository: ghcr.io/praxis-proxy/grid-operator - tag: v0.1.3 + tag: v0.1.4 gateway: - repository: ghcr.io/praxis-proxy/grid-ai-rollup - tag: v0.1.3 + repository: ghcr.io/praxis-proxy/ai + tag: 0.3.0 diff --git a/forge/src/command/down.rs b/forge/src/command/down.rs index 9f88d53b..f8855325 100644 --- a/forge/src/command/down.rs +++ b/forge/src/command/down.rs @@ -183,6 +183,8 @@ struct NetworkTeardown { name: String, /// Whether this was a dry-run skip. dry_run: bool, + /// Whether the network was removed by this operation. + removed: bool, } /// Remove the environment network if one is tracked in state. @@ -194,10 +196,19 @@ fn remove_env_network( Some(ns) if ns.phase != NetworkPhase::Gone => ns.clone(), _ => return Ok(None), }; + if !state.network_created_by_forge { + verify_pre_existing_network(ctx, state, &net.name)?; + return Ok(Some(NetworkTeardown { + name: net.name, + dry_run: false, + removed: false, + })); + } if ctx.dry_run { return Ok(Some(NetworkTeardown { name: net.name, dry_run: true, + removed: false, })); } let binary = resolve_binary(ctx, state)?; @@ -207,9 +218,29 @@ fn remove_env_network( Ok(Some(NetworkTeardown { name: net.name, dry_run: false, + removed: true, })) } +/// Verify that a network marked as pre-existing is still present before +/// reporting that it was preserved. +fn verify_pre_existing_network( + ctx: &ForgeContext<'_>, + state: &state::ForgeState, + name: &str, +) -> Result<(), ForgeError> { + if ctx.dry_run { + return Ok(()); + } + let binary = resolve_binary(ctx, state)?; + if networking::network_exists(ctx.runner, &binary, name)? { + return Ok(()); + } + Err(ForgeError::State(format!( + "pre-existing network '{name}' is no longer present; refusing to report preservation" + ))) +} + /// Get the runtime binary from state or by re-detecting. fn resolve_binary(ctx: &ForgeContext<'_>, state: &state::ForgeState) -> Result { if let Some(binary) = &state.runtime { @@ -271,7 +302,7 @@ fn render_json( if let (Some(n), Some(obj)) = (net, data.as_object_mut()) { obj.insert( "network".to_owned(), - serde_json::json!({ "name": n.name, "dryRun": n.dry_run }), + serde_json::json!({ "name": n.name, "dryRun": n.dry_run, "removed": n.removed }), ); } let envelope = output::success(data); @@ -329,7 +360,10 @@ fn format_net_text(n: &NetworkTeardown) -> String { if n.dry_run { return format!("would remove network '{}'", n.name); } - format!("removed network '{}'", n.name) + if n.removed { + return format!("removed network '{}'", n.name); + } + format!("preserved pre-existing network '{}'", n.name) } /// Format a single result as text. @@ -451,6 +485,11 @@ spec: /// Pre-populate state with a running cluster and active network. fn seed_state_with_network(state_dir: &std::path::Path) { + seed_state_with_network_ownership(state_dir, true); + } + + /// Pre-populate state with a running cluster and explicit network ownership. + fn seed_state_with_network_ownership(state_dir: &std::path::Path, created_by_forge: bool) { let mut st = state::empty(); st.runtime = Some("docker".to_owned()); st.clusters.push(ClusterState { @@ -465,6 +504,7 @@ spec: cidr: None, cluster_pools: Vec::new(), }); + st.network_created_by_forge = created_by_forge; state::save(state_dir, &st).unwrap_or_else(|_| std::process::abort()); } @@ -550,4 +590,35 @@ spec: "should report would remove network: {text}" ); } + + #[test] + fn down_preserves_pre_existing_network() { + let dir = test_dir(); + seed_state_with_network_ownership(dir.path(), false); + let config = test_config(); + let mut runner = MockRunner::new(); + runner.respond("kind", ok()); + runner.respond("docker network inspect test-net", ok()); + let ctx = ForgeContext { + runner: &runner, + config: &config, + state_dir: dir.path().to_path_buf(), + config_dir: dir.path().to_path_buf(), + format: OutputFormat::Text, + dry_run: false, + }; + let mut buf = Vec::new(); + run(&ctx, false, &mut buf).unwrap_or_else(|_| std::process::abort()); + assert!( + !runner.was_called("network rm"), + "pre-existing network must be preserved" + ); + let text = String::from_utf8_lossy(&buf); + assert!( + text.contains("preserved pre-existing network"), + "should report preservation: {text}" + ); + let state = state::load(dir.path()).unwrap_or_else(|_| std::process::abort()); + assert_eq!(state.network.as_ref().map(|n| &n.phase), Some(&NetworkPhase::Active)); + } } diff --git a/forge/src/command/up.rs b/forge/src/command/up.rs index 05b52f9e..08bc5190 100644 --- a/forge/src/command/up.rs +++ b/forge/src/command/up.rs @@ -75,9 +75,11 @@ fn ensure_network( dry_run: true, })); } + let existed_before_up = networking::network_exists(ctx.runner, binary, &net_name)?; networking::create_network(ctx.runner, binary, &net_name, env_name)?; let cidr = networking::inspect_network_cidr(ctx.runner, binary, &net_name)?; set_network_active(state, &net_name, &cidr); + state.network_created_by_forge = !existed_before_up; Ok(Some(NetworkSetup { name: net_name, dry_run: false, diff --git a/forge/src/networking.rs b/forge/src/networking.rs index 18e98631..a08efec2 100644 --- a/forge/src/networking.rs +++ b/forge/src/networking.rs @@ -96,7 +96,14 @@ pub fn remove_network( pub fn network_exists(runner: &dyn CommandRunner, binary: &str, net_name: &str) -> Result { let spec = inspect_spec(binary, net_name); let output = runner.run(&spec)?; - Ok(output.status == 0) + match output.status { + 0 => Ok(true), + 1 if output.stderr.to_ascii_lowercase().contains("not found") => Ok(false), + status => Err(ForgeError::Command { + program: format!("{binary} network inspect {net_name}"), + message: format!("exit code {status}: {}", output.stderr.trim()), + }), + } } /// Read the current IPv4 subnet from a container network. @@ -496,6 +503,26 @@ mod tests { assert!(!exists, "should report network as not existing"); } + #[test] + fn exists_rejects_runtime_failure_instead_of_treating_it_as_missing() { + let mut runner = MockRunner::new(); + runner.respond( + "docker network inspect test-net", + CommandOutput { + status: 125, + stdout: String::new(), + stderr: "docker daemon unavailable\n".to_owned(), + }, + ); + + let result = network_exists(&runner, "docker", "test-net"); + assert!(result.is_err(), "runtime failure must not be treated as absence"); + assert!( + !runner.was_called("network rm"), + "failed detection must not remove anything" + ); + } + #[test] fn inspect_network_cidr_reads_formatted_ipam_config() { let mut runner = MockRunner::new(); diff --git a/forge/src/state.rs b/forge/src/state.rs index 05794cb2..0daee979 100644 --- a/forge/src/state.rs +++ b/forge/src/state.rs @@ -48,6 +48,10 @@ pub struct ForgeState { /// Managed container network state. #[serde(default, skip_serializing_if = "Option::is_none")] pub network: Option, + /// Whether Forge created the currently tracked network during `up`. + /// A false value is the safe backward-compatible default for older state. + #[serde(default)] + pub network_created_by_forge: bool, /// Detected container runtime name, if known. #[serde(default, skip_serializing_if = "Option::is_none")] pub runtime: Option, @@ -234,6 +238,7 @@ pub fn empty() -> ForgeState { services: Vec::new(), stacks: Vec::new(), network: None, + network_created_by_forge: false, runtime: None, config_digest: None, last_operation: None, diff --git a/scripts/verify-helm-chart.sh b/scripts/verify-helm-chart.sh index c28d213c..bfd17d00 100755 --- a/scripts/verify-helm-chart.sh +++ b/scripts/verify-helm-chart.sh @@ -6,8 +6,8 @@ FAIL=0 KIND_CLUSTER="" OPERATOR_IMAGE="ghcr.io/praxis-proxy/grid-operator" -OPERATOR_TAG="${GRID_OPERATOR_CI_TAG:-v0.1.3}" -DEFAULT_GATEWAY_IMAGE="ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" +OPERATOR_TAG="${GRID_OPERATOR_CI_TAG:-v0.1.4}" +DEFAULT_GATEWAY_IMAGE="ghcr.io/praxis-proxy/ai:0.3.0" # ── Helpers ──────────────────────────────────────────────────────────── @@ -192,7 +192,7 @@ echo " Praxis Gateway Chart ($GW_DIR)" echo "======================================================================" # Common required argument for the gateway chart. The image intentionally uses -# the chart default so this path validates the released Grid rollup contract. +# the chart default so this path validates the official Praxis AI contract. GW_REQ=(--set config.existingConfigMap=test-config) # ── Helm lint ──────────────────────────────────────────────────────── diff --git a/tests/e2e/topologies/grid-combined-site/README.md b/tests/e2e/topologies/grid-combined-site/README.md index 96062f34..f850d960 100644 --- a/tests/e2e/topologies/grid-combined-site/README.md +++ b/tests/e2e/topologies/grid-combined-site/README.md @@ -11,7 +11,7 @@ cargo xtask env run-grid-combined-site-demo \ ``` The default configuration pulls -`ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3`, which contains the provider-side +`ghcr.io/praxis-proxy/ai:0.3.0`, which contains the provider-side filters used by this topology. For local development, set `GRID_XTASK_GATEWAY_IMAGE` to an AI image containing [`provider_route`](https://github.com/praxis-proxy/ai/pull/386) and set diff --git a/tests/e2e/topologies/grid-combined-site/forge.yaml b/tests/e2e/topologies/grid-combined-site/forge.yaml index a3bc1cb4..79ba9222 100644 --- a/tests/e2e/topologies/grid-combined-site/forge.yaml +++ b/tests/e2e/topologies/grid-combined-site/forge.yaml @@ -20,14 +20,14 @@ spec: region: west role: combined siteName: west - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: central stacks: [metallb, central-operator-base, vcr-backend, central-site, provider-gateway, consumer-gateway] @@ -35,14 +35,14 @@ spec: region: central role: combined siteName: central - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: east stacks: [metallb, east-operator-base, vcr-backend, east-site, provider-gateway, consumer-gateway] @@ -50,14 +50,14 @@ spec: region: east role: combined siteName: east - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" stacks: metallb: diff --git a/tests/e2e/topologies/grid-glb-demo/README.md b/tests/e2e/topologies/grid-glb-demo/README.md index 61b68144..829dcb88 100644 --- a/tests/e2e/topologies/grid-glb-demo/README.md +++ b/tests/e2e/topologies/grid-glb-demo/README.md @@ -11,7 +11,7 @@ cargo xtask env run-grid-glb-demo \ ``` The default configuration pulls -`ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3`, which contains the +`ghcr.io/praxis-proxy/ai:0.3.0`, which contains the `intelligent_route`, `provider_route`, and `credential_inject` filters used by this topology. A local AI build must be selected explicitly: diff --git a/tests/e2e/topologies/grid-glb-demo/forge.yaml b/tests/e2e/topologies/grid-glb-demo/forge.yaml index 0556e8ba..4c5bca2a 100644 --- a/tests/e2e/topologies/grid-glb-demo/forge.yaml +++ b/tests/e2e/topologies/grid-glb-demo/forge.yaml @@ -19,70 +19,70 @@ spec: properties: region: east role: provider - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: west-provider stacks: [metallb, grid-operator, west-provider-operator, west-provider-site, vcr-backend] properties: region: west role: provider - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: east-edge stacks: [metallb, grid-operator, east-edge-operator, east-edge-site, edge-gateway] properties: region: east role: edge - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: west-edge stacks: [metallb, grid-operator, west-edge-operator, west-edge-site, edge-gateway] properties: region: west role: edge - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: gtm-emulator stacks: [metallb, gtm-emulator] properties: region: global role: gtm-emulator - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" stacks: metallb: diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/README.md b/tests/e2e/topologies/grid-llmd-pool-metrics/README.md index 2d37336c..47b1e15b 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/README.md +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/README.md @@ -11,7 +11,7 @@ cargo xtask env run-grid-llmd-pool-metrics-demo \ ``` The default configuration pulls -`ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3`, which contains the provider-side +`ghcr.io/praxis-proxy/ai:0.3.0`, which contains the provider-side filters used by this topology. For local development, set `GRID_XTASK_GATEWAY_IMAGE` to an AI image containing [`provider_route`](https://github.com/praxis-proxy/ai/pull/386) and set diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml index 49d2f83e..a4ca7abd 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml @@ -20,15 +20,15 @@ spec: region: pool-a role: combined siteName: pool-a - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" eppImage: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" eppImageRepo: "ghcr.io/llm-d/llm-d-router-endpoint-picker" eppImageTag: "v0.9.0" vcrImageRepo: "ghcr.io/neuralmagic/vllm-vcr" @@ -41,15 +41,15 @@ spec: region: pool-b role: combined siteName: pool-b - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" eppImage: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" eppImageRepo: "ghcr.io/llm-d/llm-d-router-endpoint-picker" eppImageTag: "v0.9.0" vcrImageRepo: "ghcr.io/neuralmagic/vllm-vcr" @@ -495,7 +495,7 @@ spec: enabled: true image: repository: "ghcr.io/praxis-proxy/grid-overlay-sync" - tag: "v0.1.3" + tag: "v0.1.4" pullPolicy: "IfNotPresent" expectedNetwork: "grid-llmd-pool-metrics" expectedLocalSite: "{{ cluster.name }}" diff --git a/tests/e2e/topologies/grid-provider-traffic/forge.yaml b/tests/e2e/topologies/grid-provider-traffic/forge.yaml index 972f6c22..728bfb2a 100644 --- a/tests/e2e/topologies/grid-provider-traffic/forge.yaml +++ b/tests/e2e/topologies/grid-provider-traffic/forge.yaml @@ -20,14 +20,14 @@ spec: region: provider-a role: combined siteName: provider-a - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: provider-b stacks: [metallb, provider-b-operator-base, vcr-backend, provider-b-site, provider-gateway] @@ -35,14 +35,14 @@ spec: region: provider-b role: combined siteName: provider-b - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" - name: provider-c stacks: [metallb, provider-c-operator-base, vcr-backend, provider-c-site, provider-gateway] @@ -50,14 +50,14 @@ spec: region: provider-c role: combined siteName: provider-c - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "ghcr.io/praxis-proxy/ai:0.3.0" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "ghcr.io/praxis-proxy/ai" + gatewayImageTag: "0.3.0" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" stacks: metallb: diff --git a/tests/e2e/topologies/grid-single-cluster-multi-gateway/README.md b/tests/e2e/topologies/grid-single-cluster-multi-gateway/README.md index 92bcf0ff..a38ddd5f 100644 --- a/tests/e2e/topologies/grid-single-cluster-multi-gateway/README.md +++ b/tests/e2e/topologies/grid-single-cluster-multi-gateway/README.md @@ -88,23 +88,39 @@ cross-cluster discovery. Prerequisites are Docker, Kind, `kubectl`, Helm, OpenSSL, Rust, and an AI source checkout next to or otherwise accessible from this Grid checkout. The runner -uses `imagePullPolicy: Never` and requires these exact local image references: +uses `imagePullPolicy: Never` by default. The names below are local-development +defaults; release validation should provide unique references through the +`GRID_XTASK_*_IMAGE` environment variables: - `grid-operator:single-cluster-qualification` - `grid-overlay-sync:single-cluster-qualification` - `praxis-ai:single-cluster-qualification` - `ghcr.io/neuralmagic/vllm-vcr:vllm0.23` +Supported overrides are `GRID_XTASK_GATEWAY_IMAGE`, +`GRID_XTASK_OPERATOR_IMAGE`, `GRID_XTASK_OVERLAY_SYNC_IMAGE`, +`GRID_XTASK_VCR_IMAGE`, and `GRID_XTASK_IMAGE_PULL_POLICY`. Explicit image +references are materialized into the Forge configuration, loaded into Kind +when the policy is `Never`, and recorded in qualification evidence. The runner +fails before deployment if an explicit reference is malformed, missing, or +absent from the materialized configuration. + Build Forge and the Grid images from this checkout: ```console cargo build -p forge +export GRID_XTASK_GATEWAY_IMAGE=praxis-ai:single-cluster-qualification-$RUN_ID +export GRID_XTASK_OPERATOR_IMAGE=grid-operator:single-cluster-qualification-$RUN_ID +export GRID_XTASK_OVERLAY_SYNC_IMAGE=grid-overlay-sync:single-cluster-qualification-$RUN_ID +export GRID_XTASK_VCR_IMAGE=ghcr.io/neuralmagic/vllm-vcr:vllm0.23 +export GRID_XTASK_IMAGE_PULL_POLICY=Never + docker build -f deploy/operator/Containerfile \ - -t grid-operator:single-cluster-qualification . + -t "$GRID_XTASK_OPERATOR_IMAGE" . docker build -f overlay-sync/Containerfile \ - -t grid-overlay-sync:single-cluster-qualification . + -t "$GRID_XTASK_OVERLAY_SYNC_IMAGE" . ``` Build the gateway from a clean Praxis AI checkout. This qualification uses the @@ -113,7 +129,7 @@ quota filters: ```console docker build -f Containerfile \ - -t praxis-ai:single-cluster-qualification . + -t "$GRID_XTASK_GATEWAY_IMAGE" . ``` Pull the pinned simulator image, validate the topology, and run focused static diff --git a/tests/e2e/topologies/grid-token-rate-limit/README.md b/tests/e2e/topologies/grid-token-rate-limit/README.md index 5a87f61b..d29fb527 100644 --- a/tests/e2e/topologies/grid-token-rate-limit/README.md +++ b/tests/e2e/topologies/grid-token-rate-limit/README.md @@ -169,13 +169,19 @@ topology: token-rate-limit-filter,praxis-filter/basic-auth-filter ``` +> **Published-image limitation:** The standard +> `ghcr.io/praxis-proxy/ai:0.3.0` image does not contain these optional +> filters because they are experimental, so it cannot run this qualification. +> Supply a feature-enabled Praxis AI image explicitly. Grid publishes no +> alternate AI rollup. + The second entry enables Basic Auth in AI's released `praxis-filter` dependency. It does not require a Praxis source checkout, Cargo patch, Git revision, or fork pin. Build AI from its own clean source tree and committed lockfile. Basic Auth stores the qualification credential in configuration and is not the production identity mechanism proposed by Grid issue 101. -AI's current `Containerfile` does not expose a Cargo-feature build argument. +AI v0.3.0's `Containerfile` does not expose a Cargo-feature build argument. Prepare a temporary Containerfile outside the AI worktree that adds the exact feature expression to both build-stage `cargo build` commands, then label the result so the qualification can verify its contract before creating clusters: diff --git a/tests/e2e/topologies/grid-token-rate-limit/forge.yaml b/tests/e2e/topologies/grid-token-rate-limit/forge.yaml index d41073c4..42997289 100644 --- a/tests/e2e/topologies/grid-token-rate-limit/forge.yaml +++ b/tests/e2e/topologies/grid-token-rate-limit/forge.yaml @@ -20,17 +20,17 @@ spec: region: west role: combined siteName: west - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "praxis-ai:quota-features-required" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" vcrImagePullPolicy: IfNotPresent imagePullPolicy: Never - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "praxis-ai" + gatewayImageTag: "quota-features-required" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" overlaySyncImageRepo: "ghcr.io/praxis-proxy/grid-overlay-sync" - overlaySyncImageTag: "v0.1.3" + overlaySyncImageTag: "v0.1.4" - name: central stacks: [metallb, central-operator-base, central-operator-seed, vcr-backend, provider-boundary, provider-gateway, central-site, central-trust-bootstrap, site-trust-bootstrap] @@ -38,17 +38,17 @@ spec: region: central role: combined siteName: central - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "praxis-ai:quota-features-required" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" vcrImagePullPolicy: IfNotPresent imagePullPolicy: Never - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "praxis-ai" + gatewayImageTag: "quota-features-required" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" overlaySyncImageRepo: "ghcr.io/praxis-proxy/grid-overlay-sync" - overlaySyncImageTag: "v0.1.3" + overlaySyncImageTag: "v0.1.4" - name: east stacks: [metallb, east-operator-base, east-operator-seed, vcr-backend, provider-boundary, provider-gateway, east-site, east-trust-bootstrap, site-trust-bootstrap] @@ -56,17 +56,17 @@ spec: region: east role: combined siteName: east - gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + gatewayImage: "praxis-ai:quota-features-required" + operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.4" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" vcrImagePullPolicy: IfNotPresent imagePullPolicy: Never - gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" - gatewayImageTag: "v0.1.3" + gatewayImageRepo: "praxis-ai" + gatewayImageTag: "quota-features-required" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" - operatorImageTag: "v0.1.3" + operatorImageTag: "v0.1.4" overlaySyncImageRepo: "ghcr.io/praxis-proxy/grid-overlay-sync" - overlaySyncImageTag: "v0.1.3" + overlaySyncImageTag: "v0.1.4" stacks: metallb: diff --git a/tests/e2e/topologies/grid-workload-inference/README.md b/tests/e2e/topologies/grid-workload-inference/README.md index 73fa3f82..18559c47 100644 --- a/tests/e2e/topologies/grid-workload-inference/README.md +++ b/tests/e2e/topologies/grid-workload-inference/README.md @@ -15,7 +15,7 @@ cargo xtask env run-grid-glb-demo \ ``` The reused GLB configuration defaults to -`ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3`. For local development, set +`ghcr.io/praxis-proxy/ai:0.3.0`. For local development, set `GRID_XTASK_GATEWAY_IMAGE` to an AI image containing [`provider_route`](https://github.com/praxis-proxy/ai/pull/386) and set `GRID_XTASK_IMAGE_PULL_POLICY=Never` explicitly. diff --git a/xtask/src/env/combined_site_demo.rs b/xtask/src/env/combined_site_demo.rs index d98199f6..6e03ae5c 100644 --- a/xtask/src/env/combined_site_demo.rs +++ b/xtask/src/env/combined_site_demo.rs @@ -86,6 +86,9 @@ const CENTRAL_PROVIDER_BACKEND: &str = "vcr-inference-central"; /// Primary model shared by the three site-local VCR providers. const PRIMARY_MODEL: &str = "Qwen/Qwen3-0.6B"; +/// Canonical backend identity emitted by the primary VCR provider. +const PRIMARY_BACKEND_IDENTITY: &str = "vcr-backend"; + /// Combined-site overlay propagation crosses operator, SWIM, projected-volume, /// and gateway reload boundaries. Issue #21 tracks reducing this latency. const COMBINED_SITE_DATA_PLANE_WAIT: Duration = Duration::from_secs(180); @@ -93,9 +96,30 @@ const COMBINED_SITE_DATA_PLANE_WAIT: Duration = Duration::from_secs(180); /// Retry interval for serving-state convergence probes. const COMBINED_SITE_DATA_PLANE_INTERVAL: Duration = Duration::from_secs(1); +/// Stable clients must outlive every bounded full-mode lifecycle stage. +const STABLE_CLIENT_LIFETIME_SECS: &str = "7200"; + /// Makes retry probe names unique while retaining a recognizable prefix. static PROBE_SEQUENCE: AtomicU64 = AtomicU64::new(0); +/// Return a DNS-safe unique name for a test client pod. +fn unique_client_name(prefix: &str) -> String { + let sanitized: String = prefix + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || character == '-' { + character.to_ascii_lowercase() + } else { + '-' + } + }) + .collect(); + let sanitized = sanitized.trim_matches('-'); + let sequence = PROBE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let client_prefix = if sanitized.is_empty() { "client" } else { sanitized }; + format!("{}-{}", &client_prefix[..client_prefix.len().min(40)], sequence) +} + // ----------------------------------------------------------------------------- // Context // ----------------------------------------------------------------------------- @@ -471,14 +495,177 @@ fn curl_pod_overrides(pod_name: &str, curl_args: &[&str]) -> String { .to_string() } +/// Build a restricted, long-lived client pod used for request-sequence tests. +fn curl_client_pod_overrides(pod_name: &str) -> String { + serde_json::json!({ + "spec": { + "automountServiceAccountToken": false, + "securityContext": { + "runAsNonRoot": true, + "seccompProfile": { "type": "RuntimeDefault" } + }, + "containers": [{ + "name": pod_name, + "image": "curlimages/curl:8.12.1", + "command": ["sleep", STABLE_CLIENT_LIFETIME_SECS], + "securityContext": { + "runAsUser": 100, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { "drop": ["ALL"] } + } + }] + } + }) + .to_string() +} + +/// A single ready curl pod whose lifecycle is isolated from request evidence. +struct StableCurlClient { + /// Kubernetes context containing the client pod. + context: String, + /// Pod name used for the complete request sequence. + name: String, +} + +impl StableCurlClient { + /// Create the client pod and wait for its Ready condition. + fn create(context: &str, name: &str) -> Result> { + let name = unique_client_name(name); + let overrides = curl_client_pod_overrides(&name); + let output = Command::new("timeout") + .args([ + "90s", + "kubectl", + "run", + &name, + "--image=curlimages/curl:8.12.1", + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "--restart=Never", + "--overrides", + &overrides, + "--command", + "--", + "sleep", + STABLE_CLIENT_LIFETIME_SECS, + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to create stable curl client {name}: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + let wait = Command::new("timeout") + .args([ + "90s", + "kubectl", + "wait", + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "--for=condition=Ready", + &format!("pod/{name}"), + "--timeout=75s", + ]) + .output()?; + if !wait.status.success() { + drop( + Command::new("timeout") + .args([ + "30s", + "kubectl", + "delete", + "pod", + &name, + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "--ignore-not-found", + ]) + .output(), + ); + return Err(format!( + "stable curl client {name} did not become ready: {}{}", + String::from_utf8_lossy(&wait.stdout), + String::from_utf8_lossy(&wait.stderr) + ) + .into()); + } + + Ok(Self { + context: context.to_owned(), + name, + }) + } + + /// Execute one curl request without mixing in pod lifecycle output. + fn request(&self, curl_args: &[&str]) -> Result { + Command::new("timeout") + .args([ + "30s", + "kubectl", + "exec", + &self.name, + "--context", + &self.context, + "-n", + GRID_SYSTEM_NS, + "--", + "curl", + ]) + .args(curl_args) + .output() + } +} + +impl Drop for StableCurlClient { + fn drop(&mut self) { + let deleted = Command::new("timeout") + .args([ + "30s", + "kubectl", + "delete", + "pod", + &self.name, + "--context", + &self.context, + "-n", + GRID_SYSTEM_NS, + "--ignore-not-found", + "--wait=true", + ]) + .output(); + if let Ok(output) = deleted + && !output.status.success() + { + eprintln!( + "[WARN] stable curl client {} deletion failed: {}", + self.name, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + } +} + /// Run an ephemeral curl pod with restricted PodSecurity context. fn run_curl_probe(context: &str, pod_name: &str, curl_args: &[&str]) -> Result { let sequence = PROBE_SEQUENCE.fetch_add(1, Ordering::Relaxed); let prefix = pod_name.get(..pod_name.len().min(40)).unwrap_or(pod_name); let unique_pod_name = format!("{prefix}-{sequence}"); let overrides = curl_pod_overrides(&unique_pod_name, curl_args); - Command::new("kubectl") + let output = Command::new("timeout") .args([ + "45s", + "kubectl", "run", &unique_pod_name, "--image=curlimages/curl:8.12.1", @@ -486,13 +673,34 @@ fn run_curl_probe(context: &str, pod_name: &str, curl_args: &[&str]) -> Result( mut check: impl FnMut() -> Result>, ) -> Result> { let deadline = Instant::now() + COMBINED_SITE_DATA_PLANE_WAIT; + let mut last_observation = String::from("no observation captured"); loop { match check() { Ok(value) => return Ok(value), Err(error) if Instant::now() >= deadline => { return Err(format!( - "{description} did not converge within {COMBINED_SITE_DATA_PLANE_WAIT:?}: {error}" + "{description} did not converge within {COMBINED_SITE_DATA_PLANE_WAIT:?}; last observed failure: {last_observation}; final failure: {error}" ) .into()); }, - Err(_) => std::thread::park_timeout(COMBINED_SITE_DATA_PLANE_INTERVAL), + Err(error) => { + last_observation = error.to_string(); + std::thread::park_timeout(COMBINED_SITE_DATA_PLANE_INTERVAL); + }, + } + } +} + +/// Extract the most recent exact tracing field from an AI overlay log line. +fn ai_revision_from_logs(logs: &str, field: &str) -> Option { + logs.lines().rev().find_map(|line| { + let line = strip_ansi_csi(line); + let marker = format!("{field}="); + let index = line + .match_indices(&marker) + .find_map(|(index, _)| (index == 0 || line.as_bytes().get(index - 1) == Some(&b' ')).then_some(index))?; + let value = line.get(index + marker.len()..)?; + let value = value.strip_prefix('"').unwrap_or(value); + let value = value + .split(|character: char| character == '"' || character.is_whitespace()) + .next()?; + (!value.is_empty()).then(|| value.to_owned()) + }) +} + +/// Remove ANSI CSI styling emitted by tracing/logging sinks before parsing. +fn strip_ansi_csi(input: &str) -> String { + let mut output = Vec::with_capacity(input.len()); + let mut bytes = input.bytes().peekable(); + while let Some(byte) = bytes.next() { + if byte == 0x1B && bytes.peek() == Some(&b'[') { + bytes.next(); + for control in bytes.by_ref() { + if (0x40..=0x7E).contains(&control) { + break; + } + } + } else { + output.push(byte); } } + String::from_utf8_lossy(&output).into_owned() } /// Run an ephemeral curl pod with additional kubectl flags (e.g. `--labels`). @@ -522,24 +770,50 @@ fn run_curl_probe_with_flags( extra_kubectl_args: &[&str], curl_args: &[&str], ) -> Result { - let overrides = curl_pod_overrides(pod_name, curl_args); - Command::new("kubectl") + let sequence = PROBE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let prefix = pod_name.get(..pod_name.len().min(40)).unwrap_or(pod_name); + let unique_pod_name = format!("{prefix}-{sequence}"); + let overrides = curl_pod_overrides(&unique_pod_name, curl_args); + let output = Command::new("timeout") .args([ + "45s", + "kubectl", "run", - pod_name, + &unique_pod_name, "--image=curlimages/curl:8.12.1", "--context", context, "-n", GRID_SYSTEM_NS, - "--rm", "-i", "--restart=Never", "--overrides", &overrides, ]) .args(extra_kubectl_args) - .output() + .output(); + let cleanup = Command::new("timeout") + .args([ + "30s", + "kubectl", + "delete", + "pod", + &unique_pod_name, + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "--ignore-not-found", + "--wait=true", + ]) + .output()?; + if !cleanup.status.success() { + return Err(std::io::Error::other(format!( + "failed to clean ephemeral curl pod {unique_pod_name}: {}", + String::from_utf8_lossy(&cleanup.stderr).trim() + ))); + } + output } /// Return a response header value from curl `--include` output. @@ -551,6 +825,25 @@ fn response_header(output: &[u8], name: &str) -> Option { }) } +/// Parse the status code from the first HTTP response status line only. +fn response_status(output: &[u8]) -> Option { + String::from_utf8_lossy(output).lines().find_map(|line| { + let mut fields = line.split_whitespace(); + (fields.next()?.starts_with("HTTP/")).then(|| fields.next()?.parse().ok())? + }) +} + +/// Require the complete trusted attribution set for a primary response. +fn primary_response_is_trusted(output: &[u8]) -> bool { + let gateway = response_header(output, "x-grid-combined-provider-gateway"); + let ai_gateway = response_header(output, "x-ai-demo-provider-gateway"); + let backend = response_header(output, "x-ai-inference-provider"); + response_status(output) == Some(200) + && backend.as_deref() == Some(PRIMARY_BACKEND_IDENTITY) + && gateway.as_deref().is_some_and(|site| CLUSTERS.contains(&site)) + && gateway == ai_gateway +} + /// Wait for the demo environment to be ready. fn wait_for_environment_ready() -> Result> { // Wait for Grid operators to converge @@ -1041,64 +1334,161 @@ fn assert_overlay_acceptance() -> AssertionResult { } } -/// Assert local provider selection preference: west selects west, central selects central, east selects east. -fn assert_local_provider_selection() -> AssertionResult { - let start = Instant::now(); - let mut observed_facts = BTreeMap::new(); - let mut all_local = true; - - for cluster in CLUSTERS { - let context = format!("kind-grid-combined-{cluster}"); +/// Return the provider sites represented by an accepted overlay. +fn accepted_provider_sites(overlay: &OverlayData) -> BTreeSet { + overlay + .candidates + .iter() + .filter(|candidate| !candidate.site.is_empty()) + .map(|candidate| candidate.site.clone()) + .collect() +} - // Send test request to consumer gateway and check response attribution - let test_output = run_curl_probe( - &context, - &format!("route-test-{cluster}"), - &[ - "curl", - "-f", - "--include", - "-H", - "Content-Type: application/json", - "-H", - "Authorization: Bearer consumer-token", - "-H", - &format!("X-Session-Id: local-test-{cluster}"), - "-d", - r#"{"model": "Qwen/Qwen3-0.6B", "messages": [{"role":"user","content":"hello"}], "max_tokens": 16}"#, - "consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", - ], - )?; +/// Check that a sequence is a repeating round-robin cycle, allowing any +/// provider order and initial picker position but requiring every value to be +/// an accepted candidate. +fn is_round_robin_sequence(sequence: &[String], eligible: &BTreeSet) -> bool { + if eligible.is_empty() + || sequence.len() < eligible.len() * 2 + || sequence.iter().any(|site| !eligible.contains(site)) + { + return false; + } + let cycle_len = eligible.len(); + let Some(first_cycle) = sequence.get(..cycle_len) else { + return false; + }; + let first_cycle_set: BTreeSet<_> = first_cycle.iter().cloned().collect(); + first_cycle_set == *eligible + && sequence.iter().enumerate().all(|(index, site)| { + first_cycle + .get(index % cycle_len) + .is_some_and(|expected| site == expected) + }) +} - let provider_site = response_header(&test_output.stdout, "x-grid-combined-provider-gateway") - .unwrap_or_else(|| "unknown".to_owned()); - let selected_local = test_output.status.success() && provider_site == *cluster; +/// Assert accepted-overlay membership and round-robin selection across the +/// eligible provider group. Locality is intentionally not assumed. +fn assert_round_robin_provider_selection() -> AssertionResult { + let start = Instant::now(); + let mut observed_facts = BTreeMap::new(); + let overlay = read_cluster_overlay("west")?; + let eligible = accepted_provider_sites(&overlay); + let mut sequence = Vec::new(); + let mut all_attributed = true; + let client = StableCurlClient::create("kind-grid-combined-west", "route-test-client")?; - if !selected_local { - all_local = false; + let expected_revision = overlay.semantic_revision.clone(); + wait_for_combined_site_data_plane("west consumer serving accepted overlay", || { + let output = client.request(&[ + "--fail", + "--include", + "--silent", + "--show-error", + "--max-time", + "15", + "-H", + "Content-Type: application/json", + "-H", + "Authorization: Bearer consumer-token", + "-d", + r#"{"model": "Qwen/Qwen3-0.6B", "messages": [{"role":"user","content":"convergence"}], "max_tokens": 16}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ])?; + let status = response_status(&output.stdout); + let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway"); + let logs = Command::new("kubectl") + .args([ + "--context", + "kind-grid-combined-west", + "-n", + GRID_SYSTEM_NS, + "logs", + "deployment/consumer-gateway", + "-c", + "praxis", + "--tail=200", + ]) + .output()?; + let loaded_revision = + logs.status.success() && String::from_utf8_lossy(&logs.stdout).contains(expected_revision.as_str()); + if output.status.success() && status == Some(200) && provider.is_some() && loaded_revision { + Ok(()) + } else { + Err(format!( + "serving status={status:?}, provider={}, loaded_revision={loaded_revision}", + provider.as_deref().unwrap_or("missing") + ) + .into()) } + })?; + for request in 0..(eligible.len() * 3) { + let test_output = client.request(&[ + "--fail", + "--include", + "--silent", + "--show-error", + "--max-time", + "15", + "-H", + "Content-Type: application/json", + "-H", + "Authorization: Bearer consumer-token", + "-d", + r#"{"model": "Qwen/Qwen3-0.6B", "messages": [{"role":"user","content":"hello"}], "max_tokens": 16}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ])?; + + let provider_site = response_header(&test_output.stdout, "x-grid-combined-provider-gateway"); + let response_site = response_header(&test_output.stdout, "x-ai-demo-provider-gateway"); + let status = response_status(&test_output.stdout); + let attributed = test_output.status.success() + && status == Some(200) + && provider_site.as_deref().is_some_and(|site| eligible.contains(site)) + && provider_site == response_site; + if !attributed { + all_attributed = false; + } + if let Some(site) = provider_site.clone() { + sequence.push(site); + } + observed_facts.insert( + format!("request_{request}_provider_site"), + provider_site.map_or(serde_json::Value::Null, serde_json::Value::String), + ); observed_facts.insert( - format!("{cluster}_selected_provider_site"), - serde_json::Value::String(provider_site), + format!("request_{request}_attributed"), + serde_json::Value::Bool(attributed), ); observed_facts.insert( - format!("{cluster}_selected_local"), - serde_json::Value::Bool(selected_local), + format!("request_{request}_status"), + status.map_or(serde_json::Value::Null, |code| serde_json::Value::Number(code.into())), ); } - observed_facts.insert("all_sites_prefer_local".to_owned(), serde_json::Value::Bool(all_local)); + let round_robin = all_attributed && is_round_robin_sequence(&sequence, &eligible); + observed_facts.insert( + "accepted_overlay_revision".to_owned(), + serde_json::Value::String(overlay.semantic_revision), + ); + observed_facts.insert("eligible_provider_sites".to_owned(), serde_json::json!(eligible)); + observed_facts.insert("provider_sequence".to_owned(), serde_json::json!(sequence)); + observed_facts.insert( + "all_responses_trusted_and_eligible".to_owned(), + serde_json::Value::Bool(all_attributed), + ); + observed_facts.insert("round_robin_cycle".to_owned(), serde_json::Value::Bool(round_robin)); - if all_local { + if round_robin { Ok(proof_success( - "All consumer sites correctly prefer their local provider (west→west, central→central, east→east)", + "Accepted-overlay candidates are trusted and selected in a repeating round-robin cycle", observed_facts, start.elapsed(), )) } else { Ok(proof_failure( - "One or more consumer sites did not select their local provider", + "Provider selection was not a trusted round-robin cycle over accepted-overlay candidates", observed_facts, start.elapsed(), )) @@ -1779,8 +2169,42 @@ fn assert_backend_access_denial() -> AssertionResult { } } -/// Assert central provider-capacity drain triggers remote fallback detection. -fn assert_central_drain_fallback() -> AssertionResult { +/// Execute one lifecycle request through the stable central consumer client. +fn lifecycle_request( + client: &StableCurlClient, + session_id: &str, + message_text: &str, +) -> Result<(u16, String), Box> { + let session_header = format!("X-Session-Id: {session_id}"); + let body = format!(r#"{{"model":"{PRIMARY_MODEL}","messages":[{{"role":"user","content":"{message_text}"}}]}}"#); + let output = client.request(&[ + "--include", + "--silent", + "--show-error", + "--max-time", + "15", + "-H", + "Content-Type: application/json", + "-H", + &session_header, + "-d", + &body, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ])?; + let status = response_status(&output.stdout).ok_or("lifecycle response had no HTTP status")?; + let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway") + .ok_or("lifecycle response missing provider-gateway attribution")?; + let trusted_provider = response_header(&output.stdout, "x-ai-demo-provider-gateway") + .ok_or("lifecycle response missing trusted provider attribution")?; + if provider != trusted_provider { + return Err(format!("lifecycle provider attribution mismatch: {provider} != {trusted_provider}").into()); + } + Ok((status, provider)) +} + +/// Assert central provider withdrawal removes it from the accepted overlay and +/// routes subsequent requests only to remaining eligible sites. +fn assert_central_drain_fallback(client: &StableCurlClient) -> AssertionResult { let start = Instant::now(); let mut observed_facts = BTreeMap::new(); @@ -1858,41 +2282,39 @@ fn assert_central_drain_fallback() -> AssertionResult { } wait_for_site_model_absent_all_sites("central", PRIMARY_MODEL, Duration::from_secs(180))?; + let post_drain_overlay = read_cluster_overlay("central")?; + let eligible_after_drain = accepted_provider_sites(&post_drain_overlay); + let central_withdrawn = !eligible_after_drain.contains("central"); + observed_facts.insert( + "post_drain_overlay_revision".to_owned(), + serde_json::Value::String(post_drain_overlay.semantic_revision), + ); + observed_facts.insert( + "post_drain_eligible_provider_sites".to_owned(), + serde_json::json!(eligible_after_drain), + ); + observed_facts.insert( + "withdrawn_provider_absent_after_convergence".to_owned(), + serde_json::Value::Bool(central_withdrawn), + ); + if !central_withdrawn { + return Ok(proof_failure( + "Central provider remained in the accepted overlay after withdrawal convergence", + observed_facts, + start.elapsed(), + )); + } let last_observation = std::cell::RefCell::new("no request attempted".to_owned()); let remote_provider_site = wait_for_combined_site_data_plane("combined-site remote fallback", || { - let output = run_curl_probe( - central_context, - "fallback-test-central", - &[ - "curl", - "--fail-with-body", - "--silent", - "--show-error", - "--include", - "-H", - "Content-Type: application/json", - "-H", - "X-Session-Id: fallback-test-central", - "-d", - r#"{"model": "Qwen/Qwen3-0.6B", "messages": [{"role":"user","content":"hello"}], "max_tokens": 16}"#, - "consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", - ], - )?; - let provider = - response_header(&output.stdout, "x-grid-combined-provider-gateway").unwrap_or_else(|| "missing".to_owned()); - let backend = response_header(&output.stdout, "x-grid-demo-backend-provider-attribution") - .unwrap_or_else(|| "missing".to_owned()); - *last_observation.borrow_mut() = format!( - "status={}, provider_gateway={provider}, backend={backend}", - output.status - ); - if !output.status.success() { + let (status, provider) = lifecycle_request(client, "fallback-test-central", "hello")?; + *last_observation.borrow_mut() = format!("status={status}, provider_gateway={provider}"); + if status != 200 { return Err("fallback request did not return HTTP 200".into()); } - if !matches!(provider.as_str(), "west" | "east") || provider != backend { + if !eligible_after_drain.contains(&provider) || provider == "central" { return Err(format!( - "fallback selected provider_gateway={provider}, backend={backend}; expected the same remote site" + "fallback selected provider_gateway={provider}; expected an eligible non-withdrawn site" ) .into()); } @@ -1947,37 +2369,13 @@ fn deployment_scaled_to_zero(replica_state: &str) -> bool { /// This function assumes central provider is still up when called. /// The caller (run_full_scenarios) must call this BEFORE drain, /// then call assert_existing_session_after_drain and assert_new_session_after_drain. -fn assert_session_establishment() -> AssertionResult { +fn assert_session_establishment(client: &StableCurlClient) -> AssertionResult { let start = Instant::now(); let mut observed_facts = BTreeMap::new(); let session_id = "pre-drain-session"; - let central_context = "kind-grid-combined-central"; - - let initial_output = run_curl_probe( - central_context, - "session-establish", - &[ - "curl", - "-f", - "--include", - "-H", - "Content-Type: application/json", - "-H", - &format!("X-Session-Id: {session_id}"), - "-d", - r#"{"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "user", "content": "pre-drain-context"}]}"#, - "consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", - ], - )?; - - let established = initial_output.status.success(); - let mut provider_site = "unknown".to_owned(); - - if established { - provider_site = response_header(&initial_output.stdout, "x-grid-combined-provider-gateway") - .unwrap_or_else(|| "unknown".to_owned()); - } + let (status, provider_site) = lifecycle_request(client, session_id, "pre-drain-context")?; + let established = status == 200; observed_facts.insert( "session_established_before_drain".to_owned(), @@ -1990,7 +2388,7 @@ fn assert_session_establishment() -> AssertionResult { if established { Ok(proof_success( - "Session established before drain with local provider", + "Session established before drain with an accepted provider candidate", observed_facts, start.elapsed(), )) @@ -2005,19 +2403,12 @@ fn assert_session_establishment() -> AssertionResult { /// Assert the pre-drain session survives after central provider is drained. /// Must be called AFTER assert_central_drain_fallback. -fn assert_existing_session_after_drain() -> AssertionResult { +fn assert_existing_session_after_drain(client: &StableCurlClient) -> AssertionResult { let start = Instant::now(); let mut observed_facts = BTreeMap::new(); let session_id = "pre-drain-session"; - let central_context = "kind-grid-combined-central"; - - let remote_provider = wait_for_remote_session( - central_context, - "session-existing-after-drain", - session_id, - "after-drain", - ); + let remote_provider = wait_for_remote_session(client, session_id, "after-drain"); let followup_successful = remote_provider.is_ok(); let remote_provider = remote_provider.unwrap_or_else(|_| "unknown".to_owned()); @@ -2047,19 +2438,12 @@ fn assert_existing_session_after_drain() -> AssertionResult { /// Assert a new session can be established after central provider is drained. /// Must be called AFTER assert_central_drain_fallback. -fn assert_new_session_after_drain() -> AssertionResult { +fn assert_new_session_after_drain(client: &StableCurlClient) -> AssertionResult { let start = Instant::now(); let mut observed_facts = BTreeMap::new(); let new_session_id = "post-drain-new-session"; - let central_context = "kind-grid-combined-central"; - - let provider_site = wait_for_remote_session( - central_context, - "session-new-after-drain", - new_session_id, - "new-session-content", - ); + let provider_site = wait_for_remote_session(client, new_session_id, "new-session-content"); let new_session_works = provider_site.is_ok(); let provider_site = provider_site.unwrap_or_else(|_| "unknown".to_owned()); @@ -2089,45 +2473,17 @@ fn assert_new_session_after_drain() -> AssertionResult { /// Wait until a central-consumer session is served by one identified remote provider. fn wait_for_remote_session( - context: &str, - pod_name: &str, + client: &StableCurlClient, session_id: &str, message_text: &str, ) -> Result> { wait_for_combined_site_data_plane("combined-site remote session routing", || { - let session_header = format!("X-Session-Id: {session_id}"); - let body = - format!(r#"{{"model":"{PRIMARY_MODEL}","messages":[{{"role":"user","content":"{message_text}"}}]}}"#); - let output = run_curl_probe( - context, - pod_name, - &[ - "curl", - "--fail-with-body", - "--silent", - "--show-error", - "--include", - "-H", - "Content-Type: application/json", - "-H", - &session_header, - "-d", - &body, - "consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", - ], - )?; - if !output.status.success() { + let (status, provider) = lifecycle_request(client, session_id, message_text)?; + if status != 200 { return Err("session request did not return HTTP 200".into()); } - let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway") - .ok_or("session response missing provider-gateway attribution")?; - let backend = response_header(&output.stdout, "x-grid-demo-backend-provider-attribution") - .ok_or("session response missing backend attribution")?; - if !matches!(provider.as_str(), "west" | "east") || provider != backend { - return Err(format!( - "session selected provider_gateway={provider}, backend={backend}; expected the same remote site" - ) - .into()); + if !matches!(provider.as_str(), "west" | "east") { + return Err(format!("session selected provider_gateway={provider}; expected a remote site").into()); } Ok(provider) }) @@ -2185,8 +2541,8 @@ fn ensure_central_provider_restored() -> Result<(), Box> Ok(()) } -/// Assert provider restoration returns routing to local preference. -fn assert_provider_restoration() -> AssertionResult { +/// Assert provider restoration re-admits the provider to the round-robin group. +fn assert_provider_restoration(client: &StableCurlClient) -> AssertionResult { let start = Instant::now(); let mut observed_facts = BTreeMap::new(); @@ -2295,55 +2651,31 @@ fn assert_provider_restoration() -> AssertionResult { let restoration_attempt = std::cell::Cell::new(0_u64); let last_restoration_observation = std::cell::RefCell::new("no request attempted".to_owned()); - let selected_provider_site = wait_for_combined_site_data_plane("combined-site local restoration", || { + let restored_overlay = read_cluster_overlay("central")?; + let eligible = accepted_provider_sites(&restored_overlay); + let selected_provider_site = wait_for_combined_site_data_plane("combined-site restored round-robin", || { let attempt = restoration_attempt.get() + 1; restoration_attempt.set(attempt); - // Every attempt represents new traffic. Reusing a session would - // correctly preserve its remote fallback affinity after recovery. - let session_header = format!("X-Session-Id: restoration-test-central-{attempt}"); - let output = run_curl_probe( - central_context, - "restoration-test-central", - &[ - "curl", - "--fail-with-body", - "--silent", - "--show-error", - "--include", - "-H", - "Content-Type: application/json", - "-H", - &session_header, - "-d", - r#"{"model": "Qwen/Qwen3-0.6B", "messages": [{"role":"user","content":"hello"}], "max_tokens": 16}"#, - "consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", - ], - )?; - if !output.status.success() { + let (status, provider) = lifecycle_request(client, &format!("restoration-test-central-{attempt}"), "hello")?; + if status != 200 { return Err("restoration request did not return HTTP 200".into()); } - let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway") - .ok_or("restoration response missing provider-gateway attribution")?; - let backend = response_header(&output.stdout, "x-grid-demo-backend-provider-attribution") - .ok_or("restoration response missing backend attribution")?; - *last_restoration_observation.borrow_mut() = format!( - "attempt={attempt}, status={}, provider_gateway={provider}, backend={backend}", - output.status - ); - if provider != "central" || backend != "central" { + *last_restoration_observation.borrow_mut() = + format!("attempt={attempt}, status={status}, provider_gateway={provider}"); + if !eligible.contains("central") || provider != "central" { return Err(format!( - "restoration selected provider_gateway={provider}, backend={backend}; expected central" + "restoration selected provider_gateway={provider}; expected restored central candidate" ) .into()); } Ok(provider) }); - let local_preference_restored = selected_provider_site.is_ok(); + let restored_candidate_selected = selected_provider_site.is_ok(); let selected_provider_site = selected_provider_site.unwrap_or_else(|_| "none".to_owned()); observed_facts.insert( - "local_preference_restored".to_owned(), - serde_json::Value::Bool(local_preference_restored), + "round_robin_restored".to_owned(), + serde_json::Value::Bool(restored_candidate_selected), ); observed_facts.insert( "selected_provider_site".to_owned(), @@ -2358,15 +2690,15 @@ fn assert_provider_restoration() -> AssertionResult { serde_json::Value::String(last_restoration_observation.into_inner()), ); - if provider_ready && local_preference_restored { + if provider_ready && restored_candidate_selected { Ok(proof_success( - "Provider restoration successfully returned routing to local preference", + "Provider restoration re-admitted the provider to the accepted round-robin group", observed_facts, start.elapsed(), )) } else { Ok(proof_failure( - "Provider restoration failed to restore local preference", + "Provider restoration did not re-admit the provider to the accepted round-robin group", observed_facts, start.elapsed(), )) @@ -2385,38 +2717,57 @@ fn assert_rollout_convergence() -> AssertionResult { let rendered = overlay.semantic_revision != "unknown" && !overlay.semantic_revision.is_empty(); let distributed = !overlay.candidates.is_empty(); let expected_revision = overlay.semantic_revision.clone(); + let client = StableCurlClient::create(&context, &format!("rollout-convergence-{cluster}"))?; - let serving_revision = - wait_for_combined_site_data_plane(&format!("{cluster} serving current overlay revision"), || { - let output = run_curl_probe( - &context, - &format!("serving-test-{cluster}"), - &[ - "curl", - "--fail-with-body", - "--silent", - "--show-error", - "--include", - "-H", - "Content-Type: application/json", - "-d", - r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"hello"}],"max_tokens":16}"#, - "consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", - ], - )?; + let serving_revision = wait_for_combined_site_data_plane( + &format!("{cluster} serving current overlay revision"), + || { + let output = client.request(&[ + "--fail-with-body", + "--silent", + "--show-error", + "--include", + "--max-time", + "10", + "-H", + "Content-Type: application/json", + "-H", + "Authorization: Bearer consumer-token", + "-d", + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"hello"}],"max_tokens":16}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ])?; if !output.status.success() { return Err("serving request did not return HTTP 200".into()); } - let revision = response_header(&output.stdout, "x-grid-demo-backend-overlay-revision") - .ok_or("serving response missing overlay revision attribution")?; - if revision != expected_revision { + let logs = Command::new("timeout") + .args([ + "20s", + "kubectl", + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "logs", + "deployment/consumer-gateway", + "-c", + "praxis", + "--tail=200", + ]) + .output()?; + let logs_text = String::from_utf8_lossy(&logs.stdout); + let accepted_revision = ai_revision_from_logs(&logs_text, "accepted_revision"); + let revision = ai_revision_from_logs(&logs_text, "serving_revision") + .ok_or("AI logs missing serving_revision evidence")?; + if accepted_revision.as_deref() != Some(expected_revision.as_str()) || revision != expected_revision { return Err(format!( - "serving revision {revision} does not match distributed revision {expected_revision}" + "AI revisions accepted={accepted_revision:?}, serving={revision} do not match distributed revision {expected_revision}" ) .into()); } Ok(revision) - }); + }, + ); let serving = serving_revision.is_ok(); let accepted = serving; let serving_revision = serving_revision.unwrap_or_else(|_| "none".to_owned()); @@ -3419,10 +3770,10 @@ fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result return Ok(()); } - let gateway = std::env::var("GRID_XTASK_GATEWAY_IMAGE") - .unwrap_or_else(|_| "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3".to_owned()); + let gateway = + std::env::var("GRID_XTASK_GATEWAY_IMAGE").unwrap_or_else(|_| "ghcr.io/praxis-proxy/ai:0.3.0".to_owned()); let operator = std::env::var("GRID_XTASK_OPERATOR_IMAGE") - .unwrap_or_else(|_| "ghcr.io/praxis-proxy/grid-operator:v0.1.3".to_owned()); + .unwrap_or_else(|_| "ghcr.io/praxis-proxy/grid-operator:v0.1.4".to_owned()); let vcr = crate::env::image_overrides::vcr_image(); for image in [&gateway, &operator, &vcr] { @@ -3555,7 +3906,7 @@ fn apply_credential_secret(context: &str, secret_name: &str, token: &str) -> Res /// semantic revision from the `grid.praxis-proxy.io/overlay-revision` /// annotation (content-addressed, safe to compare across clusters). fn read_cluster_overlay(cluster: &str) -> Result> { - let context = format!("kind-grid-combined-{cluster}"); + let context = combined_kubectl_context(cluster); let output = Command::new("kubectl") .args([ @@ -4324,10 +4675,10 @@ fn materialize_external_provider_stack( reason = "Image override application with structured YAML manipulation; nested ifs follow YAML structure hierarchy" )] fn apply_image_overrides(config: &mut serde_yaml::Value) { - let gateway_image = std::env::var("GRID_XTASK_GATEWAY_IMAGE") - .unwrap_or_else(|_| "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3".to_owned()); + let gateway_image = + std::env::var("GRID_XTASK_GATEWAY_IMAGE").unwrap_or_else(|_| "ghcr.io/praxis-proxy/ai:0.3.0".to_owned()); let operator_image = std::env::var("GRID_XTASK_OPERATOR_IMAGE") - .unwrap_or_else(|_| "ghcr.io/praxis-proxy/grid-operator:v0.1.3".to_owned()); + .unwrap_or_else(|_| "ghcr.io/praxis-proxy/grid-operator:v0.1.4".to_owned()); let vcr_image = crate::env::image_overrides::vcr_image(); let image_pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "IfNotPresent".to_owned()); @@ -4770,11 +5121,14 @@ fn run_quick_scenarios( run_and_insert(&mut results, "overlay_acceptance", assert_overlay_acceptance); eprintln!(); - eprintln!("[SCENARIO {}] Verify local provider selection", scenario()); + eprintln!( + "[SCENARIO {}] Verify cross-site round-robin provider selection", + scenario() + ); run_and_insert( &mut results, - "local_provider_selection", - assert_local_provider_selection, + "round_robin_provider_selection", + assert_round_robin_provider_selection, ); eprintln!(); @@ -4841,13 +5195,31 @@ fn run_full_scenarios( n }; + let lifecycle_client = StableCurlClient::create("kind-grid-combined-central", "session-lifecycle-client"); + // Step 1: Establish session BEFORE drain eprintln!(); eprintln!("[SCENARIO {}] Establish session before drain", scenario()); - run_and_insert(&mut results, "session_establishment", assert_session_establishment); + match lifecycle_client { + Ok(client) => { + run_and_insert_with(&mut results, "session_establishment", || { + assert_session_establishment(&client) + }); - // Steps 2-5 are wrapped so restoration is guaranteed even if intermediate steps fail. - run_drain_session_restore_sequence(&mut results, &mut scenario); + // Steps 2-5 are wrapped so restoration is guaranteed even if intermediate steps fail. + run_drain_session_restore_sequence(&mut results, &mut scenario, &client); + }, + Err(error) => { + results.insert( + "session_establishment".to_owned(), + proof_failure( + &format!("failed to create stable lifecycle client: {error}"), + BTreeMap::new(), + Duration::ZERO, + ), + ); + }, + } // Provider lifecycle: add/remove/re-add a secondary mock provider run_provider_lifecycle_sequence( @@ -4901,34 +5273,56 @@ fn run_and_insert(results: &mut BTreeMap, name: &str, asser } } +/// Run a closure-backed assertion and insert its result. +fn run_and_insert_with(results: &mut BTreeMap, name: &str, assertion_fn: F) +where + F: FnOnce() -> AssertionResult, +{ + match run_assertion(name, assertion_fn) { + Ok(proof) => { + results.insert(name.to_owned(), proof); + }, + Err(e) => { + eprintln!(" [X] {name} failed: {e}"); + results.insert( + name.to_owned(), + proof_failure(&format!("{name} failed: {e}"), BTreeMap::new(), Duration::ZERO), + ); + }, + } +} + /// Execute the drain -> session tests -> restore sequence with guaranteed restoration. fn run_drain_session_restore_sequence( results: &mut BTreeMap, scenario: &mut dyn FnMut() -> usize, + client: &StableCurlClient, ) { // Step 2: Drain eprintln!(); eprintln!("[SCENARIO {}] Central drain and remote fallback", scenario()); - run_and_insert(results, "central_drain_fallback", assert_central_drain_fallback); + run_and_insert_with(results, "central_drain_fallback", || { + assert_central_drain_fallback(client) + }); // Step 3: Existing session after drain eprintln!(); eprintln!("[SCENARIO {}] Existing session survives drain", scenario()); - run_and_insert( - results, - "existing_session_after_drain", - assert_existing_session_after_drain, - ); + run_and_insert_with(results, "existing_session_after_drain", || { + assert_existing_session_after_drain(client) + }); // Step 4: New session after drain eprintln!(); eprintln!("[SCENARIO {}] New session after drain", scenario()); - run_and_insert(results, "new_session_after_drain", assert_new_session_after_drain); + run_and_insert_with(results, "new_session_after_drain", || { + assert_new_session_after_drain(client) + }); // Step 5: Restore (guaranteed) eprintln!(); eprintln!("[SCENARIO {}] Provider restoration", scenario()); - run_and_insert(results, "provider_restoration", assert_provider_restoration); + run_and_insert_with(results, "provider_restoration", || assert_provider_restoration(client)); if let Err(error) = ensure_central_provider_restored() { results.insert( @@ -5359,7 +5753,10 @@ spec: \x20 app.kubernetes.io/instance: {site}-secondary\n" ); - let routing_cluster = format!("vcr-{site}-provider"); + // Keep the lifecycle candidate's routing identity distinct from the + // primary provider. Stable IDs include routing identity, so reusing the + // primary cluster would make the two same-model candidates collide. + let routing_cluster = format!("vcr-{site}-provider-secondary"); let label = GRIDSITE_PROVIDER_LABEL; let inference_provider = format!( r#"apiVersion: grid.praxis-proxy.io/v1alpha1 @@ -5501,12 +5898,193 @@ fn append_secondary_mock_config( Ok(result) } +/// Add the dynamically-created secondary provider gateway to a consumer's +/// static hop and load-balancer configuration. +/// +/// The routing overlay can advertise a new candidate, but Praxis still needs +/// a corresponding named load-balancer cluster in its consumer config. Clone +/// the existing west gateway entry so its Forge-captured endpoint and TLS +/// settings remain authoritative; do not introduce a transient IP here. +fn append_secondary_consumer_config(config: &str) -> Result> { + let mut document: serde_yaml::Value = serde_yaml::from_str(config)?; + let chains = document + .get_mut("filter_chains") + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or("consumer config has no filter_chains sequence")?; + let main = chains + .iter_mut() + .find(|chain| chain.get("name").and_then(serde_yaml::Value::as_str) == Some("main")) + .ok_or("consumer config has no main filter chain")?; + let filters = main + .get_mut("filters") + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or("consumer main chain has no filters sequence")?; + + let mut added_hop = false; + let mut added_cluster = false; + for filter in filters { + let Some(filter_name) = filter.get("filter").and_then(serde_yaml::Value::as_str) else { + continue; + }; + if filter_name == "intelligent_route" { + let hops = filter + .get_mut("provider_hop_clusters") + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or("intelligent_route has no provider_hop_clusters sequence")?; + if !hops + .iter() + .any(|hop| hop.as_str() == Some("vcr-west-provider-secondary")) + { + hops.push(serde_yaml::Value::String("vcr-west-provider-secondary".to_owned())); + } + added_hop = true; + } else if filter_name == "load_balancer" { + let clusters = filter + .get_mut("clusters") + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or("load_balancer has no clusters sequence")?; + if clusters.iter().any(|cluster| { + cluster.get("name").and_then(serde_yaml::Value::as_str) == Some("vcr-west-provider-secondary") + }) { + added_cluster = true; + continue; + } + let west = clusters + .iter() + .find(|cluster| cluster.get("name").and_then(serde_yaml::Value::as_str) == Some("vcr-west-provider")) + .cloned() + .ok_or("load_balancer has no vcr-west-provider cluster to clone")?; + let mut secondary = west; + secondary + .as_mapping_mut() + .ok_or("vcr-west-provider cluster is not a YAML mapping")? + .insert( + serde_yaml::Value::String("name".to_owned()), + serde_yaml::Value::String("vcr-west-provider-secondary".to_owned()), + ); + clusters.push(secondary); + added_cluster = true; + } + } + + if !added_hop || !added_cluster { + return Err("consumer config missing intelligent_route or load_balancer secondary insertion point".into()); + } + Ok(serde_yaml::to_string(&document)?) +} + +/// Rematerialize all consumer configs after a provider is added and wait for +/// each consumer rollout before issuing requests against the new candidate. +fn rematerialize_consumers_for_secondary() -> Result<(), Box> { + for site in CLUSTERS { + let context = format!("kind-grid-combined-{site}"); + let current = kubectl::get_configmap_yaml(&context, GRID_SYSTEM_NS, "consumer-gateway-config")?; + let document: serde_yaml::Value = serde_yaml::from_str(¤t)?; + let config = document + .get("data") + .and_then(|data| data.get("praxis.yaml")) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| format!("{site}: consumer-gateway-config has no data.praxis.yaml"))?; + let rendered = append_secondary_consumer_config(config)?; + let create = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "create", + "configmap", + "consumer-gateway-config", + "--from-literal", + &format!("praxis.yaml={rendered}"), + "--dry-run=client", + "-o", + "yaml", + ]) + .output()?; + if !create.status.success() { + return Err(format!( + "{site}: failed to render secondary consumer config: {}", + String::from_utf8_lossy(&create.stderr) + ) + .into()); + } + kubectl::apply_manifest(&context, &String::from_utf8(create.stdout)?)?; + kubectl::rollout_restart_ns(&context, "consumer-gateway", GRID_SYSTEM_NS)?; + kubectl::wait_for_rollout_ns(&context, "consumer-gateway", GRID_SYSTEM_NS, site)?; + eprintln!(" [OK] {site}: consumer config rematerialized with secondary gateway cluster"); + } + Ok(()) +} + +/// Capture the exact consumer configuration before lifecycle mutation. +fn capture_consumer_configs() -> Result, Box> { + CLUSTERS + .iter() + .map(|site| { + let context = format!("kind-grid-combined-{site}"); + let current = kubectl::get_configmap_yaml(&context, GRID_SYSTEM_NS, "consumer-gateway-config")?; + let document: serde_yaml::Value = serde_yaml::from_str(¤t)?; + let config = document + .get("data") + .and_then(|data| data.get("praxis.yaml")) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| format!("{site}: consumer-gateway-config has no data.praxis.yaml"))?; + Ok(((*site).to_owned(), config.to_owned())) + }) + .collect() +} + +/// Restore captured consumer configurations exactly and only restart changed consumers. +fn restore_consumer_configs(snapshots: &BTreeMap) -> Result<(), Box> { + for site in CLUSTERS { + let Some(original) = snapshots.get(*site) else { continue }; + let context = format!("kind-grid-combined-{site}"); + let current = kubectl::get_configmap_yaml(&context, GRID_SYSTEM_NS, "consumer-gateway-config")?; + let document: serde_yaml::Value = serde_yaml::from_str(¤t)?; + let current_config = document + .get("data") + .and_then(|data| data.get("praxis.yaml")) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| format!("{site}: consumer-gateway-config has no data.praxis.yaml"))?; + if current_config == original { + continue; + } + let create = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "create", + "configmap", + "consumer-gateway-config", + "--from-literal", + &format!("praxis.yaml={original}"), + "--dry-run=client", + "-o", + "yaml", + ]) + .output()?; + if !create.status.success() { + return Err(format!( + "{site}: failed to render original consumer config: {}", + String::from_utf8_lossy(&create.stderr) + ) + .into()); + } + kubectl::apply_manifest(&context, &String::from_utf8(create.stdout)?)?; + kubectl::rollout_restart_ns(&context, "consumer-gateway", GRID_SYSTEM_NS)?; + kubectl::wait_for_rollout_ns(&context, "consumer-gateway", GRID_SYSTEM_NS, site)?; + eprintln!(" [OK] {site}: original consumer configuration restored"); + } + Ok(()) +} + /// Re-render and apply the provider gateway config for a single site. /// -/// The secondary provider shares the primary's routing cluster -/// (`routingClusterRef: vcr-{site}-provider`), so its stable_id is -/// looked up by model name in the candidates list rather than by cluster -/// key in `stable_ids`. +/// The secondary provider has its own routing cluster so its stable ID remains +/// distinct from the primary same-model candidate. fn rematerialize_site_provider_config( site: &str, external_provider: Option<&ExternalProviderDescriptor>, @@ -5542,11 +6120,7 @@ fn rematerialize_site_provider_config( } if include_secondary { - let secondary_id = overlay - .candidates - .iter() - .find(|c| c.name == SECONDARY_MODEL) - .map(|c| c.stable_id.clone()) + let secondary_id = additional_candidate_stable_id(&overlay, site, SECONDARY_MODEL, &primary_id) .ok_or_else(|| format!("{site}: secondary candidate {SECONDARY_MODEL} not in overlay candidates"))?; rendered = append_secondary_mock_config(&rendered, site, &secondary_id)?; } @@ -5589,24 +6163,41 @@ fn rematerialize_site_provider_config( clippy::disallowed_methods, reason = "Sleep is required for polling with timeout functionality" )] -fn wait_for_candidate_model_on_all_sites(model: &str, timeout: Duration) -> Result> { +fn wait_for_additional_candidate_on_all_sites( + site: &str, + model: &str, + primary_stable_id: &str, + timeout: Duration, +) -> Result> { let interval = Duration::from_secs(5); let start = Instant::now(); + let mut last_observation = String::from("no overlay observations"); while start.elapsed() < timeout { let mut ids: Vec = Vec::new(); let mut all_present = true; + let mut states = Vec::new(); for cluster in CLUSTERS { if let Ok(data) = read_cluster_overlay(cluster) { - if let Some(c) = data.candidates.iter().find(|c| c.name == model) { - ids.push(c.stable_id.clone()); + states.push(format!( + "{cluster}:{}", + data.candidates + .iter() + .map(|candidate| format!("{}@{}", candidate.name, candidate.stable_id)) + .collect::>() + .join(",") + )); + if let Some(stable_id) = additional_candidate_stable_id(&data, site, model, primary_stable_id) { + ids.push(stable_id); } else { all_present = false; } } else { + states.push(format!("{cluster}:unreadable")); all_present = false; } } + last_observation = format!("elapsed={:?}; {}", start.elapsed(), states.join("; ")); if all_present && ids.len() == CLUSTERS.len() && let Some(reference) = ids.first() @@ -5616,35 +6207,113 @@ fn wait_for_candidate_model_on_all_sites(model: &str, timeout: Duration) -> Resu } std::thread::sleep(interval); } - Err(format!("candidate model '{model}' not converged on all sites after {timeout:?}").into()) + Err(format!( + "additional candidate model '{model}' not converged on all sites after {timeout:?}; \ + primary_stable_id={primary_stable_id}; last observed state: {last_observation}" + ) + .into()) +} + +/// Return the stable ID of a lifecycle candidate, excluding the site's +/// already-present primary candidate. The same model name may legitimately be +/// advertised by multiple providers. +fn additional_candidate_stable_id( + overlay: &OverlayData, + site: &str, + model: &str, + primary_stable_id: &str, +) -> Option { + overlay + .candidates + .iter() + .find(|candidate| candidate.site == site && candidate.name == model && candidate.stable_id != primary_stable_id) + .map(|candidate| candidate.stable_id.clone()) } -/// Wait for a candidate (identified by model name) to be absent from all -/// sites' overlays. +/// Wait for a specific provider candidate to be absent from all sites' overlays. #[expect( clippy::disallowed_methods, reason = "Sleep is required for polling with timeout functionality" )] -fn wait_for_candidate_model_absent_all_sites(model: &str, timeout: Duration) -> Result<(), Box> { +fn wait_for_candidate_stable_id_absent_all_sites( + stable_id: &str, + timeout: Duration, +) -> Result<(), Box> { let interval = Duration::from_secs(5); let start = Instant::now(); + let mut last_observation = String::from("no overlay observations"); while start.elapsed() < timeout { let mut all_absent = true; + let mut states = Vec::new(); for cluster in CLUSTERS { - if let Ok(data) = read_cluster_overlay(cluster) - && data.candidates.iter().any(|c| c.name == model) - { - all_absent = false; + match read_cluster_overlay(cluster) { + Ok(data) => { + let matching = data.candidates.iter().filter(|c| c.stable_id == stable_id).count(); + states.push(format!("{cluster}:stable_id_matches={matching}")); + if matching > 0 { + all_absent = false; + } + }, + Err(error) => { + states.push(format!("{cluster}:unreadable={error}")); + all_absent = false; + }, + } + if !all_absent { break; } } + last_observation = format!("elapsed={:?}; {}", start.elapsed(), states.join("; ")); if all_absent { return Ok(()); } std::thread::sleep(interval); } - Err(format!("candidate model '{model}' still present after {timeout:?}").into()) + Err(format!( + "candidate stable_id '{stable_id}' still present after {timeout:?}; last observed state: {last_observation}" + ) + .into()) +} + +/// Wait for every candidate using a routing-cluster identity to disappear. +fn wait_for_routing_cluster_absent_all_sites( + routing_cluster: &str, + timeout: Duration, +) -> Result<(), Box> { + let interval = Duration::from_secs(5); + let start = Instant::now(); + let mut last_observation = String::from("no overlay observations"); + while start.elapsed() < timeout { + let mut absent = true; + let mut states = Vec::new(); + for cluster in CLUSTERS { + match read_cluster_overlay(cluster) { + Ok(data) => { + let matches = data + .candidates + .iter() + .filter(|candidate| candidate.cluster == routing_cluster) + .count(); + states.push(format!("{cluster}:routing_cluster_matches={matches}")); + absent &= matches == 0; + }, + Err(error) => { + absent = false; + states.push(format!("{cluster}:unreadable={error}")); + }, + } + } + last_observation = format!("elapsed={:?}; {}", start.elapsed(), states.join("; ")); + if absent { + return Ok(()); + } + std::thread::sleep(interval); + } + Err(format!( + "routing cluster '{routing_cluster}' still present after {timeout:?}; last observed state: {last_observation}" + ) + .into()) } /// Return whether an overlay contains the provider candidate for one site and model. @@ -5816,6 +6485,7 @@ fn provider_gateway_restart_args(context: &str) -> Vec<&str> { fn probe_secondary_model_with_retry( from_cluster: &str, expected_provider_site: &str, + expected_backend_cluster: &str, timeout: Duration, ) -> Result, Box> { let context = format!("kind-grid-combined-{from_cluster}"); @@ -5844,23 +6514,45 @@ fn probe_secondary_model_with_retry( ], ) { let provider_gw = response_header(&output.stdout, "x-grid-combined-provider-gateway"); + let backend_cluster = response_header(&output.stdout, "x-ai-inference-provider"); + let status = response_status(&output.stdout); let status_line = String::from_utf8_lossy(&output.stdout) .lines() .find(|line| line.starts_with("HTTP/")) .unwrap_or("missing HTTP status") .to_owned(); + let ai_gateway = response_header(&output.stdout, "x-ai-demo-provider-gateway"); + let gateway_matches = provider_gw.as_deref() == Some(expected_provider_site) && ai_gateway == provider_gw; + let backend_matches = backend_cluster.as_deref() == Some(expected_backend_cluster); last_observation = format!( - "exit_success={}, status={status_line}, provider_gateway={}", + "exit_success={}, status={status_line}, provider_gateway={}, ai_gateway={}, backend_cluster={}", output.status.success(), provider_gw.as_deref().unwrap_or("missing"), + ai_gateway.as_deref().unwrap_or("missing"), + backend_cluster.as_deref().unwrap_or("missing"), ); - if output.status.success() - && status_line.contains(" 200 ") - && provider_gw.as_deref() == Some(expected_provider_site) - { + if output.status.success() && status == Some(200) && gateway_matches && backend_matches { let mut facts = BTreeMap::new(); facts.insert("from_cluster".to_owned(), serde_json::json!(from_cluster)); + facts.insert("status".to_owned(), serde_json::json!(status)); facts.insert("provider_gateway".to_owned(), serde_json::json!(provider_gw)); + facts.insert("ai_provider_gateway".to_owned(), serde_json::json!(ai_gateway)); + facts.insert( + "expected_provider_site".to_owned(), + serde_json::json!(expected_provider_site), + ); + facts.insert( + "expected_backend_cluster".to_owned(), + serde_json::json!(expected_backend_cluster), + ); + facts.insert( + "gateway_attribution_matches".to_owned(), + serde_json::json!(gateway_matches), + ); + facts.insert( + "backend_attribution_matches".to_owned(), + serde_json::json!(backend_matches), + ); return Ok(facts); } } @@ -5893,15 +6585,29 @@ fn lifecycle_add_provider( let start = Instant::now(); let mut facts = BTreeMap::new(); + let primary_stable_id = read_cluster_overlay(site)? + .candidates + .iter() + .find(|candidate| candidate.site == site && candidate.name == PRIMARY_MODEL) + .map(|candidate| candidate.stable_id.clone()) + .ok_or_else(|| format!("{site}: primary candidate not present before secondary addition"))?; + deploy_secondary_mock_provider(site)?; - let stable_id = wait_for_candidate_model_on_all_sites(SECONDARY_MODEL, Duration::from_secs(120))?; + let stable_id = wait_for_additional_candidate_on_all_sites( + site, + SECONDARY_MODEL, + &primary_stable_id, + Duration::from_secs(120), + )?; facts.insert("secondary_stable_id".to_owned(), serde_json::json!(stable_id)); rematerialize_site_provider_config(site, external_provider, external_site, true, demo_root)?; + rematerialize_consumers_for_secondary()?; apply_provider_gateway_stack(forge_bin, resolved_config, site, external_site)?; - let probe_facts = probe_secondary_model_with_retry(site, site, COMBINED_SITE_DATA_PLANE_WAIT)?; + let probe_facts = + probe_secondary_model_with_retry(site, site, "vcr-backend-secondary", COMBINED_SITE_DATA_PLANE_WAIT)?; facts.extend(probe_facts); let primary_ctx = format!("kind-grid-combined-{site}"); @@ -5942,11 +6648,23 @@ fn lifecycle_assert_global_convergence(site: &str) -> AssertionResult { let start = Instant::now(); let mut facts = BTreeMap::new(); - let stable_id = wait_for_candidate_model_on_all_sites(SECONDARY_MODEL, Duration::from_secs(180))?; + let primary_stable_id = read_cluster_overlay(site)? + .candidates + .iter() + .find(|candidate| candidate.site == site && candidate.name == PRIMARY_MODEL) + .map(|candidate| candidate.stable_id.clone()) + .ok_or_else(|| format!("{site}: primary candidate not present during lifecycle convergence"))?; + let stable_id = wait_for_additional_candidate_on_all_sites( + site, + SECONDARY_MODEL, + &primary_stable_id, + Duration::from_secs(180), + )?; facts.insert("global_stable_id".to_owned(), serde_json::json!(stable_id)); for cluster in CLUSTERS { - let probe_facts = probe_secondary_model_with_retry(cluster, site, COMBINED_SITE_DATA_PLANE_WAIT)?; + let probe_facts = + probe_secondary_model_with_retry(cluster, site, "vcr-backend-secondary", COMBINED_SITE_DATA_PLANE_WAIT)?; facts.insert(format!("{cluster}_probe"), serde_json::json!(probe_facts)); } @@ -5965,6 +6683,7 @@ fn lifecycle_assert_global_convergence(site: &str) -> AssertionResult { )] fn lifecycle_remove_provider( site: &str, + secondary_stable_id: &str, external_provider: Option<&ExternalProviderDescriptor>, external_site: Option<&str>, forge_bin: &Path, @@ -5976,7 +6695,7 @@ fn lifecycle_remove_provider( remove_secondary_mock_provider(site)?; - wait_for_candidate_model_absent_all_sites(SECONDARY_MODEL, Duration::from_secs(180))?; + wait_for_candidate_stable_id_absent_all_sites(secondary_stable_id, Duration::from_secs(180))?; facts.insert("candidate_drained".to_owned(), serde_json::json!(true)); rematerialize_site_provider_config(site, external_provider, external_site, false, demo_root)?; @@ -6017,8 +6736,12 @@ fn lifecycle_remove_provider( )) } -/// Verify the secondary model is no longer routable from any consumer. -fn lifecycle_assert_unroutable() -> AssertionResult { +/// Verify new traffic never selects the withdrawn candidate. +/// +/// The secondary intentionally shares the primary model, so a successful +/// response is expected after removal. The assertion must inspect trusted +/// backend attribution rather than treating the shared model as unroutable. +fn lifecycle_assert_removed_candidate_not_selected() -> AssertionResult { let start = Instant::now(); let mut facts = BTreeMap::new(); @@ -6043,13 +6766,24 @@ fn lifecycle_assert_unroutable() -> AssertionResult { ], )?; let stdout = String::from_utf8_lossy(&output.stdout); - let status_line = stdout.lines().next().unwrap_or("").to_owned(); - let got_200 = status_line.contains("200"); + let status = response_status(&output.stdout); + let status_line = stdout + .lines() + .find(|line| line.starts_with("HTTP/")) + .unwrap_or("") + .to_owned(); + let gateway = response_header(&output.stdout, "x-grid-combined-provider-gateway"); + let ai_gateway = response_header(&output.stdout, "x-ai-demo-provider-gateway"); + let backend = response_header(&output.stdout, "x-ai-inference-provider"); + let trusted_primary = primary_response_is_trusted(&output.stdout); facts.insert(format!("{cluster}_status"), serde_json::json!(status_line)); - facts.insert(format!("{cluster}_got_200"), serde_json::json!(got_200)); - if got_200 { + facts.insert(format!("{cluster}_status_code"), serde_json::json!(status)); + facts.insert(format!("{cluster}_provider_gateway"), serde_json::json!(gateway)); + facts.insert(format!("{cluster}_ai_provider_gateway"), serde_json::json!(ai_gateway)); + facts.insert(format!("{cluster}_backend"), serde_json::json!(backend)); + if !trusted_primary || backend.as_deref() == Some("vcr-backend-secondary") { return Ok(proof_failure( - &format!("removed secondary model still routable from {cluster}"), + &format!("withdrawn secondary response lacked trusted primary attribution from {cluster}"), facts, start.elapsed(), )); @@ -6057,7 +6791,7 @@ fn lifecycle_assert_unroutable() -> AssertionResult { } Ok(proof_success( - "secondary model correctly unroutable from all consumers after removal", + "withdrawn secondary candidate was absent from new traffic; shared model remained routable", facts, start.elapsed(), )) @@ -6072,6 +6806,8 @@ fn lifecycle_assert_unroutable() -> AssertionResult { fn lifecycle_cleanup( results: &mut BTreeMap, site: &str, + secondary_stable_id: Option<&str>, + consumer_snapshots: &BTreeMap, external_provider: Option<&ExternalProviderDescriptor>, external_site: Option<&str>, forge_bin: &Path, @@ -6084,9 +6820,13 @@ fn lifecycle_cleanup( let cleanup_result = (|| -> Result<(), Box> { remove_secondary_mock_provider(site)?; - wait_for_candidate_model_absent_all_sites(SECONDARY_MODEL, Duration::from_secs(180))?; + if let Some(stable_id) = secondary_stable_id { + wait_for_candidate_stable_id_absent_all_sites(stable_id, Duration::from_secs(180))?; + } + wait_for_routing_cluster_absent_all_sites("vcr-west-provider-secondary", Duration::from_secs(180))?; rematerialize_site_provider_config(site, external_provider, external_site, false, demo_root)?; apply_provider_gateway_stack(forge_bin, resolved_config, site, external_site)?; + restore_consumer_configs(consumer_snapshots)?; let ctx = format!("kind-grid-combined-{site}"); let output = run_curl_probe( @@ -6153,6 +6893,21 @@ fn run_provider_lifecycle_sequence( eprintln!("=== PROVIDER LIFECYCLE (add/remove/re-add on {site}) ==="); eprintln!(); + let consumer_snapshots = match capture_consumer_configs() { + Ok(snapshots) => snapshots, + Err(error) => { + results.insert( + "provider_addition".to_owned(), + proof_failure( + &format!("could not snapshot consumer configurations: {error}"), + BTreeMap::new(), + Duration::ZERO, + ), + ); + return; + }, + }; + let baseline_deployments = record_gateway_deployment_state(); let skip = |proof_results: &mut BTreeMap, name: &str, blocker: &str| { @@ -6199,7 +6954,7 @@ fn run_provider_lifecycle_sequence( for name in [ "provider_global_convergence", "provider_removal", - "removed_provider_unroutable", + "removed_provider_not_selected", "provider_readdition", "stable_id_determinism", "gateway_restart_boundary", @@ -6210,6 +6965,8 @@ fn run_provider_lifecycle_sequence( lifecycle_cleanup( results, site, + None, + &consumer_snapshots, external_provider, external_site, forge_bin, @@ -6240,8 +6997,12 @@ fn run_provider_lifecycle_sequence( eprintln!(); eprintln!("[SCENARIO {}] Provider removal", scenario()); let removal_ok = match run_assertion("provider_removal", || { + let Some(secondary_stable_id) = add_stable_id.as_deref() else { + return Err("provider addition did not produce a secondary stable ID".into()); + }; lifecycle_remove_provider( site, + secondary_stable_id, external_provider, external_site, forge_bin, @@ -6265,7 +7026,7 @@ fn run_provider_lifecycle_sequence( if !removal_ok { for name in [ - "removed_provider_unroutable", + "removed_provider_not_selected", "provider_readdition", "stable_id_determinism", "gateway_restart_boundary", @@ -6276,6 +7037,8 @@ fn run_provider_lifecycle_sequence( lifecycle_cleanup( results, site, + add_stable_id.as_deref(), + &consumer_snapshots, external_provider, external_site, forge_bin, @@ -6288,7 +7051,11 @@ fn run_provider_lifecycle_sequence( // --- Step 4: Removed provider unroutable --- eprintln!(); eprintln!("[SCENARIO {}] Removed provider unroutable", scenario()); - run_and_insert(results, "removed_provider_unroutable", lifecycle_assert_unroutable); + run_and_insert( + results, + "removed_provider_not_selected", + lifecycle_assert_removed_candidate_not_selected, + ); // --- Step 5: Provider re-addition --- eprintln!(); @@ -6351,19 +7118,19 @@ fn run_provider_lifecycle_sequence( for cluster in CLUSTERS { let cg_key = format!("{cluster}/consumer-gateway"); - let cg_same = matches!( + let cg_changed = matches!( (baseline_deployments.get(&cg_key), post_deployments.get(&cg_key)), - (Some((b, _)), Some((p, _))) if b == p + (Some((b, _)), Some((p, _))) if b != p ); facts.insert( format!("{cluster}_consumer_gw"), - serde_json::json!(if cg_same { - "unchanged (overlay hot-reload)" + serde_json::json!(if cg_changed { + "rolled (static secondary gateway cluster added)" } else { - "RESTARTED (unexpected)" + "unchanged (unexpected: secondary gateway cluster not loaded)" }), ); - if !cg_same { + if !cg_changed { boundary_ok = false; } @@ -6413,7 +7180,7 @@ fn run_provider_lifecycle_sequence( "gateway_restart_boundary".to_owned(), if boundary_ok { proof_success( - "consumer gateways hot-reloaded overlay without restart; \ + "consumer gateways rolled to load the lifecycle gateway cluster; \ non-lifecycle provider gateways and all operators unchanged; \ lifecycle-site provider gateway rolled for static config change", facts, @@ -6442,6 +7209,8 @@ fn run_provider_lifecycle_sequence( lifecycle_cleanup( results, site, + readd_stable_id.as_deref(), + &consumer_snapshots, external_provider, external_site, forge_bin, @@ -6455,6 +7224,33 @@ fn teardown_environment(context: &CombinedSiteContext) -> Result<(), Box Result<(), Box String { + format!("grid-combined-{site}") +} + +/// Return the kubectl context corresponding to a Forge site. +fn combined_kubectl_context(site: &str) -> String { + format!("kind-{}", combined_kind_cluster_name(site)) +} + +/// Check whether a named Kind cluster exists, using a bounded command. +fn kind_cluster_exists(cluster_name: &str) -> Result> { + let listed = Command::new("timeout") + .args(["30s", "kind", "get", "clusters"]) + .output()?; + if !listed.status.success() { + return Err(format!("kind get clusters failed: {}", String::from_utf8_lossy(&listed.stderr)).into()); + } + Ok(String::from_utf8_lossy(&listed.stdout) + .lines() + .any(|line| line.trim() == cluster_name)) +} + // ----------------------------------------------------------------------------- // Main Entry Point // ----------------------------------------------------------------------------- @@ -6762,6 +7581,47 @@ mod tests { assert!(!overlay_has_site_model(&overlay, "west", "nonexistent-model")); } + #[test] + fn additional_candidate_ignores_primary_and_other_sites() { + let overlay = OverlayData { + resource_version: "1".to_owned(), + semantic_revision: "revision".to_owned(), + stable_ids: BTreeMap::new(), + candidates: vec![ + OverlayCandidate { + kind: "inference_model".to_owned(), + name: PRIMARY_MODEL.to_owned(), + site: "west".to_owned(), + cluster: "vcr-west-provider".to_owned(), + stable_id: "primary-west".to_owned(), + }, + OverlayCandidate { + kind: "inference_model".to_owned(), + name: PRIMARY_MODEL.to_owned(), + site: "central".to_owned(), + cluster: "vcr-central-provider".to_owned(), + stable_id: "primary-central".to_owned(), + }, + OverlayCandidate { + kind: "inference_model".to_owned(), + name: PRIMARY_MODEL.to_owned(), + site: "west".to_owned(), + cluster: "vcr-west-provider".to_owned(), + stable_id: "secondary-west".to_owned(), + }, + ], + }; + + assert_eq!( + additional_candidate_stable_id(&overlay, "west", PRIMARY_MODEL, "primary-west").as_deref(), + Some("secondary-west") + ); + assert_eq!( + additional_candidate_stable_id(&overlay, "central", PRIMARY_MODEL, "primary-central"), + None + ); + } + #[test] fn proof_success_creation() { let mut facts = BTreeMap::new(); @@ -6863,7 +7723,7 @@ mod tests { "swim_convergence", "external_provider_absence", "overlay_acceptance", - "local_provider_selection", + "round_robin_provider_selection", "response_attribution", "tls_certificate_validation", "authorization_replacement", @@ -6882,6 +7742,59 @@ mod tests { assert_eq!(CLUSTERS, &["west", "central", "east"]); } + fn test_overlay(sites: &[&str]) -> OverlayData { + OverlayData { + resource_version: "1".to_owned(), + semantic_revision: "test-revision".to_owned(), + stable_ids: BTreeMap::new(), + candidates: sites + .iter() + .enumerate() + .map(|(index, site)| OverlayCandidate { + kind: "inference_model".to_owned(), + name: format!("provider-{site}"), + site: (*site).to_owned(), + cluster: format!("cluster-{index}"), + stable_id: format!("stable-{index}"), + }) + .collect(), + } + } + + #[test] + fn accepted_provider_sites_come_from_overlay_candidates() { + let overlay = test_overlay(&["west", "central", "east", "west"]); + assert_eq!( + accepted_provider_sites(&overlay), + BTreeSet::from(["central".to_owned(), "east".to_owned(), "west".to_owned()]) + ); + } + + #[test] + fn round_robin_sequence_accepts_any_repeating_cycle() { + let eligible = BTreeSet::from(["central".to_owned(), "east".to_owned(), "west".to_owned()]); + let valid = ["east", "west", "central", "east", "west", "central"] + .into_iter() + .map(str::to_owned) + .collect::>(); + let invalid = ["east", "west", "east", "west", "east", "west"] + .into_iter() + .map(str::to_owned) + .collect::>(); + assert!(is_round_robin_sequence(&valid, &eligible)); + assert!(!is_round_robin_sequence(&invalid, &eligible)); + } + + #[test] + fn round_robin_sequence_rejects_unknown_provider() { + let eligible = BTreeSet::from(["west".to_owned(), "east".to_owned()]); + let sequence = ["west", "east", "unknown", "west"] + .into_iter() + .map(str::to_owned) + .collect::>(); + assert!(!is_round_robin_sequence(&sequence, &eligible)); + } + #[test] fn evidence_schema_version() { assert_eq!(EVIDENCE_SCHEMA_VERSION, "1"); @@ -6917,6 +7830,99 @@ mod tests { ); } + #[test] + fn curl_client_overrides_are_restricted_and_long_lived() { + let actual: serde_json::Value = + serde_json::from_str(&curl_client_pod_overrides("stable-client")).unwrap_or_else(|_| std::process::abort()); + assert_eq!( + actual + .pointer("/spec/securityContext/runAsNonRoot") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + actual + .pointer("/spec/securityContext/seccompProfile/type") + .and_then(serde_json::Value::as_str), + Some("RuntimeDefault") + ); + assert_eq!( + actual + .pointer("/spec/containers/0/command") + .and_then(serde_json::Value::as_array), + serde_json::json!(["sleep", STABLE_CLIENT_LIFETIME_SECS]).as_array() + ); + assert_eq!( + actual + .pointer("/spec/containers/0/securityContext/runAsUser") + .and_then(serde_json::Value::as_u64), + Some(100) + ); + assert_eq!( + actual + .pointer("/spec/containers/0/securityContext/allowPrivilegeEscalation") + .and_then(serde_json::Value::as_bool), + Some(false) + ); + } + + #[test] + fn client_names_are_unique_and_dns_safe() { + let first = unique_client_name("Rollout Client/West"); + let second = unique_client_name("Rollout Client/West"); + assert_ne!(first, second); + assert!(first.len() <= 42); + assert!( + first + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + ); + } + + #[test] + fn kind_cluster_and_context_names_are_distinct() { + assert_eq!(combined_kind_cluster_name("west"), "grid-combined-west"); + assert_eq!(combined_kubectl_context("west"), "kind-grid-combined-west"); + assert_ne!(combined_kind_cluster_name("west"), combined_kubectl_context("west")); + } + + #[test] + fn response_status_ignores_kubectl_lifecycle_text() { + let output = b"pod/probe created\nHTTP/1.1 401 Unauthorized\ncontent-type: text/plain\npod \"probe\" deleted\n"; + assert_eq!(response_status(output), Some(401)); + } + + #[test] + fn primary_attribution_requires_all_trusted_headers() { + let valid = b"HTTP/1.1 200 OK\r\nx-grid-combined-provider-gateway: west\r\nx-ai-demo-provider-gateway: west\r\nx-ai-inference-provider: vcr-backend\r\n"; + let missing = + b"HTTP/1.1 200 OK\r\nx-grid-combined-provider-gateway: west\r\nx-ai-inference-provider: vcr-backend\r\n"; + assert!(primary_response_is_trusted(valid)); + assert!(!primary_response_is_trusted(missing)); + } + + #[test] + fn response_status_reads_only_http_status_lines() { + let output = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\nHTTP/1.1 503 body text"; + assert_eq!(response_status(output), Some(200)); + assert_eq!(response_status(b"curl: (28) timeout"), None); + } + + #[test] + fn provider_attribution_requires_two_matching_trusted_headers() { + let output = + b"HTTP/1.1 200 OK\r\nx-grid-combined-provider-gateway: central\r\nx-ai-demo-provider-gateway: central\r\n"; + assert_eq!(response_status(output), Some(200)); + assert_eq!( + response_header(output, "x-grid-combined-provider-gateway").as_deref(), + Some("central") + ); + assert_eq!( + response_header(output, "x-ai-demo-provider-gateway").as_deref(), + Some("central") + ); + } + fn minimal_forge_yaml() -> String { r#"apiVersion: forge.praxis.dev/v1alpha1 kind: Environment @@ -7247,6 +8253,51 @@ spec: assert!(with_both.contains("Qwen/Qwen3-0.6B"), "primary route must be preserved",); } + #[test] + #[expect(clippy::indexing_slicing, reason = "test assertions on the fixed-shape YAML fixture")] + fn append_secondary_consumer_config_clones_forge_gateway_settings() { + let config = r#" +filter_chains: + - name: main + filters: + - filter: intelligent_route + provider_hop_clusters: [vcr-west-provider] + - filter: load_balancer + clusters: + - name: vcr-west-provider + endpoints: ["captured-ip:8443"] + tls: + sni: west.grid.internal +admin: + address: 127.0.0.1:9901 +"#; + let rendered = append_secondary_consumer_config(config).unwrap(); + let value: serde_yaml::Value = serde_yaml::from_str(&rendered).unwrap(); + let filters = &value["filter_chains"][0]["filters"]; + assert_eq!(filters[0]["provider_hop_clusters"].as_sequence().unwrap().len(), 2); + let clusters = filters[1]["clusters"].as_sequence().unwrap(); + assert_eq!(clusters.len(), 2); + assert_eq!(clusters[1]["name"].as_str(), Some("vcr-west-provider-secondary")); + assert_eq!(clusters[1]["endpoints"][0].as_str(), Some("captured-ip:8443")); + assert_eq!(clusters[1]["tls"]["sni"].as_str(), Some("west.grid.internal")); + } + + #[test] + fn ai_revision_from_logs_reads_latest_exact_field() { + let logs = r#" +overlay reloaded accepted_revision=old serving_revision=old +overlay reloaded accepted_revision="new" serving_revision="new" previous_serving_revision="old" +"#; + assert_eq!(ai_revision_from_logs(logs, "accepted_revision"), Some("new".to_owned())); + assert_eq!(ai_revision_from_logs(logs, "serving_revision"), Some("new".to_owned())); + } + + #[test] + fn ai_revision_from_logs_does_not_match_previous_serving_revision() { + let logs = "overlay reloaded previous_serving_revision=old"; + assert_eq!(ai_revision_from_logs(logs, "serving_revision"), None); + } + #[test] fn provider_config_rematerialization_requires_gateway_restart() { assert_eq!( diff --git a/xtask/src/env/forge_config.rs b/xtask/src/env/forge_config.rs index 3ca72dbf..54f95806 100644 --- a/xtask/src/env/forge_config.rs +++ b/xtask/src/env/forge_config.rs @@ -9,9 +9,47 @@ use super::image_overrides; /// Render a Forge environment with the explicitly selected demo images. pub(crate) fn materialize(source: &Path, output: Option<&Path>) -> Result> { + let images = ImageOverrides { + gateway: image_overrides::gateway_image(), + operator: image_overrides::operator_image(), + overlay_sync: image_overrides::overlay_sync_image(), + vcr: image_overrides::vcr_image(), + pull_policy: image_overrides::image_pull_policy(), + }; + if images.pull_policy == "Never" + && (std::env::var_os("GRID_XTASK_GATEWAY_IMAGE").is_none() + || std::env::var_os("GRID_XTASK_OPERATOR_IMAGE").is_none() + || std::env::var_os("GRID_XTASK_OVERLAY_SYNC_IMAGE").is_none()) + { + return Err("GRID_XTASK_GATEWAY_IMAGE, GRID_XTASK_OPERATOR_IMAGE, and GRID_XTASK_OVERLAY_SYNC_IMAGE are required when GRID_XTASK_IMAGE_PULL_POLICY=Never".into()); + } + materialize_with_images(source, output, &images) +} + +/// Image values to inject into a Forge configuration. +#[derive(Debug, Clone)] +pub(crate) struct ImageOverrides { + /// Gateway image reference. + pub(crate) gateway: String, + /// Grid operator image reference. + pub(crate) operator: String, + /// Overlay-sync image reference. + pub(crate) overlay_sync: String, + /// VCR image reference. + pub(crate) vcr: String, + /// Kubernetes image pull policy. + pub(crate) pull_policy: String, +} + +/// Render a Forge environment with an explicit image set. +pub(crate) fn materialize_with_images( + source: &Path, + output: Option<&Path>, + images: &ImageOverrides, +) -> Result> { let content = fs::read_to_string(source)?; let mut config: serde_yaml::Value = serde_yaml::from_str(&content)?; - apply_image_overrides(&mut config)?; + apply_image_values(&mut config, images)?; let destination = output.map_or_else( || { source.with_file_name(format!( @@ -25,25 +63,20 @@ pub(crate) fn materialize(source: &Path, output: Option<&Path>) -> Result Result<(), Box> { - let pull_policy = image_overrides::image_pull_policy(); - let gateway = image_overrides::gateway_image(); - let operator = image_overrides::operator_image(); - let overlay_sync = image_overrides::overlay_sync_image(); - let vcr = image_overrides::vcr_image(); - - if pull_policy == "Never" - && (std::env::var_os("GRID_XTASK_GATEWAY_IMAGE").is_none() - || std::env::var_os("GRID_XTASK_OPERATOR_IMAGE").is_none() - || std::env::var_os("GRID_XTASK_OVERLAY_SYNC_IMAGE").is_none()) - { - return Err("GRID_XTASK_GATEWAY_IMAGE, GRID_XTASK_OPERATOR_IMAGE, and GRID_XTASK_OVERLAY_SYNC_IMAGE are required when GRID_XTASK_IMAGE_PULL_POLICY=Never".into()); - } +fn apply_image_values( + config: &mut serde_yaml::Value, + images: &ImageOverrides, +) -> Result<(), Box> { + let pull_policy = images.pull_policy.clone(); + let gateway = images.gateway.clone(); + let operator = images.operator.clone(); + let overlay_sync = images.overlay_sync.clone(); + let vcr = images.vcr.clone(); let (gateway_repo, gateway_tag) = parse_image_ref(&gateway); let (operator_repo, operator_tag) = parse_image_ref(&operator); diff --git a/xtask/src/env/glb.rs b/xtask/src/env/glb.rs index 9aa977c0..3d689225 100644 --- a/xtask/src/env/glb.rs +++ b/xtask/src/env/glb.rs @@ -1875,6 +1875,14 @@ fn check_one_backend_network_boundary( target, mode, )?; + if allowed.phase != "Running" && allowed.phase != "Succeeded" { + return Err(format!( + "{provider}/{instance}: allowed probe did not reach Running; phase={}, diagnostics={}", + allowed.phase, + safe_truncate_str(&allowed.diagnostics, 512) + ) + .into()); + } if allowed.phase != "Succeeded" || !allowed.logs.contains("tcp-probe=connected") { return Err(format!( "{provider}/{instance}: allowed backend probe did not connect (phase={}, logs={})", @@ -1885,6 +1893,14 @@ fn check_one_backend_network_boundary( } let denied = run_probe_pod(context, &denied_name, None, target, mode)?; + if denied.phase != "Running" && denied.phase != "Succeeded" && denied.phase != "Failed" { + return Err(format!( + "{provider}/{instance}: unlabeled probe did not reach Running; this is not NetworkPolicy evidence; phase={}, diagnostics={}", + denied.phase, + safe_truncate_str(&denied.diagnostics, 512) + ) + .into()); + } if denied.phase != "Failed" || !(denied.logs.contains("tcp-probe=timeout") || denied.logs.contains("tcp-probe=connect-failed")) { @@ -1905,6 +1921,57 @@ struct NetworkPolicyProbe { phase: String, /// Bounded probe output. logs: String, + /// Pod status and scheduling events captured when the probe cannot finish. + diagnostics: String, +} + +/// Capture bounded pod status and scheduling events for a non-terminal probe. +#[expect( + clippy::too_many_lines, + reason = "pod status and event diagnostics are captured as one bounded evidence record" +)] +fn probe_diagnostics(context: &str, name: &str) -> String { + let status = Command::new("timeout") + .args([ + "15s", + "kubectl", + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "pod", + name, + "-o", + "json", + ]) + .output(); + let events = Command::new("timeout") + .args([ + "15s", + "kubectl", + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + "events", + "--field-selector", + &format!("involvedObject.name={name}"), + "-o", + "wide", + ]) + .output(); + let render = |output: Result| match output { + Ok(output) => format!( + "stdout={} stderr={} status={}", + safe_truncate_str(String::from_utf8_lossy(&output.stdout).trim(), 1200), + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 600), + output.status + ), + Err(error) => format!("command error={error}"), + }; + format!("pod={}; events={}", render(status), render(events)) } /// Delete a fixed-name probe pod without failing cleanup. @@ -1951,6 +2018,7 @@ fn run_probe_pod( // command. VCR replaces that image, so use a small public curl image and // preserve the same terminal evidence contract for the policy assertion. let probe_image = "curlimages/curl:8.10.1"; + let probe_script = "if curl -sS --connect-timeout 2 --max-time 5 -o /dev/null -w '%{http_code}' \"http://$1\" >/dev/null 2>&1; then echo tcp-probe=connected; exit 0; else echo tcp-probe=connect-failed; exit 1; fi"; command.args([ "--context", context, @@ -1965,17 +2033,31 @@ fn run_probe_pod( ), "--restart=Never", ]); + let overrides = serde_json::json!({ + "apiVersion": "v1", + "spec": { + "securityContext": { + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": {"type": "RuntimeDefault"} + }, + "containers": [{ + "name": name, + "image": probe_image, + "imagePullPolicy": crate::env::image_overrides::demo_image_pull_policy(mode), + "command": ["sh", "-c", probe_script], + "args": ["curl-probe", target], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]} + } + }] + } + }); + command.args(["--overrides", &overrides.to_string()]); if let Some(value) = labels { command.arg(format!("--labels={value}")); } - command.args([ - "--", - "sh", - "-c", - "if curl -sS --connect-timeout 2 --max-time 5 -o /dev/null -w '%{http_code}' \"http://$1\" >/dev/null 2>&1; then echo tcp-probe=connected; exit 0; else echo tcp-probe=connect-failed; exit 1; fi", - "curl-probe", - target, - ]); let output = command.output()?; if !output.status.success() { return Err(format!( @@ -1992,7 +2074,11 @@ fn run_probe_pod( break phase; } if Instant::now() >= deadline { - return Err(format!("probe pod {name} did not finish; last phase={phase}").into()); + return Ok(NetworkPolicyProbe { + phase, + logs: String::new(), + diagnostics: probe_diagnostics(context, name), + }); } thread::sleep(Duration::from_millis(250)); }; @@ -2008,6 +2094,7 @@ fn run_probe_pod( Ok(NetworkPolicyProbe { phase, logs: safe_truncate_str(logs.trim(), 512), + diagnostics: String::new(), }) } diff --git a/xtask/src/env/glb_demo.rs b/xtask/src/env/glb_demo.rs index e1c98ac1..53c5710b 100644 --- a/xtask/src/env/glb_demo.rs +++ b/xtask/src/env/glb_demo.rs @@ -2234,7 +2234,8 @@ fn load_local_images_if_required( let operator = image_overrides::demo_operator_image(ingress_mode); let gateway = image_overrides::demo_gateway_image(ingress_mode); let vcr = image_overrides::vcr_image(); - for image in [&operator, &gateway, &vcr] { + let probe = "curlimages/curl:8.10.1"; + for image in [&operator, &gateway, &vcr, &probe.to_owned()] { require_local_image(image)?; } let gateway_clusters = match ingress_mode { @@ -2249,6 +2250,7 @@ fn load_local_images_if_required( } for cluster in PROVIDER_CLUSTERS { run_forge(forge, config, &["cluster", "load-image", cluster, &vcr])?; + run_forge(forge, config, &["cluster", "load-image", cluster, probe])?; } Ok(()) } diff --git a/xtask/src/env/image_overrides.rs b/xtask/src/env/image_overrides.rs index 9c88a415..36283501 100644 --- a/xtask/src/env/image_overrides.rs +++ b/xtask/src/env/image_overrides.rs @@ -55,19 +55,19 @@ const DEFAULT_OPERATOR_IMAGE: &str = "grid-operator:latest"; const DEFAULT_OVERLAY_SYNC_IMAGE: &str = "grid-overlay-sync:latest"; /// Default gateway image used by the GLB demo. -const DEFAULT_GLB_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3"; +const DEFAULT_GLB_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/ai:0.3.0"; /// Default VCR image used by forge-based demos. const DEFAULT_VCR_IMAGE: &str = "ghcr.io/neuralmagic/vllm-vcr:vllm0.23"; /// Default operator image used by the GLB demo. -const DEFAULT_GLB_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.3"; +const DEFAULT_GLB_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.4"; /// Default gateway image for workload-inference demos. -const DEFAULT_WORKLOAD_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3"; +const DEFAULT_WORKLOAD_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/ai:0.3.0"; /// Default operator image for workload-inference demos. -const DEFAULT_WORKLOAD_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.3"; +const DEFAULT_WORKLOAD_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.4"; /// Default image pull policy for workload-inference demos (registry-backed). const DEFAULT_WORKLOAD_IMAGE_PULL_POLICY: &str = "IfNotPresent"; @@ -188,16 +188,13 @@ mod tests { assert_eq!(DEFAULT_MOCK_EPP_IMAGE, "localhost/praxis-ai-mock-epp:latest"); assert_eq!(DEFAULT_OPERATOR_IMAGE, "grid-operator:latest"); assert_eq!(DEFAULT_OVERLAY_SYNC_IMAGE, "grid-overlay-sync:latest"); - assert_eq!(DEFAULT_GLB_GATEWAY_IMAGE, "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3"); - assert_eq!(DEFAULT_GLB_OPERATOR_IMAGE, "ghcr.io/praxis-proxy/grid-operator:v0.1.3"); + assert_eq!(DEFAULT_GLB_GATEWAY_IMAGE, "ghcr.io/praxis-proxy/ai:0.3.0"); + assert_eq!(DEFAULT_GLB_OPERATOR_IMAGE, "ghcr.io/praxis-proxy/grid-operator:v0.1.4"); assert_eq!(DEFAULT_IMAGE_PULL_POLICY, "Never"); - assert_eq!( - DEFAULT_WORKLOAD_GATEWAY_IMAGE, - "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" - ); + assert_eq!(DEFAULT_WORKLOAD_GATEWAY_IMAGE, "ghcr.io/praxis-proxy/ai:0.3.0"); assert_eq!( DEFAULT_WORKLOAD_OPERATOR_IMAGE, - "ghcr.io/praxis-proxy/grid-operator:v0.1.3" + "ghcr.io/praxis-proxy/grid-operator:v0.1.4" ); assert_eq!(DEFAULT_WORKLOAD_IMAGE_PULL_POLICY, "IfNotPresent"); } diff --git a/xtask/src/env/kubectl.rs b/xtask/src/env/kubectl.rs index bf3ef2a0..a71bd22e 100644 --- a/xtask/src/env/kubectl.rs +++ b/xtask/src/env/kubectl.rs @@ -83,17 +83,18 @@ pub(crate) fn wait_for_rollout( /// Returns an error if the `kubectl` process cannot be spawned or exits /// with a non-zero status. pub(crate) fn rollout_restart(context: &str, deployment: &str) -> Result<(), Box> { + rollout_restart_ns(context, deployment, ROLLOUT_NAMESPACE) +} + +/// Restart a `Deployment` rollout in an explicit namespace. +pub(crate) fn rollout_restart_ns( + context: &str, + deployment: &str, + namespace: &str, +) -> Result<(), Box> { let resource = format!("deployment/{deployment}"); let status = Command::new("kubectl") - .args([ - "--context", - context, - "-n", - ROLLOUT_NAMESPACE, - "rollout", - "restart", - &resource, - ]) + .args(["--context", context, "-n", namespace, "rollout", "restart", &resource]) .status()?; if !status.success() { return Err(format!("kubectl rollout restart {deployment} failed").into()); diff --git a/xtask/src/env/llmd_pool_metrics_demo.rs b/xtask/src/env/llmd_pool_metrics_demo.rs index 980845de..656be73d 100644 --- a/xtask/src/env/llmd_pool_metrics_demo.rs +++ b/xtask/src/env/llmd_pool_metrics_demo.rs @@ -119,10 +119,10 @@ const GRID_NETWORK_NAME: &str = "grid-llmd-pool-metrics"; /// combined-site demo. Both require the `peer_identity_trust`, /// `provider_route`, `credential_inject`, and `intelligent_route` filters /// which are built into the published Grid AI rollup. -const DEFAULT_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3"; +const DEFAULT_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/ai:0.3.0"; /// Default operator image tag. -const DEFAULT_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.3"; +const DEFAULT_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.4"; /// Default EPP image reference required by this demo. const DEFAULT_EPP_IMAGE: &str = "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0"; @@ -131,7 +131,7 @@ const DEFAULT_EPP_IMAGE: &str = "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9 const DEFAULT_VCR_IMAGE: &str = "ghcr.io/neuralmagic/vllm-vcr:vllm0.23"; /// Default overlay-sync sidecar image tag. -const DEFAULT_OVERLAY_SYNC_IMAGE: &str = "ghcr.io/praxis-proxy/grid-overlay-sync:v0.1.3"; +const DEFAULT_OVERLAY_SYNC_IMAGE: &str = "ghcr.io/praxis-proxy/grid-overlay-sync:v0.1.4"; /// Default nginx image for the metrics TLS reverse proxy sidecar. const DEFAULT_NGINX_IMAGE: &str = "docker.io/library/nginx:1.27.4-alpine"; @@ -528,7 +528,14 @@ fn prepare_setup( let images = resolve_images(metrics_transport)?; verify_images(&images)?; - let resolved_config = materialize_config(forge_config, metrics_transport, scoring_flavor, images.nginx.as_deref())?; + let resolved_config = materialize_config_with_images( + forge_config, + metrics_transport, + scoring_flavor, + images.nginx.as_deref(), + Some(&images), + )?; + verify_materialized_images(&resolved_config, &images)?; let forge_bin = glb::resolve_forge_binary() .ok_or("praxis-forge binary not found")? .into(); @@ -2189,8 +2196,8 @@ fn wait_for_overlay_convergence() -> Result<(), Box> { /// Load pre-built images into Kind clusters in local-image mode. /// -/// Uses the forge-expected tags (created by [`tag_images_for_forge`]) since -/// the forge manifests and Helm values reference those names. +/// Uses the exact resolved image references so `imagePullPolicy: Never` +/// references the image that was actually loaded into each Kind node. fn load_images_into_clusters(context: &DemoContext) -> Result<(), Box> { if uses_registry_images() { eprintln!(" [OK] Registry image mode: skipped local Kind image loading"); @@ -2198,11 +2205,11 @@ fn load_images_into_clusters(context: &DemoContext) -> Result<(), Box = vec![ - "grid-operator:llmd-pool-metrics-demo", - "grid-overlay-sync:llmd-pool-metrics-demo", - "praxis-ai:llmd-pool-metrics-demo", - "llm-d-epp:llmd-pool-metrics-demo", - "vllm-vcr:llmd-pool-metrics-demo", + &context.images.operator, + &context.images.overlay_sync, + &context.images.gateway, + &context.images.epp, + &context.images.vcr, ]; if let Some(nginx) = &context.images.nginx { tags.push(nginx); @@ -2334,15 +2341,92 @@ fn run_forge_stack( /// When `scoring_flavor` is `KvCachePressure`, additionally swaps both /// sites' `GridNetwork.spec.scoringPolicy.strategy` from the template's /// default `queueDepth` to `kvCachePressure`. +#[cfg(test)] fn materialize_config( forge_config: &Path, metrics_transport: MetricsTransport, scoring_flavor: ScoringFlavor, nginx_image: Option<&str>, +) -> Result> { + materialize_config_with_images(forge_config, metrics_transport, scoring_flavor, nginx_image, None) +} + +/// Materialize a Forge config and inject all explicitly selected images. +/// +/// The llm-d topology has image values in cluster properties and in the +/// overlay-sync sidecar values. Keeping this injection here ensures the image +/// references used by Forge, Kind loading, and the environment variables are +/// identical before any cluster is created. +fn materialize_config_with_images( + forge_config: &Path, + metrics_transport: MetricsTransport, + scoring_flavor: ScoringFlavor, + nginx_image: Option<&str>, + images: Option<&ResolvedImages>, ) -> Result> { let dir = forge_config.parent().unwrap_or_else(|| Path::new(".")); let resolved = dir.join(".forge.resolved.yaml"); let mut result = fs::read_to_string(forge_config)?; + if let Some(images) = images { + let image_pull_policy = + std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "IfNotPresent".to_owned()); + if image_pull_policy != "IfNotPresent" { + result = checked_replace( + &result, + "imagePullPolicy: IfNotPresent", + &format!("imagePullPolicy: {image_pull_policy}"), + 2, + "image pull policy", + )?; + } + for (role, selected, default) in [ + ("gateway", images.gateway.as_str(), DEFAULT_GATEWAY_IMAGE), + ("operator", images.operator.as_str(), DEFAULT_OPERATOR_IMAGE), + ("epp", images.epp.as_str(), DEFAULT_EPP_IMAGE), + ("vcr", images.vcr.as_str(), DEFAULT_VCR_IMAGE), + ("overlay-sync", images.overlay_sync.as_str(), DEFAULT_OVERLAY_SYNC_IMAGE), + ] { + let (repository, tag) = split_image_reference(selected)?; + let (default_repository, default_tag) = split_image_reference(default)?; + if selected != default { + if role != "overlay-sync" { + result = checked_replace(&result, default, selected, 2, &format!("{role} image reference"))?; + } + let property = match role { + "gateway" => "gatewayImage", + "operator" => "operatorImage", + "epp" => "eppImage", + "vcr" => "vcrImage", + "overlay-sync" => "overlay-sync", + _ => unreachable!(), + }; + let repo_property = if role == "overlay-sync" { + "repository".to_owned() + } else { + format!("{property}Repo") + }; + let tag_property = if role == "overlay-sync" { + "tag".to_owned() + } else { + format!("{property}Tag") + }; + result = checked_replace( + &result, + &format!("{repo_property}: \"{default_repository}\""), + &format!("{repo_property}: \"{repository}\""), + if role == "overlay-sync" { 1 } else { 2 }, + &format!("{role} image repository"), + )?; + result = checked_replace( + &result, + &format!("{tag_property}: \"{default_tag}\""), + &format!("{tag_property}: \"{tag}\""), + if role == "overlay-sync" { 1 } else { 2 }, + &format!("{role} image tag"), + )?; + } + } + } for cluster in CLUSTERS { let provider_name = format!("llmd-{cluster}-provider"); let candidate_id = fnv1a_hex8(&format!("inference_model/{VCR_MODEL}/{cluster}/{provider_name}")); @@ -2442,6 +2526,67 @@ fn materialize_config( Ok(resolved) } +/// Split a registry image reference into repository and tag. +fn split_image_reference(image: &str) -> Result<(&str, &str), Box> { + let (repository, tag) = image + .rsplit_once(':') + .ok_or_else(|| format!("image reference {image:?} has no tag"))?; + if repository.is_empty() || tag.is_empty() || tag.contains('/') { + return Err(format!("image reference {image:?} must contain a repository and tag").into()); + } + Ok((repository, tag)) +} + +/// Verify every selected image occurs in the resolved Forge source. +fn verify_materialized_images(path: &Path, images: &ResolvedImages) -> Result<(), Box> { + let content = fs::read_to_string(path)?; + let expected = [ + ("gateway", images.gateway.as_str()), + ("operator", images.operator.as_str()), + ("epp", images.epp.as_str()), + ("vcr", images.vcr.as_str()), + ("overlay-sync", images.overlay_sync.as_str()), + ]; + for (role, image) in expected { + let (repository, tag) = split_image_reference(image)?; + let (repo_property, tag_property, expected_count) = if role == "overlay-sync" { + ("repository", "tag", 1) + } else { + let property = match role { + "gateway" => "gatewayImage", + "operator" => "operatorImage", + "epp" => "eppImage", + "vcr" => "vcrImage", + _ => unreachable!(), + }; + // These fields occur once per cluster. + (property, property, 2) + }; + let repository_key = if role == "overlay-sync" { + repo_property.to_owned() + } else { + format!("{repo_property}Repo") + }; + let tag_key = if role == "overlay-sync" { + tag_property.to_owned() + } else { + format!("{tag_property}Tag") + }; + let repository_count = content.matches(&format!("{repository_key}: \"{repository}\"")).count(); + let tag_count = content.matches(&format!("{tag_key}: \"{tag}\"")).count(); + if repository_count != expected_count || tag_count != expected_count { + return Err(format!( + "resolved Forge config is missing {role} image {image:?} (repository matches: {repository_count}, tag matches: {tag_count})" + ) + .into()); + } + } + if !uses_registry_images() && !content.contains("imagePullPolicy: Never") { + return Err("resolved Forge config does not set imagePullPolicy: Never for local image mode".into()); + } + Ok(()) +} + /// Replace `needle` in `content`, failing if the match count differs from `expected`. fn checked_replace( content: &str, diff --git a/xtask/src/env/single_cluster_multi_gateway_qualification.rs b/xtask/src/env/single_cluster_multi_gateway_qualification.rs index 8a2f1318..86d7acfa 100644 --- a/xtask/src/env/single_cluster_multi_gateway_qualification.rs +++ b/xtask/src/env/single_cluster_multi_gateway_qualification.rs @@ -11,7 +11,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, - fs, + env, fs, path::{Path, PathBuf}, process::{Child, Command, Output, Stdio}, thread, @@ -82,6 +82,7 @@ struct Evidence { struct Cleanup { forge: PathBuf, config: PathBuf, + resolved_config: PathBuf, enabled: bool, } @@ -97,6 +98,7 @@ impl Drop for Cleanup { .status(), ); } + drop(fs::remove_file(&self.resolved_config)); } } @@ -235,12 +237,85 @@ fn capture_grid_state() -> Result { } /// Images required by the topology's `Never` pull policy. -const QUALIFICATION_IMAGES: [&str; 4] = [ - "grid-operator:single-cluster-qualification", - "grid-overlay-sync:single-cluster-qualification", - "praxis-ai:single-cluster-qualification", - "ghcr.io/neuralmagic/vllm-vcr:vllm0.23", -]; +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct ResolvedImages { + gateway: String, + operator: String, + overlay_sync: String, + vcr: String, + pull_policy: String, +} + +const DEFAULT_GATEWAY_IMAGE: &str = "praxis-ai:single-cluster-qualification"; +const DEFAULT_OPERATOR_IMAGE: &str = "grid-operator:single-cluster-qualification"; +const DEFAULT_OVERLAY_SYNC_IMAGE: &str = "grid-overlay-sync:single-cluster-qualification"; +const DEFAULT_VCR_IMAGE: &str = "ghcr.io/neuralmagic/vllm-vcr:vllm0.23"; + +fn resolved_images() -> Result { + resolve_images( + env::var("GRID_XTASK_GATEWAY_IMAGE").ok(), + env::var("GRID_XTASK_OPERATOR_IMAGE").ok(), + env::var("GRID_XTASK_OVERLAY_SYNC_IMAGE").ok(), + env::var("GRID_XTASK_VCR_IMAGE").ok(), + env::var("GRID_XTASK_IMAGE_PULL_POLICY").ok(), + ) +} + +fn resolve_images( + gateway: Option, + operator: Option, + overlay_sync: Option, + vcr: Option, + pull_policy: Option, +) -> Result { + let images = ResolvedImages { + gateway: gateway.unwrap_or_else(|| DEFAULT_GATEWAY_IMAGE.to_owned()), + operator: operator.unwrap_or_else(|| DEFAULT_OPERATOR_IMAGE.to_owned()), + overlay_sync: overlay_sync.unwrap_or_else(|| DEFAULT_OVERLAY_SYNC_IMAGE.to_owned()), + vcr: vcr.unwrap_or_else(|| DEFAULT_VCR_IMAGE.to_owned()), + pull_policy: pull_policy.unwrap_or_else(|| "Never".to_owned()), + }; + if !matches!(images.pull_policy.as_str(), "Never" | "IfNotPresent" | "Always") { + return Err(format!( + "GRID_XTASK_IMAGE_PULL_POLICY has invalid value {:?}", + images.pull_policy + )); + } + for (name, image) in [ + ("gateway", images.gateway.as_str()), + ("operator", images.operator.as_str()), + ("overlay-sync", images.overlay_sync.as_str()), + ("vcr", images.vcr.as_str()), + ] { + validate_image_reference(name, image)?; + } + Ok(images) +} + +fn validate_image_reference(name: &str, image: &str) -> Result<(), String> { + if image.is_empty() || image.chars().any(char::is_whitespace) || image.starts_with('/') { + return Err(format!("{name} image reference is malformed: {image:?}")); + } + if image.contains('@') { + let (repository, digest) = image + .split_once('@') + .ok_or_else(|| format!("{name} image reference is malformed: {image:?}"))?; + if repository.is_empty() || !digest.starts_with("sha256:") || digest.len() <= "sha256:".len() { + return Err(format!("{name} image reference is malformed: {image:?}")); + } + } else { + let last_slash = image.rfind('/').unwrap_or(0); + let Some(colon) = image.rfind(':') else { + return Err(format!( + "{name} image reference must include a tag or digest: {image:?}" + )); + }; + if colon <= last_slash || colon == image.len() - 1 { + return Err(format!("{name} image reference is malformed: {image:?}")); + } + } + Ok(()) +} /// Match a Docker reference against the repository and tag columns from `crictl`. fn node_has_image(listing: &str, image: &str) -> bool { @@ -279,16 +354,31 @@ fn discover_node(kind_cluster: &str) -> Result { clippy::too_many_lines, reason = "each image is inspected, loaded, and verified before deployment" )] -fn load_and_verify_images() -> Result<(), String> { - let identity = cluster_identity(); - let node = discover_node(&identity.kind_cluster)?; - for image in QUALIFICATION_IMAGES { +fn load_and_verify_images(images: &ResolvedImages) -> Result, String> { + let image_names = [ + ("gateway", images.gateway.as_str()), + ("operator", images.operator.as_str()), + ("overlay_sync", images.overlay_sync.as_str()), + ("vcr", images.vcr.as_str()), + ]; + let mut metadata = BTreeMap::new(); + for (role, image) in image_names { let mut inspect = Command::new("docker"); inspect.args(["image", "inspect", image]); let output = command_output(&mut inspect, Duration::from_secs(30))?; if !output.status.success() { return Err(format!("required local image is missing: {image}")); } + let details: serde_json::Value = serde_json::from_slice(&output.stdout) + .map_err(|error| format!("parse docker metadata for {image}: {error}"))?; + metadata.insert(role.to_owned(), details); + } + if images.pull_policy != "Never" { + return Ok(metadata); + } + let identity = cluster_identity(); + let node = discover_node(&identity.kind_cluster)?; + for (_, image) in image_names { let mut load = Command::new("kind"); load.args(["load", "docker-image", image, "--name", &identity.kind_cluster]); let load_output = command_output(&mut load, QUALIFICATION_TIMEOUT)?; @@ -312,7 +402,48 @@ fn load_and_verify_images() -> Result<(), String> { )); } } - Ok(()) + Ok(metadata) +} + +/// Materialize the topology using the resolved image references. +#[expect( + clippy::too_many_lines, + reason = "materialization validates the complete image contract at one boundary" +)] +fn materialize_config(forge_config: &Path, evidence_dir: &Path, images: &ResolvedImages) -> Result { + let output = forge_config.parent().unwrap_or_else(|| Path::new(".")).join(format!( + ".grid-single-cluster-multi-gateway-{}.resolved.yaml", + std::process::id() + )); + let selected = super::forge_config::ImageOverrides { + gateway: images.gateway.clone(), + operator: images.operator.clone(), + overlay_sync: images.overlay_sync.clone(), + vcr: images.vcr.clone(), + pull_policy: images.pull_policy.clone(), + }; + let resolved = super::forge_config::materialize_with_images(forge_config, Some(&output), &selected) + .map_err(|error| format!("materialize Forge configuration: {error}"))?; + fs::copy(&resolved, evidence_dir.join("resolved-forge.yaml")) + .map_err(|error| format!("copy materialized Forge configuration to evidence: {error}"))?; + let content = + fs::read_to_string(&resolved).map_err(|error| format!("read materialized Forge configuration: {error}"))?; + for (role, image) in [ + ("gateway", images.gateway.as_str()), + ("operator", images.operator.as_str()), + ("overlay-sync", images.overlay_sync.as_str()), + ("vcr", images.vcr.as_str()), + ] { + if !content.contains(image) { + return Err(format!( + "materialized Forge configuration does not contain {role} image {image:?}" + )); + } + } + if !content.contains(&format!("imagePullPolicy: {}", images.pull_policy)) { + return Err("materialized Forge configuration does not contain the selected image pull policy".to_owned()); + } + Ok(resolved) } /// Apply one Forge stack and preserve its complete bounded result. @@ -780,13 +911,17 @@ pub(crate) fn run(forge_config: &Path, options: &Options) -> Result<(), Box Result<(), Box { + let images_ready = match load_and_verify_images(&images) { + Ok(metadata) => { + observations.insert( + "image_metadata".to_owned(), + serde_json::to_value(metadata).unwrap_or(serde_json::Value::Null), + ); scenarios.push(scenario( "image-loading", "PASS", @@ -840,7 +983,7 @@ pub(crate) fn run(forge_config: &Path, options: &Options) -> Result<(), Box scenarios.push(scenario(format!("stack/{stack}").as_str(), "PASS", "stack completed")), Err(error) => { stacks_ready = false; @@ -1156,12 +1299,13 @@ pub(crate) fn run(forge_config: &Path, options: &Options) -> Result<(), Box