From d74279f2ec28ca778adfa3ecf5e4f9eb6c75f867 Mon Sep 17 00:00:00 2001 From: Revanth Reddy Airre Date: Thu, 20 Aug 2026 22:40:38 -0700 Subject: [PATCH 1/8] fix: update llm-d EPP image (#86) Signed-off-by: Revanth Reddy Airre Co-authored-by: Brent Salisbury --- .../grid-llmd-pool-metrics/forge.yaml | 12 +++--- .../resources/pool-a/epp-deployment-mtls.yaml | 2 +- .../resources/pool-a/epp-deployment.yaml | 2 +- .../resources/pool-b/epp-deployment-mtls.yaml | 2 +- .../resources/pool-b/epp-deployment.yaml | 2 +- xtask/src/env/llmd_pool_metrics_demo.rs | 42 ++++++++++++------- 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml index ad34e5f..49d2f83 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml @@ -22,15 +22,15 @@ spec: siteName: pool-a gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" - eppImage: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + eppImage: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" gatewayImageTag: "v0.1.3" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" operatorImageTag: "v0.1.3" - eppImageRepo: "ghcr.io/llm-d/llm-d-inference-scheduler" - eppImageTag: "v0.8.0" + eppImageRepo: "ghcr.io/llm-d/llm-d-router-endpoint-picker" + eppImageTag: "v0.9.0" vcrImageRepo: "ghcr.io/neuralmagic/vllm-vcr" vcrImageTag: "vllm0.23" poolName: pool-a @@ -43,15 +43,15 @@ spec: siteName: pool-b gatewayImage: "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" operatorImage: "ghcr.io/praxis-proxy/grid-operator:v0.1.3" - eppImage: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + eppImage: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" vcrImage: "ghcr.io/neuralmagic/vllm-vcr:vllm0.23" imagePullPolicy: IfNotPresent gatewayImageRepo: "ghcr.io/praxis-proxy/grid-ai-rollup" gatewayImageTag: "v0.1.3" operatorImageRepo: "ghcr.io/praxis-proxy/grid-operator" operatorImageTag: "v0.1.3" - eppImageRepo: "ghcr.io/llm-d/llm-d-inference-scheduler" - eppImageTag: "v0.8.0" + eppImageRepo: "ghcr.io/llm-d/llm-d-router-endpoint-picker" + eppImageTag: "v0.9.0" vcrImageRepo: "ghcr.io/neuralmagic/vllm-vcr" vcrImageTag: "vllm0.23" poolName: pool-b diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml index 5737956..31c0781 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment-mtls.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-a diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml index 294929f..4fc1949 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-a/epp-deployment.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-a diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml index 62ad34c..02172d3 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment-mtls.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-b diff --git a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml index 231b14f..8e9b8e4 100644 --- a/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml +++ b/tests/e2e/topologies/grid-llmd-pool-metrics/resources/pool-b/epp-deployment.yaml @@ -25,7 +25,7 @@ spec: type: RuntimeDefault containers: - name: epp - image: "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" + image: "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" imagePullPolicy: "IfNotPresent" args: - --pool-name=pool-b diff --git a/xtask/src/env/llmd_pool_metrics_demo.rs b/xtask/src/env/llmd_pool_metrics_demo.rs index 11f148f..6c23245 100644 --- a/xtask/src/env/llmd_pool_metrics_demo.rs +++ b/xtask/src/env/llmd_pool_metrics_demo.rs @@ -125,7 +125,7 @@ const DEFAULT_GATEWAY_IMAGE: &str = "ghcr.io/praxis-proxy/grid-ai-rollup:v0.1.3" const DEFAULT_OPERATOR_IMAGE: &str = "ghcr.io/praxis-proxy/grid-operator:v0.1.3"; /// Default EPP image reference required by this demo. -const DEFAULT_EPP_IMAGE: &str = "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0"; +const DEFAULT_EPP_IMAGE: &str = "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0"; /// Default vllm-vcr image reference required by this demo. const DEFAULT_VCR_IMAGE: &str = "ghcr.io/neuralmagic/vllm-vcr:vllm0.23"; @@ -836,11 +836,14 @@ fn proof_provenance(mtls: bool) -> ProofResult { let deadline = Instant::now() + Duration::from_secs(30); while Instant::now() < deadline { if let Ok(metrics_text) = kubectl_exec_epp_metrics(cluster, mtls) { - let has_kv = metrics_text.contains("inference_pool_average_kv_cache_utilization") + let has_kv = metrics_text.contains("llm_d_epp_average_kv_cache_utilization") + || metrics_text.contains("inference_pool_average_kv_cache_utilization") || metrics_text.contains("llm_d_router_epp_average_kv_cache_utilization"); - let has_queue = metrics_text.contains("inference_pool_average_queue_size") + let has_queue = metrics_text.contains("llm_d_epp_average_queue_size") + || metrics_text.contains("inference_pool_average_queue_size") || metrics_text.contains("llm_d_router_epp_average_queue_size"); - let has_ready = metrics_text.contains("inference_pool_ready_pods") + let has_ready = metrics_text.contains("llm_d_epp_ready_endpoints") + || metrics_text.contains("inference_pool_ready_pods") || metrics_text.contains("llm_d_router_epp_ready_endpoints"); if has_kv && has_queue && has_ready { observations.push(format!("{cluster}: all 3 EPP pool metrics present")); @@ -1392,17 +1395,16 @@ fn scrape_epp_metrics(cluster: &str, mtls: bool) -> EppMetrics { /// [`scrape_epp_metrics`], separated out so metric-name-fallback behavior is /// unit-testable without a live EPP). /// -/// Both `queue_size` and `kv_cache` fall back from the `inference_pool_*` -/// series to the `llm_d_router_epp_*` series symmetrically -- an EPP build -/// that only exposes the latter must not silently read `kv_cache` as a -/// permanent 0.0, which would make a `kvCachePressure` run's pressure phase -/// never announce despite real KV pressure driving the flip. +/// Both `queue_size` and `kv_cache` prefer the canonical `llm_d_epp_*` +/// series and fall back symmetrically to the legacy metric names. fn parse_epp_metrics(text: &str) -> EppMetrics { EppMetrics { - queue_size: extract_prom_value(text, "inference_pool_average_queue_size") + queue_size: extract_prom_value(text, "llm_d_epp_average_queue_size") + .or_else(|| extract_prom_value(text, "inference_pool_average_queue_size")) .or_else(|| extract_prom_value(text, "llm_d_router_epp_average_queue_size")) .unwrap_or(0.0), - kv_cache: extract_prom_value(text, "inference_pool_average_kv_cache_utilization") + kv_cache: extract_prom_value(text, "llm_d_epp_average_kv_cache_utilization") + .or_else(|| extract_prom_value(text, "inference_pool_average_kv_cache_utilization")) .or_else(|| extract_prom_value(text, "llm_d_router_epp_average_kv_cache_utilization")) .unwrap_or(0.0), } @@ -3838,14 +3840,26 @@ inference_pool_average_kv_cache_utilization{name="pool-a"} 0.35 #[test] #[expect(clippy::float_cmp, reason = "exact literal round-trips in test assertions")] - fn parse_epp_metrics_prefers_primary_metric_names() { - let text = "inference_pool_average_queue_size{name=\"pool-a\"} 4.5\n\ - inference_pool_average_kv_cache_utilization{name=\"pool-a\"} 0.35\n"; + fn parse_epp_metrics_prefers_llm_d_epp_metric_names() { + let text = "llm_d_epp_average_queue_size{name=\"pool-a\"} 4.5\n\ + llm_d_epp_average_kv_cache_utilization{name=\"pool-a\"} 0.35\n\ + inference_pool_average_queue_size{name=\"pool-a\"} 7.0\n\ + inference_pool_average_kv_cache_utilization{name=\"pool-a\"} 0.70\n"; let epp = parse_epp_metrics(text); assert_eq!(epp.queue_size, 4.5); assert_eq!(epp.kv_cache, 0.35); } + #[test] + #[expect(clippy::float_cmp, reason = "exact literal round-trips in test assertions")] + fn parse_epp_metrics_falls_back_to_inference_pool_metric_names() { + let text = "inference_pool_average_queue_size{name=\"pool-a\"} 5.0\n\ + inference_pool_average_kv_cache_utilization{name=\"pool-a\"} 0.40\n"; + let epp = parse_epp_metrics(text); + assert_eq!(epp.queue_size, 5.0); + assert_eq!(epp.kv_cache, 0.40); + } + #[test] #[expect(clippy::float_cmp, reason = "exact literal round-trips in test assertions")] fn parse_epp_metrics_falls_back_to_llm_d_router_metric_names() { From 1e761cedfafd8ca3424844fe1d31e899ebddcfcb Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sun, 23 Aug 2026 21:52:50 -0400 Subject: [PATCH 2/8] docs(architecture): clarify SWIM is site-to-site and single-site GridSite behavior (#94) Single-site/combined deployments legitimately run a one-node SWIM mesh with zero peers, so the local GridSite stays Pending with reason AwaitingDiscovery. That is expected and does not block local routing: local InferenceProviders are eligible regardless of GridSite phase; only remote CRDT provider records are phase-gated. Add a 'Single-Site and Combined Deployments' section to the architecture overview and a single-site note to the GridSite lifecycle so single-cluster users do not mistake Pending for a discovery failure or try to add SWIM peers/replicas to fix it. Signed-off-by: Brent Salisbury --- docs/architecture/crds.md | 13 +++++++++++++ docs/architecture/overview.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index aa68a4a..9995c41 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -399,6 +399,19 @@ prove that a Praxis gateway has loaded the latest routing config or authorized provider-side traffic. Data-plane readiness is verified separately at request time by provider gateway filters. +**Single-site / combined-site deployments:** Site discovery is driven by *remote* +SWIM peers - the `Pending -> Discovered` transition above requires a remote peer +observed Alive. A single or combined cluster has no peers, so its own `GridSite` +stays `Pending` with reason `AwaitingDiscovery`. **This is expected and does not +block local serving:** local `InferenceProvider`s are eligible regardless of +`GridSite.status.phase`; only *remote* CRDT provider records are phase-gated (see +"Routing eligibility" above). Do not add SWIM seeds or extra operator replicas to +try to force the site `Active` - there is no second site to discover, and a lone +operator legitimately runs a single-node mesh with zero peers. The `Active` phase +and its mTLS gateway probe (`spec.egress` + `spec.trust`) apply to reaching +*remote* sites, or a manually-configured peer gateway endpoint. See +[Architecture Overview -> Single-Site and Combined Deployments](overview.md#single-site-and-combined-deployments). + See [Routing eligibility](routing.md#routing-eligibility) for the full gating rule. Example status — Mutual TLS verified: diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 9c19ad2..f2287f2 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -287,6 +287,36 @@ key group, but stronger sender/origin binding is still hardening work. Do not treat distributed CRDT state as fully security-sensitive routing input until that work is complete. +## Single-Site and Combined Deployments + +SWIM membership is **site-granular**: each Grid operator is a single SWIM node +carrying its site's identity, and SWIM members are *other sites'* operators - not +the gateways, providers, or pods inside a site. Seeds (`GridNetwork.spec.seeds`) +point at other sites, and the operator filters out its own address, so a lone +site legitimately forms a **single-node mesh with zero peers**. + +```text +Multi-site Single / combined site +-------------------------------- ------------------------------ +site-a operator -- SWIM -- site-b one operator -- SWIM (self only) + | | | + local providers local providers several gateways / providers + (NOT SWIM members) +``` + +Two things that commonly surprise people on a single or combined cluster: + +- **Zero SWIM peers is expected, not a failure.** There is no second site to + discover, so do not add SWIM seeds or extra operator replicas to "make + discovery work." +- **Local routing does not require `GridSite.status.phase == Active`.** Local + `InferenceProvider`s are eligible regardless of GridSite phase; only *remote* + (cross-site CRDT) provider records are phase-gated. So a single-site deployment + routes to its local providers even while its own `GridSite` is `Pending`. + +See the [GridSite lifecycle](crds.md#gridsite) for the phase machine and this +single-site behavior. + ## Routing Overlays For each gateway reference on a `GridNetwork`, the operator writes a From 7e659cdf65d3b20fcfb20a20111f2ae571b2272a Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sun, 23 Aug 2026 22:27:02 -0400 Subject: [PATCH 3/8] fix(operator): let a standalone single-site GridNetwork reach Active (#95) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit determine_phase only returned Active via phase_hint(), which requires at least one Alive SWIM peer. A single-site / combined deployment legitimately has zero peers (peers are other sites, not intra-site gateways/pods), so its GridNetwork was pinned at Initializing forever even though the local control plane was fully operational and already rendering overlays for local providers. Treat a standalone network (no seeds configured) with the SWIM runtime up and TLS trust material as Active. Networks that DO configure seeds still stay Initializing until a peer is observed, so multi-site behavior is unchanged. Peer connectivity remains reported separately via status.connectedSites. Note: the oversized self-site SWIM broadcast is unrelated — it already soft-fails (logged and dropped in swim node + reconcile), so it never blocked the phase. Adds regression tests: determine_phase_standalone_single_site_reaches_active and determine_phase_seeded_but_peerless_stays_initializing. Signed-off-by: Brent Salisbury --- .github/workflows/helm.yaml | 37 ++++++++++++++ .github/workflows/operator-image.yaml | 41 +++++++++++++++ operator/src/controller/grid_network.rs | 68 +++++++++++++++++++++++-- 3 files changed, 142 insertions(+), 4 deletions(-) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index 67ec089..313b8d5 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -28,6 +28,41 @@ env: HELM_UNITTEST_VERSION: v0.8.2 jobs: + # ------------------------------------------------------------------------------ + # Avoid building runtime images for documentation-only pull requests. + # ------------------------------------------------------------------------------ + + changes: + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runtime: ${{ steps.changed.outputs.runtime }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Detect runtime changes + id: changed + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + + if [ "${EVENT_NAME}" != "pull_request" ]; then + echo "runtime=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + changed_files="$(git diff --name-only "${BASE_SHA}...${GITHUB_SHA}")" + if printf '%s\n' "${changed_files}" | grep -qvE '(^docs/|(^|/)README[^/]*$|\.md$)'; then + echo "runtime=true" >> "${GITHUB_OUTPUT}" + else + echo "runtime=false" >> "${GITHUB_OUTPUT}" + fi + # ---------------------------------------------------------------------------- # Static chart validation (lint, template, schema, CRD sync, package) # ---------------------------------------------------------------------------- @@ -89,6 +124,8 @@ jobs: # ---------------------------------------------------------------------------- kind: + needs: changes + if: needs.changes.outputs.runtime == 'true' runs-on: ubuntu-24.04 permissions: contents: read diff --git a/.github/workflows/operator-image.yaml b/.github/workflows/operator-image.yaml index fed4b9f..3a557f9 100644 --- a/.github/workflows/operator-image.yaml +++ b/.github/workflows/operator-image.yaml @@ -24,11 +24,48 @@ env: OVERLAY_SYNC_IMAGE_NAME: ghcr.io/praxis-proxy/grid-overlay-sync jobs: + # ------------------------------------------------------------------------------ + # Avoid building runtime images for documentation-only pull requests. + # ------------------------------------------------------------------------------ + + changes: + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runtime: ${{ steps.changed.outputs.runtime }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Detect runtime changes + id: changed + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + + if [ "${EVENT_NAME}" != "pull_request" ]; then + echo "runtime=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + changed_files="$(git diff --name-only "${BASE_SHA}...${GITHUB_SHA}")" + if printf '%s\n' "${changed_files}" | grep -qvE '(^docs/|(^|/)README[^/]*$|\.md$)'; then + echo "runtime=true" >> "${GITHUB_OUTPUT}" + else + echo "runtime=false" >> "${GITHUB_OUTPUT}" + fi + # ---------------------------------------------------------------------------- # Build the operator container image # ---------------------------------------------------------------------------- operator-image: + needs: changes + if: needs.changes.outputs.runtime == 'true' runs-on: ubuntu-24.04 permissions: contents: read @@ -78,6 +115,8 @@ jobs: # ---------------------------------------------------------------------------- mock-providers-image: + needs: changes + if: needs.changes.outputs.runtime == 'true' runs-on: ubuntu-24.04 permissions: contents: read @@ -127,6 +166,8 @@ jobs: # ---------------------------------------------------------------------------- overlay-sync-image: + needs: changes + if: needs.changes.outputs.runtime == 'true' runs-on: ubuntu-24.04 permissions: contents: read diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index e64b206..9507ebc 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -1430,12 +1430,30 @@ fn determine_phase(network: &GridNetwork, grid_id: &str, membership: Option<&Mem { return hint; } - // Existing static phase logic (no live membership yet). + // No live phase hint. `phase_hint` returns `Some` only when at least one + // Alive/Degraded peer exists, so we reach here when the network has no peers + // yet — either the SWIM runtime is not up (`membership` is `None`) or it is + // up but no peers have joined (`Some`, empty snapshot). let has_tls = network.spec.tls.ca_secret_ref.is_some(); - if has_tls { - GridNetworkPhase::Initializing + if !has_tls { + return GridNetworkPhase::Pending; + } + // A single-site / combined deployment legitimately has zero SWIM peers — + // peers are other *sites*, not intra-site gateways or pods. When no seeds + // are configured, this network is standalone, so a running SWIM runtime + // (`membership.is_some()`) with TLS trust material is a locally operational + // control plane and reports `Active` instead of pinning `Initializing` + // forever. Peer connectivity is reported separately via + // `status.connectedSites`. + // + // When seeds ARE configured the network expects peers, so a peerless + // snapshot stays `Initializing` until at least one peer is observed (handled + // by `phase_hint` above). `membership.is_none()` means the SWIM runtime is + // not up yet, which also stays `Initializing`. + if membership.is_some() && network.spec.seeds.is_empty() { + GridNetworkPhase::Active } else { - GridNetworkPhase::Pending + GridNetworkPhase::Initializing } } @@ -2707,6 +2725,48 @@ mod tests { ); } + #[test] + fn determine_phase_standalone_single_site_reaches_active() { + // Single-site / combined deployment: no seeds, no SWIM peers. With TLS + // trust material and the SWIM runtime up (Some, but empty membership), + // the local control plane is operational and reports Active rather than + // staying Initializing forever. + let mut network = base_network(); + network.spec.tls.ca_secret_ref = Some(crate::crd::grid_network::SecretRef { + name: "ca".to_owned(), + namespace: "default".to_owned(), + key: None, + }); + network.spec.seeds.clear(); + let empty = MembershipSnapshot::default(); + let phase = determine_phase(&network, "some-id", Some(&empty)); + assert_eq!( + phase, + GridNetworkPhase::Active, + "peerless single-site (no seeds) with SWIM up and TLS must reach Active" + ); + } + + #[test] + fn determine_phase_seeded_but_peerless_stays_initializing() { + // With seeds configured the network expects peers; until at least one is + // observed it must stay Initializing and not prematurely claim Active. + let mut network = base_network(); + network.spec.tls.ca_secret_ref = Some(crate::crd::grid_network::SecretRef { + name: "ca".to_owned(), + namespace: "default".to_owned(), + key: None, + }); + network.spec.seeds = vec!["grid.peer:7946".to_owned()]; + let empty = MembershipSnapshot::default(); + let phase = determine_phase(&network, "some-id", Some(&empty)); + assert_eq!( + phase, + GridNetworkPhase::Initializing, + "seeded network with no peers observed yet must stay Initializing" + ); + } + #[test] fn connected_sites_is_zero_without_membership() { // Verify the update_status path: no membership → connected_sites = 0. From f39cd091959ea6130bed5dd097f80580d79ecab6 Mon Sep 17 00:00:00 2001 From: Jordi Gil Date: Tue, 25 Aug 2026 15:27:38 -0400 Subject: [PATCH 4/8] feat: AgentToolProvider reconciler for cross-cluster MCP tool federation (grid#41) (#56) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(operator): add MCP tools/list probe client with bounded telemetry (grid#41) Adds the live-probe half of the AgentToolProvider reconciler: an rmcp-based Streamable HTTP client that calls tools/list against spec.endpoint, with SSRF protection (blocks loopback/link-local/cloud-metadata targets), TLS material resolution, and bearer-token auth from a Secret. Also wires grid_mcp_probe_total{outcome} and grid_mcp_probe_duration_seconds metrics, following grid#9's bounded-cardinality convention: TlsConfigInvalid's carried reason string is deliberately collapsed to a single label so metric cardinality stays fixed regardless of cluster misconfiguration variety. Signed-off-by: Jordi Gil * feat(operator): add AgentToolProvider reconciler (grid#41) Wires the AgentToolProvider CRD to the live MCP probe client added in the previous commit: resolves siteSelector matches against GridNetwork/GridSite, runs the probe, and maps its outcome to status.phase/status.reason/ status.discoveredTools. Mirrors InferenceProvider's reconciliation shape, including its documented cross-resource watch limitation as a follow-up. Data-plane MCP tool-catalog aggregation and cross-cluster tools/call routing (praxis-ai#155, #205, #173) are out of scope here — this is the Grid-side control plane only, per grid#41's stated scope. Signed-off-by: Jordi Gil * feat(charts,mock-providers,xtask): AgentToolProvider Helm wiring, mock MCP server, and E2E check (grid#41) - charts/grid-operator: install the AgentToolProvider CRD and grant its RBAC verbs, with helm-unittest coverage for the new ClusterRole rule; wires charts/grid-operator into the CI helm-unittest job (only charts/grid-site ran before). - mock-providers: new --mcp-server mode built on the real rmcp server SDK (Streamable HTTP), so the E2E check below probes actual MCP wire protocol behavior, not a hand-rolled JSON-RPC approximation. Disables rmcp's default DNS-rebinding Host-header allowlist (localhost/127.0.0.1/::1 only), since this mock is reached over its in-cluster Service DNS name or NodePort address, never loopback. - xtask: cargo xtask env verify-agenttoolprovider-convergence deploys the mock as a real in-cluster NodePort service (the operator under test runs out-of-cluster, so it can't resolve in-cluster .svc DNS names) and proves Pending -> Available with discoveredTools populated end-to-end, plus the unreachable-endpoint failure path landing on Unavailable with a reason. Verified twice against a real kind cluster. Signed-off-by: Jordi Gil * test(operator): close AgentToolProvider TLS/Secret and GridNetwork test gaps (grid#41) Backfills the coverage gaps found during grid#56's GA readiness audit: - attach_tls_ca/attach_tls_client_identity/read_tls_material now have real unit-test coverage against a mocked kube::Client (tower::service_fn), which the module's own doc comment had falsely claimed already existed. - Along the way, found and fixed a real bug: reqwest::Certificate::from_pem and reqwest::Identity::from_pem don't reject malformed/empty PEM input, unlike the rustls::pki_types-based validation InferenceProvider's build_tls_client_config already uses. attach_tls_ca and attach_tls_client_identity now eagerly validate PEM material the same strict way before handing it to reqwest, so EndpointTlsMaterialInvalid/ EndpointTlsIdentityMismatch are reachable in practice, not just in name. - Also found (but did not fix here, to keep this change scoped) a pre-existing bug in the shared endpoint_tls.rs/secret.rs Secret-read path that also affects InferenceProvider: a key entirely absent from an existing Secret's data is misreported as SecretMissing instead of KeyMissing. Filed as grid#58, with a test that locks in current behavior and points at the issue. - static_config_failure_reason (GridNetworkNotFound/ProviderConfigInvalid) now has direct unit-test coverage via the same kube::Client mocking pattern, proving wrong an existing code comment that claimed this "cannot be unit-tested without a live cluster or a mock Kubernetes server." - Fixed doc drift: overview.md's claim that AgentToolProvider "does not currently run full controllers" (it does, as of grid#56), crds.md's Degraded phase (unreachable by design for this CRD, per phase_and_reason_from_probe's own doc comment) and missing reason/ observedGeneration status fields, and two stale "once PR 2 lands" code comments left over from the original 3-PR plan that landed as one. Signed-off-by: Jordi Gil * fix(mock-providers): resolve private-intra-doc-link rustdoc error `router`'s doc comment linked to `FixedToolsServer`, a private struct -- the link only resolved locally because of --document-private-items and failed CI's -D rustdoc::private-intra-doc-links check. Drop the link, keep the type name as plain text with a note that it's module-private. Signed-off-by: Jordi Gil * fix(operator): enforce one combined probe timeout, not per-phase budgets Addresses grid#56 review feedback (pull/56#pullrequestreview, praxis-bot): - PROBE_TIMEOUT was documented as a single combined budget for the live MCP probe, but resolve_endpoint_for_probe (DNS), the connect/handshake, and tools/list each independently got up to the full timeout, and TLS Secret material reads via the Kubernetes API had no timeout at all. Worst-case wall-clock time could exceed 3x the documented 10s budget, plus unbounded Kubernetes API latency. Fixed by wrapping the whole probe sequence in a single outer tokio::time::timeout in probe_agent_tool_provider, so all phases now share one real budget. - auth_header_map silently dropped the Authorization header when a bearer token contained characters invalid in an HTTP header value, with no log signal -- the resulting probe failure would look like an auth/response problem with no trace back to the real cause. Added a tracing::warn! on that branch. - validate_probe_url already blocks IPv6 link-local (fe80::/10) and unique-local (fd00::/8) addresses via is_ssrf_sensitive, but had no test coverage proving it (only the IPv4 link-local and IPv6 loopback cases were tested). Added the two missing regression tests. Signed-off-by: Jordi Gil * test(operator): cover reconcile() end-to-end against a mocked kube::Client All existing agent_tool_provider tests exercise resolve_phase_and_sites's constituent resolve_*/pure-logic functions in isolation; none drove the public reconcile() entrypoint itself, so nothing proved the resolved (phase, reason, matchingSites, discoveredTools) tuple actually reaches the Kubernetes API as the status PATCH body a real controller sends. Add a PATCH-capturing mock kube::Client and two reconcile()-level tests covering the two code paths that need no live MCP probe: a config-invalid provider (fully short-circuited, no Kubernetes calls at all) and a provider whose gridNetworkRef doesn't resolve (exercises the live GridNetwork GET). Both mutation-tested by hand against the reason strings they assert on to confirm they fail on regression, not just pass by construction. Signed-off-by: Jordi Gil * fix(operator): trust only the configured CA for AgentToolProvider probes Addresses grid#56 review feedback (nerdalert): attach_tls_ca used reqwest::ClientBuilder::add_root_certificate, which merges the configured CA into reqwest's platform trust store rather than replacing it -- reqwest's own docs mark this method deprecated in favor of tls_certs_merge()/tls_certs_only() for exactly this ambiguity. EndpointTlsConfig's doc comment promises the opposite: "the scraper trusts only this CA -- system root certificates are not consulted." With add_root_certificate, a publicly trusted certificate could still satisfy a probe explicitly configured to trust only a private CA. Switch to tls_certs_only([ca_cert]), which disables the platform trust store and uses only the supplied CA, matching the documented invariant and mirroring how metrics_scraper::build_tls_client_config already builds its rustls::RootCertStore from empty rather than merging into the platform's. Signed-off-by: Jordi Gil * fix(operator): fail AgentToolProvider probe closed on unencodable auth token Addresses grid#56 review feedback (nerdalert): auth_header_map already logged a warning when a resolved spec.auth bearer token contained characters invalid in an HTTP header value (see c2b724e), but still returned an empty header map and let the probe proceed unauthenticated. An MCP endpoint that permits anonymous tools/list could then be marked Available without the configured credential ever being exercised -- the diagnosable warning masked a silent open-fails-open path. auth_header_map now returns Result<_, McpProbeOutcome>, mapping an unencodable token to the new AuthConfigInvalid outcome (status.reason McpAuthTokenInvalid) instead of an empty header map, and run_probe_session propagates that failure instead of sending an unauthenticated request. Regenerated the AgentToolProvider CRD manifests (deploy/, charts/, tests/e2e/) for the new documented reason. Signed-off-by: Jordi Gil * fix(operator): bound and normalize discovered tool names before persisting Addresses grid#56 review feedback (nerdalert): Every tool name a probed MCP server returns was copied into status.discoveredTools with no count, name-length, or total-size limit, and in server-returned order. A server advertising an implausibly large or long-named tool catalog could grow the Kubernetes status object without bound; a server merely reordering an unchanged catalog would trigger an unnecessary status patch on every later reconcile. Add bound_and_normalize_discovered_tools: truncates each name to 256 bytes (at a UTF-8 char boundary), deduplicates and sorts (order is not semantically meaningful), then truncates the deduplicated list to 500 entries. Applied to every successful probe's tool list before it reaches McpProbeOutcome::Success. This bounds what this reconciler persists and holds in memory; it does not bound the raw HTTP response rmcp/reqwest buffer before parsing it into a tools/list result -- rmcp's JSON response path (unlike its SSE path, which honors max_sse_event_size) has no such cap in the version this workspace depends on. Tracked as a separate follow-up rather than folded into this fix, since closing it would mean bypassing rmcp's higher-level session API for a manual, byte-capped read. Signed-off-by: Jordi Gil --------- Signed-off-by: Jordi Gil --- .github/workflows/helm.yaml | 2 +- Cargo.lock | 763 +++++- Cargo.toml | 10 + .../grid-operator/crds/agenttoolprovider.yaml | 295 +++ .../templates/clusterrole-crd.yaml | 2 + .../tests/clusterrole-crd_test.yaml | 28 + deploy/crds/agenttoolprovider.yaml | 295 +++ docs/architecture/crds.md | 34 +- docs/architecture/overview.md | 10 +- mock-providers/Cargo.toml | 1 + mock-providers/src/lib.rs | 2 + mock-providers/src/main.rs | 83 +- mock-providers/src/mcp.rs | 120 + operator/Cargo.toml | 5 + operator/src/bin/generate_crds.rs | 6 +- operator/src/controller.rs | 5 + .../src/controller/agent_tool_provider.rs | 1284 ++++++++++ operator/src/crd/agent_tool_provider.rs | 142 +- operator/src/main.rs | 59 +- operator/src/metrics.rs | 106 + operator/src/resources.rs | 2 + operator/src/resources/mcp_probe.rs | 2056 +++++++++++++++++ scripts/generate-deployment-crds.sh | 7 +- scripts/verify-helm-chart.sh | 11 +- xtask/src/env.rs | 144 ++ xtask/src/env/kind.rs | 146 ++ xtask/src/env/operator.rs | 146 ++ 27 files changed, 5705 insertions(+), 59 deletions(-) create mode 100644 charts/grid-operator/crds/agenttoolprovider.yaml create mode 100644 charts/grid-operator/tests/clusterrole-crd_test.yaml create mode 100644 deploy/crds/agenttoolprovider.yaml create mode 100644 mock-providers/src/mcp.rs create mode 100644 operator/src/controller/agent_tool_provider.rs create mode 100644 operator/src/resources/mcp_probe.rs diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index 313b8d5..75ae8b6 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -92,7 +92,7 @@ jobs: run: helm plugin install https://github.com/helm-unittest/helm-unittest --version "${HELM_UNITTEST_VERSION}" - name: Chart template unit tests - run: helm unittest charts/grid-site + run: helm unittest charts/grid-site charts/grid-operator - name: Validate charts (static) run: ./scripts/verify-helm-chart.sh diff --git a/Cargo.lock b/Cargo.lock index aa6d92e..f85c590 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,7 +20,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -65,6 +65,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -139,7 +148,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -151,7 +160,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -185,7 +194,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] @@ -269,6 +289,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bincode" version = "2.0.1" @@ -347,6 +373,29 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + [[package]] name = "cipher" version = "0.4.4" @@ -388,7 +437,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -403,6 +452,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -437,6 +496,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crdt" version = "0.1.3" @@ -492,7 +560,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", ] [[package]] @@ -503,7 +571,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -531,7 +599,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -581,7 +649,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.118", ] [[package]] @@ -602,7 +670,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -620,7 +688,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -646,7 +714,7 @@ checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -712,7 +780,7 @@ checksum = "1f59e967f3f675997e4a4a6b99d2a75148d59d64c46211b78b4f34ebb951b273" dependencies = [ "bincode", "bytes", - "rand", + "rand 0.9.4", "serde", "tracing", ] @@ -813,7 +881,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -887,6 +955,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -1069,13 +1138,16 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -1083,12 +1155,139 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1108,6 +1307,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1142,7 +1347,56 @@ checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.118", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", ] [[package]] @@ -1197,7 +1451,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b326f5219dd55872a72c1b6ddd1b830b8334996c667449c29391d657d78d5e" dependencies = [ - "base64", + "base64 0.22.1", "jiff", "serde", "serde_json", @@ -1222,7 +1476,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcaf2d1f1a91e1805d4cd82e8333c022767ae8ffd65909bbef6802733a7dd40" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "either", "futures", @@ -1281,7 +1535,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn", + "syn 2.0.118", ] [[package]] @@ -1329,6 +1583,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -1395,6 +1655,7 @@ dependencies = [ "axum", "clap", "http", + "rmcp", "rustls", "serde", "serde_json", @@ -1507,6 +1768,8 @@ dependencies = [ "k8s-openapi", "kube", "prometheus", + "reqwest", + "rmcp", "rustls", "schemars", "scoring", @@ -1564,13 +1827,19 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pem" version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -1610,7 +1879,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1639,7 +1908,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1655,7 +1924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -1675,6 +1944,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1765,6 +2043,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -1793,6 +2082,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rcgen" version = "0.14.8" @@ -1833,7 +2128,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1865,6 +2160,45 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -1879,6 +2213,37 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmcp" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8dddc5b1924b9a59fba420166160ca2c4663a4e01803e52eda33070f56d63c8" +dependencies = [ + "async-trait", + "base64 0.23.1", + "bytes", + "chrono", + "futures", + "http", + "http-body", + "http-body-util", + "pastey", + "pin-project-lite", + "rand 0.10.2", + "reqwest", + "schemars", + "serde", + "serde_json", + "sse-stream", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-util", + "tower-service", + "tracing", + "uuid", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -1946,6 +2311,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -1975,6 +2367,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -1990,6 +2391,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ + "chrono", "dyn-clone", "ref-cast", "schemars_derive", @@ -2006,7 +2408,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.118", ] [[package]] @@ -2099,7 +2501,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2110,7 +2512,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2189,7 +2591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2218,6 +2620,22 @@ dependencies = [ "libc", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -2240,6 +2658,25 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sse-stream" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" +dependencies = [ + "bytes", + "futures-util", + "http-body", + "http-body-util", + "pin-project-lite", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" version = "0.11.1" @@ -2261,7 +2698,7 @@ dependencies = [ "bytes", "crdt", "foca", - "rand", + "rand 0.9.4", "serde", "serde_json", "thiserror 2.0.18", @@ -2280,11 +2717,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -2294,7 +2745,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2336,7 +2787,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2347,7 +2798,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2389,6 +2840,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tokio" version = "1.52.3" @@ -2413,7 +2874,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2426,6 +2887,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -2504,9 +2976,10 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "base64", + "base64 0.22.1", "bitflags 2.13.0", "bytes", + "futures-util", "http", "http-body", "mime", @@ -2515,6 +2988,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -2549,7 +3023,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2656,6 +3130,24 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2685,6 +3177,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2722,6 +3224,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.126" @@ -2741,7 +3253,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -2754,6 +3266,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2770,18 +3314,80 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2879,6 +3485,12 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + [[package]] name = "x509-parser" version = "0.18.1" @@ -2904,7 +3516,7 @@ dependencies = [ "certs", "clap", "pem", - "rand", + "rand 0.9.4", "serde", "serde_json", "serde_yaml", @@ -2925,6 +3537,29 @@ dependencies = [ "time", ] +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.54" @@ -2942,7 +3577,28 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", ] [[package]] @@ -2951,6 +3607,39 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 86e28a4..9e07e8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,16 @@ k8s-openapi = { version = "0.27.1", features = ["v1_32"] } kube = { version = "3.1.0", features = ["client", "derive", "runtime", "rustls-tls"] } prometheus = "0.14" rcgen = "0.14.8" +# rustls-no-provider (not "rustls"): the plain "rustls" feature pulls in +# aws-lc-rs as reqwest's crypto provider, which conflicts at runtime with the +# ring provider main.rs installs process-wide for kube/hyper-rustls — see +# rustls::crypto::CryptoProvider::install_default() in main.rs. +reqwest = { version = "0.13.4", default-features = false, features = ["rustls-no-provider", "json", "stream"] } +rmcp = { version = "3.1.1", default-features = false, features = [ + "client", + "transport-streamable-http-client-reqwest", + "reqwest-tls-no-provider", +] } rustls = { version = "0.23", default-features = false, features = ["logging", "std", "tls12"] } schemars = "1.2.1" sha2 = "0.10" diff --git a/charts/grid-operator/crds/agenttoolprovider.yaml b/charts/grid-operator/crds/agenttoolprovider.yaml new file mode 100644 index 0000000..2ea2833 --- /dev/null +++ b/charts/grid-operator/crds/agenttoolprovider.yaml @@ -0,0 +1,295 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: agenttoolproviders.grid.praxis-proxy.io +spec: + group: grid.praxis-proxy.io + names: + categories: [] + kind: AgentToolProvider + plural: agenttoolproviders + shortNames: [] + singular: agenttoolprovider + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.protocol + name: Protocol + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for AgentToolProviderSpec via `CustomResource` + properties: + spec: + description: Specification for an [`AgentToolProvider`]. + properties: + accessPolicy: + default: + siteSelector: + matchLabels: {} + description: Which sites can consume these tools. + properties: + siteSelector: + default: + matchLabels: {} + description: Which sites can route to this provider. + properties: + matchLabels: + additionalProperties: + type: string + default: {} + description: Label key-value pairs that must match. + type: object + type: object + type: object + auth: + description: Authentication configuration. + nullable: true + properties: + manual: + default: false + description: |- + Whether the user manages credentials manually. + + When true, the operator does not inject credentials + and the user is responsible for configuring auth. + type: boolean + secretRef: + description: Reference to a Secret containing the credential. + nullable: true + properties: + key: + description: |- + Key within the Secret's `data` map. + + Required when the Secret holds multiple keys (e.g. credential references + in `InferenceProvider.spec.auth.secretRef`). Omit only when the entire + Secret is consumed (e.g. TLS `ca_secret_ref`). + minLength: 1 + nullable: true + type: string + name: + description: Secret name. + minLength: 1 + type: string + namespace: + description: Secret namespace. + minLength: 1 + type: string + required: + - name + - namespace + type: object + strategy: + description: How credentials are presented to the provider. + enum: + - api_key + - bearer_token + - custom + - mtls_only + - oauth2 + - service_account + - sigv4 + type: string + required: + - strategy + type: object + endpoint: + description: HTTP endpoint of the MCP server. + type: string + gridNetworkRef: + description: |- + Name of the [`GridNetwork`] this provider belongs to. + + [`GridNetwork`]: crate::crd::grid_network::GridNetwork + type: string + protocol: + default: mcp + description: Protocol used (only "mcp" initially). + type: string + siteSelector: + default: + matchLabels: {} + description: Which sites host this provider. + properties: + matchLabels: + additionalProperties: + type: string + default: {} + description: Label key-value pairs that must match. + type: object + type: object + tls: + description: |- + TLS configuration for the operator's own MCP `tools/list` probe. + + Reuses [`EndpointTlsConfig`] from [`InferenceProvider`] (CA trust and + optional mTLS client identity for the probe connection). When absent, + the probe uses native root certificates and no client certificate. + + [`InferenceProvider`]: crate::crd::inference_provider::InferenceProvider + nullable: true + properties: + caSecretRef: + description: |- + Reference to a Secret containing the CA certificate PEM for server + verification. + + The Secret must contain the PEM-encoded CA certificate under the key + `ca.crt` (or the key specified by `key`). When this CA is + set, the scraper trusts **only** this CA — system root certificates + are not consulted. + properties: + key: + description: |- + Key within the Secret's `data` map. + + Required when the Secret holds multiple keys (e.g. credential references + in `InferenceProvider.spec.auth.secretRef`). Omit only when the entire + Secret is consumed (e.g. TLS `ca_secret_ref`). + minLength: 1 + nullable: true + type: string + name: + description: Secret name. + minLength: 1 + type: string + namespace: + description: Secret namespace. + minLength: 1 + type: string + required: + - name + - namespace + type: object + clientCertificateSecretRef: + description: |- + Reference to a Secret containing the client certificate and private + key for mutual TLS. + + When set, the scraper presents this identity during the TLS handshake. + The Secret must contain `tls.crt` (or `certificate_key`) and + `tls.key` (or `private_key_key`) in PEM format. + + When absent, the scraper performs one-way TLS only (server verification + with the CA from `ca_secret_ref`, no client certificate). + nullable: true + properties: + certificateKey: + default: tls.crt + description: Key within `Secret.data` holding the PEM-encoded client certificate. + minLength: 1 + type: string + name: + description: Secret name. + minLength: 1 + type: string + namespace: + description: Secret namespace. + minLength: 1 + type: string + privateKeyKey: + default: tls.key + description: Key within `Secret.data` holding the PEM-encoded private key. + minLength: 1 + type: string + required: + - name + - namespace + type: object + required: + - caSecretRef + type: object + tools: + default: [] + description: Tool definitions (auto-discovered if omitted). + items: + description: Metadata for a single MCP tool. + properties: + description: + description: Human-readable description. + nullable: true + type: string + name: + description: Tool name. + type: string + required: + - name + type: object + type: array + required: + - endpoint + - gridNetworkRef + type: object + status: + description: |- + Observed status of an [`AgentToolProvider`]. + + # Stable `reason` values + + `reason` is `None` while the provider is healthy. When set, it is one + of the following stable, machine-readable strings — following the + same naming convention as [`InferenceProvider`]'s `MetricsTls*`/ + `HealthCheckTls*` reasons: + + | Reason | Meaning | + |--------|---------| + | `ProviderConfigInvalid` | `spec.endpoint` or `spec.gridNetworkRef` is blank or whitespace-only. | + | `GridNetworkNotFound` | The `GridNetwork` referenced by `spec.gridNetworkRef` does not exist. | + | `McpEndpointUnreachable` | The MCP probe could not connect (transport failure, timeout, DNS error). | + | `McpToolsListInvalidResponse` | The endpoint responded but the `tools/list` response was malformed. | + | `McpAuthRejected` | The MCP server rejected the configured `spec.auth` credentials. | + | `McpAuthTokenInvalid` | The resolved `spec.auth` bearer token contains characters that cannot be sent as an HTTP header value; the probe fails closed rather than proceeding unauthenticated. | + | `EndpointTlsSecretMissing` | `spec.tls`'s referenced Secret does not exist in the cluster. | + | `EndpointTlsKeyMissing` | `spec.tls`'s referenced Secret exists but is missing the expected key. | + | `EndpointTlsMaterialInvalid` | `spec.tls`'s certificate or key material could not be parsed. | + + [`InferenceProvider`]: crate::crd::inference_provider::InferenceProvider + nullable: true + properties: + discoveredTools: + default: [] + description: Tools discovered via MCP `tools/list`. + items: + type: string + type: array + matchingSites: + default: [] + description: Sites matched by the site selector. + items: + type: string + type: array + observedGeneration: + default: 0 + description: Last observed generation. + format: int64 + type: integer + phase: + default: Pending + description: Current phase. + enum: + - Pending + - Available + - Degraded + - Unavailable + type: string + reason: + description: |- + Machine-readable reason for the current phase, `None` when healthy. + + See the type-level doc comment for the table of stable values. + nullable: true + type: string + type: object + required: + - spec + title: AgentToolProvider + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/grid-operator/templates/clusterrole-crd.yaml b/charts/grid-operator/templates/clusterrole-crd.yaml index 09cd63b..3ce3fed 100644 --- a/charts/grid-operator/templates/clusterrole-crd.yaml +++ b/charts/grid-operator/templates/clusterrole-crd.yaml @@ -8,6 +8,7 @@ metadata: rules: - apiGroups: ["grid.praxis-proxy.io"] resources: + - agenttoolproviders - gridnetworks - inferenceproviders verbs: ["get", "list", "watch", "patch"] @@ -17,6 +18,7 @@ rules: verbs: ["get", "list", "watch", "patch", "create", "update"] - apiGroups: ["grid.praxis-proxy.io"] resources: + - agenttoolproviders/status - gridnetworks/status - gridsites/status - inferenceproviders/status diff --git a/charts/grid-operator/tests/clusterrole-crd_test.yaml b/charts/grid-operator/tests/clusterrole-crd_test.yaml new file mode 100644 index 0000000..c59b5b4 --- /dev/null +++ b/charts/grid-operator/tests/clusterrole-crd_test.yaml @@ -0,0 +1,28 @@ +suite: ClusterRole CRD template +templates: + - templates/clusterrole-crd.yaml +tests: + - it: grants agenttoolproviders the same verbs as gridnetworks and inferenceproviders + asserts: + - contains: + path: rules[0].resources + content: agenttoolproviders + - equal: + path: rules[0].verbs + value: ["get", "list", "watch", "patch"] + + - it: grants agenttoolproviders/status get and patch, mirroring the other status subresources + asserts: + - contains: + path: rules[2].resources + content: agenttoolproviders/status + - equal: + path: rules[2].verbs + value: ["get", "patch"] + + - it: renders nothing when rbac.create is false + set: + rbac.create: false + asserts: + - hasDocuments: + count: 0 diff --git a/deploy/crds/agenttoolprovider.yaml b/deploy/crds/agenttoolprovider.yaml new file mode 100644 index 0000000..2ea2833 --- /dev/null +++ b/deploy/crds/agenttoolprovider.yaml @@ -0,0 +1,295 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: agenttoolproviders.grid.praxis-proxy.io +spec: + group: grid.praxis-proxy.io + names: + categories: [] + kind: AgentToolProvider + plural: agenttoolproviders + shortNames: [] + singular: agenttoolprovider + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.protocol + name: Protocol + type: string + - jsonPath: .status.phase + name: Phase + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for AgentToolProviderSpec via `CustomResource` + properties: + spec: + description: Specification for an [`AgentToolProvider`]. + properties: + accessPolicy: + default: + siteSelector: + matchLabels: {} + description: Which sites can consume these tools. + properties: + siteSelector: + default: + matchLabels: {} + description: Which sites can route to this provider. + properties: + matchLabels: + additionalProperties: + type: string + default: {} + description: Label key-value pairs that must match. + type: object + type: object + type: object + auth: + description: Authentication configuration. + nullable: true + properties: + manual: + default: false + description: |- + Whether the user manages credentials manually. + + When true, the operator does not inject credentials + and the user is responsible for configuring auth. + type: boolean + secretRef: + description: Reference to a Secret containing the credential. + nullable: true + properties: + key: + description: |- + Key within the Secret's `data` map. + + Required when the Secret holds multiple keys (e.g. credential references + in `InferenceProvider.spec.auth.secretRef`). Omit only when the entire + Secret is consumed (e.g. TLS `ca_secret_ref`). + minLength: 1 + nullable: true + type: string + name: + description: Secret name. + minLength: 1 + type: string + namespace: + description: Secret namespace. + minLength: 1 + type: string + required: + - name + - namespace + type: object + strategy: + description: How credentials are presented to the provider. + enum: + - api_key + - bearer_token + - custom + - mtls_only + - oauth2 + - service_account + - sigv4 + type: string + required: + - strategy + type: object + endpoint: + description: HTTP endpoint of the MCP server. + type: string + gridNetworkRef: + description: |- + Name of the [`GridNetwork`] this provider belongs to. + + [`GridNetwork`]: crate::crd::grid_network::GridNetwork + type: string + protocol: + default: mcp + description: Protocol used (only "mcp" initially). + type: string + siteSelector: + default: + matchLabels: {} + description: Which sites host this provider. + properties: + matchLabels: + additionalProperties: + type: string + default: {} + description: Label key-value pairs that must match. + type: object + type: object + tls: + description: |- + TLS configuration for the operator's own MCP `tools/list` probe. + + Reuses [`EndpointTlsConfig`] from [`InferenceProvider`] (CA trust and + optional mTLS client identity for the probe connection). When absent, + the probe uses native root certificates and no client certificate. + + [`InferenceProvider`]: crate::crd::inference_provider::InferenceProvider + nullable: true + properties: + caSecretRef: + description: |- + Reference to a Secret containing the CA certificate PEM for server + verification. + + The Secret must contain the PEM-encoded CA certificate under the key + `ca.crt` (or the key specified by `key`). When this CA is + set, the scraper trusts **only** this CA — system root certificates + are not consulted. + properties: + key: + description: |- + Key within the Secret's `data` map. + + Required when the Secret holds multiple keys (e.g. credential references + in `InferenceProvider.spec.auth.secretRef`). Omit only when the entire + Secret is consumed (e.g. TLS `ca_secret_ref`). + minLength: 1 + nullable: true + type: string + name: + description: Secret name. + minLength: 1 + type: string + namespace: + description: Secret namespace. + minLength: 1 + type: string + required: + - name + - namespace + type: object + clientCertificateSecretRef: + description: |- + Reference to a Secret containing the client certificate and private + key for mutual TLS. + + When set, the scraper presents this identity during the TLS handshake. + The Secret must contain `tls.crt` (or `certificate_key`) and + `tls.key` (or `private_key_key`) in PEM format. + + When absent, the scraper performs one-way TLS only (server verification + with the CA from `ca_secret_ref`, no client certificate). + nullable: true + properties: + certificateKey: + default: tls.crt + description: Key within `Secret.data` holding the PEM-encoded client certificate. + minLength: 1 + type: string + name: + description: Secret name. + minLength: 1 + type: string + namespace: + description: Secret namespace. + minLength: 1 + type: string + privateKeyKey: + default: tls.key + description: Key within `Secret.data` holding the PEM-encoded private key. + minLength: 1 + type: string + required: + - name + - namespace + type: object + required: + - caSecretRef + type: object + tools: + default: [] + description: Tool definitions (auto-discovered if omitted). + items: + description: Metadata for a single MCP tool. + properties: + description: + description: Human-readable description. + nullable: true + type: string + name: + description: Tool name. + type: string + required: + - name + type: object + type: array + required: + - endpoint + - gridNetworkRef + type: object + status: + description: |- + Observed status of an [`AgentToolProvider`]. + + # Stable `reason` values + + `reason` is `None` while the provider is healthy. When set, it is one + of the following stable, machine-readable strings — following the + same naming convention as [`InferenceProvider`]'s `MetricsTls*`/ + `HealthCheckTls*` reasons: + + | Reason | Meaning | + |--------|---------| + | `ProviderConfigInvalid` | `spec.endpoint` or `spec.gridNetworkRef` is blank or whitespace-only. | + | `GridNetworkNotFound` | The `GridNetwork` referenced by `spec.gridNetworkRef` does not exist. | + | `McpEndpointUnreachable` | The MCP probe could not connect (transport failure, timeout, DNS error). | + | `McpToolsListInvalidResponse` | The endpoint responded but the `tools/list` response was malformed. | + | `McpAuthRejected` | The MCP server rejected the configured `spec.auth` credentials. | + | `McpAuthTokenInvalid` | The resolved `spec.auth` bearer token contains characters that cannot be sent as an HTTP header value; the probe fails closed rather than proceeding unauthenticated. | + | `EndpointTlsSecretMissing` | `spec.tls`'s referenced Secret does not exist in the cluster. | + | `EndpointTlsKeyMissing` | `spec.tls`'s referenced Secret exists but is missing the expected key. | + | `EndpointTlsMaterialInvalid` | `spec.tls`'s certificate or key material could not be parsed. | + + [`InferenceProvider`]: crate::crd::inference_provider::InferenceProvider + nullable: true + properties: + discoveredTools: + default: [] + description: Tools discovered via MCP `tools/list`. + items: + type: string + type: array + matchingSites: + default: [] + description: Sites matched by the site selector. + items: + type: string + type: array + observedGeneration: + default: 0 + description: Last observed generation. + format: int64 + type: integer + phase: + default: Pending + description: Current phase. + enum: + - Pending + - Available + - Degraded + - Unavailable + type: string + reason: + description: |- + Machine-readable reason for the current phase, `None` when healthy. + + See the type-level doc comment for the table of stable values. + nullable: true + type: string + type: object + required: + - spec + title: AgentToolProvider + type: object + served: true + storage: true + subresources: + status: {} diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 9995c41..325307a 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -703,10 +703,40 @@ spec: grid.praxis-proxy.io/site: cluster-a ``` -**Phases**: Pending → Available → Degraded → Unavailable +**Phases**: Pending → Available → Unavailable + +`Degraded` is not currently reachable for this CRD: unlike +`InferenceProvider`'s metrics-scrape path, the MCP `tools/list` probe has no +partial-success state to represent — it either succeeds (`Available`) or +fails outright (`Unavailable`), mirroring `phase_and_reason_from_probe`'s and +`phase_from_matching`'s explicit design (both are tested to never emit +`Degraded`). **Status fields**: `discoveredTools` (auto-populated -from MCP `tools/list`), `matchingSites` +from MCP `tools/list`; a failed probe preserves the previous list rather than +clearing it), `matchingSites`, `reason` (machine-readable; see reason codes +below), `observedGeneration` + +#### AgentToolProvider reason codes + +| Reason | Phase | Meaning | +|---|---|---| +| `ProviderConfigInvalid` | Unavailable | Static spec validation failed (e.g. malformed `siteSelector`) before any `GridNetwork`/site/probe work runs. | +| `GridNetworkNotFound` | Unavailable | `spec.gridNetworkRef` does not resolve to an existing `GridNetwork`. | +| `CredentialSecretMissing` | Unavailable | `spec.auth.secretRef` does not resolve to an accessible Secret. | +| `McpEndpointUnreachable` | Unavailable | The MCP endpoint could not be reached: transport failure, DNS error, timeout, or a blocked (SSRF-sensitive) address. | +| `McpToolsListInvalidResponse` | Unavailable | The endpoint was reached but the `tools/list` exchange failed or returned an unparseable response. | +| `McpAuthRejected` | Unavailable | The MCP server rejected the configured `spec.auth` credentials (HTTP 401/403). | +| `McpAuthTokenInvalid` | Unavailable | The resolved `spec.auth` bearer token contains characters that cannot be sent as an HTTP header value; the probe fails closed rather than proceeding unauthenticated. | +| `EndpointTlsSecretMissing` | Unavailable | A referenced TLS Secret does not exist (or the requested key is absent — see [`grid#58`](https://github.com/praxis-proxy/grid/issues/58) for a known misclassification of the latter). | +| `EndpointTlsKeyMissing` | Unavailable | The expected key exists in the Secret but its value is empty. | +| `EndpointTlsMaterialInvalid` | Unavailable | CA certificate PEM material could not be parsed. | +| `EndpointTlsIdentityMismatch` | Unavailable | Client certificate or private key PEM material could not be parsed. | + +An empty `reason` with `phase: Available` means `SitesMatched`; an empty +`reason` with `phase: Pending` means `AwaitingSiteMatch` — both are +telemetry-only labels (`grid_mcp_probe_total`, Events), not persisted to +`status.reason` itself. ## AgentToAgentProvider diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index f2287f2..dab7907 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -222,9 +222,13 @@ The implemented inference path uses three cluster-scoped CRDs: | `InferenceProvider` | Declares model capacity: model name, backend kind, endpoint, health config, auth strategy, access policy, and provider status. | `AgentToolProvider` and `AgentToAgentProvider` are schema direction for MCP and -A2A. Their resource types exist, but the operator does not currently run full -controllers, distribute their state, score them, or render complete routed paths -for them. Inference is the mature reconciled path today. +A2A. `AgentToolProvider` has a running reconciler that resolves `siteSelector` +matches and live-probes the endpoint's MCP `tools/list` contract, but does not +yet distribute discovered tools across sites via SWIM/CRDT, score them, or +render a routed data-plane path — those remain grid-local only. +`AgentToAgentProvider`'s resource type exists, but the operator does not yet +run a controller for it at all. Inference is the mature reconciled path +today. See [CRDs](crds.md) for field-level details. diff --git a/mock-providers/Cargo.toml b/mock-providers/Cargo.toml index 0448133..178b390 100644 --- a/mock-providers/Cargo.toml +++ b/mock-providers/Cargo.toml @@ -10,6 +10,7 @@ publish = false axum = { workspace = true } clap = { workspace = true } http = { workspace = true } +rmcp = { workspace = true, features = ["server", "transport-streamable-http-server"] } rustls = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/mock-providers/src/lib.rs b/mock-providers/src/lib.rs index 6959e4f..e18d10a 100644 --- a/mock-providers/src/lib.rs +++ b/mock-providers/src/lib.rs @@ -15,6 +15,8 @@ pub mod anthropic; pub mod bedrock; /// Shared HTTP response utilities. mod common; +/// Mock MCP (Model Context Protocol) `tools/list` server. +pub mod mcp; /// Mock `OpenAI` chat completions and Responses API. pub mod openai; /// Mock Google Vertex AI `generateContent` API. diff --git a/mock-providers/src/main.rs b/mock-providers/src/main.rs index ba0dfec..2a2b75a 100644 --- a/mock-providers/src/main.rs +++ b/mock-providers/src/main.rs @@ -29,8 +29,8 @@ struct Cli { #[arg( short, long, - required_unless_present_any = ["tcp_probe", "http_probe", "tls_probe_server"], - conflicts_with_all = ["tcp_probe", "http_probe", "tls_probe_server"] + required_unless_present_any = ["tcp_probe", "http_probe", "tls_probe_server", "mcp_server"], + conflicts_with_all = ["tcp_probe", "http_probe", "tls_probe_server", "mcp_server"] )] provider: Option, @@ -38,10 +38,24 @@ struct Cli { #[arg(long, default_value = "8080")] port: u16, + /// Run a mock MCP (Model Context Protocol) `tools/list` server instead + /// of an AI provider mock. + #[arg(long, conflicts_with_all = ["provider", "tcp_probe", "http_probe", "tls_probe_server"])] + mcp_server: bool, + + /// Comma-separated tool names the MCP mock reports from `tools/list`. + #[arg(long, default_value = "search", requires = "mcp_server")] + mcp_tools: String, + + /// If set, the MCP mock rejects `tools/list` calls whose bearer token + /// does not match this value. + #[arg(long, requires = "mcp_server")] + mcp_bearer: Option, + /// Run a TLS-only probe server that accepts mTLS connections. #[arg( long, - conflicts_with_all = ["provider", "tcp_probe", "http_probe"], + conflicts_with_all = ["provider", "tcp_probe", "http_probe", "mcp_server"], requires_all = ["tls_cert", "tls_key", "tls_ca"] )] tls_probe_server: bool, @@ -125,8 +139,13 @@ async fn main() { return; } + if cli.mcp_server { + run_mcp_server(cli.port, &cli.mcp_tools, cli.mcp_bearer.as_deref()).await; + return; + } + let Some(provider) = cli.provider else { - eprintln!("either --provider, --tcp-probe, or --http-probe is required"); + eprintln!("either --provider, --mcp-server, --tcp-probe, or --http-probe is required"); std::process::exit(2); }; let state = app_state(); @@ -281,6 +300,30 @@ async fn run_tls_probe_server( } } +/// Run the mock MCP `tools/list` server (see `mock_providers::mcp`). +async fn run_mcp_server(port: u16, tools_csv: &str, required_bearer: Option<&str>) { + let tools: Vec = tools_csv + .split(',') + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(str::to_owned) + .collect(); + let router = mock_providers::mcp::router(tools, required_bearer.map(str::to_owned)); + + let addr = format!("0.0.0.0:{port}"); + eprintln!("mock-mcp-server listening on {addr}"); + + let listener = tokio::net::TcpListener::bind(&addr).await.unwrap_or_else(|error| { + eprintln!("failed to bind {addr}: {error}"); + std::process::exit(1); + }); + + axum::serve(listener, router).await.unwrap_or_else(|error| { + eprintln!("server error: {error}"); + std::process::exit(1); + }); +} + /// Run a single TCP probe for `NetworkPolicy` verification. async fn run_tcp_probe(target: &str, timeout: Duration) { if target.is_empty() || target.len() > 512 || timeout.is_zero() || timeout > Duration::from_secs(30) { @@ -430,6 +473,38 @@ mod tests { ); } + #[test] + fn mcp_server_mode_parses_without_provider() { + let cli = Cli::try_parse_from([ + "mock-providers", + "--mcp-server", + "--port", + "9091", + "--mcp-tools", + "search,read_file", + "--mcp-bearer", + "s3cr3t", + ]) + .unwrap_or_else(|_| std::process::abort()); + assert!(cli.provider.is_none()); + assert!(cli.mcp_server); + assert_eq!(cli.mcp_tools, "search,read_file"); + assert_eq!(cli.mcp_bearer.as_deref(), Some("s3cr3t")); + } + + #[test] + fn mcp_server_conflicts_with_provider() { + let result = Cli::try_parse_from(["mock-providers", "--provider", "openai", "--mcp-server"]); + assert!(result.is_err(), "--mcp-server must conflict with --provider"); + } + + #[test] + fn mcp_tools_defaults_to_search_when_unset() { + let cli = Cli::try_parse_from(["mock-providers", "--mcp-server"]).unwrap_or_else(|_| std::process::abort()); + assert_eq!(cli.mcp_tools, "search"); + assert!(cli.mcp_bearer.is_none()); + } + #[test] fn tls_probe_server_mode_parses_without_provider() { let cli = Cli::try_parse_from([ diff --git a/mock-providers/src/mcp.rs b/mock-providers/src/mcp.rs new file mode 100644 index 0000000..e72ec87 --- /dev/null +++ b/mock-providers/src/mcp.rs @@ -0,0 +1,120 @@ +//! Mock MCP (Model Context Protocol) `tools/list` server. +//! +//! Built on the real `rmcp` server SDK (the same crate `AgentToolProvider`'s +//! live probe uses on the client side — see +//! `operator/src/resources/mcp_probe.rs`), not a hand-rolled JSON-RPC +//! responder. This guarantees the mock speaks the actual Streamable HTTP +//! wire protocol (handshake, session semantics, SSE framing) rather than an +//! approximation that happens to satisfy one client implementation. +//! +//! Used by `cargo xtask env verify-agenttoolprovider-convergence` as a real, +//! deployed-in-cluster MCP endpoint for `AgentToolProvider`'s probe to +//! discover tools from — mirroring how `openai`/`anthropic`/etc. already +//! serve as real in-cluster mocks for `InferenceProvider`. + +use rmcp::{ + ServerHandler, + model::{ListToolsResult, PaginatedRequestParams, ServerCapabilities, ServerInfo, Tool}, + service::RequestContext, + transport::streamable_http_server::{ + StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager, + }, +}; + +/// A minimal MCP server that answers `tools/list` with a fixed set of tool +/// names, optionally requiring a specific bearer token. +#[derive(Clone)] +struct FixedToolsServer { + /// Tool names returned by `tools/list`, in order. + tools: Vec, + /// If set, `tools/list` calls must carry this exact bearer token. + required_bearer: Option, +} + +impl ServerHandler for FixedToolsServer { + fn get_info(&self) -> ServerInfo { + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) + } + + async fn list_tools( + &self, + _request: Option, + context: RequestContext, + ) -> Result { + if let Some(expected) = &self.required_bearer { + // rmcp threads the raw incoming `http::request::Parts` (headers + // included) into RequestContext::extensions — no axum middleware + // needed to see what the probe actually sent. + let got = context + .extensions + .get::() + .and_then(|parts| parts.headers.get(http::header::AUTHORIZATION)) + .and_then(|value| value.to_str().ok()); + if got != Some(format!("Bearer {expected}").as_str()) { + return Err(rmcp::ErrorData::invalid_request("missing or wrong bearer token", None)); + } + } + let tools = self + .tools + .iter() + .map(|name| { + let mut tool = Tool::default(); + tool.name = name.clone().into(); + tool + }) + .collect(); + Ok(ListToolsResult::with_all_items(tools)) + } +} + +/// Build an `axum::Router` serving a `FixedToolsServer` (private to this +/// module) at `/mcp`. +/// +/// `required_bearer: None` means the server accepts any (or no) +/// `Authorization` header — used to validate `AgentToolProvider`'s +/// unauthenticated healthy-probe path in the E2E convergence check. +pub fn router(tools: Vec, required_bearer: Option) -> axum::Router { + let handler = FixedToolsServer { tools, required_bearer }; + // `rmcp`'s default `allowed_hosts` (`localhost`/`127.0.0.1`/`::1`) is a + // DNS-rebinding guard aimed at servers bound to a developer's own + // loopback interface. This mock is deployed in-cluster and reached by + // real probe clients over its Service DNS name or `NodePort` address — + // neither of which is loopback — so the default would 403 every + // legitimate probe. Disabling it is safe here: this is a disposable test + // fixture with no browser-facing surface, not a public deployment. + let config = StreamableHttpServerConfig::default().disable_allowed_hosts(); + let service: StreamableHttpService = + StreamableHttpService::new(move || Ok(handler.clone()), std::sync::Arc::default(), config); + axum::Router::new().nest_service("/mcp", service) +} + +#[cfg(test)] +mod tests { + use tower::ServiceExt as _; + + use super::*; + + #[tokio::test] + async fn unauthenticated_request_to_mcp_path_is_routed_not_404() { + // A full protocol round trip needs a real rmcp client (covered by + // the operator's own integration tests against this exact server + // shape); this test only proves the router wiring itself — that + // `/mcp` is a live route, not a typo'd path silently 404ing. + let app = router(vec!["search".to_owned()], None); + let response = app + .oneshot( + http::Request::builder() + .method("GET") + .uri("/mcp") + .body(axum::body::Body::empty()) + .unwrap_or_else(|_| std::process::abort()), + ) + .await + .unwrap_or_else(|_| std::process::abort()); + assert_ne!( + response.status(), + http::StatusCode::NOT_FOUND, + "the /mcp path must be routed to the MCP service, not fall through to a 404" + ); + } +} diff --git a/operator/Cargo.toml b/operator/Cargo.toml index a71eb33..4e8fa86 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -24,6 +24,8 @@ k8s-openapi = { workspace = true } kube = { workspace = true } prometheus = { workspace = true } schemars = { workspace = true } +reqwest = { workspace = true } +rmcp = { workspace = true } rustls = { workspace = true } sha2 = { workspace = true } serde = { workspace = true } @@ -39,6 +41,9 @@ uuid = { workspace = true } zeroize = { workspace = true } [dev-dependencies] +# Server-side rmcp features, only for the mcp_probe integration tests, which +# spin up a real Streamable HTTP MCP server to probe against. +rmcp = { workspace = true, features = ["server", "transport-streamable-http-server"] } tower = { workspace = true } [lints] diff --git a/operator/src/bin/generate_crds.rs b/operator/src/bin/generate_crds.rs index 1b8d8ec..8cd8fe3 100644 --- a/operator/src/bin/generate_crds.rs +++ b/operator/src/bin/generate_crds.rs @@ -17,10 +17,14 @@ //! independently. use kube::CustomResourceExt as _; -use operator::crd::{grid_network::GridNetwork, grid_site::GridSite, inference_provider::InferenceProvider}; +use operator::crd::{ + agent_tool_provider::AgentToolProvider, grid_network::GridNetwork, grid_site::GridSite, + inference_provider::InferenceProvider, +}; fn main() { let crds = [ + serde_json::to_value(AgentToolProvider::crd()), serde_json::to_value(GridNetwork::crd()), serde_json::to_value(GridSite::crd()), serde_json::to_value(InferenceProvider::crd()), diff --git a/operator/src/controller.rs b/operator/src/controller.rs index d23624d..23c5751 100644 --- a/operator/src/controller.rs +++ b/operator/src/controller.rs @@ -1,5 +1,10 @@ //! Kubernetes controllers for the Grid Operator. +/// [`AgentToolProvider`] controller. +/// +/// [`AgentToolProvider`]: crate::crd::agent_tool_provider::AgentToolProvider +pub mod agent_tool_provider; + /// [`GridNetwork`] controller. /// /// [`GridNetwork`]: crate::crd::grid_network::GridNetwork diff --git a/operator/src/controller/agent_tool_provider.rs b/operator/src/controller/agent_tool_provider.rs new file mode 100644 index 0000000..0fd5f1d --- /dev/null +++ b/operator/src/controller/agent_tool_provider.rs @@ -0,0 +1,1284 @@ +//! [`AgentToolProvider`] controller. +//! +//! Reconciles [`AgentToolProvider`] resources: validates the static spec, +//! resolves the referenced [`GridNetwork`], resolves matching [`GridSite`]s +//! via the site selector, live-probes the endpoint's MCP `tools/list` +//! contract, and sets `status.phase`, `status.matchingSites`, +//! `status.discoveredTools`, `status.reason`, and `status.observedGeneration`. +//! +//! Structured exactly like [`inference_provider`](crate::controller::inference_provider): +//! static validation short-circuits first, then `GridNetwork`/site +//! resolution, then the live probe outcome merges on top. +//! +//! [`AgentToolProvider`]: crate::crd::agent_tool_provider::AgentToolProvider +//! [`GridNetwork`]: crate::crd::grid_network::GridNetwork +//! [`GridSite`]: crate::crd::grid_site::GridSite + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use k8s_openapi::api::core::v1::ObjectReference; +use kube::{ + Client, Resource as _, + api::{Api, ListParams, Patch, PatchParams}, + runtime::{ + controller::Action, + events::{Event, EventType, Recorder, Reporter}, + }, +}; +use tracing::info; + +use crate::{ + crd::{ + agent_tool_provider::{AgentToolProvider, AgentToolProviderStatus}, + grid_network::GridNetwork, + grid_site::GridSite, + inference_provider::ProviderPhase, + }, + error::OperatorError, + resources::{ + credentials::{self, CredentialPlan, CredentialResolver as _, KubernetesSecretResolver}, + mcp_probe, + }, +}; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/// Requeue interval after a successful reconciliation. +/// +/// Matches [`inference_provider`](crate::controller::inference_provider)'s +/// default — no `healthCheck.interval`-equivalent config exists on +/// [`AgentToolProviderSpec`](crate::crd::agent_tool_provider::AgentToolProviderSpec) yet. +const REQUEUE_INTERVAL: Duration = Duration::from_secs(300); + +/// Total bounded wall-clock budget for the live MCP probe: DNS resolution, +/// TLS Secret material reads, connect/handshake, and the `tools/list` call +/// combined — enforced as a single outer timeout in +/// [`mcp_probe::probe_agent_tool_provider`], not summed/multiplied across +/// phases. +/// +/// [`AgentToolProviderSpec`](crate::crd::agent_tool_provider::AgentToolProviderSpec) +/// has no `healthCheck.timeout`-equivalent field yet, so this is a fixed +/// constant rather than a per-resource override — revisit if a future CRD +/// revision adds one. +const PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Field manager name for server-side apply. +const FIELD_MANAGER: &str = "grid-operator"; + +// --------------------------------------------------------------------------- +// Reconcile +// --------------------------------------------------------------------------- + +/// Reconcile an [`AgentToolProvider`] resource. +/// +/// # Errors +/// +/// Returns [`OperatorError`] on Kubernetes API errors. +pub async fn reconcile(provider: Arc, client: Arc) -> Result { + let name = provider + .metadata + .name + .as_deref() + .unwrap_or_else(|| std::process::abort()); + + info!(name, "reconciling AgentToolProvider"); + + let reporter = Reporter { + controller: "agent-tool-provider-controller".into(), + instance: None, + }; + let object_ref = provider.object_ref(&()); + let recorder = Recorder::new(client.as_ref().clone(), reporter); + + let (phase, matching_sites, reason, discovered_tools) = + Box::pin(resolve_phase_and_sites(&provider, &client)).await?; + let generation = provider.metadata.generation.unwrap_or(0); + update_status( + &provider, + &client, + phase, + matching_sites, + discovered_tools, + generation, + reason, + &recorder, + &object_ref, + ) + .await?; + + Ok(Action::requeue(REQUEUE_INTERVAL)) +} + +/// Error policy for the [`AgentToolProvider`] controller. +pub fn error_policy(_provider: Arc, error: &OperatorError, _ctx: Arc) -> Action { + tracing::error!(%error, "AgentToolProvider reconciliation failed"); + Action::requeue(Duration::from_secs(30)) +} + +// --------------------------------------------------------------------------- +// Static validation +// --------------------------------------------------------------------------- + +/// Validate the static configuration of a provider (no Kubernetes API calls). +/// +/// Returns `Some(reason)` if the provider has a configuration error that +/// immediately maps to `Unavailable`, or `None` if static validation passes. +/// +/// Business rule: a provider must have both a non-blank `endpoint` and a +/// non-blank `gridNetworkRef` to be considered configured. The +/// `gridNetworkRef` *existence* check (does the referenced `GridNetwork` +/// actually exist) requires a Kubernetes API call and is not part of this +/// pure function — see resolution in `resolve_phase_and_sites`. +pub(crate) fn validate_provider_config(provider: &AgentToolProvider) -> Option<&'static str> { + if provider.spec.endpoint.trim().is_empty() { + return Some("blank endpoint"); + } + if provider.spec.grid_network_ref.trim().is_empty() { + return Some("blank gridNetworkRef"); + } + None +} + +// --------------------------------------------------------------------------- +// Site resolution and matching +// --------------------------------------------------------------------------- + +/// Compute the provider phase from site matching results. +/// +/// Business rule: a provider is only actionable once at least one site +/// matches its selector. Returns [`ProviderPhase::Pending`] when no sites +/// match, and [`ProviderPhase::Available`] when at least one site matches. +/// +/// This function never returns [`ProviderPhase::Degraded`] or +/// [`ProviderPhase::Unavailable`] — those are only reachable via the +/// config/`GridNetwork`-missing short-circuits in `resolve_phase_and_sites`, +/// or the live probe outcome merge (`phase_and_reason_from_probe`). +pub(crate) fn phase_from_matching(matching: &[String]) -> ProviderPhase { + if matching.is_empty() { + ProviderPhase::Pending + } else { + ProviderPhase::Available + } +} + +/// Apply `siteSelector.matchLabels` against the supplied sites. +/// +/// An empty `matchLabels` matches all sites. All configured key-value pairs +/// must match (AND semantics); extra labels on the site are ignored. +/// Returns a deterministically sorted list of matching site names. +/// +/// Network filtering (by `spec.gridNetworkRef`) is the caller's +/// responsibility — this function does not filter by network, mirroring +/// [`inference_provider::sites_matching_selector`](crate::controller::inference_provider::sites_matching_selector). +pub(crate) fn sites_matching_selector(provider: &AgentToolProvider, sites: &[GridSite]) -> Vec { + let selector = &provider.spec.site_selector.match_labels; + + let mut names: Vec = sites + .iter() + .filter(|site| { + let site_labels = site.metadata.labels.as_ref(); + selector + .iter() + .all(|(k, v)| site_labels.is_some_and(|labels| labels.get(k).is_some_and(|sv| sv == v))) + }) + .filter_map(|site| site.metadata.name.clone()) + .collect(); + + names.sort(); + names +} + +/// List all [`GridSite`]s whose `spec.gridNetworkRef` matches `network_ref`. +/// +/// Network filtering is applied here so that [`sites_matching_selector`] +/// only sees sites from the correct network. +async fn list_sites_for_network(client: &Client, network_ref: &str) -> Result, OperatorError> { + let api: Api = Api::all(client.clone()); + let all = api.list(&ListParams::default()).await?; + Ok(all + .items + .into_iter() + .filter(|s| s.spec.grid_network_ref == network_ref) + .collect()) +} + +/// Static config validation plus the `GridNetwork`-existence check, both of +/// which short-circuit straight to `Unavailable` before any site or probe +/// work runs. +/// +/// Unlike `inference_provider.rs` (which leaves status.reason as None for +/// its equivalent checks), `AgentToolProvider` populates a stable reason +/// here: grid#9 requires transition evidence with bounded cardinality +/// across Events/metrics/logs, and a stable reason string is the shared +/// label all three channels key off in `update_status`. +/// +/// # Errors +/// +/// Returns [`OperatorError`] on Kubernetes API failures. +async fn static_config_failure_reason( + provider: &AgentToolProvider, + client: &Client, + name: &str, +) -> Result, OperatorError> { + if let Some(config_error) = validate_provider_config(provider) { + tracing::warn!(name, reason = config_error, "AgentToolProvider config invalid"); + return Ok(Some("ProviderConfigInvalid")); + } + + let network_ref = &provider.spec.grid_network_ref; + let network_api: Api = Api::all(client.clone()); + if network_api.get_opt(network_ref).await?.is_none() { + tracing::warn!(name, network = %network_ref, "referenced GridNetwork not found"); + return Ok(Some("GridNetworkNotFound")); + } + + Ok(None) +} + +/// Determine the provider phase, matching sites, optional failure reason, +/// and discovered tool names. +/// +/// Returns `(ProviderPhase, sorted_matching_site_names, Option, discovered_tools)`. +/// +/// # Errors +/// +/// Returns [`OperatorError`] on Kubernetes API failures. +async fn resolve_phase_and_sites( + provider: &AgentToolProvider, + client: &Client, +) -> Result<(ProviderPhase, Vec, Option, Vec), OperatorError> { + let name = provider.metadata.name.as_deref().unwrap_or("?"); + let previous_tools = provider + .status + .as_ref() + .map_or_else(Vec::new, |s| s.discovered_tools.clone()); + + if let Some(reason) = static_config_failure_reason(provider, client, name).await? { + return Ok(( + ProviderPhase::Unavailable, + Vec::new(), + Some(reason.to_owned()), + previous_tools, + )); + } + + let sites = list_sites_for_network(client, &provider.spec.grid_network_ref).await?; + let matching = sites_matching_selector(provider, &sites); + let site_phase = phase_from_matching(&matching); + + if site_phase != ProviderPhase::Available { + // No sites match (yet) — nothing to probe. Mirrors + // inference_provider's semantics: the health probe never runs + // before the resource has anything to reach. + return Ok((site_phase, matching, None, previous_tools)); + } + + Box::pin(probe_and_merge(provider, client, name, matching, previous_tools)).await +} + +/// Outcome of resolving `spec.auth` into a probe-ready credential, before +/// the live probe itself runs. +enum CredentialProbeInput { + /// Credentials (if any) resolved cleanly; the probe may proceed. + Ready(Option), + /// Config or Secret resolution failed; carries the stable `status.reason`. + Failed(String), +} + +/// Resolve `spec.auth` into a [`CredentialProbeInput`]: parse the plan, +/// verify the referenced Secret is accessible, and (for `bearer_token`) +/// resolve the token value. +/// +/// Split out of [`probe_and_merge`] purely to keep both functions within the +/// project's complexity lints — this is the credential half of what was +/// previously one larger function. +/// +/// # Errors +/// +/// Returns [`OperatorError`] on Kubernetes API failures. +async fn resolve_probe_credentials( + provider: &AgentToolProvider, + client: &Client, + name: &str, +) -> Result { + let plan = match credentials::credential_plan_from_auth(provider.spec.auth.as_ref()) { + Ok(plan) => plan, + Err(_parse_err) => { + let cr = credentials::credential_failure_reason_for_auth(provider.spec.auth.as_ref()); + tracing::warn!(name, reason = cr.as_str(), "AgentToolProvider auth config invalid"); + return Ok(CredentialProbeInput::Failed(cr.as_str().to_owned())); + }, + }; + + if let Some(cr) = credentials::verify_credential_accessible(client, &plan).await? { + tracing::warn!( + name, + reason = cr.as_str(), + "AgentToolProvider credential Secret inaccessible" + ); + return Ok(CredentialProbeInput::Failed(cr.as_str().to_owned())); + } + + let CredentialPlan::Bearer(bearer_ref) = &plan else { + return Ok(CredentialProbeInput::Ready(None)); + }; + + let resolver = KubernetesSecretResolver::new(client.clone()); + match resolver.resolve(bearer_ref).await { + Ok(token) => Ok(CredentialProbeInput::Ready(Some(token))), + Err(error) => { + tracing::warn!(name, %error, "AgentToolProvider bearer token resolution failed"); + Ok(CredentialProbeInput::Failed("CredentialSecretMissing".to_owned())) + }, + } +} + +/// Resolve credentials and run the live MCP probe, merging its outcome into +/// the final `(phase, matching_sites, reason, discovered_tools)` tuple. +/// +/// Split out of [`resolve_phase_and_sites`] to keep that function within the +/// project's complexity lints once credential resolution and probing are +/// both inline. +/// +/// # Errors +/// +/// Returns [`OperatorError`] on Kubernetes API failures. +async fn probe_and_merge( + provider: &AgentToolProvider, + client: &Client, + name: &str, + matching: Vec, + previous_tools: Vec, +) -> Result<(ProviderPhase, Vec, Option, Vec), OperatorError> { + let token = match Box::pin(resolve_probe_credentials(provider, client, name)).await? { + CredentialProbeInput::Ready(token) => token, + CredentialProbeInput::Failed(reason) => { + return Ok((ProviderPhase::Unavailable, matching, Some(reason), previous_tools)); + }, + }; + + let probe_started = Instant::now(); + let outcome = Box::pin(mcp_probe::probe_agent_tool_provider( + client, + mcp_probe::ProbeRequest { + endpoint: &provider.spec.endpoint, + timeout: PROBE_TIMEOUT, + tls_config: provider.spec.tls.as_ref(), + provider_identity: name, + auth_token: token.as_ref(), + }, + )) + .await; + crate::metrics::record_mcp_probe(mcp_probe::mcp_probe_outcome_label(&outcome), probe_started.elapsed()); + + let (probe_phase, probe_reason) = mcp_probe::phase_and_reason_from_probe(&outcome); + let discovered = mcp_probe::discovered_tools_after_probe(&previous_tools, &outcome); + + Ok((probe_phase, matching, probe_reason, discovered)) +} + +// --------------------------------------------------------------------------- +// Status Update +// --------------------------------------------------------------------------- + +/// Patch the [`AgentToolProvider`] status subresource. +/// +/// # Errors +/// +/// Returns [`OperatorError`] on Kubernetes API errors. +#[expect( + clippy::too_many_arguments, + reason = "all parameters are distinct reconcile outputs plus telemetry sinks; no logical grouping reduces them" +)] +#[expect( + clippy::large_stack_frames, + reason = "async future with kube API types; mirrors grid_site.rs precedent" +)] +#[expect( + clippy::too_many_lines, + reason = "status construction, patch-if-changed short-circuit, and the transition-telemetry call each need to \ + stay inline for the flow to read top-to-bottom; the telemetry body itself is already split out into \ + emit_transition_telemetry" +)] +async fn update_status( + provider: &AgentToolProvider, + client: &Client, + phase: ProviderPhase, + matching_sites: Vec, + discovered_tools: Vec, + observed_generation: i64, + reason: Option, + recorder: &Recorder, + object_ref: &ObjectReference, +) -> Result<(), OperatorError> { + let name = provider + .metadata + .name + .as_deref() + .unwrap_or_else(|| std::process::abort()); + + let existing = provider.status.as_ref(); + let current_phase = existing.map(|s| &s.phase); + let phase_changed = current_phase != Some(&phase); + let reason_changed = existing.map(|s| &s.reason) != Some(&reason); + + let api: Api = Api::all(client.clone()); + let status = AgentToolProviderStatus { + discovered_tools, + matching_sites, + observed_generation, + phase, + reason, + }; + + if !agent_tool_provider_status_needs_update(existing, &status) { + return Ok(()); + } + + let patch = serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "AgentToolProvider", + "status": status + }); + + api.patch_status(name, &PatchParams::apply(FIELD_MANAGER).force(), &Patch::Apply(patch)) + .await?; + + info!(name, "updated AgentToolProvider status"); + + // Events, metrics, and transition-level logs only fire on a real phase + // or reason transition — never on a matchingSites/discoveredTools-only + // patch — so a converged provider being repeatedly re-listed doesn't + // spam the Event feed or inflate the phase-transition counter. + if is_real_transition(phase_changed, reason_changed) { + emit_transition_telemetry(name, current_phase, &status, recorder, object_ref).await; + } + + Ok(()) +} + +/// Emit the Event, metric, and transition-level log for a real phase or +/// reason transition. +/// +/// Split out of [`update_status`] purely to keep that function within the +/// project's line/complexity lints — this is the transition-telemetry half +/// of what was previously one larger function, called only once, from the +/// `is_real_transition` branch. +async fn emit_transition_telemetry( + name: &str, + current_phase: Option<&ProviderPhase>, + status: &AgentToolProviderStatus, + recorder: &Recorder, + object_ref: &ObjectReference, +) { + let label = telemetry_reason_label(&status.phase, status.reason.as_deref()); + let from_label = current_phase.map_or("None", phase_label); + let to_label = phase_label(&status.phase); + + tracing::info!(name, previous_phase = ?current_phase, phase = ?status.phase, reason = label, "AgentToolProvider phase transition"); + + if let Err(e) = recorder + .publish( + &Event { + type_: event_type_for_reason(label), + reason: label.to_owned(), + note: status.reason.clone(), + action: "Reconcile".to_owned(), + secondary: None, + }, + object_ref, + ) + .await + { + tracing::warn!(error = %e, "failed to publish AgentToolProvider event"); + } + + crate::metrics::record_agent_tool_provider_phase_transition(from_label, to_label, label); +} + +/// Render a [`ProviderPhase`] as its bounded-cardinality metric/log label. +fn phase_label(phase: &ProviderPhase) -> &'static str { + match phase { + ProviderPhase::Pending => "Pending", + ProviderPhase::Available => "Available", + ProviderPhase::Degraded => "Degraded", + ProviderPhase::Unavailable => "Unavailable", + } +} + +/// Synthesize a bounded telemetry reason label for Events, metrics, and logs. +/// +/// Business rule: when `status.reason` is set (an unhealthy phase with a +/// diagnostic code — see [`AgentToolProviderStatus`]'s doc comment), that +/// code *is* the telemetry label, keeping a single source of truth between +/// what a user reads on the CR and what appears in Events/metrics. When +/// `status.reason` is `None` (a healthy phase), this synthesizes one of two +/// bounded labels from the phase alone, since [`AgentToolProviderStatus`] +/// deliberately never sets `reason` while healthy. +fn telemetry_reason_label<'reason>(phase: &ProviderPhase, status_reason: Option<&'reason str>) -> &'reason str { + match status_reason { + Some(reason) => reason, + None if *phase == ProviderPhase::Available => "SitesMatched", + None => "AwaitingSiteMatch", + } +} + +/// Map a telemetry reason label to a Kubernetes [`EventType`]. +/// +/// [`Normal`] for the two healthy-phase labels synthesized by +/// [`telemetry_reason_label`]; [`Warning`] for every diagnostic +/// `status.reason` code (config, `GridNetwork`, and probe/TLS failures), +/// including any future code not yet in this list, +/// since an unrecognized reason is safer treated as a `Warning` than +/// silently downgraded to `Normal`. +/// +/// [`Normal`]: EventType::Normal +/// [`Warning`]: EventType::Warning +fn event_type_for_reason(reason: &str) -> EventType { + match reason { + "SitesMatched" | "AwaitingSiteMatch" => EventType::Normal, + _ => EventType::Warning, + } +} + +/// Whether a phase or reason change is a "real" transition worth surfacing +/// via Event, metric, and transition-level log — as opposed to a status +/// patch driven solely by `matchingSites`/`discoveredTools` churn. +fn is_real_transition(phase_changed: bool, reason_changed: bool) -> bool { + phase_changed || reason_changed +} + +/// Return whether the status subresource differs from the desired status. +/// +/// Business rule: the status subresource is only patched when `phase`, +/// `reason`, `matchingSites`, or `discoveredTools` materially changed — +/// never on a no-op reconcile. +fn agent_tool_provider_status_needs_update( + current: Option<&AgentToolProviderStatus>, + desired: &AgentToolProviderStatus, +) -> bool { + current != Some(desired) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing, reason = "tests")] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn test_provider(endpoint: &str, grid_network_ref: &str) -> AgentToolProvider { + serde_json::from_value(serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "AgentToolProvider", + "metadata": { "name": "prov" }, + "spec": { + "gridNetworkRef": grid_network_ref, + "endpoint": endpoint + } + })) + .unwrap_or_else(|_| std::process::abort()) + } + + // ----------------------------------------------------------------------- + // validate_provider_config — static validation + // ----------------------------------------------------------------------- + + #[test] + fn blank_endpoint_maps_to_unavailable() { + let provider = test_provider("", "net"); + let err = validate_provider_config(&provider); + assert!(err.is_some(), "blank endpoint must fail static validation"); + assert!( + err.unwrap_or_else(|| std::process::abort()).contains("endpoint"), + "error must mention endpoint" + ); + } + + #[test] + fn whitespace_only_endpoint_maps_to_unavailable() { + let provider = test_provider(" ", "net"); + assert!( + validate_provider_config(&provider).is_some(), + "whitespace-only endpoint must fail static validation" + ); + } + + #[test] + fn blank_grid_network_ref_maps_to_unavailable() { + let provider = test_provider("http://tools:8080", ""); + let err = validate_provider_config(&provider); + assert!(err.is_some(), "blank gridNetworkRef must fail static validation"); + assert!( + err.unwrap_or_else(|| std::process::abort()).contains("gridNetworkRef"), + "error must mention gridNetworkRef" + ); + } + + #[test] + fn whitespace_only_grid_network_ref_maps_to_unavailable() { + let provider = test_provider("http://tools:8080", " "); + assert!( + validate_provider_config(&provider).is_some(), + "whitespace-only gridNetworkRef must fail static validation" + ); + } + + #[test] + fn valid_config_passes_static_validation() { + let provider = test_provider("http://tools:8080", "net"); + assert!( + validate_provider_config(&provider).is_none(), + "valid provider config must pass static validation" + ); + } + + // ----------------------------------------------------------------------- + // static_config_failure_reason — async wiring around validate_provider_config + // and the GridNetwork-existence check, against a mocked Kubernetes API. + // ----------------------------------------------------------------------- + + /// Build a `kube::Client` backed by an in-memory `GridNetwork` map keyed + /// by name, mirroring `mcp_probe::tests::mock_kube_client_with_secrets`' + /// pattern for a different resource type. + #[expect( + clippy::too_many_lines, + reason = "test mock builder: 404-vs-200 branches are the whole point" + )] + fn mock_kube_client_with_grid_networks(networks: HashMap<&'static str, GridNetwork>) -> Client { + let service = tower::service_fn(move |req: http::Request| { + let networks = networks.clone(); + async move { + let name = req.uri().path().rsplit('/').next().unwrap_or_default().to_owned(); + let response = networks.get(name.as_str()).map_or_else( + || { + let not_found = serde_json::json!({ + "kind": "Status", + "apiVersion": "v1", + "status": "Failure", + "message": format!("gridnetworks.grid.praxis-proxy.io \"{name}\" not found"), + "reason": "NotFound", + "code": 404, + }); + http::Response::builder() + .status(404) + .body(kube::client::Body::from( + serde_json::to_vec(¬_found).unwrap_or_else(|_| std::process::abort()), + )) + .unwrap_or_else(|_| std::process::abort()) + }, + |network| { + http::Response::builder() + .status(200) + .body(kube::client::Body::from( + serde_json::to_vec(network).unwrap_or_else(|_| std::process::abort()), + )) + .unwrap_or_else(|_| std::process::abort()) + }, + ); + Ok::<_, std::convert::Infallible>(response) + } + }); + Client::new(service, "default") + } + + fn test_grid_network(name: &str) -> GridNetwork { + serde_json::from_value(serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "GridNetwork", + "metadata": { "name": name }, + "spec": {} + })) + .unwrap_or_else(|_| std::process::abort()) + } + + /// A `kube::Client` that panics if a request is ever sent through it — + /// used to prove `static_config_failure_reason` short-circuits on a + /// config error *before* making any Kubernetes API call. + fn unused_kube_client() -> Client { + let service = tower::service_fn(|_req: http::Request| async { + Ok::<_, std::convert::Infallible>(http::Response::new(kube::client::Body::empty())) + }); + Client::new(service, "default") + } + + #[tokio::test] + async fn static_config_failure_reason_short_circuits_on_invalid_config_without_calling_kubernetes() { + let provider = test_provider("", "net"); + let result = static_config_failure_reason(&provider, &unused_kube_client(), "prov").await; + assert_eq!( + result.unwrap_or_else(|_| std::process::abort()), + Some("ProviderConfigInvalid"), + "an invalid static config must short-circuit to ProviderConfigInvalid before any GridNetwork lookup" + ); + } + + #[tokio::test] + async fn static_config_failure_reason_returns_grid_network_not_found_when_absent() { + let provider = test_provider("http://tools:8080", "absent-net"); + let client = mock_kube_client_with_grid_networks(HashMap::new()); + let result = static_config_failure_reason(&provider, &client, "prov").await; + assert_eq!( + result.unwrap_or_else(|_| std::process::abort()), + Some("GridNetworkNotFound"), + "a gridNetworkRef that doesn't resolve to an existing GridNetwork must yield GridNetworkNotFound" + ); + } + + #[tokio::test] + async fn static_config_failure_reason_returns_none_when_grid_network_exists() { + let provider = test_provider("http://tools:8080", "net"); + let client = mock_kube_client_with_grid_networks(HashMap::from([("net", test_grid_network("net"))])); + let result = static_config_failure_reason(&provider, &client, "prov").await; + assert_eq!( + result.unwrap_or_else(|_| std::process::abort()), + None, + "a valid config with an existing GridNetwork must pass both static checks" + ); + } + + // ----------------------------------------------------------------------- + // phase_from_matching — pure phase logic + // ----------------------------------------------------------------------- + + #[test] + fn no_matching_sites_yields_pending() { + let phase = phase_from_matching(&[]); + assert_eq!(phase, ProviderPhase::Pending, "empty matching → Pending"); + } + + #[test] + fn one_matching_site_yields_available() { + let phase = phase_from_matching(&["site-a".to_owned()]); + assert_eq!(phase, ProviderPhase::Available, "one match → Available"); + } + + #[test] + fn multiple_matching_sites_yields_available() { + let phase = phase_from_matching(&["site-a".to_owned(), "site-b".to_owned()]); + assert_eq!(phase, ProviderPhase::Available, "multiple matches → Available"); + } + + #[test] + fn phase_from_matching_never_emits_degraded_or_unavailable() { + let empty_phase = phase_from_matching(&[]); + let some_phase = phase_from_matching(&["site-x".to_owned()]); + assert_ne!( + empty_phase, + ProviderPhase::Degraded, + "Degraded unreachable from site matching alone" + ); + assert_ne!( + some_phase, + ProviderPhase::Degraded, + "Degraded unreachable from site matching alone" + ); + assert_ne!( + empty_phase, + ProviderPhase::Unavailable, + "site matching alone never yields Unavailable" + ); + assert_ne!( + some_phase, + ProviderPhase::Unavailable, + "site matching alone never yields Unavailable" + ); + } + + // ----------------------------------------------------------------------- + // sites_matching_selector — selector matching + // ----------------------------------------------------------------------- + + fn test_site(name: &str, network: &str) -> GridSite { + serde_json::from_value(serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "GridSite", + "metadata": { "name": name }, + "spec": { "gridNetworkRef": network } + })) + .unwrap_or_else(|_| std::process::abort()) + } + + fn test_site_with_labels(name: &str, network: &str, labels: &[(&str, &str)]) -> GridSite { + let labels_map: serde_json::Map = labels + .iter() + .map(|(k, v)| (k.to_string(), serde_json::Value::String(v.to_string()))) + .collect(); + serde_json::from_value(serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "GridSite", + "metadata": { "name": name, "labels": labels_map }, + "spec": { "gridNetworkRef": network } + })) + .unwrap_or_else(|_| std::process::abort()) + } + + fn test_provider_with_selector(network: &str, selector: &[(&str, &str)]) -> AgentToolProvider { + let match_labels: serde_json::Map = selector + .iter() + .map(|(k, v)| (k.to_string(), serde_json::Value::String(v.to_string()))) + .collect(); + serde_json::from_value(serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "AgentToolProvider", + "metadata": { "name": "prov" }, + "spec": { + "gridNetworkRef": network, + "endpoint": "http://tools:8080", + "siteSelector": { "matchLabels": match_labels } + } + })) + .unwrap_or_else(|_| std::process::abort()) + } + + #[test] + fn empty_selector_matches_all_passed_sites() { + let provider = test_provider("http://tools:8080", "net"); + let sites = vec![test_site("site-a", "net"), test_site("site-b", "net")]; + let matching = sites_matching_selector(&provider, &sites); + assert_eq!( + matching, + vec!["site-a", "site-b"], + "empty selector must match all pre-filtered sites" + ); + } + + #[test] + fn label_selector_matches_only_matching_labels() { + let provider = test_provider_with_selector("net", &[("hw", "gpu")]); + let sites = vec![ + test_site_with_labels("gpu-site", "net", &[("hw", "gpu")]), + test_site_with_labels("cpu-site", "net", &[("hw", "cpu")]), + ]; + let matching = sites_matching_selector(&provider, &sites); + assert_eq!(matching, vec!["gpu-site"], "only gpu-site should match"); + } + + #[test] + fn matching_sites_are_sorted_deterministically() { + let provider = test_provider("http://tools:8080", "net"); + let sites = vec![ + test_site("zebra-site", "net"), + test_site("alpha-site", "net"), + test_site("mango-site", "net"), + ]; + let matching = sites_matching_selector(&provider, &sites); + assert_eq!( + matching, + vec!["alpha-site", "mango-site", "zebra-site"], + "matching sites must be sorted alphabetically" + ); + } + + #[test] + fn no_matching_sites_returns_empty() { + let provider = test_provider_with_selector("net", &[("hw", "gpu")]); + let sites = vec![test_site_with_labels("cpu-site", "net", &[("hw", "cpu")])]; + let matching = sites_matching_selector(&provider, &sites); + assert!(matching.is_empty(), "no matching sites should return empty"); + } + + #[test] + fn multi_key_selector_requires_all_keys_to_match() { + let provider = test_provider_with_selector("net", &[("hw", "gpu"), ("region", "us-east")]); + let both = test_site_with_labels( + "full-match", + "net", + &[("hw", "gpu"), ("region", "us-east"), ("extra", "ignored")], + ); + let partial = test_site_with_labels("partial", "net", &[("hw", "gpu")]); + let sites = vec![both, partial]; + let matching = sites_matching_selector(&provider, &sites); + assert_eq!( + matching, + vec!["full-match"], + "multi-key selector requires ALL keys to match (AND semantics)" + ); + } + + #[test] + fn empty_selector_with_no_sites_returns_empty() { + let provider = test_provider("http://tools:8080", "net"); + let matching = sites_matching_selector(&provider, &[]); + assert!( + matching.is_empty(), + "passing an empty sites slice must return an empty result" + ); + } + + // ----------------------------------------------------------------------- + // agent_tool_provider_status_needs_update — patch-if-changed decision + // ----------------------------------------------------------------------- + + fn baseline_status() -> AgentToolProviderStatus { + AgentToolProviderStatus { + discovered_tools: vec!["search".to_owned()], + matching_sites: vec!["site-a".to_owned()], + observed_generation: 2, + phase: ProviderPhase::Available, + reason: None, + } + } + + #[test] + fn no_op_reconcile_does_not_need_update() { + let baseline = baseline_status(); + assert!( + !agent_tool_provider_status_needs_update(Some(&baseline), &baseline), + "identical current and desired status must never require a patch" + ); + } + + #[test] + fn phase_change_needs_update() { + let baseline = baseline_status(); + let changed = AgentToolProviderStatus { + phase: ProviderPhase::Degraded, + ..baseline.clone() + }; + assert!( + agent_tool_provider_status_needs_update(Some(&baseline), &changed), + "a phase change must require a status patch" + ); + } + + #[test] + fn reason_change_needs_update() { + let baseline = baseline_status(); + let changed = AgentToolProviderStatus { + reason: Some("McpEndpointUnreachable".to_owned()), + ..baseline.clone() + }; + assert!( + agent_tool_provider_status_needs_update(Some(&baseline), &changed), + "a reason change must require a status patch, even when phase is unchanged" + ); + } + + #[test] + fn matching_sites_change_needs_update() { + let baseline = baseline_status(); + let changed = AgentToolProviderStatus { + matching_sites: vec!["site-a".to_owned(), "site-b".to_owned()], + ..baseline.clone() + }; + assert!( + agent_tool_provider_status_needs_update(Some(&baseline), &changed), + "a matchingSites change must require a status patch" + ); + } + + #[test] + fn discovered_tools_change_needs_update() { + let baseline = baseline_status(); + let changed = AgentToolProviderStatus { + discovered_tools: vec!["search".to_owned(), "fetch".to_owned()], + ..baseline.clone() + }; + assert!( + agent_tool_provider_status_needs_update(Some(&baseline), &changed), + "a discoveredTools change must require a status patch" + ); + } + + #[test] + fn absent_current_status_needs_update() { + let desired = baseline_status(); + assert!( + agent_tool_provider_status_needs_update(None, &desired), + "an AgentToolProvider with no prior status must always be patched on first reconcile" + ); + } + + // ----------------------------------------------------------------------- + // telemetry_reason_label — synthesize a bounded telemetry label + // ----------------------------------------------------------------------- + + #[test] + fn telemetry_label_passes_through_status_reason_when_set() { + assert_eq!( + telemetry_reason_label(&ProviderPhase::Unavailable, Some("ProviderConfigInvalid")), + "ProviderConfigInvalid", + "an explicit status.reason must be used verbatim as the telemetry label" + ); + } + + #[test] + fn telemetry_label_passes_through_grid_network_not_found_reason() { + assert_eq!( + telemetry_reason_label(&ProviderPhase::Unavailable, Some("GridNetworkNotFound")), + "GridNetworkNotFound" + ); + } + + #[test] + fn telemetry_label_synthesizes_sites_matched_for_available_with_no_reason() { + assert_eq!( + telemetry_reason_label(&ProviderPhase::Available, None), + "SitesMatched", + "Available with no status.reason (the healthy case) must synthesize a bounded label" + ); + } + + #[test] + fn telemetry_label_synthesizes_awaiting_site_match_for_pending_with_no_reason() { + assert_eq!( + telemetry_reason_label(&ProviderPhase::Pending, None), + "AwaitingSiteMatch", + "Pending with no status.reason must synthesize a bounded label distinct from Available's" + ); + } + + #[test] + fn telemetry_label_falls_back_to_awaiting_site_match_for_any_other_healthy_phase() { + assert_eq!( + telemetry_reason_label(&ProviderPhase::Degraded, None), + "AwaitingSiteMatch", + "any non-Available phase with no explicit reason falls back to the Pending-style label" + ); + } + + // ----------------------------------------------------------------------- + // event_type_for_reason — bounded Event severity mapping + // ----------------------------------------------------------------------- + + #[test] + fn sites_matched_reason_is_a_normal_event() { + assert!(matches!(event_type_for_reason("SitesMatched"), EventType::Normal)); + } + + #[test] + fn awaiting_site_match_reason_is_a_normal_event() { + assert!(matches!(event_type_for_reason("AwaitingSiteMatch"), EventType::Normal)); + } + + #[test] + fn provider_config_invalid_reason_is_a_warning_event() { + assert!(matches!( + event_type_for_reason("ProviderConfigInvalid"), + EventType::Warning + )); + } + + #[test] + fn grid_network_not_found_reason_is_a_warning_event() { + assert!(matches!( + event_type_for_reason("GridNetworkNotFound"), + EventType::Warning + )); + } + + #[test] + fn unrecognized_reason_defaults_to_warning_event() { + assert!(matches!( + event_type_for_reason("SomeFutureProbeReason"), + EventType::Warning + )); + } + + // ----------------------------------------------------------------------- + // is_real_transition — gates Event emission, metric recording, and + // transition-level logging so a no-op reconcile never fires any of them + // ----------------------------------------------------------------------- + + #[test] + fn no_change_is_not_a_real_transition() { + assert!( + !is_real_transition(false, false), + "neither phase nor reason changed: not a real transition" + ); + } + + #[test] + fn phase_change_alone_is_a_real_transition() { + assert!( + is_real_transition(true, false), + "a phase change alone must count as a real transition" + ); + } + + #[test] + fn reason_change_alone_is_a_real_transition() { + assert!( + is_real_transition(false, true), + "a reason change alone (phase steady) must still count as a real transition" + ); + } + + #[test] + fn both_changing_is_a_real_transition() { + assert!(is_real_transition(true, true)); + } + + // ----------------------------------------------------------------------- + // reconcile — full orchestration against a mocked Kubernetes API. + // + // Everything above exercises resolve_phase_and_sites/update_status's + // constituent resolve_*/pure-logic functions in isolation. These tests + // drive the public reconcile() entrypoint itself end-to-end, proving the + // resolved (phase, reason, matchingSites, discoveredTools) tuple actually + // reaches the Kubernetes API as the status PATCH body a real controller + // would send — the seam none of the functions above individually cover. + // ----------------------------------------------------------------------- + + /// A `kube::Client` that serves `GridNetwork` GETs from an in-memory map + /// and captures every status-subresource PATCH body sent through it into + /// `captured`. Any other request (a missing `GridNetwork`, the `Event` + /// POST from `Recorder::publish`, a `GridSite` LIST) 404s: `update_status` + /// and `emit_transition_telemetry` both already tolerate a failed event + /// publish by design (logged, not propagated — see `emit_transition_telemetry`), + /// and the two scenarios below never reach `GridSite` listing at all, + /// since both short-circuit inside `static_config_failure_reason`. + #[expect( + clippy::too_many_lines, + reason = "test mock builder: PATCH-capture vs GridNetwork-GET vs catch-all-404 branches are the whole point" + )] + fn mock_kube_client_capturing_status_patch( + networks: HashMap<&'static str, GridNetwork>, + captured: Arc>>, + ) -> Client { + let service = tower::service_fn(move |req: http::Request| { + let networks = networks.clone(); + let captured = Arc::clone(&captured); + async move { + if req.method() == http::Method::PATCH && req.uri().path().ends_with("/status") { + let name = req + .uri() + .path() + .trim_end_matches("/status") + .rsplit('/') + .next() + .unwrap_or_default() + .to_owned(); + let bytes = http_body_util::BodyExt::collect(req.into_body()) + .await + .unwrap_or_else(|_| std::process::abort()) + .to_bytes(); + let body: serde_json::Value = + serde_json::from_slice(&bytes).unwrap_or_else(|_| std::process::abort()); + *captured.lock().unwrap_or_else(|_| std::process::abort()) = Some(body.clone()); + + let echo = serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "AgentToolProvider", + "metadata": { "name": name }, + "spec": { "gridNetworkRef": "net", "endpoint": "http://tools:8080" }, + "status": body["status"], + }); + return Ok::<_, std::convert::Infallible>( + http::Response::builder() + .status(200) + .body(kube::client::Body::from( + serde_json::to_vec(&echo).unwrap_or_else(|_| std::process::abort()), + )) + .unwrap_or_else(|_| std::process::abort()), + ); + } + + if req.method() == http::Method::GET { + let name = req.uri().path().rsplit('/').next().unwrap_or_default().to_owned(); + if let Some(network) = networks.get(name.as_str()) { + return Ok(http::Response::builder() + .status(200) + .body(kube::client::Body::from( + serde_json::to_vec(network).unwrap_or_else(|_| std::process::abort()), + )) + .unwrap_or_else(|_| std::process::abort())); + } + } + + let not_found = serde_json::json!({ + "kind": "Status", + "apiVersion": "v1", + "status": "Failure", + "message": "not found", + "reason": "NotFound", + "code": 404, + }); + Ok(http::Response::builder() + .status(404) + .body(kube::client::Body::from( + serde_json::to_vec(¬_found).unwrap_or_else(|_| std::process::abort()), + )) + .unwrap_or_else(|_| std::process::abort())) + } + }); + Client::new(service, "default") + } + + #[tokio::test] + async fn reconcile_patches_unavailable_provider_config_invalid_without_any_grid_network_lookup() { + let provider = test_provider("", "net"); + let captured = Arc::new(std::sync::Mutex::new(None)); + let client = mock_kube_client_capturing_status_patch(HashMap::new(), Arc::clone(&captured)); + + let action = Box::pin(reconcile(Arc::new(provider), Arc::new(client))) + .await + .unwrap_or_else(|_| std::process::abort()); + assert_eq!( + action, + Action::requeue(REQUEUE_INTERVAL), + "reconcile must always requeue on a successful (non-erroring) pass, even when the provider is Unavailable" + ); + + let patched = captured + .lock() + .unwrap_or_else(|_| std::process::abort()) + .clone() + .expect("reconcile must PATCH the status subresource for a first-seen config-invalid provider"); + assert_eq!( + patched["status"]["phase"], "Unavailable", + "a blank endpoint must surface as Unavailable all the way through to the persisted status" + ); + assert_eq!( + patched["status"]["reason"], "ProviderConfigInvalid", + "the specific static-validation failure reason must reach the persisted status" + ); + assert_eq!( + patched["status"]["matchingSites"], + serde_json::json!([]), + "a config-invalid provider must never report matching sites" + ); + } + + #[tokio::test] + async fn reconcile_patches_unavailable_grid_network_not_found_when_referenced_network_absent() { + let provider = test_provider("http://tools:8080", "missing-net"); + let captured = Arc::new(std::sync::Mutex::new(None)); + let client = mock_kube_client_capturing_status_patch(HashMap::new(), Arc::clone(&captured)); + + let action = Box::pin(reconcile(Arc::new(provider), Arc::new(client))) + .await + .unwrap_or_else(|_| std::process::abort()); + assert_eq!( + action, + Action::requeue(REQUEUE_INTERVAL), + "reconcile must requeue even when the referenced GridNetwork cannot be found" + ); + + let patched = captured + .lock() + .unwrap_or_else(|_| std::process::abort()) + .clone() + .expect("reconcile must PATCH the status subresource once the GridNetwork lookup 404s"); + assert_eq!( + patched["status"]["phase"], "Unavailable", + "an unresolvable gridNetworkRef must surface as Unavailable through the full reconcile path" + ); + assert_eq!( + patched["status"]["reason"], "GridNetworkNotFound", + "the GridNetwork-lookup failure reason must reach the persisted status, proving reconcile actually \ + performed the live GET rather than short-circuiting on static config alone" + ); + } +} diff --git a/operator/src/crd/agent_tool_provider.rs b/operator/src/crd/agent_tool_provider.rs index fa8b2c7..9afcec2 100644 --- a/operator/src/crd/agent_tool_provider.rs +++ b/operator/src/crd/agent_tool_provider.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use super::{ auth::{AccessPolicy, AuthConfig, SelectorConfig}, - inference_provider::ProviderPhase, + inference_provider::{EndpointTlsConfig, ProviderPhase}, }; // --------------------------------------------------------------------------- @@ -52,6 +52,16 @@ pub struct AgentToolProviderSpec { #[serde(default)] pub site_selector: SelectorConfig, + /// TLS configuration for the operator's own MCP `tools/list` probe. + /// + /// Reuses [`EndpointTlsConfig`] from [`InferenceProvider`] (CA trust and + /// optional mTLS client identity for the probe connection). When absent, + /// the probe uses native root certificates and no client certificate. + /// + /// [`InferenceProvider`]: crate::crd::inference_provider::InferenceProvider + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tls: Option, + /// Tool definitions (auto-discovered if omitted). #[serde(default)] pub tools: Vec, @@ -72,7 +82,28 @@ pub struct ToolInfo { // --------------------------------------------------------------------------- /// Observed status of an [`AgentToolProvider`]. -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] +/// +/// # Stable `reason` values +/// +/// `reason` is `None` while the provider is healthy. When set, it is one +/// of the following stable, machine-readable strings — following the +/// same naming convention as [`InferenceProvider`]'s `MetricsTls*`/ +/// `HealthCheckTls*` reasons: +/// +/// | Reason | Meaning | +/// |--------|---------| +/// | `ProviderConfigInvalid` | `spec.endpoint` or `spec.gridNetworkRef` is blank or whitespace-only. | +/// | `GridNetworkNotFound` | The `GridNetwork` referenced by `spec.gridNetworkRef` does not exist. | +/// | `McpEndpointUnreachable` | The MCP probe could not connect (transport failure, timeout, DNS error). | +/// | `McpToolsListInvalidResponse` | The endpoint responded but the `tools/list` response was malformed. | +/// | `McpAuthRejected` | The MCP server rejected the configured `spec.auth` credentials. | +/// | `McpAuthTokenInvalid` | The resolved `spec.auth` bearer token contains characters that cannot be sent as an HTTP header value; the probe fails closed rather than proceeding unauthenticated. | +/// | `EndpointTlsSecretMissing` | `spec.tls`'s referenced Secret does not exist in the cluster. | +/// | `EndpointTlsKeyMissing` | `spec.tls`'s referenced Secret exists but is missing the expected key. | +/// | `EndpointTlsMaterialInvalid` | `spec.tls`'s certificate or key material could not be parsed. | +/// +/// [`InferenceProvider`]: crate::crd::inference_provider::InferenceProvider +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AgentToolProviderStatus { /// Tools discovered via MCP `tools/list`. @@ -90,6 +121,12 @@ pub struct AgentToolProviderStatus { /// Current phase. #[serde(default)] pub phase: ProviderPhase, + + /// Machine-readable reason for the current phase, `None` when healthy. + /// + /// See the type-level doc comment for the table of stable values. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, } // --------------------------------------------------------------------------- @@ -106,6 +143,8 @@ fn default_protocol() -> String { // --------------------------------------------------------------------------- #[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, reason = "tests")] mod tests { use super::*; @@ -120,4 +159,103 @@ mod tests { assert_eq!(spec.protocol, "mcp", "default protocol"); assert_eq!(spec.tools.len(), 1, "tool count"); } + + // ----------------------------------------------------------------------- + // spec.tls — absent must default to None, present must round-trip + // ----------------------------------------------------------------------- + + #[test] + fn spec_tls_absent_defaults_to_none() { + let json = serde_json::json!({ + "gridNetworkRef": "production", + "endpoint": "http://tools:8080" + }); + let spec: AgentToolProviderSpec = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); + assert!(spec.tls.is_none(), "absent spec.tls must deserialize to None"); + } + + #[test] + fn spec_tls_with_ca_only_round_trips() { + let json = serde_json::json!({ + "gridNetworkRef": "production", + "endpoint": "https://tools:8443", + "tls": { + "caSecretRef": { "name": "tools-ca", "namespace": "grid-system" } + } + }); + let spec: AgentToolProviderSpec = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); + let tls = spec.tls.unwrap_or_else(|| std::process::abort()); + assert_eq!(tls.ca_secret_ref.name, "tools-ca", "caSecretRef.name must round-trip"); + assert_eq!( + tls.ca_secret_ref.namespace, "grid-system", + "caSecretRef.namespace must round-trip" + ); + assert!( + tls.client_certificate_secret_ref.is_none(), + "absent clientCertificateSecretRef must be None" + ); + } + + #[test] + fn spec_tls_with_client_cert_round_trips() { + let json = serde_json::json!({ + "gridNetworkRef": "production", + "endpoint": "https://tools:8443", + "tls": { + "caSecretRef": { "name": "tools-ca", "namespace": "grid-system" }, + "clientCertificateSecretRef": { "name": "tools-client-cert", "namespace": "grid-system" } + } + }); + let spec: AgentToolProviderSpec = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); + let tls = spec.tls.unwrap_or_else(|| std::process::abort()); + let client_ref = tls + .client_certificate_secret_ref + .unwrap_or_else(|| std::process::abort()); + assert_eq!(client_ref.name, "tools-client-cert", "client cert name must round-trip"); + assert_eq!( + client_ref.certificate_key, "tls.crt", + "certificateKey must default to tls.crt" + ); + assert_eq!( + client_ref.private_key_key, "tls.key", + "privateKeyKey must default to tls.key" + ); + } + + // ----------------------------------------------------------------------- + // status.reason — absent must default to None, present must round-trip, + // and must be omitted from serialized output when None (not written as + // an explicit null onto the status subresource). + // ----------------------------------------------------------------------- + + #[test] + fn status_reason_absent_defaults_to_none() { + let json = serde_json::json!({}); + let status: AgentToolProviderStatus = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); + assert!(status.reason.is_none(), "absent status.reason must default to None"); + } + + #[test] + fn status_reason_round_trips_when_present() { + let json = serde_json::json!({ "reason": "McpEndpointUnreachable" }); + let status: AgentToolProviderStatus = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); + assert_eq!( + status.reason.as_deref(), + Some("McpEndpointUnreachable"), + "status.reason must round-trip" + ); + } + + #[test] + fn status_reason_none_is_omitted_from_serialized_output() { + let status = AgentToolProviderStatus::default(); + let value = serde_json::to_value(&status).unwrap_or_else(|_| std::process::abort()); + assert!( + !value + .as_object() + .unwrap_or_else(|| std::process::abort()) + .contains_key("reason"), + "None reason must be omitted, not serialized as an explicit null" + ); + } } diff --git a/operator/src/main.rs b/operator/src/main.rs index f6f1e6a..305cd82 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -59,10 +59,14 @@ use kube::{ use operator::{ cli::Cli, controller::{ + agent_tool_provider, grid_network::{self, OperatorCtx}, grid_site, inference_provider, }, - crd::{grid_network::GridNetwork, grid_site::GridSite, inference_provider::InferenceProvider}, + crd::{ + agent_tool_provider::AgentToolProvider, grid_network::GridNetwork, grid_site::GridSite, + inference_provider::InferenceProvider, + }, gateway, swim_runtime::{self, RevisionLease, SwimConfig}, }; @@ -72,11 +76,35 @@ use operator::{ // --------------------------------------------------------------------------- #[tokio::main] -#[expect(clippy::large_stack_frames, reason = "top-level binary with tokio runtime")] +#[expect( + clippy::large_stack_frames, + clippy::too_many_lines, + reason = "top-level binary with tokio runtime; startup sequence (crypto provider, CLI parsing, \ + SWIM bootstrap, controller fan-out) reads clearer sequential than split further" +)] async fn main() { tracing_subscriber::fmt::init(); tracing::info!("starting grid-operator"); + // Explicit process-wide rustls crypto-provider choice. + // + // `InferenceProvider`'s probe (`inference_provider.rs`) builds a + // `hyper-rustls` client via `.with_native_roots()`, which relies on + // `rustls` auto-detecting a single process-wide `CryptoProvider`. The + // `AgentToolProvider` MCP probe (PR 2 of grid#41) links in `reqwest` + // (via `rmcp`'s reqwest-backed transport) using its `rustls-no-provider` + // feature specifically to avoid pulling in `aws-lc-rs` alongside `ring` + // (see the workspace `Cargo.toml` comment on the `reqwest`/`rmcp` + // entries) — but that feature means `reqwest` will no longer install a + // default provider on our behalf either, so `Client::builder().build()` + // panics with "No rustls crypto provider is configured" unless one is + // installed explicitly first. Installing `ring` here, once, up front, + // covers both `hyper-rustls` and `reqwest` for every reconciler in this + // binary, regardless of which one runs first. + if rustls::crypto::ring::default_provider().install_default().is_err() { + tracing::warn!("rustls default CryptoProvider already installed; continuing"); + } + let config = Cli::parse(); let client = match Client::try_default().await { @@ -103,6 +131,7 @@ async fn main() { run_network_controller(client.clone(), Arc::clone(&ctx)), run_site_controller(client.clone()), run_provider_controller(client.clone()), + run_agent_tool_provider_controller(client.clone()), run_metrics_server(), ); @@ -526,6 +555,32 @@ async fn run_provider_controller(client: Client) -> Result<(), Box Result<(), Box> { + let api = Api::::all(client.clone()); + + Controller::new(api, watcher::Config::default()) + .run( + agent_tool_provider::reconcile, + agent_tool_provider::error_policy, + Arc::new(client), + ) + .for_each(|result| async { + match result { + Ok((obj, _action)) => tracing::info!(%obj, "reconciled AgentToolProvider"), + Err(e) => tracing::error!(error = ?e, "AgentToolProvider watch error"), + } + }) + .await; + + Ok(()) +} + /// Serve Prometheus metrics and health endpoints. async fn run_metrics_server() -> Result<(), Box> { let addr = std::env::var("GRID_METRICS_ADDR").unwrap_or_else(|_| "0.0.0.0:9090".to_owned()); diff --git a/operator/src/metrics.rs b/operator/src/metrics.rs index 064cd2c..86992c3 100644 --- a/operator/src/metrics.rs +++ b/operator/src/metrics.rs @@ -22,6 +22,12 @@ static REGISTRY: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PHASE_TRANSITIONS.clone())) .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(AGENT_TOOL_PROVIDER_PHASE_TRANSITIONS.clone())) + .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(MCP_PROBE_TOTAL.clone())) + .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(MCP_PROBE_DURATION.clone())) + .unwrap_or_else(|_| std::process::abort()); r }); @@ -52,6 +58,49 @@ static PHASE_TRANSITIONS: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()) }); +/// `AgentToolProvider` phase transitions by source phase, target phase, and reason. +/// +/// Kept as a distinct metric (rather than reusing [`PHASE_TRANSITIONS`]) so +/// dashboards can alert on each CRD's convergence independently — see grid#9. +static AGENT_TOOL_PROVIDER_PHASE_TRANSITIONS: LazyLock = LazyLock::new(|| { + IntCounterVec::new( + Opts::new( + "grid_agent_tool_provider_phase_transition_total", + "AgentToolProvider phase transitions", + ), + &["from_phase", "to_phase", "reason"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + +/// Total `AgentToolProvider` MCP `tools/list` probe attempts by outcome. +/// +/// The `outcome` label comes from +/// [`mcp_probe::mcp_probe_outcome_label`](crate::resources::mcp_probe::mcp_probe_outcome_label), +/// which is deliberately bounded to the fixed `McpProbeOutcome` variant set +/// — never the free-form reason string `TlsConfigInvalid` carries — so this +/// metric's cardinality stays fixed regardless of how many distinct Secret +/// misconfigurations occur in the cluster. +static MCP_PROBE_TOTAL: LazyLock = LazyLock::new(|| { + IntCounterVec::new( + Opts::new( + "grid_mcp_probe_total", + "Total AgentToolProvider MCP tools/list probe attempts", + ), + &["outcome"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + +/// `AgentToolProvider` MCP `tools/list` probe duration in seconds. +static MCP_PROBE_DURATION: LazyLock = LazyLock::new(|| { + Histogram::with_opts(HistogramOpts::new( + "grid_mcp_probe_duration_seconds", + "AgentToolProvider MCP tools/list probe duration", + )) + .unwrap_or_else(|_| std::process::abort()) +}); + // --------------------------------------------------------------------------- // Public API // --------------------------------------------------------------------------- @@ -67,6 +116,19 @@ pub(crate) fn record_phase_transition(from: &str, to: &str, reason: &str) { PHASE_TRANSITIONS.with_label_values(&[from, to, reason]).inc(); } +/// Record an `AgentToolProvider` phase transition. +pub(crate) fn record_agent_tool_provider_phase_transition(from: &str, to: &str, reason: &str) { + AGENT_TOOL_PROVIDER_PHASE_TRANSITIONS + .with_label_values(&[from, to, reason]) + .inc(); +} + +/// Record a completed `AgentToolProvider` MCP `tools/list` probe attempt. +pub(crate) fn record_mcp_probe(outcome: &str, duration: Duration) { + MCP_PROBE_TOTAL.with_label_values(&[outcome]).inc(); + MCP_PROBE_DURATION.observe(duration.as_secs_f64()); +} + /// Gather all registered metrics for serialization. pub(crate) fn gather_metrics() -> Vec { REGISTRY.gather() @@ -103,6 +165,18 @@ mod tests { assert!(val >= 1, "transition counter should be >= 1, got {val}"); } + #[test] + fn record_agent_tool_provider_phase_transition_increments_counter() { + record_agent_tool_provider_phase_transition("Pending", "Available", "SitesMatched"); + let val = AGENT_TOOL_PROVIDER_PHASE_TRANSITIONS + .with_label_values(&["Pending", "Available", "SitesMatched"]) + .get(); + assert!( + val >= 1, + "agent tool provider transition counter should be >= 1, got {val}" + ); + } + #[test] fn probe_duration_records_observation() { record_probe("ConnectTimeout", "mtls", Duration::from_millis(100)); @@ -110,6 +184,38 @@ mod tests { assert!(count >= 1, "histogram should have at least 1 observation"); } + #[test] + fn record_mcp_probe_increments_counter_by_outcome() { + record_mcp_probe("Success", Duration::from_millis(12)); + let val = MCP_PROBE_TOTAL.with_label_values(&["Success"]).get(); + assert!(val >= 1, "mcp probe counter should be >= 1, got {val}"); + } + + #[test] + fn record_mcp_probe_records_duration_observation() { + record_mcp_probe("Unreachable", Duration::from_millis(250)); + let count = MCP_PROBE_DURATION.get_sample_count(); + assert!( + count >= 1, + "mcp probe duration histogram should have at least 1 observation" + ); + } + + #[test] + fn encode_metrics_includes_mcp_probe_metrics() { + record_mcp_probe("AuthRejected", Duration::from_millis(5)); + let buf = encode_metrics(); + let text = String::from_utf8(buf).unwrap_or_else(|_| std::process::abort()); + assert!( + text.contains("grid_mcp_probe_total"), + "output should contain mcp probe counter" + ); + assert!( + text.contains("grid_mcp_probe_duration_seconds"), + "output should contain mcp probe duration histogram" + ); + } + #[test] fn encode_metrics_produces_prometheus_text() { record_probe("ConnectionFailed", "mtls", Duration::from_millis(1)); diff --git a/operator/src/resources.rs b/operator/src/resources.rs index 5e5c841..5638a52 100644 --- a/operator/src/resources.rs +++ b/operator/src/resources.rs @@ -48,5 +48,7 @@ pub mod trust_bundle; pub(crate) mod endpoint_tls; /// Typed gateway probe outcome and phase-transition contracts. pub(crate) mod gateway_probe; +/// Live MCP `tools/list` probe for [`AgentToolProvider`](crate::crd::agent_tool_provider::AgentToolProvider). +pub(crate) mod mcp_probe; /// TLS gateway probe — bounded handshake and peer certificate extraction. pub(crate) mod tls_probe; diff --git a/operator/src/resources/mcp_probe.rs b/operator/src/resources/mcp_probe.rs new file mode 100644 index 0000000..38d30f2 --- /dev/null +++ b/operator/src/resources/mcp_probe.rs @@ -0,0 +1,2056 @@ +//! Live MCP `tools/list` probe for `AgentToolProvider`. +//! +//! Adapts `praxis-ai`'s `mcp_client::list_tools()` pattern (an `rmcp`-based +//! Streamable HTTP transport, plus its SSRF/DNS-pinning protections) for use +//! as a controller-initiated discovery probe rather than a request-time proxy +//! call. `rmcp`'s transport already handles both MCP protocol generations +//! (legacy handshake-based and modern stateless) transparently, so none of +//! that detection logic is reimplemented here. +//! +//! Split into three layers, deliberately: +//! +//! - **Pure decision logic** (outcome classification, phase/reason mapping, discovered-tools preservation, tool-name +//! extraction, header/TLS attachment decisions): unit-tested with hand-built fixtures, no network. +//! - **Mockable Kubernetes I/O** (`attach_tls_ca`/`attach_tls_client_identity`/`read_tls_material`'s +//! `Api::::get_opt` calls and PEM parsing): unit-tested against a `tower::service_fn`-backed `kube::Client` +//! (see `mock_kube_client_with_secrets` in `mod tests`) — this is genuine Secret I/O, but deterministic and mockable +//! without a real API server, so it stays at the unit tier rather than the integration tier below. +//! - **Real network I/O** (`rmcp`/`reqwest` error introspection, the actual live probe): covered by the integration +//! tier against a real local HTTP listener (`mod integration_tests`), not unit-tested, since it exercises third-party +//! wire behavior rather than this crate's own branching. + +use std::{ + collections::HashMap, + net::{IpAddr, Ipv4Addr, SocketAddr}, + time::Duration, +}; + +use http::{HeaderName, HeaderValue}; +use rmcp::{ + ServiceExt as _, + transport::{ + StreamableHttpClientTransport, + streamable_http_client::{StreamableHttpClientTransportConfig, StreamableHttpError}, + }, +}; +use rustls::pki_types::pem::PemObject as _; + +use crate::{ + crd::inference_provider::EndpointTlsConfig, + resources::{ + credentials::BearerToken, + endpoint_tls::{read_secret_bytes_for_tls, secret_ref_from_client_cert}, + }, +}; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/// Alibaba Cloud instance metadata service IPv4 endpoint. +/// +/// Not covered by [`Ipv4Addr::is_link_local`] (unlike AWS/GCP/Azure's shared +/// `169.254.169.254`, which the 169.254.0.0/16 link-local check already +/// blocks) since Alibaba's metadata service sits in RFC 6598 shared address +/// space, not the link-local range. +const ALIBABA_CLOUD_METADATA_V4: Ipv4Addr = Ipv4Addr::new(100, 100, 100, 200); + +/// Stable `status.reason` for any of `attach_tls_client_identity`'s three +/// client-identity failure modes (unparseable cert, unparseable key, or a +/// cert/key pair `reqwest::Identity` itself refuses to build from). +/// +/// A single shared reason rather than three variants: none of the three are +/// distinguishable in a way that would change what an operator does next +/// (fix the referenced client certificate/key Secret), so splitting them +/// would add `status.reason` cardinality without adding diagnostic value. +const ENDPOINT_TLS_IDENTITY_MISMATCH: &str = "EndpointTlsIdentityMismatch"; + +/// Maximum number of tool names persisted to `status.discoveredTools` from +/// a single probe. +/// +/// Bounds the Kubernetes status object's size against a server advertising +/// an implausibly large tool catalog; ordinary MCP servers advertise a +/// handful to a few dozen tools. Applied after deduplication, so it only +/// discards genuinely distinct names beyond this limit. +const MAX_DISCOVERED_TOOLS: usize = 500; + +/// Maximum length, in bytes, of a single tool name persisted to +/// `status.discoveredTools`. +/// +/// Bounds per-entry size against a server advertising implausibly long +/// tool names. Truncation lands on a UTF-8 character boundary so it never +/// produces invalid UTF-8. +const MAX_TOOL_NAME_LEN: usize = 256; + +// --------------------------------------------------------------------------- +// McpProbeOutcome +// --------------------------------------------------------------------------- + +/// Outcome of a single MCP `tools/list` probe attempt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum McpProbeOutcome { + /// `tools/list` succeeded; carries the discovered tool names. + Success(Vec), + + /// The endpoint could not be reached: transport failure, DNS error, + /// timeout, or a blocked (SSRF-sensitive) address. + Unreachable, + + /// The endpoint was reached but the `tools/list` exchange itself failed + /// or returned a response that could not be parsed. + InvalidResponse, + + /// The MCP server rejected the configured `spec.auth` credentials + /// (HTTP 401/403-equivalent). + AuthRejected, + + /// The resolved `spec.auth` bearer token contains characters that + /// cannot be encoded into an HTTP header value, so no request was ever + /// sent. Fails closed rather than silently proceeding unauthenticated + /// — an endpoint that permits anonymous `tools/list` could otherwise + /// be marked `Available` without ever exercising the configured + /// credential. + AuthConfigInvalid, + + /// `spec.tls`'s referenced Secret material could not be resolved into a + /// usable client certificate/CA bundle. Carries the stable status.reason + /// string (`EndpointTls*`) rather than a fixed variant, since the exact + /// failure (missing Secret, missing key, unparseable PEM) is only known + /// once [`endpoint_tls::read_secret_bytes_for_tls`](crate::resources::endpoint_tls::read_secret_bytes_for_tls) + /// runs. + TlsConfigInvalid(String), +} + +/// Map a [`McpProbeOutcome`] to the resulting [`ProviderPhase`] and, for +/// failure outcomes, the stable `status.reason` string documented on +/// [`AgentToolProviderStatus`](crate::crd::agent_tool_provider::AgentToolProviderStatus). +/// +/// Business rule: a successful probe always yields `Available` with no +/// reason (healthy); every failure outcome maps to `Unavailable` with its +/// own stable, machine-readable reason — mirroring how +/// `inference_provider::phase_from_probe` merges a health-probe outcome on +/// top of the site-matching phase, but simpler here since there is no +/// separate `Degraded` outcome for this probe. +/// +/// [`ProviderPhase`]: crate::crd::inference_provider::ProviderPhase +pub(crate) fn phase_and_reason_from_probe( + outcome: &McpProbeOutcome, +) -> (crate::crd::inference_provider::ProviderPhase, Option) { + use crate::crd::inference_provider::ProviderPhase; + + match outcome { + McpProbeOutcome::Success(_) => (ProviderPhase::Available, None), + McpProbeOutcome::Unreachable => (ProviderPhase::Unavailable, Some("McpEndpointUnreachable".to_owned())), + McpProbeOutcome::InvalidResponse => ( + ProviderPhase::Unavailable, + Some("McpToolsListInvalidResponse".to_owned()), + ), + McpProbeOutcome::AuthRejected => (ProviderPhase::Unavailable, Some("McpAuthRejected".to_owned())), + McpProbeOutcome::AuthConfigInvalid => (ProviderPhase::Unavailable, Some("McpAuthTokenInvalid".to_owned())), + McpProbeOutcome::TlsConfigInvalid(reason) => (ProviderPhase::Unavailable, Some(reason.clone())), + } +} + +/// Map a probe outcome to the bounded `outcome` label used by +/// `grid_mcp_probe_total{outcome}` (see `metrics::record_mcp_probe`). +/// +/// Deliberately collapses [`McpProbeOutcome::TlsConfigInvalid`]'s carried +/// reason string to a single fixed label: that string can vary per Secret +/// misconfiguration and is unbounded-ish, so folding it into a metric label +/// would risk unbounded label cardinality — the same concern `grid#9` +/// documents for the phase-transition metrics. +pub(crate) fn mcp_probe_outcome_label(outcome: &McpProbeOutcome) -> &'static str { + match outcome { + McpProbeOutcome::Success(_) => "Success", + McpProbeOutcome::Unreachable => "Unreachable", + McpProbeOutcome::InvalidResponse => "InvalidResponse", + McpProbeOutcome::AuthRejected => "AuthRejected", + McpProbeOutcome::AuthConfigInvalid => "AuthConfigInvalid", + McpProbeOutcome::TlsConfigInvalid(_) => "TlsConfigInvalid", + } +} + +/// Determine the `discoveredTools` value to persist after a probe attempt. +/// +/// Business rule: a failed probe must never wipe a previously-discovered +/// tool list — only a successful probe overwrites it, with the freshly +/// discovered set (which may itself be empty, if the server genuinely +/// advertises zero tools). +pub(crate) fn discovered_tools_after_probe(previous: &[String], outcome: &McpProbeOutcome) -> Vec { + match outcome { + McpProbeOutcome::Success(tools) => tools.clone(), + McpProbeOutcome::Unreachable + | McpProbeOutcome::InvalidResponse + | McpProbeOutcome::AuthRejected + | McpProbeOutcome::AuthConfigInvalid + | McpProbeOutcome::TlsConfigInvalid(_) => previous.to_vec(), + } +} + +/// Extract tool names from `rmcp`'s `tools/list` result. +/// +/// Pure mapping — no validation of tool schemas or descriptions, since only +/// the name is surfaced on `status.discoveredTools`. +pub(crate) fn discovered_tool_names(tools: &[rmcp::model::Tool]) -> Vec { + tools.iter().map(|tool| tool.name.clone().into_owned()).collect() +} + +/// Truncate `name` to at most [`MAX_TOOL_NAME_LEN`] bytes, landing on a +/// UTF-8 character boundary so truncation never produces invalid UTF-8. +fn truncate_tool_name(name: String) -> String { + if name.len() <= MAX_TOOL_NAME_LEN { + return name; + } + let mut end = MAX_TOOL_NAME_LEN; + while end > 0 && !name.is_char_boundary(end) { + end -= 1; + } + let mut truncated = name; + truncated.truncate(end); + truncated +} + +/// Bound and normalize a raw list of discovered tool names before it is +/// persisted to `status.discoveredTools`. +/// +/// Applies, in order: (1) per-name truncation to [`MAX_TOOL_NAME_LEN`] +/// bytes, (2) deduplication and sorting — tool order is not semantically +/// meaningful, and a server returning the same catalog in a different +/// order must not trigger a status patch on a later reconcile — and (3) +/// truncation of the deduplicated list to at most [`MAX_DISCOVERED_TOOLS`] +/// entries. Keeps both the persisted Kubernetes status object and this +/// reconciler's own memory use bounded against a server advertising an +/// implausibly large or malformed tool catalog. +pub(crate) fn bound_and_normalize_discovered_tools(names: Vec) -> Vec { + let mut names: Vec = names.into_iter().map(truncate_tool_name).collect(); + names.sort(); + names.dedup(); + names.truncate(MAX_DISCOVERED_TOOLS); + names +} + +/// Classify a post-connect `tools/list` call failure into a [`McpProbeOutcome`]. +/// +/// `status` is the HTTP status code observed on the failing exchange, when +/// one was observable (see `observed_status_from_service_error`) — `None` +/// when the failure was not HTTP-status-shaped (e.g. a deserialize or +/// protocol-level error). +/// +/// Business rule: 401/403 map to [`AuthRejected`](McpProbeOutcome::AuthRejected); +/// every other status, or no status at all, maps to +/// [`InvalidResponse`](McpProbeOutcome::InvalidResponse) — the connection to +/// the endpoint was already established by this point (this function is +/// only reached post-connect), so a failure here is a protocol/response +/// problem, not an unreachable endpoint. +pub(crate) fn classify_list_tools_failure(status: Option) -> McpProbeOutcome { + match status { + Some(401 | 403) => McpProbeOutcome::AuthRejected, + _ => McpProbeOutcome::InvalidResponse, + } +} + +/// Whether the probe should build a custom TLS-configured client rather +/// than using native root certificates. +/// +/// Trivial in isolation, but named and tested like the rest of this +/// module's business rules per the project's decision-logic-first testing +/// convention — `spec.tls` presence is the single source of truth for this +/// choice, so this function's name is deliberately the whole rule. +pub(crate) fn should_use_custom_tls(tls_config: Option<&EndpointTlsConfig>) -> bool { + tls_config.is_some() +} + +/// Build the outbound `Authorization` header for the probe request. +/// +/// Returns an empty map when `token` is `None` (`spec.auth` absent, manual, +/// or not yet resolved) — the probe request carries no `Authorization` +/// header, matching an unauthenticated MCP server. The token value is +/// wrapped in [`BearerToken`], which suppresses `Debug` output, so it is +/// never visible if this map is accidentally logged. +/// +/// # Errors +/// +/// Returns [`McpProbeOutcome::AuthConfigInvalid`] if `token` is `Some` but +/// its value cannot be encoded into an HTTP header value. Fails closed +/// rather than silently omitting the header: an endpoint that permits +/// anonymous `tools/list` could otherwise be probed successfully — and +/// marked `Available` — without the configured credential ever being +/// exercised. +pub(crate) fn auth_header_map( + token: Option<&BearerToken>, +) -> Result, McpProbeOutcome> { + let mut headers = HashMap::new(); + let Some(token) = token else { + return Ok(headers); + }; + let bearer = format!("Bearer {}", token.expose_secret()); + let value = HeaderValue::from_str(&bearer).map_err(|_invalid_header_value| { + tracing::warn!( + "bearer token contains characters invalid in an HTTP header value; failing the probe closed rather than \ + proceeding unauthenticated" + ); + McpProbeOutcome::AuthConfigInvalid + })?; + headers.insert(http::header::AUTHORIZATION, value); + Ok(headers) +} + +// --------------------------------------------------------------------------- +// SSRF validation (synchronous, DNS-resolution-free portion) +// --------------------------------------------------------------------------- + +/// Result of validating an `AgentToolProvider`'s `spec.endpoint` before +/// attempting to probe it. +/// +/// Covers everything that can be decided without a DNS lookup. Hostname +/// resolution and per-address SSRF pinning (mirroring +/// `praxis-ai`'s `resolve_hostname_ssrf`) happen in the async probe itself, +/// since they require I/O and are covered by the integration tier instead. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum McpUrlValidation { + /// The URL passed all synchronous checks. + Ok, + /// The URL could not be parsed. + InvalidUrl, + /// The scheme is neither `http` nor `https`. + UnsupportedScheme, + /// The URL's authority embeds `user:pass@` credentials. + EmbeddedCredentials, + /// The URL has no host component. + MissingHost, + /// The host is a blocked hostname or resolves (as a literal IP) to a + /// loopback, link-local, unique-local, unspecified, or known cloud + /// metadata address. + BlockedHost, +} + +/// Validate an `AgentToolProvider`'s `spec.endpoint` before probing it. +/// +/// Business rule: only `http`/`https` URLs with a host, no embedded +/// credentials, and no SSRF-sensitive literal-IP host are eligible for a +/// live probe. Hostnames that are not literal IPs are deferred to the +/// async probe's DNS resolution step — this function cannot resolve them. +pub(crate) fn validate_probe_url(url: &str) -> McpUrlValidation { + // `http::Uri`'s parser treats an empty authority (`http:///path` or + // bare `http://`) as a hard parse error rather than a URI with a blank + // host, so that case is detected here before attempting to parse — + // otherwise it would be misreported as InvalidUrl instead of MissingHost. + if let Some(rest) = url.strip_prefix("http://").or_else(|| url.strip_prefix("https://")) + && (rest.is_empty() || rest.starts_with('/')) + { + return McpUrlValidation::MissingHost; + } + let Ok(uri) = url.parse::() else { + return McpUrlValidation::InvalidUrl; + }; + match uri.scheme_str() { + Some("http" | "https") => {}, + _ => return McpUrlValidation::UnsupportedScheme, + } + if uri.authority().is_some_and(|a| a.as_str().contains('@')) { + return McpUrlValidation::EmbeddedCredentials; + } + let Some(host) = uri.host() else { + return McpUrlValidation::MissingHost; + }; + let host = host.trim_matches(|c| c == '[' || c == ']'); + if is_blocked_hostname(host) { + return McpUrlValidation::BlockedHost; + } + if let Ok(ip) = host.parse::() + && is_ssrf_sensitive(&normalize_mapped_ipv4(ip)) + { + return McpUrlValidation::BlockedHost; + } + McpUrlValidation::Ok +} + +/// Hostnames that resolve to loopback without a DNS lookup. +fn is_blocked_hostname(host: &str) -> bool { + let lower = host.to_ascii_lowercase(); + lower == "localhost" || lower.ends_with(".localhost") +} + +/// Loopback, link-local, unspecified, unique-local, and known cloud +/// metadata addresses are SSRF-sensitive. +fn is_ssrf_sensitive(ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + v4.is_loopback() || v4.is_link_local() || v4.is_unspecified() || *v4 == ALIBABA_CLOUD_METADATA_V4 + }, + IpAddr::V6(v6) => { + let [a, b, ..] = v6.octets(); + v6.is_loopback() || v6.is_unspecified() || (a == 0xFE && (b & 0xC0) == 0x80) || (a & 0xFE) == 0xFC + }, + } +} + +/// Normalize an IPv4-mapped IPv6 address (`::ffff:a.b.c.d`) to its IPv4 +/// form before SSRF checks, closing the bypass where a mapped address +/// would otherwise skip the IPv4 loopback/link-local checks entirely. +fn normalize_mapped_ipv4(ip: IpAddr) -> IpAddr { + match ip { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4), + v4 @ IpAddr::V4(_) => v4, + } +} + +/// Check DNS-resolved addresses against the SSRF block list. +/// +/// Used by the async probe after resolving a hostname, mirroring +/// `praxis-ai`'s `check_resolved_addrs` — kept here (rather than inline in +/// the probe function) so both the literal-IP path +/// ([`validate_probe_url`]) and the resolved-hostname path share the same +/// [`is_ssrf_sensitive`] rule. +pub(crate) fn check_resolved_addrs(addrs: &[SocketAddr]) -> bool { + addrs + .iter() + .all(|addr| !is_ssrf_sensitive(&normalize_mapped_ipv4(addr.ip()))) +} + +// --------------------------------------------------------------------------- +// Error introspection glue (not unit-tested; see module doc) +// --------------------------------------------------------------------------- + +/// Extract the HTTP status code observed on a failed `tools/list` call, when +/// the underlying transport error carries one. +/// +/// Downcasts `rmcp`'s boxed transport error back to the concrete +/// `StreamableHttpError` this crate's transport always +/// produces. Returns `None` for any error shape that isn't HTTP-status-like +/// (deserialize errors, closed transports, etc.) — callers treat `None` as +/// "no distinguishing status observed" via [`classify_list_tools_failure`]. +#[expect(clippy::wildcard_enum_match_arm, reason = "external type with many variants")] +pub(crate) fn observed_status_from_service_error(error: &rmcp::ServiceError) -> Option { + let rmcp::ServiceError::TransportSend(dyn_err) = error else { + return None; + }; + let transport_err = dyn_err.error.downcast_ref::>()?; + match transport_err { + StreamableHttpError::AuthRequired(_) => Some(http::StatusCode::UNAUTHORIZED.as_u16()), + StreamableHttpError::InsufficientScope(_) => Some(http::StatusCode::FORBIDDEN.as_u16()), + StreamableHttpError::Client(reqwest_err) => reqwest_err.status().map(|s| s.as_u16()), + _ => None, + } +} + +// --------------------------------------------------------------------------- +// Live probe orchestration (I/O; covered by the integration/E2E tiers) +// --------------------------------------------------------------------------- + +/// Hostname and DNS-resolved addresses pinned for connect-time use, +/// eliminating the DNS-rebinding TOCTOU window between SSRF validation and +/// the actual connection. +struct ResolvedEndpoint { + /// Present for DNS-resolved hostnames; absent for literal IPs (nothing + /// to pin — the literal address itself already passed [`validate_probe_url`]). + hostname: Option, + /// Validated socket addresses from DNS resolution. + addrs: Vec, +} + +/// Resolve `endpoint`'s host, applying the DNS-resolved-address half of the +/// SSRF check (the literal-IP half already ran in [`validate_probe_url`]). +/// +/// Fails closed: DNS resolution failure, timeout, or any resolved address +/// being SSRF-sensitive returns [`McpProbeOutcome::Unreachable`]. +async fn resolve_endpoint_for_probe(endpoint: &str, timeout: Duration) -> Result { + let uri: http::Uri = endpoint.parse().map_err(|_parse_err| McpProbeOutcome::Unreachable)?; + let host = uri.host().ok_or(McpProbeOutcome::Unreachable)?; + let host = host.trim_matches(|c| c == '[' || c == ']'); + + if host.parse::().is_ok() { + // Literal IP: already validated by validate_probe_url, nothing to resolve/pin. + return Ok(ResolvedEndpoint { + hostname: None, + addrs: Vec::new(), + }); + } + + let port = uri + .port_u16() + .unwrap_or_else(|| if uri.scheme_str() == Some("https") { 443 } else { 80 }); + let addrs: Vec = tokio::time::timeout(timeout, tokio::net::lookup_host((host, port))) + .await + .map_err(|_elapsed| McpProbeOutcome::Unreachable)? + .map_err(|_dns_err| McpProbeOutcome::Unreachable)? + .collect(); + + if !check_resolved_addrs(&addrs) { + return Err(McpProbeOutcome::Unreachable); + } + + Ok(ResolvedEndpoint { + hostname: Some(host.to_owned()), + addrs, + }) +} + +/// Build the `reqwest::Client` used for the probe: address-pinned per +/// [`resolve_endpoint_for_probe`], and carrying `spec.tls`'s CA/client +/// identity material when configured. +/// +/// # Errors +/// +/// Returns [`McpProbeOutcome::TlsConfigInvalid`] if `tls_config` is `Some` +/// but its referenced Secret material cannot be resolved into a usable +/// certificate/identity, or [`McpProbeOutcome::Unreachable`] if the +/// underlying `reqwest::Client` fails to build. +async fn build_probe_client( + kube_client: &kube::Client, + tls_config: Option<&EndpointTlsConfig>, + provider_identity: &str, + resolved: &ResolvedEndpoint, +) -> Result { + let mut builder = reqwest::Client::builder() + .pool_max_idle_per_host(0) + .no_proxy() + .redirect(reqwest::redirect::Policy::none()); + + if let Some(hostname) = &resolved.hostname { + builder = builder.resolve_to_addrs(hostname, &resolved.addrs); + } + + if should_use_custom_tls(tls_config) { + let tls = tls_config.unwrap_or_else(|| std::process::abort()); + builder = attach_tls_material(builder, kube_client, tls, provider_identity).await?; + } + + builder.build().map_err(|_build_err| McpProbeOutcome::Unreachable) +} + +/// Read `tls`'s referenced Secret material and attach it to `builder` as a +/// root CA and, when configured, a client identity. +/// +/// Delegates to [`attach_tls_ca`] and [`attach_tls_client_identity`], split +/// out purely to keep each function within the project's complexity/line +/// lints — the CA and client-identity halves have no shared state beyond +/// the builder itself. +async fn attach_tls_material( + builder: reqwest::ClientBuilder, + kube_client: &kube::Client, + tls: &EndpointTlsConfig, + provider_identity: &str, +) -> Result { + let builder = Box::pin(attach_tls_ca(builder, kube_client, tls, provider_identity)).await?; + let Some(client_ref) = &tls.client_certificate_secret_ref else { + return Ok(builder); + }; + Box::pin(attach_tls_client_identity( + builder, + kube_client, + client_ref, + provider_identity, + )) + .await +} + +/// Read one piece of TLS material (CA cert, client cert, or client key) +/// from a Secret, mapping a resolution failure to the stable +/// `EndpointTls*` `status.reason` family. +/// +/// Shared by [`attach_tls_ca`] and [`attach_tls_client_identity`] so the +/// Secret-read-plus-reason-mapping logic exists exactly once rather than +/// once per material kind. +async fn read_tls_material( + kube_client: &kube::Client, + secret_ref: &crate::crd::grid_network::SecretRef, + key_name: &str, + provider_identity: &str, + material_desc: &str, +) -> Result, McpProbeOutcome> { + read_secret_bytes_for_tls(kube_client, secret_ref, key_name, provider_identity, material_desc) + .await + .map_err(|(reason, msg)| { + tracing::warn!(provider_identity, error = %msg, material_desc, "AgentToolProvider probe TLS material invalid"); + McpProbeOutcome::TlsConfigInvalid(reason.as_status_reason("Endpoint")) + }) +} + +/// Structurally validate that `pem` decodes to at least one well-formed +/// certificate. +/// +/// `reqwest::Certificate::from_pem` is too lenient to be a validation gate +/// by itself: it accepts empty input and PEM blocks with undecodable base64 +/// content without returning `Err` (only genuine third-party wire behavior, +/// like a TLS handshake against real malformed material, would eventually +/// surface a problem — far too late for a reconcile-time `status.reason`). +/// This reuses the same strict `rustls::pki_types` parsing +/// [`metrics_scraper::build_tls_client_config`](crate::metrics_scraper::build_tls_client_config) +/// already relies on for `InferenceProvider`, so both TLS paths reject +/// malformed CA material identically rather than diverging silently. +fn validate_pem_certificates(pem: &[u8]) -> Result<(), String> { + let certs = rustls::pki_types::CertificateDer::pem_slice_iter(pem) + .collect::, _>>() + .map_err(|e| e.to_string())?; + if certs.is_empty() { + return Err("PEM contains no certificates".to_owned()); + } + Ok(()) +} + +/// Structurally validate that `pem` decodes to a well-formed private key. +/// +/// Same rationale as [`validate_pem_certificates`]: `reqwest::Identity::from_pem` +/// alone is not a reliable validation gate for malformed key material. +fn validate_pem_private_key(pem: &[u8]) -> Result<(), String> { + rustls::pki_types::PrivateKeyDer::from_pem_slice(pem) + .map(|_key| ()) + .map_err(|e| e.to_string()) +} + +/// Read `tls.ca_secret_ref`'s CA certificate and add it to `builder` as a +/// trusted root. +async fn attach_tls_ca( + builder: reqwest::ClientBuilder, + kube_client: &kube::Client, + tls: &EndpointTlsConfig, + provider_identity: &str, +) -> Result { + let ca_key = tls.ca_secret_ref.key.as_deref().unwrap_or("ca.crt"); + let ca_pem = Box::pin(read_tls_material( + kube_client, + &tls.ca_secret_ref, + ca_key, + provider_identity, + "CA", + )) + .await?; + if let Err(e) = validate_pem_certificates(&ca_pem) { + tracing::warn!(provider_identity, error = %e, "AgentToolProvider probe CA PEM unparseable"); + return Err(McpProbeOutcome::TlsConfigInvalid( + "EndpointTlsMaterialInvalid".to_owned(), + )); + } + // `reqwest::Certificate::from_pem` itself no longer needs to be a + // validation gate — `validate_pem_certificates` above already is — + // but building the actual `Certificate` reqwest will use is still + // required, and kept as defense-in-depth should a future reqwest + // version regain stricter parsing of its own. + let ca_cert = reqwest::Certificate::from_pem(&ca_pem).map_err(|e| { + tracing::warn!(provider_identity, error = %e, "AgentToolProvider probe CA PEM unparseable"); + McpProbeOutcome::TlsConfigInvalid("EndpointTlsMaterialInvalid".to_owned()) + })?; + // `tls_certs_only` (not `add_root_certificate`, which merges this CA + // into reqwest's platform trust store) so a publicly trusted + // certificate cannot satisfy a probe explicitly configured to trust + // only this private CA -- `spec.tls.caSecretRef` documents the + // supplied CA as the endpoint's only trusted root. + Ok(builder.tls_certs_only([ca_cert])) +} + +/// Read `client_ref`'s certificate and private key and attach them to +/// `builder` as the mTLS client identity. +#[expect( + clippy::too_many_lines, + reason = "sequential cert+key reads, eager rustls PEM validation for each, then the reqwest Identity build" +)] +async fn attach_tls_client_identity( + builder: reqwest::ClientBuilder, + kube_client: &kube::Client, + client_ref: &crate::crd::inference_provider::ClientCertificateSecretRef, + provider_identity: &str, +) -> Result { + let cert_ref = secret_ref_from_client_cert(client_ref); + let mut identity_pem = Box::pin(read_tls_material( + kube_client, + &cert_ref, + &client_ref.certificate_key, + provider_identity, + "client cert", + )) + .await?; + let key_pem = Box::pin(read_tls_material( + kube_client, + &cert_ref, + &client_ref.private_key_key, + provider_identity, + "client key", + )) + .await?; + if let Err(e) = validate_pem_certificates(&identity_pem) { + tracing::warn!(provider_identity, error = %e, "AgentToolProvider probe client certificate unparseable"); + return Err(McpProbeOutcome::TlsConfigInvalid( + ENDPOINT_TLS_IDENTITY_MISMATCH.to_owned(), + )); + } + if let Err(e) = validate_pem_private_key(&key_pem) { + tracing::warn!(provider_identity, error = %e, "AgentToolProvider probe client key unparseable"); + return Err(McpProbeOutcome::TlsConfigInvalid( + ENDPOINT_TLS_IDENTITY_MISMATCH.to_owned(), + )); + } + identity_pem.extend_from_slice(&key_pem); + // As in `attach_tls_ca`: the strict `rustls::pki_types` validation above + // is the real gate; this call still has to happen to build the + // `Identity` reqwest will actually use. + let identity = reqwest::Identity::from_pem(&identity_pem).map_err(|e| { + tracing::warn!(provider_identity, error = %e, "AgentToolProvider probe client identity unparseable"); + McpProbeOutcome::TlsConfigInvalid(ENDPOINT_TLS_IDENTITY_MISMATCH.to_owned()) + })?; + Ok(builder.identity(identity)) +} + +/// Parameters for a single live MCP probe attempt. +/// +/// Grouped into one struct (rather than individual arguments) purely to +/// keep [`probe_agent_tool_provider`]'s signature within the project's +/// `too_many_arguments` lint — `kube_client` stays a separate parameter +/// since callers already hold it as a long-lived `&kube::Client` distinct +/// from this per-attempt request data. +pub(crate) struct ProbeRequest<'request> { + /// `spec.endpoint` — the MCP server's HTTP(S) URL. + pub(crate) endpoint: &'request str, + /// Total wall-clock budget for the whole probe: DNS resolution, TLS + /// Secret material reads, connect/handshake, and the `tools/list` call + /// combined. Enforced by a single outer `tokio::time::timeout` in + /// [`probe_agent_tool_provider`] — the per-phase timeouts inside it are + /// defensive inner bounds, not independent budgets, so this value is + /// never multiplied across phases. + pub(crate) timeout: Duration, + /// `spec.tls`, when the probe should use a custom CA/client identity + /// instead of the platform trust store. + pub(crate) tls_config: Option<&'request EndpointTlsConfig>, + /// The `AgentToolProvider`'s name, for log/tracing attribution only. + pub(crate) provider_identity: &'request str, + /// The resolved bearer token from `spec.auth`, when configured. + pub(crate) auth_token: Option<&'request BearerToken>, +} + +/// Run a live MCP `tools/list` probe against `request.endpoint`. +/// +/// Validates the URL (SSRF/scheme/format), resolves and pins DNS addresses, +/// builds an address-pinned `reqwest::Client` (with `spec.tls` material when +/// configured), then delegates the connect/`list_tools` exchange to +/// [`run_probe_session`]. +/// +/// Only the first page of results is fetched — `AgentToolProvider` has no +/// documented need for multi-page tool catalogs at this scope, and every +/// reconcile re-probes regardless (see the CRD's staleness-note doc comment). +/// +/// The whole sequence — DNS resolution, TLS Secret reads, connect/handshake, +/// and `tools/list` — is bounded by one outer `request.timeout`, so a slow +/// Kubernetes API (TLS Secret fetch has no timeout of its own) or a peer +/// that stalls at one phase cannot push total probe latency past the +/// documented budget by combining several unbounded or independently-bounded +/// phases. +pub(crate) async fn probe_agent_tool_provider( + kube_client: &kube::Client, + request: ProbeRequest<'_>, +) -> McpProbeOutcome { + match tokio::time::timeout( + request.timeout, + probe_agent_tool_provider_unbounded(kube_client, request), + ) + .await + { + Ok(outcome) => outcome, + Err(_elapsed) => McpProbeOutcome::Unreachable, + } +} + +/// The actual probe sequence, without its own overall deadline — +/// [`probe_agent_tool_provider`] is the only caller and supplies the single +/// outer `tokio::time::timeout` that bounds this function's total runtime. +async fn probe_agent_tool_provider_unbounded(kube_client: &kube::Client, request: ProbeRequest<'_>) -> McpProbeOutcome { + if validate_probe_url(request.endpoint) != McpUrlValidation::Ok { + tracing::warn!( + provider_identity = request.provider_identity, + endpoint = request.endpoint, + "AgentToolProvider probe endpoint failed SSRF/format validation" + ); + return McpProbeOutcome::Unreachable; + } + + let resolved = match resolve_endpoint_for_probe(request.endpoint, request.timeout).await { + Ok(resolved) => resolved, + Err(outcome) => return outcome, + }; + + let client = match build_probe_client(kube_client, request.tls_config, request.provider_identity, &resolved).await { + Ok(client) => client, + Err(outcome) => return outcome, + }; + + Box::pin(run_probe_session( + client, + request.endpoint, + request.timeout, + request.auth_token, + )) + .await +} + +/// Connect to the MCP server over Streamable HTTP and call `tools/list`, +/// bounding both steps by `timeout`. +/// +/// Split out of [`probe_agent_tool_provider`] purely to keep both functions +/// within the project's complexity/line lints — this is the +/// connect-and-call half of what was previously one larger function. +async fn run_probe_session( + client: reqwest::Client, + endpoint: &str, + timeout: Duration, + auth_token: Option<&BearerToken>, +) -> McpProbeOutcome { + let mut transport_config = StreamableHttpClientTransportConfig::with_uri(endpoint); + let headers = match auth_header_map(auth_token) { + Ok(headers) => headers, + Err(outcome) => return outcome, + }; + if !headers.is_empty() { + transport_config = transport_config.custom_headers(headers); + } + let transport = StreamableHttpClientTransport::with_client(client, transport_config); + + let running = match tokio::time::timeout(timeout, Box::pin(().serve(transport))).await { + Err(_elapsed) => return McpProbeOutcome::Unreachable, + Ok(Err(_init_err)) => return McpProbeOutcome::Unreachable, + Ok(Ok(running)) => running, + }; + + match tokio::time::timeout(timeout, Box::pin(running.list_tools(None))).await { + Err(_elapsed) => McpProbeOutcome::Unreachable, + Ok(Err(service_err)) => classify_list_tools_failure(observed_status_from_service_error(&service_err)), + Ok(Ok(page)) => { + McpProbeOutcome::Success(bound_and_normalize_discovered_tools(discovered_tool_names(&page.tools))) + }, + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, reason = "tests")] +mod tests { + use super::*; + use crate::crd::inference_provider::ProviderPhase; + + // ----------------------------------------------------------------------- + // phase_and_reason_from_probe + // ----------------------------------------------------------------------- + + #[test] + fn success_outcome_yields_available_with_no_reason() { + let (phase, reason) = phase_and_reason_from_probe(&McpProbeOutcome::Success(vec!["search".to_owned()])); + assert_eq!( + phase, + ProviderPhase::Available, + "a successful probe must yield Available" + ); + assert!( + reason.is_none(), + "a successful probe must clear any prior status.reason" + ); + } + + #[test] + fn unreachable_outcome_yields_unavailable_with_stable_reason() { + let (phase, reason) = phase_and_reason_from_probe(&McpProbeOutcome::Unreachable); + assert_eq!(phase, ProviderPhase::Unavailable); + assert_eq!(reason.as_deref(), Some("McpEndpointUnreachable")); + } + + #[test] + fn invalid_response_outcome_yields_unavailable_with_stable_reason() { + let (phase, reason) = phase_and_reason_from_probe(&McpProbeOutcome::InvalidResponse); + assert_eq!(phase, ProviderPhase::Unavailable); + assert_eq!(reason.as_deref(), Some("McpToolsListInvalidResponse")); + } + + #[test] + fn auth_rejected_outcome_yields_unavailable_with_stable_reason() { + let (phase, reason) = phase_and_reason_from_probe(&McpProbeOutcome::AuthRejected); + assert_eq!(phase, ProviderPhase::Unavailable); + assert_eq!(reason.as_deref(), Some("McpAuthRejected")); + } + + #[test] + fn auth_config_invalid_outcome_yields_unavailable_with_stable_reason() { + let (phase, reason) = phase_and_reason_from_probe(&McpProbeOutcome::AuthConfigInvalid); + assert_eq!(phase, ProviderPhase::Unavailable); + assert_eq!(reason.as_deref(), Some("McpAuthTokenInvalid")); + } + + #[test] + fn tls_config_invalid_outcome_yields_unavailable_with_its_carried_reason() { + let (phase, reason) = phase_and_reason_from_probe(&McpProbeOutcome::TlsConfigInvalid( + "EndpointTlsSecretMissing".to_owned(), + )); + assert_eq!(phase, ProviderPhase::Unavailable); + assert_eq!( + reason.as_deref(), + Some("EndpointTlsSecretMissing"), + "TlsConfigInvalid's stable reason comes from the variant itself, not a fixed string" + ); + } + + // ----------------------------------------------------------------------- + // mcp_probe_outcome_label — bounded telemetry label for grid_mcp_probe_total{outcome} + // ----------------------------------------------------------------------- + + #[test] + fn success_outcome_label_is_success() { + assert_eq!( + mcp_probe_outcome_label(&McpProbeOutcome::Success(vec!["search".to_owned()])), + "Success" + ); + } + + #[test] + fn unreachable_outcome_label_is_unreachable() { + assert_eq!(mcp_probe_outcome_label(&McpProbeOutcome::Unreachable), "Unreachable"); + } + + #[test] + fn invalid_response_outcome_label_is_invalid_response() { + assert_eq!( + mcp_probe_outcome_label(&McpProbeOutcome::InvalidResponse), + "InvalidResponse" + ); + } + + #[test] + fn auth_rejected_outcome_label_is_auth_rejected() { + assert_eq!(mcp_probe_outcome_label(&McpProbeOutcome::AuthRejected), "AuthRejected"); + } + + #[test] + fn auth_config_invalid_outcome_label_is_auth_config_invalid() { + assert_eq!( + mcp_probe_outcome_label(&McpProbeOutcome::AuthConfigInvalid), + "AuthConfigInvalid" + ); + } + + #[test] + fn tls_config_invalid_outcome_label_is_bounded_regardless_of_carried_reason() { + // The label must stay bounded/enum-shaped even though the variant + // itself carries an unbounded-ish String — two different carried + // reasons must still map to the exact same label, never leaking the + // inner string into a metric label (unbounded cardinality risk). + let a = mcp_probe_outcome_label(&McpProbeOutcome::TlsConfigInvalid( + "EndpointTlsSecretMissing".to_owned(), + )); + let b = mcp_probe_outcome_label(&McpProbeOutcome::TlsConfigInvalid("EndpointTlsKeyMissing".to_owned())); + assert_eq!(a, "TlsConfigInvalid"); + assert_eq!(a, b, "the label must not vary with the carried reason string"); + } + + // ----------------------------------------------------------------------- + // discovered_tools_after_probe — preserve-on-failure business rule + // ----------------------------------------------------------------------- + + #[test] + fn successful_probe_overwrites_discovered_tools() { + let previous = vec!["old-tool".to_owned()]; + let outcome = McpProbeOutcome::Success(vec!["new-tool".to_owned()]); + assert_eq!( + discovered_tools_after_probe(&previous, &outcome), + vec!["new-tool".to_owned()], + "a successful probe must overwrite discoveredTools with the freshly discovered set" + ); + } + + #[test] + fn successful_probe_with_zero_tools_overwrites_to_empty() { + let previous = vec!["old-tool".to_owned()]; + let outcome = McpProbeOutcome::Success(vec![]); + assert!( + discovered_tools_after_probe(&previous, &outcome).is_empty(), + "a successful probe genuinely advertising zero tools must still overwrite, not preserve stale entries" + ); + } + + #[test] + fn unreachable_probe_preserves_previously_discovered_tools() { + let previous = vec!["search".to_owned(), "fetch".to_owned()]; + assert_eq!( + discovered_tools_after_probe(&previous, &McpProbeOutcome::Unreachable), + previous, + "a probe failure must never wipe a previously-discovered tool list" + ); + } + + #[test] + fn invalid_response_probe_preserves_previously_discovered_tools() { + let previous = vec!["search".to_owned()]; + assert_eq!( + discovered_tools_after_probe(&previous, &McpProbeOutcome::InvalidResponse), + previous, + "a probe failure must never wipe a previously-discovered tool list" + ); + } + + #[test] + fn auth_rejected_probe_preserves_previously_discovered_tools() { + let previous = vec!["search".to_owned()]; + assert_eq!( + discovered_tools_after_probe(&previous, &McpProbeOutcome::AuthRejected), + previous, + "a probe failure must never wipe a previously-discovered tool list" + ); + } + + #[test] + fn auth_config_invalid_probe_preserves_previously_discovered_tools() { + let previous = vec!["search".to_owned()]; + assert_eq!( + discovered_tools_after_probe(&previous, &McpProbeOutcome::AuthConfigInvalid), + previous, + "a probe failure must never wipe a previously-discovered tool list" + ); + } + + #[test] + fn failed_probe_with_no_prior_tools_stays_empty() { + let previous: Vec = vec![]; + assert!( + discovered_tools_after_probe(&previous, &McpProbeOutcome::Unreachable).is_empty(), + "a first-ever probe failure with nothing previously discovered must remain empty, not panic or fabricate" + ); + } + + // ----------------------------------------------------------------------- + // discovered_tool_names — pure extraction from rmcp::model::Tool + // ----------------------------------------------------------------------- + + fn test_tool(name: &str) -> rmcp::model::Tool { + // rmcp::model::Tool is #[non_exhaustive]; build via Default then + // mutate the one field this test cares about. + let mut tool = rmcp::model::Tool::default(); + tool.name = name.to_owned().into(); + tool + } + + #[test] + fn extracts_names_from_multiple_tools_in_order() { + let tools = vec![test_tool("search"), test_tool("fetch")]; + assert_eq!( + discovered_tool_names(&tools), + vec!["search".to_owned(), "fetch".to_owned()], + "tool names must be extracted in the order the server returned them" + ); + } + + #[test] + fn extracts_empty_vec_from_zero_tools() { + assert!( + discovered_tool_names(&[]).is_empty(), + "zero tools must yield an empty name list, not error" + ); + } + + // ----------------------------------------------------------------------- + // classify_list_tools_failure — post-connect failure classification + // ----------------------------------------------------------------------- + + #[test] + fn http_401_after_connect_is_auth_rejected() { + assert_eq!(classify_list_tools_failure(Some(401)), McpProbeOutcome::AuthRejected); + } + + #[test] + fn http_403_after_connect_is_auth_rejected() { + assert_eq!(classify_list_tools_failure(Some(403)), McpProbeOutcome::AuthRejected); + } + + #[test] + fn http_500_after_connect_is_invalid_response_not_auth_rejected() { + assert_eq!( + classify_list_tools_failure(Some(500)), + McpProbeOutcome::InvalidResponse, + "a non-auth HTTP failure after a successful connect is a response problem, not an unreachable endpoint" + ); + } + + #[test] + fn no_observed_status_after_connect_is_invalid_response() { + assert_eq!( + classify_list_tools_failure(None), + McpProbeOutcome::InvalidResponse, + "a protocol/deserialize failure with no HTTP status is still a response problem post-connect" + ); + } + + // ----------------------------------------------------------------------- + // should_use_custom_tls + // ----------------------------------------------------------------------- + + #[test] + fn absent_spec_tls_does_not_use_custom_tls() { + assert!( + !should_use_custom_tls(None), + "absent spec.tls must use native root certificates" + ); + } + + #[test] + fn present_spec_tls_uses_custom_tls() { + let tls = EndpointTlsConfig { + ca_secret_ref: crate::crd::grid_network::SecretRef { + name: "ca".to_owned(), + namespace: "ns".to_owned(), + key: None, + }, + client_certificate_secret_ref: None, + }; + assert!( + should_use_custom_tls(Some(&tls)), + "present spec.tls must trigger the custom TLS path" + ); + } + + // ----------------------------------------------------------------------- + // auth_header_map + // ----------------------------------------------------------------------- + + #[test] + fn absent_token_yields_no_authorization_header() { + let headers = auth_header_map(None).expect("no token must never fail"); + assert!( + headers.is_empty(), + "no resolved bearer token must mean no Authorization header at all" + ); + } + + #[test] + fn present_token_yields_bearer_authorization_header() { + let token = BearerToken::new("s3cr3t".to_owned()); + let headers = auth_header_map(Some(&token)).expect("a well-formed token must not fail"); + assert_eq!( + headers.get(&http::header::AUTHORIZATION).map(|v| v.to_str().unwrap()), + Some("Bearer s3cr3t"), + "a resolved bearer token must be attached as a standard Bearer Authorization header" + ); + } + + #[test] + fn token_with_invalid_header_characters_fails_closed() { + let token = BearerToken::new("s3cr3t\nwith-newline".to_owned()); + assert_eq!( + auth_header_map(Some(&token)), + Err(McpProbeOutcome::AuthConfigInvalid), + "a token that cannot be encoded as an HTTP header value must fail the probe closed, \ + not proceed unauthenticated" + ); + } + + // ----------------------------------------------------------------------- + // bound_and_normalize_discovered_tools + // ----------------------------------------------------------------------- + + #[test] + fn small_valid_list_passes_through_sorted() { + let names = vec!["fetch".to_owned(), "search".to_owned()]; + assert_eq!( + bound_and_normalize_discovered_tools(names), + vec!["fetch".to_owned(), "search".to_owned()], + "an already-small, already-sorted list must pass through unchanged" + ); + } + + #[test] + fn out_of_order_names_are_sorted() { + let names = vec!["search".to_owned(), "fetch".to_owned()]; + assert_eq!( + bound_and_normalize_discovered_tools(names), + vec!["fetch".to_owned(), "search".to_owned()], + "tool order is not semantically meaningful and must be normalized to avoid \ + unnecessary status churn on later reconciles" + ); + } + + #[test] + fn duplicate_names_are_deduplicated() { + let names = vec!["search".to_owned(), "fetch".to_owned(), "search".to_owned()]; + assert_eq!( + bound_and_normalize_discovered_tools(names), + vec!["fetch".to_owned(), "search".to_owned()], + "duplicate tool names must be collapsed to one entry" + ); + } + + #[test] + fn overly_long_name_is_truncated_at_a_char_boundary() { + let long_name = "€".repeat(MAX_TOOL_NAME_LEN); // multi-byte codepoint, byte length != char count + let result = bound_and_normalize_discovered_tools(vec![long_name]); + assert_eq!(result.len(), 1); + let truncated = result.first().expect("one entry must remain"); + assert!( + truncated.len() <= MAX_TOOL_NAME_LEN, + "a name longer than the byte limit must be truncated to at most {MAX_TOOL_NAME_LEN} bytes" + ); + assert!( + truncated.is_char_boundary(truncated.len()), + "truncation must never split a multi-byte UTF-8 codepoint" + ); + } + + #[test] + fn name_at_exactly_the_limit_is_not_truncated() { + let name = "a".repeat(MAX_TOOL_NAME_LEN); + let result = bound_and_normalize_discovered_tools(vec![name.clone()]); + assert_eq!( + result, + vec![name], + "a name exactly at the byte limit must not be altered" + ); + } + + #[test] + fn tool_count_beyond_the_limit_is_truncated() { + let names: Vec = (0..MAX_DISCOVERED_TOOLS + 10).map(|i| format!("tool-{i:05}")).collect(); + let result = bound_and_normalize_discovered_tools(names); + assert_eq!( + result.len(), + MAX_DISCOVERED_TOOLS, + "a catalog advertising more than {MAX_DISCOVERED_TOOLS} distinct tools must be truncated \ + to keep the persisted status object bounded" + ); + } + + #[test] + fn empty_list_stays_empty() { + assert!( + bound_and_normalize_discovered_tools(vec![]).is_empty(), + "an empty catalog must remain empty, not panic or fabricate entries" + ); + } + + // ----------------------------------------------------------------------- + // validate_probe_url — synchronous SSRF/scheme/format validation + // ----------------------------------------------------------------------- + + #[test] + fn valid_https_url_passes() { + assert_eq!( + validate_probe_url("https://tools.grid-system.svc:8443/mcp"), + McpUrlValidation::Ok + ); + } + + #[test] + fn valid_http_url_passes() { + assert_eq!(validate_probe_url("http://tools:8080/mcp"), McpUrlValidation::Ok); + } + + #[test] + fn unparseable_url_is_invalid() { + assert_eq!(validate_probe_url("not a url \n"), McpUrlValidation::InvalidUrl); + } + + #[test] + fn ftp_scheme_is_unsupported() { + assert_eq!( + validate_probe_url("ftp://tools:21/mcp"), + McpUrlValidation::UnsupportedScheme + ); + } + + #[test] + fn websocket_scheme_is_unsupported() { + assert_eq!( + validate_probe_url("ws://tools:8080/mcp"), + McpUrlValidation::UnsupportedScheme + ); + } + + #[test] + fn embedded_credentials_are_rejected() { + assert_eq!( + validate_probe_url("https://user:pass@tools:8443/mcp"), + McpUrlValidation::EmbeddedCredentials + ); + } + + #[test] + fn localhost_hostname_is_blocked() { + assert_eq!( + validate_probe_url("http://localhost:8080/mcp"), + McpUrlValidation::BlockedHost + ); + } + + #[test] + fn localhost_subdomain_is_blocked() { + assert_eq!( + validate_probe_url("http://tools.localhost:8080/mcp"), + McpUrlValidation::BlockedHost + ); + } + + #[test] + fn loopback_literal_ip_is_blocked() { + assert_eq!( + validate_probe_url("http://127.0.0.1:8080/mcp"), + McpUrlValidation::BlockedHost + ); + } + + #[test] + fn ipv6_loopback_literal_is_blocked() { + assert_eq!( + validate_probe_url("http://[::1]:8080/mcp"), + McpUrlValidation::BlockedHost + ); + } + + #[test] + fn ipv6_link_local_literal_is_blocked() { + assert_eq!( + validate_probe_url("http://[fe80::1]:8080/mcp"), + McpUrlValidation::BlockedHost, + "IPv6 link-local addresses must be blocked" + ); + } + + #[test] + fn ipv6_unique_local_literal_is_blocked() { + assert_eq!( + validate_probe_url("http://[fd00::1]:8080/mcp"), + McpUrlValidation::BlockedHost, + "IPv6 unique-local (ULA) addresses must be blocked" + ); + } + + #[test] + fn link_local_literal_ip_is_blocked() { + assert_eq!( + validate_probe_url("http://169.254.169.254:80/mcp"), + McpUrlValidation::BlockedHost, + "169.254.169.254 (AWS/GCP/Azure metadata) must be blocked as link-local" + ); + } + + #[test] + fn alibaba_cloud_metadata_literal_ip_is_blocked() { + assert_eq!( + validate_probe_url("http://100.100.100.200/mcp"), + McpUrlValidation::BlockedHost + ); + } + + #[test] + fn unspecified_literal_ip_is_blocked() { + assert_eq!( + validate_probe_url("http://0.0.0.0:8080/mcp"), + McpUrlValidation::BlockedHost + ); + } + + #[test] + fn ipv4_mapped_ipv6_loopback_is_blocked() { + assert_eq!( + validate_probe_url("http://[::ffff:127.0.0.1]:8080/mcp"), + McpUrlValidation::BlockedHost, + "an IPv4-mapped IPv6 loopback address must not bypass the IPv4 loopback check" + ); + } + + #[test] + fn regular_cluster_service_literal_ip_passes() { + // A normal in-cluster ClusterIP is neither loopback nor link-local — + // this validation must not block ordinary in-cluster addresses. + assert_eq!(validate_probe_url("http://10.96.0.42:8080/mcp"), McpUrlValidation::Ok); + } + + #[test] + fn missing_host_is_rejected() { + assert_eq!(validate_probe_url("http:///mcp"), McpUrlValidation::MissingHost); + } + + // ----------------------------------------------------------------------- + // check_resolved_addrs + // ----------------------------------------------------------------------- + + #[test] + fn all_safe_resolved_addrs_pass() { + let addrs: Vec = vec!["10.96.0.42:8080".parse().unwrap(), "10.96.0.43:8080".parse().unwrap()]; + assert!( + check_resolved_addrs(&addrs), + "ordinary cluster addresses must pass DNS-resolved SSRF checks" + ); + } + + #[test] + fn any_unsafe_resolved_addr_fails_the_whole_set() { + let addrs: Vec = vec!["10.96.0.42:8080".parse().unwrap(), "127.0.0.1:8080".parse().unwrap()]; + assert!( + !check_resolved_addrs(&addrs), + "a hostname resolving to even one SSRF-sensitive address must fail closed for the whole set" + ); + } + + #[test] + fn empty_resolved_addrs_passes_vacuously() { + assert!( + check_resolved_addrs(&[]), + "an empty address list has nothing unsafe to find" + ); + } + + // ----------------------------------------------------------------------- + // observed_status_from_service_error + // ----------------------------------------------------------------------- + + #[test] + fn non_transport_service_error_yields_no_status() { + let err = rmcp::ServiceError::TransportClosed; + assert_eq!( + observed_status_from_service_error(&err), + None, + "a non-transport ServiceError variant carries no HTTP status to extract" + ); + } + + // ----------------------------------------------------------------------- + // TLS Secret resolution — attach_tls_ca / attach_tls_client_identity / + // read_tls_material, against a mocked Kubernetes API. + // + // Uses a real `tower::service_fn`-backed `kube::Client` (no network) so + // these functions' actual `Api::::get_opt` calls, PEM parsing, + // and reason-mapping are the thing under test — not a hand-built + // `McpProbeOutcome` fixture standing in for them. + // ----------------------------------------------------------------------- + + /// Build a `kube::Client` backed by an in-memory Secret map keyed by + /// Secret name, so a real `Api::::get_opt` round-trips through + /// this module's own `TlsFailureReason`-mapping logic. + #[expect( + clippy::too_many_lines, + reason = "test mock builder: 404-vs-200 branches are the whole point" + )] + fn mock_kube_client_with_secrets( + secrets: HashMap<&'static str, k8s_openapi::api::core::v1::Secret>, + ) -> kube::Client { + let service = tower::service_fn(move |req: http::Request| { + let secrets = secrets.clone(); + async move { + let name = req.uri().path().rsplit('/').next().unwrap_or_default().to_owned(); + let response = secrets.get(name.as_str()).map_or_else( + || { + let not_found = serde_json::json!({ + "kind": "Status", + "apiVersion": "v1", + "status": "Failure", + "message": format!("secrets \"{name}\" not found"), + "reason": "NotFound", + "code": 404, + }); + http::Response::builder() + .status(404) + .body(kube::client::Body::from( + serde_json::to_vec(¬_found).unwrap_or_else(|_| std::process::abort()), + )) + .unwrap_or_else(|_| std::process::abort()) + }, + |secret| { + http::Response::builder() + .status(200) + .body(kube::client::Body::from( + serde_json::to_vec(secret).unwrap_or_else(|_| std::process::abort()), + )) + .unwrap_or_else(|_| std::process::abort()) + }, + ); + Ok::<_, std::convert::Infallible>(response) + } + }); + kube::Client::new(service, "default") + } + + /// Build a Secret with a single `data` key. + fn secret_with_key(key: &str, value: &[u8]) -> k8s_openapi::api::core::v1::Secret { + let mut data = std::collections::BTreeMap::new(); + data.insert(key.to_owned(), k8s_openapi::ByteString(value.to_vec())); + k8s_openapi::api::core::v1::Secret { + data: Some(data), + ..Default::default() + } + } + + fn test_secret_ref(name: &str) -> crate::crd::grid_network::SecretRef { + crate::crd::grid_network::SecretRef { + name: name.to_owned(), + namespace: "default".to_owned(), + key: None, + } + } + + // ----------------------------------------------------------------------- + // validate_pem_certificates / validate_pem_private_key — pure decision + // logic, no Kubernetes I/O + // ----------------------------------------------------------------------- + + #[test] + fn validate_pem_certificates_accepts_a_real_certificate() { + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + assert!( + validate_pem_certificates(ca.cert_pem.as_bytes()).is_ok(), + "a real, well-formed certificate PEM must validate" + ); + } + + #[test] + fn validate_pem_certificates_rejects_empty_input() { + assert!( + validate_pem_certificates(b"").is_err(), + "empty input contains no certificates and must be rejected — this is exactly what \ + reqwest::Certificate::from_pem fails to reject on its own" + ); + } + + #[test] + fn validate_pem_certificates_rejects_undecodable_base64() { + assert!( + validate_pem_certificates(b"-----BEGIN CERTIFICATE-----\nnot valid base64!!!\n-----END CERTIFICATE-----\n") + .is_err(), + "undecodable base64 inside PEM markers must be rejected" + ); + } + + #[test] + fn validate_pem_private_key_accepts_a_real_key() { + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + assert!( + validate_pem_private_key(ca.key_pem.as_bytes()).is_ok(), + "a real, well-formed private key PEM must validate" + ); + } + + #[test] + fn validate_pem_private_key_rejects_garbage() { + assert!( + validate_pem_private_key(b"not a private key").is_err(), + "garbage input must be rejected" + ); + } + + /// Installs the process-wide `rustls` crypto provider these tests need + /// before any `reqwest::Certificate`/`reqwest::Identity` PEM parsing — + /// see `probe_via_pipeline_for_tests` in `integration_tests` for why. + fn install_test_crypto_provider() { + drop(rustls::crypto::ring::default_provider().install_default()); + } + + #[tokio::test] + async fn read_tls_material_returns_bytes_when_secret_and_key_present() { + let client = + mock_kube_client_with_secrets(HashMap::from([("ca-secret", secret_with_key("ca.crt", b"ca-bytes"))])); + let result = read_tls_material(&client, &test_secret_ref("ca-secret"), "ca.crt", "test-provider", "CA").await; + assert_eq!(result, Ok(b"ca-bytes".to_vec()), "must return the exact stored bytes"); + } + + #[tokio::test] + async fn read_tls_material_secret_missing_yields_endpoint_tls_secret_missing() { + let client = mock_kube_client_with_secrets(HashMap::new()); + let result = read_tls_material(&client, &test_secret_ref("absent"), "ca.crt", "test-provider", "CA").await; + assert_eq!( + result, + Err(McpProbeOutcome::TlsConfigInvalid("EndpointTlsSecretMissing".to_owned())), + "a missing Secret must surface as EndpointTlsSecretMissing" + ); + } + + /// Covers the fix landed for + /// : the shared + /// `resources::secret::read_secret_bytes`/`endpoint_tls::read_secret_bytes_for_tls` + /// pipeline (used by `InferenceProvider`'s metrics/health-check TLS too, + /// not specific to `AgentToolProvider`) now distinguishes a Secret that + /// exists but lacks the requested key (`KeyMissing`) from a Secret that + /// does not exist at all (`SecretMissing`), rather than collapsing both + /// into the latter. + #[tokio::test] + async fn read_tls_material_key_absent_from_data_yields_endpoint_tls_key_missing() { + let client = mock_kube_client_with_secrets(HashMap::from([( + "ca-secret", + secret_with_key("wrong-key", b"ca-bytes"), + )])); + let result = read_tls_material(&client, &test_secret_ref("ca-secret"), "ca.crt", "test-provider", "CA").await; + assert_eq!( + result, + Err(McpProbeOutcome::TlsConfigInvalid("EndpointTlsKeyMissing".to_owned())), + "a key absent from an existing Secret's data must surface as EndpointTlsKeyMissing, \ + not EndpointTlsSecretMissing (grid#58)" + ); + } + + #[tokio::test] + async fn read_tls_material_key_present_but_empty_yields_endpoint_tls_key_missing() { + let client = mock_kube_client_with_secrets(HashMap::from([("ca-secret", secret_with_key("ca.crt", b""))])); + let result = read_tls_material(&client, &test_secret_ref("ca-secret"), "ca.crt", "test-provider", "CA").await; + assert_eq!( + result, + Err(McpProbeOutcome::TlsConfigInvalid("EndpointTlsKeyMissing".to_owned())), + "a key present in Secret.data with an empty value must also surface as EndpointTlsKeyMissing, \ + the same as a key absent entirely (grid#58)" + ); + } + + #[tokio::test] + async fn attach_tls_ca_succeeds_with_valid_ca_pem() { + install_test_crypto_provider(); + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + let client = mock_kube_client_with_secrets(HashMap::from([( + "ca-secret", + secret_with_key("ca.crt", ca.cert_pem.as_bytes()), + )])); + let tls = EndpointTlsConfig { + ca_secret_ref: test_secret_ref("ca-secret"), + client_certificate_secret_ref: None, + }; + let result = attach_tls_ca(reqwest::Client::builder(), &client, &tls, "test-provider").await; + drop(result.expect("a valid CA PEM must attach cleanly")); + } + + #[tokio::test] + async fn attach_tls_ca_malformed_pem_yields_material_invalid() { + install_test_crypto_provider(); + // Must have valid PEM *markers* with undecodable content inside: bytes + // with no `-----BEGIN CERTIFICATE-----` block at all are silently + // treated by `reqwest::Certificate::from_pem` as "zero certificates + // found" rather than a parse error, so they would not exercise this + // failure path. + let client = mock_kube_client_with_secrets(HashMap::from([( + "ca-secret", + secret_with_key( + "ca.crt", + b"-----BEGIN CERTIFICATE-----\nnot valid base64 content!!!\n-----END CERTIFICATE-----\n", + ), + )])); + let tls = EndpointTlsConfig { + ca_secret_ref: test_secret_ref("ca-secret"), + client_certificate_secret_ref: None, + }; + let result = attach_tls_ca(reqwest::Client::builder(), &client, &tls, "test-provider").await; + assert_eq!( + result.unwrap_err(), + McpProbeOutcome::TlsConfigInvalid("EndpointTlsMaterialInvalid".to_owned()), + "unparseable CA PEM must surface as EndpointTlsMaterialInvalid" + ); + } + + #[tokio::test] + async fn attach_tls_ca_empty_pem_yields_material_invalid() { + install_test_crypto_provider(); + let client = mock_kube_client_with_secrets(HashMap::from([("ca-secret", secret_with_key("ca.crt", b""))])); + let tls = EndpointTlsConfig { + ca_secret_ref: test_secret_ref("ca-secret"), + client_certificate_secret_ref: None, + }; + let result = attach_tls_ca(reqwest::Client::builder(), &client, &tls, "test-provider").await; + // An empty key value is caught earlier by read_tls_material's own + // "key present but empty" check, before validate_pem_certificates + // ever runs — this asserts that ordering explicitly, since it's + // easy to accidentally invert. + assert_eq!( + result.unwrap_err(), + McpProbeOutcome::TlsConfigInvalid("EndpointTlsKeyMissing".to_owned()), + "an empty CA Secret value is caught by read_tls_material before PEM validation runs" + ); + } + + #[tokio::test] + async fn attach_tls_ca_missing_secret_propagates_read_tls_material_reason() { + install_test_crypto_provider(); + let client = mock_kube_client_with_secrets(HashMap::new()); + let tls = EndpointTlsConfig { + ca_secret_ref: test_secret_ref("absent"), + client_certificate_secret_ref: None, + }; + let result = attach_tls_ca(reqwest::Client::builder(), &client, &tls, "test-provider").await; + assert_eq!( + result.unwrap_err(), + McpProbeOutcome::TlsConfigInvalid("EndpointTlsSecretMissing".to_owned()), + "attach_tls_ca must propagate read_tls_material's Secret-missing reason unchanged" + ); + } + + #[tokio::test] + async fn attach_tls_client_identity_succeeds_with_matching_cert_and_key() { + install_test_crypto_provider(); + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + let site = certs::generate_site_cert(&ca, "test-client").unwrap_or_else(|_| std::process::abort()); + let mut data = std::collections::BTreeMap::new(); + data.insert( + "tls.crt".to_owned(), + k8s_openapi::ByteString(site.cert_pem.into_bytes()), + ); + data.insert("tls.key".to_owned(), k8s_openapi::ByteString(site.key_pem.into_bytes())); + let secret = k8s_openapi::api::core::v1::Secret { + data: Some(data), + ..Default::default() + }; + let client = mock_kube_client_with_secrets(HashMap::from([("client-cert", secret)])); + let client_ref = crate::crd::inference_provider::ClientCertificateSecretRef { + name: "client-cert".to_owned(), + namespace: "default".to_owned(), + certificate_key: "tls.crt".to_owned(), + private_key_key: "tls.key".to_owned(), + }; + let result = + attach_tls_client_identity(reqwest::Client::builder(), &client, &client_ref, "test-provider").await; + drop(result.expect("a valid, matching cert/key pair must attach cleanly")); + } + + #[tokio::test] + async fn attach_tls_client_identity_unparseable_key_yields_identity_mismatch() { + install_test_crypto_provider(); + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + let site = certs::generate_site_cert(&ca, "test-client").unwrap_or_else(|_| std::process::abort()); + let mut data = std::collections::BTreeMap::new(); + data.insert( + "tls.crt".to_owned(), + k8s_openapi::ByteString(site.cert_pem.into_bytes()), + ); + data.insert( + "tls.key".to_owned(), + k8s_openapi::ByteString(b"not a private key".to_vec()), + ); + let secret = k8s_openapi::api::core::v1::Secret { + data: Some(data), + ..Default::default() + }; + let client = mock_kube_client_with_secrets(HashMap::from([("client-cert", secret)])); + let client_ref = crate::crd::inference_provider::ClientCertificateSecretRef { + name: "client-cert".to_owned(), + namespace: "default".to_owned(), + certificate_key: "tls.crt".to_owned(), + private_key_key: "tls.key".to_owned(), + }; + let result = + attach_tls_client_identity(reqwest::Client::builder(), &client, &client_ref, "test-provider").await; + assert_eq!( + result.unwrap_err(), + McpProbeOutcome::TlsConfigInvalid(ENDPOINT_TLS_IDENTITY_MISMATCH.to_owned()), + "unparseable key material must surface as EndpointTlsIdentityMismatch" + ); + } + + #[tokio::test] + async fn attach_tls_material_ca_only_when_no_client_cert_configured() { + install_test_crypto_provider(); + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + let client = mock_kube_client_with_secrets(HashMap::from([( + "ca-secret", + secret_with_key("ca.crt", ca.cert_pem.as_bytes()), + )])); + let tls = EndpointTlsConfig { + ca_secret_ref: test_secret_ref("ca-secret"), + client_certificate_secret_ref: None, + }; + let result = attach_tls_material(reqwest::Client::builder(), &client, &tls, "test-provider").await; + drop(result.expect("CA-only TLS config must attach cleanly")); + } + + #[tokio::test] + async fn attach_tls_material_attaches_both_ca_and_client_identity() { + install_test_crypto_provider(); + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + let site = certs::generate_site_cert(&ca, "test-client").unwrap_or_else(|_| std::process::abort()); + let mut client_data = std::collections::BTreeMap::new(); + client_data.insert( + "tls.crt".to_owned(), + k8s_openapi::ByteString(site.cert_pem.into_bytes()), + ); + client_data.insert("tls.key".to_owned(), k8s_openapi::ByteString(site.key_pem.into_bytes())); + let client_secret = k8s_openapi::api::core::v1::Secret { + data: Some(client_data), + ..Default::default() + }; + let kube_client = mock_kube_client_with_secrets(HashMap::from([ + ("ca-secret", secret_with_key("ca.crt", ca.cert_pem.as_bytes())), + ("client-cert", client_secret), + ])); + let tls = EndpointTlsConfig { + ca_secret_ref: test_secret_ref("ca-secret"), + client_certificate_secret_ref: Some(crate::crd::inference_provider::ClientCertificateSecretRef { + name: "client-cert".to_owned(), + namespace: "default".to_owned(), + certificate_key: "tls.crt".to_owned(), + private_key_key: "tls.key".to_owned(), + }), + }; + let result = attach_tls_material(reqwest::Client::builder(), &kube_client, &tls, "test-provider").await; + drop(result.expect("CA + client identity TLS config must attach cleanly")); + } + + #[tokio::test] + async fn attach_tls_material_propagates_client_identity_secret_missing() { + install_test_crypto_provider(); + let ca = certs::generate_ca("test-ca").unwrap_or_else(|_| std::process::abort()); + let kube_client = mock_kube_client_with_secrets(HashMap::from([( + "ca-secret", + secret_with_key("ca.crt", ca.cert_pem.as_bytes()), + )])); + let tls = EndpointTlsConfig { + ca_secret_ref: test_secret_ref("ca-secret"), + client_certificate_secret_ref: Some(crate::crd::inference_provider::ClientCertificateSecretRef { + name: "absent-client-cert".to_owned(), + namespace: "default".to_owned(), + certificate_key: "tls.crt".to_owned(), + private_key_key: "tls.key".to_owned(), + }), + }; + let result = attach_tls_material(reqwest::Client::builder(), &kube_client, &tls, "test-provider").await; + assert_eq!( + result.unwrap_err(), + McpProbeOutcome::TlsConfigInvalid("EndpointTlsSecretMissing".to_owned()), + "a missing client-cert Secret must fail the whole attach_tls_material call, not be silently skipped" + ); + } +} + +/// Integration tier: [`probe_agent_tool_provider`] against a real Streamable +/// HTTP MCP server over a local TCP listener — no mocks below the socket. +/// +/// The pure decision logic (URL validation, outcome-to-phase mapping, +/// discovered-tools preservation) and the TLS/Secret material attachment +/// path (`attach_tls_material` and friends, against a mocked `kube::Client`) +/// are already covered at the unit tier above; these tests instead exercise +/// the actual network path: DNS/address resolution, `reqwest` client +/// construction, the `rmcp` client/server handshake, and header propagation. +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, reason = "tests")] +mod integration_tests { + use rmcp::{ + ServerHandler, + model::{ListToolsResult, PaginatedRequestParams, ServerCapabilities, ServerInfo, Tool}, + service::RequestContext, + transport::streamable_http_server::{ + StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager, + }, + }; + + use super::{ + BearerToken, McpProbeOutcome, ProbeRequest, build_probe_client, resolve_endpoint_for_probe, run_probe_session, + }; + + /// Runs the same resolve → build-client → session pipeline as + /// [`super::probe_agent_tool_provider`], skipping only its + /// `validate_probe_url` SSRF/format gate. + /// + /// That gate is already exhaustively covered by the fast, synchronous + /// unit tests above (loopback, `localhost`, link-local, cloud metadata, + /// etc. are all proven blocked there) — and a local test server + /// necessarily binds to loopback, so calling through the public + /// entry point here would only prove the gate blocks our own test + /// fixture, not that the resolve/connect/probe pipeline behind it + /// actually works end-to-end. This helper exercises that real, + /// previously-untested pipeline instead. + async fn probe_via_pipeline_for_tests(kube_client: &kube::Client, request: ProbeRequest<'_>) -> McpProbeOutcome { + // reqwest's `rustls-no-provider` feature (see the workspace + // Cargo.toml comment) means the *application* must install a + // process-wide crypto provider before building any `reqwest::Client` + // — `main.rs` does this once for the real binary; test binaries have + // no equivalent entry point, so each call here does it instead. + // Idempotent: a second install attempt just returns `Err`, which is + // exactly what happens when multiple tests in this binary race here. + drop(rustls::crypto::ring::default_provider().install_default()); + + let resolved = match resolve_endpoint_for_probe(request.endpoint, request.timeout).await { + Ok(resolved) => resolved, + Err(outcome) => return outcome, + }; + let client = + match build_probe_client(kube_client, request.tls_config, request.provider_identity, &resolved).await { + Ok(client) => client, + Err(outcome) => return outcome, + }; + run_probe_session(client, request.endpoint, request.timeout, request.auth_token).await + } + + /// A minimal MCP server that answers `tools/list` with a fixed set of + /// tool names, optionally requiring a specific bearer token. + #[derive(Clone)] + struct FixedToolsServer { + tools: Vec, + required_bearer: Option, + } + + impl ServerHandler for FixedToolsServer { + fn get_info(&self) -> ServerInfo { + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) + } + + async fn list_tools( + &self, + _request: Option, + context: RequestContext, + ) -> Result { + if let Some(expected) = &self.required_bearer { + // rmcp threads the raw incoming `http::request::Parts` (headers + // included) into RequestContext::extensions — no axum + // middleware needed to see what the probe actually sent. + let got = context + .extensions + .get::() + .and_then(|parts| parts.headers.get(http::header::AUTHORIZATION)) + .and_then(|value| value.to_str().ok()); + if got != Some(format!("Bearer {expected}").as_str()) { + return Err(rmcp::ErrorData::invalid_request("missing or wrong bearer token", None)); + } + } + let tools = self + .tools + .iter() + .map(|name| { + let mut tool = Tool::default(); + tool.name = name.clone().into(); + tool + }) + .collect(); + Ok(ListToolsResult::with_all_items(tools)) + } + } + + /// Spawn a real [`FixedToolsServer`] on a local TCP listener and return + /// its base MCP endpoint URL (`http://127.0.0.1:/mcp`). + /// + /// The spawned `axum::serve` task is never explicitly cancelled — it is + /// dropped along with the `#[tokio::test]` runtime when each test + /// function returns, mirroring the fire-and-forget test-server pattern + /// already used by `metrics_scraper`/`tls_probe` in this crate. + async fn spawn_mcp_server(tools: &[&str], required_bearer: Option<&str>) -> String { + let handler = FixedToolsServer { + tools: tools.iter().map(|t| (*t).to_owned()).collect(), + required_bearer: required_bearer.map(str::to_owned), + }; + let config = StreamableHttpServerConfig::default(); + let service: StreamableHttpService = + StreamableHttpService::new(move || Ok(handler.clone()), std::sync::Arc::default(), config); + let router = axum::Router::new().nest_service("/mcp", service); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + drop(axum::serve(listener, router).await); + }); + + format!("http://{addr}/mcp") + } + + /// A `kube::Client` that panics if a request is ever sent through it. + /// + /// Every test in this module uses `tls_config: None`, so `kube_client` + /// is never dereferenced by [`probe_agent_tool_provider`] — it is only + /// used on the `spec.tls`-configured path, covered by the + /// `attach_tls_material`/`attach_tls_ca`/`attach_tls_client_identity`/ + /// `read_tls_material` unit tests in `mod tests` above (against a + /// mocked `kube::Client`, where Secret I/O is the thing under test). + fn unused_kube_client() -> kube::Client { + let service = tower::service_fn(|_req: http::Request| async { + Ok::<_, std::convert::Infallible>(http::Response::new(kube::client::Body::empty())) + }); + kube::Client::new(service, "default") + } + + #[tokio::test] + async fn probe_against_real_server_discovers_tools_sorted() { + let endpoint = spawn_mcp_server(&["read_file", "list_directory"], None).await; + let kube_client = unused_kube_client(); + + let outcome = probe_via_pipeline_for_tests( + &kube_client, + ProbeRequest { + endpoint: &endpoint, + timeout: std::time::Duration::from_secs(5), + tls_config: None, + provider_identity: "it-discovers-tools", + auth_token: None, + }, + ) + .await; + + assert_eq!( + outcome, + McpProbeOutcome::Success(vec!["list_directory".to_owned(), "read_file".to_owned()]), + "a real tools/list round trip must surface the server's tool names, normalized to sorted order" + ); + } + + #[tokio::test] + async fn probe_against_closed_port_is_unreachable() { + // Bind then immediately drop the listener: the port is free again + // but nothing is listening, so connect must fail — a real refused + // connection, not a stubbed-out unit-test double. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + let endpoint = format!("http://{addr}/mcp"); + let kube_client = unused_kube_client(); + + let outcome = probe_via_pipeline_for_tests( + &kube_client, + ProbeRequest { + endpoint: &endpoint, + timeout: std::time::Duration::from_secs(2), + tls_config: None, + provider_identity: "it-closed-port", + auth_token: None, + }, + ) + .await; + + assert_eq!( + outcome, + McpProbeOutcome::Unreachable, + "a real connection-refused failure must classify as Unreachable" + ); + } + + #[tokio::test] + async fn probe_with_correct_bearer_token_succeeds() { + let endpoint = spawn_mcp_server(&["search"], Some("s3cr3t-token")).await; + let kube_client = unused_kube_client(); + let token = BearerToken::new("s3cr3t-token".to_owned()); + + let outcome = probe_via_pipeline_for_tests( + &kube_client, + ProbeRequest { + endpoint: &endpoint, + timeout: std::time::Duration::from_secs(5), + tls_config: None, + provider_identity: "it-correct-bearer", + auth_token: Some(&token), + }, + ) + .await; + + assert_eq!( + outcome, + McpProbeOutcome::Success(vec!["search".to_owned()]), + "the resolved bearer token must reach the server as a real Authorization header" + ); + } + + #[tokio::test] + async fn probe_with_missing_bearer_token_is_auth_rejected() { + let endpoint = spawn_mcp_server(&["search"], Some("s3cr3t-token")).await; + let kube_client = unused_kube_client(); + + let outcome = probe_via_pipeline_for_tests( + &kube_client, + ProbeRequest { + endpoint: &endpoint, + timeout: std::time::Duration::from_secs(5), + tls_config: None, + provider_identity: "it-missing-bearer", + auth_token: None, + }, + ) + .await; + + assert_eq!( + outcome, + McpProbeOutcome::InvalidResponse, + "an MCP-level error response (no HTTP-status-coded rejection) classifies as InvalidResponse, \ + not AuthRejected — that distinction is covered at the unit tier by classify_list_tools_failure" + ); + } +} diff --git a/scripts/generate-deployment-crds.sh b/scripts/generate-deployment-crds.sh index 2d63051..2505def 100755 --- a/scripts/generate-deployment-crds.sh +++ b/scripts/generate-deployment-crds.sh @@ -13,9 +13,10 @@ echo "Generating Grid CRDs..." mkdir -p "$CRD_DIR" # Generate CRDs and split into individual YAML files -cargo run -p operator --bin generate_crds | jq -r '.items[0]' | yq eval -P > "$CRD_DIR/gridnetwork.yaml" -cargo run -p operator --bin generate_crds | jq -r '.items[1]' | yq eval -P > "$CRD_DIR/gridsite.yaml" -cargo run -p operator --bin generate_crds | jq -r '.items[2]' | yq eval -P > "$CRD_DIR/inferenceprovider.yaml" +cargo run -p operator --bin generate_crds | jq -r '.items[0]' | yq eval -P > "$CRD_DIR/agenttoolprovider.yaml" +cargo run -p operator --bin generate_crds | jq -r '.items[1]' | yq eval -P > "$CRD_DIR/gridnetwork.yaml" +cargo run -p operator --bin generate_crds | jq -r '.items[2]' | yq eval -P > "$CRD_DIR/gridsite.yaml" +cargo run -p operator --bin generate_crds | jq -r '.items[3]' | yq eval -P > "$CRD_DIR/inferenceprovider.yaml" echo "CRDs generated in $CRD_DIR:" ls -la "$CRD_DIR" diff --git a/scripts/verify-helm-chart.sh b/scripts/verify-helm-chart.sh index 0ae20b2..c28d213 100755 --- a/scripts/verify-helm-chart.sh +++ b/scripts/verify-helm-chart.sh @@ -75,7 +75,7 @@ fi # ── CRD synchronization ───────────────────────────────────────────── echo "" echo "=== CRD synchronization ===" -for crd in gridnetwork gridsite inferenceprovider; do +for crd in agenttoolprovider gridnetwork gridsite inferenceprovider; do if diff -q "$DEPLOY_CRDS/${crd}.yaml" "$CHART_DIR/crds/${crd}.yaml" >/dev/null 2>&1; then pass "crd sync: ${crd}.yaml" else @@ -167,7 +167,8 @@ TGZ=$(echo "$PKG_OUT" | grep -oP '/tmp/\S+\.tgz') if [ -f "$TGZ" ]; then pass "helm package: $(basename "$TGZ") ($(stat -c%s "$TGZ") bytes)" CONTENTS=$(tar tzf "$TGZ" 2>&1) - for f in Chart.yaml values.yaml values.schema.json templates/deployment.yaml crds/gridnetwork.yaml; do + for f in Chart.yaml values.yaml values.schema.json templates/deployment.yaml crds/agenttoolprovider.yaml \ + crds/gridnetwork.yaml crds/gridsite.yaml crds/inferenceprovider.yaml; do if echo "$CONTENTS" | grep -q "$f"; then pass "package contains: $f" else @@ -537,7 +538,8 @@ if [ "${KIND:-}" = "1" ] || [ "${1:-}" = "--kind" ]; then fail "kind: operator install" fi - for crd in gridnetworks.grid.praxis-proxy.io gridsites.grid.praxis-proxy.io inferenceproviders.grid.praxis-proxy.io; do + for crd in agenttoolproviders.grid.praxis-proxy.io gridnetworks.grid.praxis-proxy.io gridsites.grid.praxis-proxy.io \ + inferenceproviders.grid.praxis-proxy.io; do if kubectl --context "$KCTX" get crd "$crd" >/dev/null 2>&1; then pass "kind: crd $crd established" else @@ -617,7 +619,8 @@ CR_EOF fail "kind: operator uninstall" fi - for crd in gridnetworks.grid.praxis-proxy.io gridsites.grid.praxis-proxy.io inferenceproviders.grid.praxis-proxy.io; do + for crd in agenttoolproviders.grid.praxis-proxy.io gridnetworks.grid.praxis-proxy.io gridsites.grid.praxis-proxy.io \ + inferenceproviders.grid.praxis-proxy.io; do if kubectl --context "$KCTX" get crd "$crd" >/dev/null 2>&1; then pass "kind: crd $crd retained after uninstall" else diff --git a/xtask/src/env.rs b/xtask/src/env.rs index 66ae5e3..947eb4a 100644 --- a/xtask/src/env.rs +++ b/xtask/src/env.rs @@ -817,6 +817,29 @@ pub(crate) enum Action { site: Option, }, + /// Verify `AgentToolProvider` convergence: `Pending` -> `Available` with `discoveredTools` + /// populated against a real mock MCP server, plus the unreachable-endpoint failure path. + /// + /// Deploys a real (not local-loopback) mock MCP `tools/list` server in-cluster — + /// `AgentToolProvider`'s probe deliberately blocks loopback/link-local targets via + /// SSRF protection, unlike `GridSite`'s gateway probe — applies a `GridNetwork` + + /// `GridSite` + healthy `AgentToolProvider`, and asserts the phase transitions to + /// `Available` with the mock's tool names in `status.discoveredTools`. A second + /// `AgentToolProvider` pointed at a nonexistent Service confirms the failure path: + /// `Unavailable` with a populated `status.reason`. + /// + /// Requires a kind cluster with Grid CRDs and the `grid-mock-providers` image + /// loaded (or pullable). Safe to rerun. + VerifyAgenttoolproviderConvergence { + /// Path to the environment config file. + #[arg(short, long, default_value = "tests/env/operator-routing.toml")] + config: PathBuf, + + /// Kind cluster context to run against (first provider site by default). + #[arg(long)] + site: Option, + }, + /// Verify lost-peer staleness: remote provider marked stale when SWIM peer is killed. /// /// Starts a primary (east) and a joining (west) SWIM operator, publishes a @@ -1068,6 +1091,9 @@ pub(crate) fn run(action: &Action) -> Result<(), Box> { Action::VerifySiteJoinDiscovery { config } => env_verify_site_join_discovery(config), Action::VerifyGridsiteRotation { config, site } => env_verify_gridsite_rotation(config, site.as_deref()), Action::VerifyGridsiteConvergence { config, site } => env_verify_gridsite_convergence(config, site.as_deref()), + Action::VerifyAgenttoolproviderConvergence { config, site } => { + env_verify_agenttoolprovider_convergence(config, site.as_deref()) + }, Action::VerifyFailoverUnderLostPeer { config } => env_verify_failover_under_lost_peer(config), Action::VerifyStaleGcTtl { config } => env_verify_stale_gc_ttl(config), Action::VerifyOperatorInstallRbac { config, site } => env_verify_operator_install_rbac(config, site.as_deref()), @@ -6721,6 +6747,124 @@ fn env_verify_gridsite_convergence(config: &Path, site: Option<&str>) -> Result< Ok(()) } +// --------------------------------------------------------------------------- +// AgentToolProvider convergence verifier (grid#41) +// --------------------------------------------------------------------------- + +/// Verify `AgentToolProvider` convergence against a real mock MCP server. +/// +/// Proves the full stack end-to-end: real cluster, real controller binary, +/// real (mock) MCP server, actual `status.phase`/`status.discoveredTools` +/// convergence — not a one-off manual check. See `grid#41`'s plan +/// ("E2E tier"), which this command was scoped to satisfy permanently. +/// +/// Steps: +/// 1. Install CRDs, clean up any stale prior run's resources. +/// 2. Deploy a real mock MCP server in-cluster (a Service is required — the probe's SSRF protection deliberately blocks +/// loopback/link-local targets, so a locally-spawned process is not a legitimate substitute here, unlike +/// `GridSite`'s gateway probe) and discover its `NodePort` address: the operator under test runs out-of-cluster +/// (step 4), so it cannot resolve the mock's in-cluster `.svc` DNS name. +/// 3. Apply `GridNetwork` + `GridSite` + a healthy `AgentToolProvider`. +/// 4. Spawn the operator locally and wait for `Pending` -> `Available` with `discoveredTools` matching the mock's +/// configured tool list. +/// 5. Apply a second `AgentToolProvider` pointed at a nonexistent Service and confirm it lands on `Unavailable` with a +/// populated `status.reason`. +#[expect(clippy::too_many_lines, reason = "multi-step E2E orchestration")] +fn env_verify_agenttoolprovider_convergence( + config: &Path, + site: Option<&str>, +) -> Result<(), Box> { + use operator::{AGENT_TOOL_TEST_PROVIDER_HEALTHY, AGENT_TOOL_TEST_PROVIDER_UNREACHABLE, STATUS_POLL_TIMEOUT}; + + let cfg = EnvConfig::from_file(config)?; + let site_name = resolve_operator_site_name(&cfg, site)?; + let context = resolve_operator_context(&cfg, site)?; + let cluster_name = kind::cluster_name_from_config(site_name); + eprintln!("verify-agenttoolprovider-convergence: context={context}"); + + // ── Step 1: setup ──────────────────────────────────────────────────── + eprintln!("verify-agenttoolprovider-convergence: [1/5] setup"); + operator::install_grid_crds(&context)?; + operator::cleanup_agent_tool_provider_test_resources(&context); + kind::delete_mock_mcp_server(&context); + + // ── Step 2: deploy the real mock MCP server ───────────────────────── + eprintln!("verify-agenttoolprovider-convergence: [2/5] deploying mock MCP server..."); + let mock_tools = ["read_file", "list_directory"]; + kind::deploy_mock_mcp_server(&context, &cluster_name, &mock_tools.join(","), None)?; + // The operator under test runs as a local out-of-cluster process (see + // `spawn_operator` below), so it cannot resolve in-cluster `.svc` DNS + // names — it must reach the mock over the kind node's NodePort address, + // the same pattern `discover_provider_cluster_endpoint` uses for the + // provider gateway. + let mock_node_port = kind::service_node_port(&context, kind::MOCK_MCP_SVC, "default") + .ok_or("mock MCP server Service has no NodePort assigned")?; + let mock_node_ip = kind::kind_node_ip(&context)?; + let mock_endpoint = format!("http://{mock_node_ip}:{mock_node_port}/mcp"); + + // ── Step 3: apply GridNetwork + GridSite + healthy AgentToolProvider ─ + eprintln!("verify-agenttoolprovider-convergence: [3/5] applying fixtures..."); + operator::apply_agent_tool_provider_network_fixtures(&context)?; + operator::apply_agent_tool_provider(&context, AGENT_TOOL_TEST_PROVIDER_HEALTHY, &mock_endpoint)?; + // A Service name that cannot resolve — DNS failure, not a refused + // connection, exercising the same Unreachable classification either way. + operator::apply_agent_tool_provider( + &context, + AGENT_TOOL_TEST_PROVIDER_UNREACHABLE, + "http://mock-mcp-server-does-not-exist.default.svc:8080/mcp", + )?; + + let op = operator::spawn_operator(&context)?; + let mut op_guard = ProcGuard(Some(op), "agent-tool-provider-operator"); + + let result: Result<(), Box> = (|| { + // ── Step 4: healthy path — Pending -> Available, tools discovered ─ + eprintln!("verify-agenttoolprovider-convergence: [4/5] waiting for healthy convergence..."); + operator::wait_for_agent_tool_provider_phase( + &context, + AGENT_TOOL_TEST_PROVIDER_HEALTHY, + "Available", + STATUS_POLL_TIMEOUT, + )?; + let mut expected_tools: Vec = mock_tools.iter().map(|t| (*t).to_owned()).collect(); + expected_tools.sort(); + let discovered = + operator::read_agent_tool_provider_discovered_tools(&context, AGENT_TOOL_TEST_PROVIDER_HEALTHY)?; + if discovered != expected_tools { + return Err(format!("discoveredTools mismatch: expected {expected_tools:?}, got {discovered:?}").into()); + } + eprintln!(" [OK] discoveredTools = {discovered:?}"); + + // ── Step 5: unreachable path — Unavailable + populated reason ──── + eprintln!("verify-agenttoolprovider-convergence: [5/5] waiting for unreachable-endpoint failure..."); + operator::wait_for_agent_tool_provider_phase( + &context, + AGENT_TOOL_TEST_PROVIDER_UNREACHABLE, + "Unavailable", + STATUS_POLL_TIMEOUT, + )?; + let reason = operator::read_agent_tool_provider_reason(&context, AGENT_TOOL_TEST_PROVIDER_UNREACHABLE)?; + if reason.is_empty() { + return Err("expected a populated status.reason for the unreachable-endpoint AgentToolProvider".into()); + } + eprintln!(" [OK] unreachable endpoint reason = {reason:?}"); + Ok(()) + })(); + + if let Some(c) = op_guard.0.take() { + operator::kill_operator(c); + } + operator::cleanup_agent_tool_provider_test_resources(&context); + kind::delete_mock_mcp_server(&context); + result?; + + eprintln!( + "verify-agenttoolprovider-convergence: PASS — Pending -> Available with discoveredTools populated \ + against a real mock MCP server, and the unreachable-endpoint path lands on Unavailable with a reason" + ); + Ok(()) +} + // --------------------------------------------------------------------------- // Fingerprint trust promotion validation // --------------------------------------------------------------------------- diff --git a/xtask/src/env/kind.rs b/xtask/src/env/kind.rs index 2732e9b..969855e 100644 --- a/xtask/src/env/kind.rs +++ b/xtask/src/env/kind.rs @@ -50,6 +50,18 @@ pub(crate) const MOCK_CLOUD_PORT: u16 = 8080; /// Kubernetes namespace for provider backend deployments. const NAMESPACE: &str = "default"; +/// Kubernetes Deployment and Service name for the mock MCP `tools/list` server. +/// +/// Deployed as a real in-cluster Service (not a locally-spawned process) so +/// that `AgentToolProvider`'s SSRF-hardened probe — which deliberately +/// blocks loopback/link-local targets, unlike `GridSite`'s gateway probe — +/// has a legitimate, non-blocked address to discover tools from. See +/// `cargo xtask env verify-agenttoolprovider-convergence`. +pub(crate) const MOCK_MCP_SVC: &str = "mock-mcp-server"; + +/// Port used by the mock MCP server. +pub(crate) const MOCK_MCP_PORT: u16 = 8080; + // --------------------------------------------------------------------------- // Public API // --------------------------------------------------------------------------- @@ -488,6 +500,140 @@ spec: ) } +// --------------------------------------------------------------------------- +// Mock MCP server (AgentToolProvider probe target) +// --------------------------------------------------------------------------- + +/// Deploy the mock MCP `tools/list` server into `cluster_name`, reusing the +/// same `grid-mock-providers` image as the AI-provider mocks (just a +/// different `--mcp-server` CLI mode — see `mock-providers/src/mcp.rs`). +/// +/// `tools_csv` and `required_bearer` are forwarded verbatim as the mock's +/// `--mcp-tools`/`--mcp-bearer` arguments. +pub(crate) fn deploy_mock_mcp_server( + context: &str, + cluster_name: &str, + tools_csv: &str, + required_bearer: Option<&str>, +) -> Result<(), Box> { + let mock_provider_img = image_overrides::mock_provider_image(); + + if image_overrides::should_skip_kind_image_loading() { + eprintln!(" skipping loading {mock_provider_img} (pull policy is not Never)"); + } else { + eprintln!(" loading {mock_provider_img} into {cluster_name}..."); + run_cmd( + "kind", + &["load", "docker-image", &mock_provider_img, "--name", cluster_name], + ) + .map_err(|e| { + format!( + "{e}\n\ + hint: build the image first with:\n\ + \x20 docker build -t {mock_provider_img} -f mock-providers/Containerfile ." + ) + })?; + } + + eprintln!(" deploying {MOCK_MCP_SVC} to {cluster_name}..."); + kubectl::apply_manifest(context, &mock_mcp_server_deployment_yaml(tools_csv, required_bearer))?; + kubectl::apply_manifest(context, &mock_mcp_server_service_yaml())?; + kubectl::wait_for_rollout(context, MOCK_MCP_SVC, cluster_name)?; + Ok(()) +} + +/// Delete the mock MCP server Deployment and Service if they exist. +/// +/// Best-effort: errors are ignored, matching [`delete_mock_api_provider`]. +pub(crate) fn delete_mock_mcp_server(context: &str) { + for kind in ["deployment", "service"] { + drop( + Command::new("kubectl") + .args([ + "--context", + context, + "-n", + NAMESPACE, + "delete", + kind, + MOCK_MCP_SVC, + "--ignore-not-found", + ]) + .status(), + ); + } +} + +/// Generate the Deployment YAML for the mock MCP server. +#[expect(clippy::too_many_lines, reason = "readable multiline Kubernetes Deployment YAML")] +fn mock_mcp_server_deployment_yaml(tools_csv: &str, required_bearer: Option<&str>) -> String { + let mock_provider_img = image_overrides::mock_provider_image(); + let pull_policy = image_overrides::image_pull_policy(); + let mut yaml = format!( + r#"apiVersion: apps/v1 +kind: Deployment +metadata: + name: {MOCK_MCP_SVC} + namespace: {NAMESPACE} + labels: + app: {MOCK_MCP_SVC} +spec: + replicas: 1 + selector: + matchLabels: + app: {MOCK_MCP_SVC} + template: + metadata: + labels: + app: {MOCK_MCP_SVC} + spec: + containers: + - name: mock-mcp + image: {mock_provider_img} + imagePullPolicy: {pull_policy} + args: + - "--mcp-server" + - "--port" + - "{MOCK_MCP_PORT}" + - "--mcp-tools" + - "{tools_csv}" +"# + ); + if let Some(token) = required_bearer { + write!(yaml, " - \"--mcp-bearer\"\n - \"{token}\"\n") + .unwrap_or_else(|_| std::process::abort()); + } + write!(yaml, " ports:\n - containerPort: {MOCK_MCP_PORT}\n") + .unwrap_or_else(|_| std::process::abort()); + yaml +} + +/// Generate the Service YAML for the mock MCP server. +/// +/// `NodePort` (not `ClusterIP`): the `AgentToolProvider` E2E check spawns the +/// operator as a local out-of-cluster process (see `spawn_operator`), which +/// cannot resolve in-cluster `.svc` DNS names or reach `ClusterIP`s — only +/// the kind node's `NodePort`-mapped address is reachable from the host. No +/// `nodePort` is pinned; callers discover the assigned port via +/// [`service_node_port`]. +fn mock_mcp_server_service_yaml() -> String { + format!( + "apiVersion: v1 +kind: Service +metadata: + name: {MOCK_MCP_SVC} + namespace: {NAMESPACE} +spec: + type: NodePort + selector: + app: {MOCK_MCP_SVC} + ports: + - port: {MOCK_MCP_PORT} + targetPort: {MOCK_MCP_PORT} +" + ) +} + /// Deploy the cloud-managed mock provider into a cluster and wait for readiness. /// /// Represents a `cloud_managed` API endpoint in the full-grid routing validation. diff --git a/xtask/src/env/operator.rs b/xtask/src/env/operator.rs index 625163d..413a48a 100644 --- a/xtask/src/env/operator.rs +++ b/xtask/src/env/operator.rs @@ -38,6 +38,22 @@ pub(crate) const TEST_GATEWAY_NAME: &str = "op-e2e-gw"; pub(crate) const TEST_GATEWAY_NS: &str = "default"; /// Name of the `InferenceProvider` with a valid endpoint (expected: reconciles to `Pending`). pub(crate) const TEST_PROVIDER_HEALTHY: &str = "op-e2e-healthy"; + +/// Name of the test `GridNetwork` used by the `AgentToolProvider` convergence check. +/// +/// Distinct from [`TEST_NETWORK`] so this E2E command can be re-run +/// independently of (and concurrently with) the `InferenceProvider`-focused +/// fixtures without name collisions. +pub(crate) const AGENT_TOOL_TEST_NETWORK: &str = "op-e2e-agent-tool-net"; +/// Name of the test `GridSite` referencing [`AGENT_TOOL_TEST_NETWORK`]. +/// +/// Its mere existence is sufficient for site-matching: `AgentToolProvider`'s +/// default (empty) `siteSelector` matches every `GridSite` in the network. +pub(crate) const AGENT_TOOL_TEST_SITE: &str = "op-e2e-agent-tool-site"; +/// Name of the `AgentToolProvider` pointed at a real, reachable mock MCP server. +pub(crate) const AGENT_TOOL_TEST_PROVIDER_HEALTHY: &str = "op-e2e-agent-tool-healthy"; +/// Name of the `AgentToolProvider` pointed at a deliberately-unreachable endpoint. +pub(crate) const AGENT_TOOL_TEST_PROVIDER_UNREACHABLE: &str = "op-e2e-agent-tool-unreachable"; /// Name of the `InferenceProvider` with a blank endpoint (expected: reconciles to `Unavailable`). pub(crate) const TEST_PROVIDER_INVALID: &str = "op-e2e-invalid"; /// Name of the `InferenceProvider` whose health probe returns non-2xx (expected: `Degraded`). @@ -1015,6 +1031,136 @@ fn provider_fixture_json( }) } +// --------------------------------------------------------------------------- +// AgentToolProvider convergence fixtures (grid#41) +// --------------------------------------------------------------------------- + +/// Apply the minimal `GridNetwork` + `GridSite` fixtures the +/// `AgentToolProvider` convergence check needs. +/// +/// Neither resource needs a `gatewayRef` or labels: `AgentToolProvider`'s +/// default `siteSelector` matches every `GridSite` referencing the network, +/// so a bare `GridSite` is enough to clear site-matching once the network +/// exists. +pub(crate) fn apply_agent_tool_provider_network_fixtures(context: &str) -> Result<(), Box> { + let network = serde_json::to_string_pretty(&serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "GridNetwork", + "metadata": { "name": AGENT_TOOL_TEST_NETWORK }, + "spec": {} + })) + .unwrap_or_else(|e| { + eprintln!("AgentToolProvider GridNetwork fixture serialization failed: {e}"); + std::process::exit(1); + }); + let site = serde_json::to_string_pretty(&serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "GridSite", + "metadata": { "name": AGENT_TOOL_TEST_SITE }, + "spec": { "gridNetworkRef": AGENT_TOOL_TEST_NETWORK } + })) + .unwrap_or_else(|e| { + eprintln!("AgentToolProvider GridSite fixture serialization failed: {e}"); + std::process::exit(1); + }); + kubectl::apply_manifest(context, &network)?; + kubectl::apply_manifest(context, &site)?; + Ok(()) +} + +/// Apply an `AgentToolProvider` fixture pointed at `endpoint`. +pub(crate) fn apply_agent_tool_provider( + context: &str, + name: &str, + endpoint: &str, +) -> Result<(), Box> { + let manifest = serde_json::to_string_pretty(&serde_json::json!({ + "apiVersion": "grid.praxis-proxy.io/v1alpha1", + "kind": "AgentToolProvider", + "metadata": { "name": name }, + "spec": { + "gridNetworkRef": AGENT_TOOL_TEST_NETWORK, + "endpoint": endpoint + } + })) + .unwrap_or_else(|e| { + eprintln!("AgentToolProvider fixture serialization failed: {e}"); + std::process::exit(1); + }); + kubectl::apply_manifest(context, &manifest)?; + Ok(()) +} + +/// Delete the `AgentToolProvider` convergence test's `GridNetwork`, +/// `GridSite`, and `AgentToolProvider` resources if they exist. +/// +/// Best-effort: errors are ignored, matching [`cleanup_validation_resources`]. +pub(crate) fn cleanup_agent_tool_provider_test_resources(context: &str) { + for (kind, name) in [ + ("agenttoolproviders", AGENT_TOOL_TEST_PROVIDER_HEALTHY), + ("agenttoolproviders", AGENT_TOOL_TEST_PROVIDER_UNREACHABLE), + ("gridsites", AGENT_TOOL_TEST_SITE), + ("gridnetworks", AGENT_TOOL_TEST_NETWORK), + ] { + drop( + Command::new("kubectl") + .args(["--context", context, "delete", kind, name, "--ignore-not-found"]) + .status(), + ); + } +} + +#[expect( + clippy::disallowed_methods, + reason = "synchronous poll loop in xtask; no async runtime available" +)] +/// Poll until an `AgentToolProvider`'s `status.phase` equals `expected_phase`. +pub(crate) fn wait_for_agent_tool_provider_phase( + context: &str, + name: &str, + expected_phase: &str, + timeout: Duration, +) -> Result<(), Box> { + let start = Instant::now(); + loop { + let observed = + kubectl_jsonpath(context, &format!("agenttoolproviders/{name}"), "{.status.phase}").unwrap_or_default(); + + if observed == expected_phase { + eprintln!(" [OK] {name} phase = {observed}"); + return Ok(()); + } + + if start.elapsed() >= timeout { + return Err( + format!("timeout waiting for {name} phase={expected_phase}; last observed: {observed:?}").into(), + ); + } + eprintln!(" waiting for {name} phase={expected_phase} (observed={observed:?})..."); + std::thread::sleep(POLL_INTERVAL); + } +} + +/// Read an `AgentToolProvider`'s `status.discoveredTools` as a sorted `Vec`. +pub(crate) fn read_agent_tool_provider_discovered_tools( + context: &str, + name: &str, +) -> Result, Box> { + let raw = kubectl_jsonpath( + context, + &format!("agenttoolproviders/{name}"), + "{.status.discoveredTools[*]}", + )?; + let mut tools: Vec = raw.split_whitespace().map(str::to_owned).collect(); + tools.sort(); + Ok(tools) +} + +/// Read an `AgentToolProvider`'s `status.reason`. +pub(crate) fn read_agent_tool_provider_reason(context: &str, name: &str) -> Result> { + kubectl_jsonpath(context, &format!("agenttoolproviders/{name}"), "{.status.reason}") +} + // --------------------------------------------------------------------------- // Operator binary freshness // --------------------------------------------------------------------------- From 267e8837b7384cbb91f177ca8c14c2af65c026b2 Mon Sep 17 00:00:00 2001 From: Jordi Gil Date: Wed, 26 Aug 2026 00:08:37 -0400 Subject: [PATCH 5/8] fix(swim): verify ECDSA P-256 signatures on pinned-origin state broadcasts (grid#75, partial) (#97) * feat(swim): verify ECDSA P-256 signatures on pinned state broadcasts Adds sign/verify primitives (swim::signing, backed by ring), a signature field on the StateBroadcast wire extension, and a pinned-identity trust store gate in StateBroadcastHandler::receive_item: an origin with a pinned public key must supply a valid signature or the broadcast is rejected before merge. An origin with no pinned entry passes through unchanged, so pinning can roll out one site at a time instead of requiring a synchronized cutover. Deliberately decoupled from key sourcing. The trust store takes raw public key bytes from any caller, so it works whether grid#75 lands on reusing the site's mTLS certificate or a dedicated signing key -- that question is still open and tracked there, along with the CRD/secret/rotation plumbing needed to populate the trust store from live GridSite state. Signed-off-by: Jordi Gil * fix(swim): log rejection reason for pinned-origin signature failures verify_signature_if_pinned() rejected unsigned, unverifiable, and invalid broadcasts with no operational signal at all, unlike the sibling make_room_for() eviction path a few lines below it. Once a trust store is populated, a forged or replayed broadcast for a pinned origin would be silently dropped with no way to tell "nothing is attacking us" from "something is being rejected and nobody can see it" -- exactly the gap nerdalert's grid#75 review asked to close with bounded rejection metrics that never log payload contents. Add a tracing::warn! on each of the three rejection branches, carrying only origin_site and a closed-set reason tag (missing_signature / signable_encode / signature_invalid), never the broadcast body. Signed-off-by: Jordi Gil * test(swim): cover Display messages for signing and broadcast errors This codebase has a standing convention of testing thiserror Display output directly (node::tests::state_broadcast_error_formats_correctly asserts on .to_string()). The six error variants introduced for broadcast signing -- InvalidKey, SigningFailed, VerificationError:: Invalid, MissingSignature, SignatureInvalid, SignableEncode -- had no such test, which is exactly why cargo llvm-cov's function-coverage column looked worse than the underlying decision logic actually is. SigningFailed and SignableEncode are constructed directly rather than triggered through the real code path: their real trigger (an RNG failure inside ring's sign, or a bincode encode failure with no writer I/O) has no reachable path through this crate's public API, matching how this same file already treats PoisonError::into_inner recovery as an accepted, untriggerable defensive branch. Signed-off-by: Jordi Gil * fix(swim): recover gateway/cert from pre-signature broadcasts during rolling update `BroadcastExtension` gained a third field (`signature`) in the prior commit on this branch. bincode is not self-describing, so decoding a two-field payload from a not-yet-upgraded peer against the current three-field struct fails partway through the missing field rather than falling back to `#[serde(default)]` -- the `?` inside serde's generated `visit_seq` propagates that error first. The existing bare-`String` fallback then silently misdecodes the raw bytes into garbage instead of recovering `gateway_address`/`site_cert_pem`, or erroring. Add a two-field `PreSignatureBroadcastExtension` fallback tier between the current struct decode and the bare-String fallback, so broadcasts from peers still running the pre-signature wire format decode correctly during a rolling update. Extract the extension fallback chain into `decode_extension` to keep `decode` under the line-count lint. Regression test encodes a payload using the exact two-field wire format and asserts both fields, and `signature: None`, survive decode. Signed-off-by: Jordi Gil * fix(swim): bound signature replay window and scope signatures to their GridNetwork Addresses grid#97 review feedback (nerdalert) on anti-replay and domain separation, the two findings left partially open after the first pass (key rotation and trust-state migration were already closed). Anti-replay: add StateBroadcast.signed_at_ms (epoch millis), covered by signable_bytes() so a captured signature can't be re-attached to a forged timestamp. verify_signature_if_pinned() now rejects a pinned origin's broadcast whose timestamp is missing or falls outside [now - MAX_BROADCAST_AGE_MS, now + MAX_CLOCK_SKEW_AHEAD_MS] (5 minutes / 30 seconds), grounded in OWASP ASVS 12.3.5's replay-resistant PKI-auth requirement. This bounds the replay window; it does not eliminate replay, since nothing here is persisted across a process restart -- documented explicitly rather than overclaiming full compliance. Domain separation: add StateBroadcast.grid_id (Option), also covered by signable_bytes(), so a signature valid for one GridNetwork cannot be replayed as valid for another sharing the same cluster's SwimHandle. Elevated to a required fix (not optional hardening) once issue #48 confirmed that multiple GridNetworks per cluster is a supported, documented tenant/trust-domain isolation guarantee. Both new fields reuse the bincode-is-not-self-describing fallback pattern already established for `signature`: BroadcastExtension grows to five fields, with a PreTimestampBroadcastExtension tier preserving interop with peers running the prior three-field wire format. Also corrects two ASVS citations from the first audit pass that didn't hold up against the literal ASVS 5.0.0 control text: trust-state migration's V11.1.1 (about a documented key-management policy, not revalidating merged state) and domain separation's V11.2.1 (about vetted crypto libraries, not payload scoping) are both reframed as general engineering principles rather than dedicated control IDs. New tests: 4 freshness-window boundary checks, 3 receive_item timestamp rejection cases (missing/stale/future), signed_at_ms and grid_id wire round-trips plus a legacy-decode-tier case, 2 new error Display-message tests, and grid_id-specific signable-bytes/cross-grid-signature tests. Fixed node.rs's pin_origin_rotation_update_does_not_purge_already_authenticated_state to attach signed_at_ms, since it already signed a broadcast for a pinned origin. Signed-off-by: Jordi Gil * fix(operator): thread GridNetwork identity into published state broadcasts Companion to the swim-crate domain-separation fix on this branch: publish_real_provider_state (grid_network controller) now passes its already-resolved resolve_grid_id(network) value into StateBroadcast::with_grid_id(), so this GridNetwork's signed state cannot be replayed as valid for a different GridNetwork sharing the same cluster's SwimHandle (issue #48). publish_gateway_address_broadcast intentionally keeps grid_id: None, documented inline, since it can fire before a node has joined any GridNetwork. Signed-off-by: Jordi Gil --------- Signed-off-by: Jordi Gil --- Cargo.lock | 3 + Cargo.toml | 2 + operator/src/controller/grid_network.rs | 11 +- operator/src/swim_runtime.rs | 5 + swim/Cargo.toml | 3 + swim/src/lib.rs | 7 +- swim/src/node.rs | 245 ++++- swim/src/signing.rs | 151 +++ swim/src/state_broadcast.rs | 1254 ++++++++++++++++++++++- 9 files changed, 1652 insertions(+), 29 deletions(-) create mode 100644 swim/src/signing.rs diff --git a/Cargo.lock b/Cargo.lock index f85c590..ab4d2a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2699,11 +2699,14 @@ dependencies = [ "crdt", "foca", "rand 0.9.4", + "rcgen", + "ring", "serde", "serde_json", "thiserror 2.0.18", "tokio", "tracing", + "x509-parser", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9e07e8b..732a4ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ rcgen = "0.14.8" # ring provider main.rs installs process-wide for kube/hyper-rustls — see # rustls::crypto::CryptoProvider::install_default() in main.rs. reqwest = { version = "0.13.4", default-features = false, features = ["rustls-no-provider", "json", "stream"] } +ring = "0.17" rmcp = { version = "3.1.1", default-features = false, features = [ "client", "transport-streamable-http-client-reqwest", @@ -63,6 +64,7 @@ tracing = "0.1.44" rand = { version = "0.9", features = ["small_rng"] } tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] } uuid = { version = "1.23.4", features = ["v4"] } +x509-parser = "0.18" zeroize = "1.8" [profile.release] diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 9507ebc..8826651 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -399,7 +399,7 @@ pub async fn reconcile(network: Arc, ctx: Arc) -> Resu // Publish real InferenceProvider-derived CRDT state so peers learn this site's providers. let distributed_provider_count = if let Some(swim) = ctx.swim.as_ref().filter(|handle| handle.is_running()) { - publish_real_provider_state(swim, name, &providers, &raw_metrics); + publish_real_provider_state(swim, name, &grid_id, &providers, &raw_metrics); count_remote_provider_records(swim, name) } else { 0 @@ -1569,9 +1569,15 @@ fn provider_revision(provider: &InferenceProvider) -> u64 { /// remote sites can learn which providers exist and avoid routing to unhealthy /// ones. The routing overlay layer already filters `Unavailable` providers /// from local routing decisions. +/// +/// `grid_id` (the caller's already-[`resolve_grid_id`]d value) is attached to +/// the broadcast so a signature over this `GridNetwork`'s state cannot be +/// replayed as valid for a different `GridNetwork` sharing the same +/// cluster's `SwimHandle` — see [`swim::StateBroadcast::grid_id`]. fn publish_real_provider_state( swim: &SwimHandle, network_name: &str, + grid_id: &str, providers: &[InferenceProvider], raw_metrics: &HashMap, ) { @@ -1609,7 +1615,8 @@ fn publish_real_provider_state( // Use the highest provider revision as this origin's broadcast revision. // Duplicate unchanged broadcasts are idempotent; newer Kubernetes writes // advance resourceVersion and therefore advance the broadcast revision. - let bc = StateBroadcast::new(site_name.to_owned(), max_revision, snap, gateway_address); + let bc = StateBroadcast::new(site_name.to_owned(), max_revision, snap, gateway_address) + .with_grid_id(Some(grid_id.to_owned())); if let Err(e) = swim.publish_state_broadcast(bc) { tracing::debug!(error = %e, "CRDT broadcast channel unavailable — runtime not yet receiving"); } diff --git a/operator/src/swim_runtime.rs b/operator/src/swim_runtime.rs index d53404c..505de6c 100644 --- a/operator/src/swim_runtime.rs +++ b/operator/src/swim_runtime.rs @@ -1215,6 +1215,11 @@ fn canonical_state_payload(broadcast: &swim::StateBroadcast) -> Result, /// that joins before it has any local `GridNetwork` can still advertise its /// data-plane gateway address to peers. The broadcast carries an empty CRDT /// snapshot and only updates the peer gateway-address map. +/// +/// Deliberately leaves [`swim::StateBroadcast::grid_id`] as `None`: this +/// broadcast can fire before the node has joined any `GridNetwork`, so no +/// `grid_id` is available yet here. `publish_real_provider_state` in the +/// `grid_network` controller attaches `grid_id` once a `GridNetwork` exists. fn publish_gateway_address_broadcast(node: &mut SwimNode, site_name: &str, revision: u64, gateway_address: &str) { let broadcast = swim::StateBroadcast::new( site_name.to_owned(), diff --git a/swim/Cargo.toml b/swim/Cargo.toml index 53379c9..8d1b9a4 100644 --- a/swim/Cargo.toml +++ b/swim/Cargo.toml @@ -14,13 +14,16 @@ bytes = { workspace = true } crdt = { path = "../crdt" } foca = { workspace = true } rand = { workspace = true } +ring = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["sync"] } tracing = { workspace = true } [dev-dependencies] +rcgen = { workspace = true } serde_json = { workspace = true } +x509-parser = { workspace = true } [lints] workspace = true diff --git a/swim/src/lib.rs b/swim/src/lib.rs index 6f54b30..81624a6 100644 --- a/swim/src/lib.rs +++ b/swim/src/lib.rs @@ -26,6 +26,8 @@ pub mod identity; pub mod node; /// Foca runtime adapter with accumulated output. pub mod runtime; +/// ECDSA P-256 sign/verify primitives for SWIM state broadcasts. +pub mod signing; /// State snapshot payloads for SWIM custom broadcasts. pub mod state_broadcast; @@ -33,7 +35,8 @@ pub use event::MemberEvent; pub use identity::NodeId; pub use node::SwimNode; pub use runtime::{AccumulatedOutput, GridRuntime}; +pub use signing::{SigningError, VerificationError}; pub use state_broadcast::{ - STATE_BROADCAST_VERSION, STATE_BROADCAST_VERSION_V1, StateBroadcast, StateBroadcastError, StateBroadcastHandler, - StateBroadcastKey, + MAX_PINNED_KEYS_PER_ORIGIN, STATE_BROADCAST_VERSION, STATE_BROADCAST_VERSION_V1, StateBroadcast, + StateBroadcastError, StateBroadcastHandler, StateBroadcastKey, TooManyPinnedKeys, TrustStore, }; diff --git a/swim/src/node.rs b/swim/src/node.rs index e1f6c4c..0fa3198 100644 --- a/swim/src/node.rs +++ b/swim/src/node.rs @@ -16,7 +16,10 @@ use tokio::sync::watch; use crate::{ AccumulatedOutput, GridRuntime, NodeId, runtime::TimerEvent, - state_broadcast::{DEFAULT_MAX_RETAINED_ORIGINS, OriginStateHandle, StateBroadcast, StateBroadcastHandler}, + state_broadcast::{ + DEFAULT_MAX_RETAINED_ORIGINS, MAX_PINNED_KEYS_PER_ORIGIN, OriginStateHandle, StateBroadcast, + StateBroadcastHandler, TooManyPinnedKeys, TrustStore, + }, }; // --------------------------------------------------------------------------- @@ -66,6 +69,10 @@ pub struct SwimNode { /// Immediate control path for coordinated per-origin state eviction. origin_state: OriginStateHandle, + + /// Sender for the pinned-identity trust store read by the broadcast + /// handler inside `foca`. See [`SwimNode::pin_origin`]. + trust_store_tx: watch::Sender, } impl SwimNode { @@ -108,6 +115,7 @@ impl SwimNode { let state_rx = handler.subscribe(); let gateway_addrs_rx = handler.subscribe_gateway_addrs(); let cert_pems_rx = handler.subscribe_cert_pems(); + let trust_store_tx = handler.trust_store_sender(); Self { foca: foca::Foca::with_custom_broadcast(identity, grid_config(), rng, codec, handler), @@ -116,6 +124,7 @@ impl SwimNode { gateway_addrs_rx, cert_pems_rx, origin_state, + trust_store_tx, } } @@ -124,6 +133,51 @@ impl SwimNode { self.origin_state.remove_origin(origin); } + /// Pin `origin` to a bounded set of accepted raw ECDSA P-256 public keys. + /// + /// If `origin` has no existing pin (or its existing pin is empty), + /// immediately purges any state already merged from that origin via + /// [`evict_origin`](Self::evict_origin) before installing the new pin — + /// so state accepted from `origin` while it was unauthenticated cannot + /// silently remain trusted once signature enforcement begins for it. + /// Updating an *existing* non-empty pin (e.g. adding a next key during + /// rotation, or dropping a retired one) does not purge state, since + /// that update never crosses the unauthenticated-to-authenticated + /// boundary. + /// + /// # Errors + /// + /// Returns [`TooManyPinnedKeys`] if `keys` has more than + /// [`MAX_PINNED_KEYS_PER_ORIGIN`] entries, without changing the trust + /// store. + pub fn pin_origin(&self, origin: String, keys: Vec>) -> Result<(), TooManyPinnedKeys> { + if keys.len() > MAX_PINNED_KEYS_PER_ORIGIN { + return Err(TooManyPinnedKeys { + origin, + supplied: keys.len(), + }); + } + let was_unpinned = self.trust_store_tx.borrow().get(&origin).is_none_or(Vec::is_empty); + if was_unpinned { + self.evict_origin(&origin); + } + self.trust_store_tx.send_modify(|store| { + store.insert(origin, keys); + }); + Ok(()) + } + + /// Remove `origin`'s pin, returning it to unenforced (pass-through) status. + /// + /// Does not purge `origin`'s currently merged state — unpinning is a + /// deliberate relaxation, not a security event, and the state was + /// already accepted under whatever enforcement applied when it arrived. + pub fn unpin_origin(&self, origin: &str) { + self.trust_store_tx.send_modify(|store| { + store.remove(origin); + }); + } + /// Feed an incoming UDP packet to foca. /// /// Returns accumulated side effects: outbound messages, scheduled timers, @@ -375,6 +429,195 @@ mod tests { .unwrap_or_else(|_| std::process::abort()); } + // ----------------------------------------------------------------------- + // Trust-store pinning + // ----------------------------------------------------------------------- + + /// Generate an ECDSA P-256 signing key plus the raw SPKI EC point a + /// verifier needs, independent of *how* a real deployment would source + /// or pin this key material (grid#75, still open). + fn generate_signing_key_and_pubkey() -> (Vec, Vec) { + let key_pair = rcgen::KeyPair::generate().unwrap_or_else(|_| std::process::abort()); + let pkcs8_der = key_pair.serialize_der(); + let params = rcgen::CertificateParams::new(vec!["spike.grid.internal".to_owned()]) + .unwrap_or_else(|_| std::process::abort()); + let cert = params.self_signed(&key_pair).unwrap_or_else(|_| std::process::abort()); + let (_, parsed) = x509_parser::parse_x509_certificate(cert.der()).unwrap_or_else(|_| std::process::abort()); + let raw_pubkey = parsed.public_key().subject_public_key.as_ref().to_vec(); + (pkcs8_der, raw_pubkey) + } + + /// Return the current wall-clock time in milliseconds since the Unix + /// epoch, for constructing test broadcasts with a fresh `signed_at_ms`. + fn now_ms() -> u64 { + u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_else(|_| std::process::abort()) + .as_millis(), + ) + .unwrap_or_else(|_| std::process::abort()) + } + + #[test] + fn pin_origin_rejects_more_than_the_max_pinned_keys() { + let (node, _) = make_node("site-b", 19_230); + + let result = node.pin_origin("site-p".to_owned(), vec![vec![1], vec![2], vec![3]]); + + assert!( + matches!(&result, Err(TooManyPinnedKeys { origin, supplied }) if origin == "site-p" && *supplied == 3), + "pinning 3 keys (over MAX_PINNED_KEYS_PER_ORIGIN=2) must be rejected, got {result:?}" + ); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "establishes membership, publishes unpinned state, then pins and re-checks purge in one proof" + )] + fn pin_origin_purges_previously_unauthenticated_state_from_that_origin() { + let id_a = local_id("site-a", 19_231); + let id_b = local_id("site-b", 19_232); + let (mut node_a, _) = make_node("site-a", 19_231); + let (mut node_b, _) = make_node("site-b", 19_232); + establish_membership(&mut node_a, &mut node_b, &id_a, &id_b); + + // A publishes unsigned state; B has no pin for A yet, so it merges. + node_a + .publish_state_broadcast(&StateBroadcast::new( + "site-a".to_owned(), + 1, + provider_snap("site-a", 0.4), + None, + )) + .unwrap_or_else(|_| std::process::abort()); + for msg in &node_a.gossip().messages { + if msg.addr == id_b.socket_addr() { + drop(node_b.handle_data(&msg.data)); + } + } + assert!( + node_b + .state_snapshot() + .provider("net", "site-a", "provider-1") + .is_some(), + "B must have accepted A's unsigned, unpinned state" + ); + + let (_key, pubkey) = generate_signing_key_and_pubkey(); + node_b + .pin_origin("site-a".to_owned(), vec![pubkey]) + .unwrap_or_else(|_| std::process::abort()); + + assert!( + node_b + .state_snapshot() + .provider("net", "site-a", "provider-1") + .is_none(), + "pinning a previously-unpinned origin for the first time must purge its unauthenticated state" + ); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "establishes membership, pins, signs+publishes, and re-pins with a rotated key set in one proof" + )] + fn pin_origin_rotation_update_does_not_purge_already_authenticated_state() { + let id_a = local_id("site-a", 19_233); + let id_b = local_id("site-b", 19_234); + let (mut node_a, _) = make_node("site-a", 19_233); + let (mut node_b, _) = make_node("site-b", 19_234); + establish_membership(&mut node_a, &mut node_b, &id_a, &id_b); + + let (signing_key, current_pubkey) = generate_signing_key_and_pubkey(); + node_b + .pin_origin("site-a".to_owned(), vec![current_pubkey.clone()]) + .unwrap_or_else(|_| std::process::abort()); + + let unsigned = StateBroadcast::new("site-a".to_owned(), 1, provider_snap("site-a", 0.5), None) + .with_signed_at(Some(now_ms())); + let signature = crate::signing::sign_ecdsa_p256( + &signing_key, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + node_a + .publish_state_broadcast(&unsigned.with_signature(Some(signature))) + .unwrap_or_else(|_| std::process::abort()); + for msg in &node_a.gossip().messages { + if msg.addr == id_b.socket_addr() { + drop(node_b.handle_data(&msg.data)); + } + } + assert!( + node_b + .state_snapshot() + .provider("net", "site-a", "provider-1") + .is_some(), + "B must have accepted A's correctly signed, pinned state" + ); + + // Rotation: add a next key alongside the current one. This is an + // update to an existing non-empty pin, not a first-time pin. + let (_next_key, next_pubkey) = generate_signing_key_and_pubkey(); + node_b + .pin_origin("site-a".to_owned(), vec![current_pubkey, next_pubkey]) + .unwrap_or_else(|_| std::process::abort()); + + assert!( + node_b + .state_snapshot() + .provider("net", "site-a", "provider-1") + .is_some(), + "rotating an already-pinned origin's key set must not purge its already-authenticated state" + ); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "establishes membership, publishes state, then unpins and re-checks retention in one proof" + )] + fn unpin_origin_does_not_purge_existing_state() { + let id_a = local_id("site-a", 19_235); + let id_b = local_id("site-b", 19_236); + let (mut node_a, _) = make_node("site-a", 19_235); + let (mut node_b, _) = make_node("site-b", 19_236); + establish_membership(&mut node_a, &mut node_b, &id_a, &id_b); + + node_a + .publish_state_broadcast(&StateBroadcast::new( + "site-a".to_owned(), + 1, + provider_snap("site-a", 0.6), + None, + )) + .unwrap_or_else(|_| std::process::abort()); + for msg in &node_a.gossip().messages { + if msg.addr == id_b.socket_addr() { + drop(node_b.handle_data(&msg.data)); + } + } + assert!( + node_b + .state_snapshot() + .provider("net", "site-a", "provider-1") + .is_some() + ); + + node_b.unpin_origin("site-a"); + + assert!( + node_b + .state_snapshot() + .provider("net", "site-a", "provider-1") + .is_some(), + "unpinning an origin must not purge its currently merged state" + ); + } + // ----------------------------------------------------------------------- // Real foca CRDT broadcast propagation // ----------------------------------------------------------------------- diff --git a/swim/src/signing.rs b/swim/src/signing.rs new file mode 100644 index 0000000..6db611c --- /dev/null +++ b/swim/src/signing.rs @@ -0,0 +1,151 @@ +//! ECDSA P-256 sign/verify primitives for SWIM state broadcasts. +//! +//! Deliberately decoupled from *where* key material comes from: callers +//! supply raw PKCS8 DER (for signing) or a raw uncompressed EC point (for +//! verification) and this module only performs the cryptographic operation. +//! Sourcing and pinning key material against a [`GridSite`] identity is an +//! operator-level concern tracked separately. +//! +//! [`GridSite`]: https://github.com/praxis-proxy/grid/issues/75 + +use ring::{ + rand::SystemRandom, + signature::{ECDSA_P256_SHA256_ASN1, ECDSA_P256_SHA256_ASN1_SIGNING, EcdsaKeyPair, UnparsedPublicKey}, +}; + +/// Errors produced while signing a payload. +#[derive(Debug, thiserror::Error)] +pub enum SigningError { + /// The supplied PKCS8 DER key material was rejected by the signing backend. + #[error("invalid PKCS8 signing key")] + InvalidKey, + + /// Signing failed for a reason the backend does not disclose in detail. + #[error("signing operation failed")] + SigningFailed, +} + +/// Errors produced while verifying a payload's signature. +#[derive(Debug, thiserror::Error)] +pub enum VerificationError { + /// The signature does not match the payload under the supplied public key. + #[error("signature verification failed")] + Invalid, +} + +/// Sign `payload` with an ECDSA P-256 key supplied as PKCS8 DER. +/// +/// Returns the signature in ASN.1 DER form (~70-72 bytes for P-256). +/// +/// # Errors +/// +/// Returns [`SigningError::InvalidKey`] if `pkcs8_der` is not a valid ECDSA +/// P-256 PKCS8 key, or [`SigningError::SigningFailed`] if the underlying RNG +/// or signing operation fails. +pub fn sign_ecdsa_p256(pkcs8_der: &[u8], payload: &[u8]) -> Result, SigningError> { + let rng = SystemRandom::new(); + let key_pair = EcdsaKeyPair::from_pkcs8(&ECDSA_P256_SHA256_ASN1_SIGNING, pkcs8_der, &rng) + .map_err(|_key_rejected| SigningError::InvalidKey)?; + let signature = key_pair + .sign(&rng, payload) + .map_err(|_signing_failed| SigningError::SigningFailed)?; + Ok(signature.as_ref().to_vec()) +} + +/// Verify `signature` over `payload` under an ECDSA P-256 public key +/// supplied as a raw uncompressed EC point (the `subjectPublicKey` bit +/// string contents of an X.509 SPKI structure, sec1 uncompressed form). +/// +/// # Errors +/// +/// Returns [`VerificationError::Invalid`] when the signature does not +/// verify, including when `raw_pubkey` is malformed. +pub fn verify_ecdsa_p256(raw_pubkey: &[u8], payload: &[u8], signature: &[u8]) -> Result<(), VerificationError> { + let verifier = UnparsedPublicKey::new(&ECDSA_P256_SHA256_ASN1, raw_pubkey); + verifier + .verify(payload, signature) + .map_err(|_unspecified| VerificationError::Invalid) +} + +#[cfg(test)] +mod tests { + use rcgen::KeyPair; + + use super::*; + + // Test Utilities + + /// Generate an ECDSA P-256 keypair plus the raw SPKI EC point ring + /// expects for verification, mirroring how a `GridSite`'s certificate + /// key material is produced by `certs::generate`. + fn generate_key_and_raw_pubkey() -> (Vec, Vec) { + let key_pair = KeyPair::generate().unwrap_or_else(|_| std::process::abort()); + let pkcs8_der = key_pair.serialize_der(); + let params = rcgen::CertificateParams::new(vec!["spike.grid.internal".to_owned()]) + .unwrap_or_else(|_| std::process::abort()); + let cert = params.self_signed(&key_pair).unwrap_or_else(|_| std::process::abort()); + let (_, parsed) = x509_parser::parse_x509_certificate(cert.der()).unwrap_or_else(|_| std::process::abort()); + let raw_pubkey = parsed.public_key().subject_public_key.as_ref().to_vec(); + (pkcs8_der, raw_pubkey) + } + + #[test] + fn sign_then_verify_round_trips_for_a_matching_key_and_payload() { + let (pkcs8_der, raw_pubkey) = generate_key_and_raw_pubkey(); + let payload = b"origin-site-a|revision=1"; + + let signature = sign_ecdsa_p256(&pkcs8_der, payload).unwrap_or_else(|_| std::process::abort()); + + verify_ecdsa_p256(&raw_pubkey, payload, &signature).unwrap_or_else(|_| std::process::abort()); + } + + #[test] + fn verify_rejects_a_signature_over_a_different_payload() { + let (pkcs8_der, raw_pubkey) = generate_key_and_raw_pubkey(); + let signature = sign_ecdsa_p256(&pkcs8_der, b"revision=1").unwrap_or_else(|_| std::process::abort()); + + let result = verify_ecdsa_p256(&raw_pubkey, b"revision=2", &signature); + + assert!(matches!(result, Err(VerificationError::Invalid))); + } + + #[test] + fn verify_rejects_a_signature_from_an_unrelated_key() { + let (pkcs8_der, _) = generate_key_and_raw_pubkey(); + let (_, other_raw_pubkey) = generate_key_and_raw_pubkey(); + let payload = b"origin-site-a|revision=1"; + let signature = sign_ecdsa_p256(&pkcs8_der, payload).unwrap_or_else(|_| std::process::abort()); + + let result = verify_ecdsa_p256(&other_raw_pubkey, payload, &signature); + + assert!(matches!(result, Err(VerificationError::Invalid))); + } + + #[test] + fn sign_rejects_malformed_pkcs8_key_material() { + let result = sign_ecdsa_p256(b"not-a-real-key", b"payload"); + + assert!(matches!(result, Err(SigningError::InvalidKey))); + } + + #[test] + fn invalid_key_error_message_is_descriptive() { + assert!(SigningError::InvalidKey.to_string().contains("PKCS8")); + } + + #[test] + fn signing_failed_error_message_is_descriptive() { + // SigningFailed only arises from an RNG failure inside ring's sign + // call, which cannot be triggered deterministically through this + // module's public API. Constructed directly to verify the message + // wording, matching this codebase's `..._error_formats_correctly` + // convention for defensive error variants (see e.g. + // `node::tests::state_broadcast_error_formats_correctly`). + assert!(SigningError::SigningFailed.to_string().contains("signing")); + } + + #[test] + fn verification_invalid_error_message_is_descriptive() { + assert!(VerificationError::Invalid.to_string().contains("verification")); + } +} diff --git a/swim/src/state_broadcast.rs b/swim/src/state_broadcast.rs index 4ec27e2..df712d5 100644 --- a/swim/src/state_broadcast.rs +++ b/swim/src/state_broadcast.rs @@ -8,6 +8,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, sync::{Arc, Mutex, MutexGuard, PoisonError}, + time::{Duration, SystemTime, UNIX_EPOCH}, }; use crdt::GridStateSnapshot; @@ -34,6 +35,40 @@ pub const STATE_BROADCAST_VERSION: u16 = STATE_BROADCAST_VERSION_V1; /// Default hard bound for distinct origins retained by one broadcast handler. pub const DEFAULT_MAX_RETAINED_ORIGINS: usize = 1_024; +/// Domain-separation prefix mixed into every state-broadcast signature. +/// +/// Binds a signature to this exact protocol, message type, and wire-format +/// version, so it can never be replayed as valid input to a different +/// signing context even if the same ECDSA key were ever reused elsewhere. +/// +/// Scopes a signature to *this protocol*, not to a particular +/// `GridNetwork`: that narrower scoping is [`StateBroadcast::grid_id`]'s +/// job, since a node can publish a broadcast before joining any +/// `GridNetwork` and so cannot always supply one. Anything broader than a +/// single cluster's `GridNetwork`s — cross-deployment or per-peer mesh +/// identity — remains out of scope for this constant and for `grid_id` +/// alike. +const SIGNATURE_DOMAIN: &[u8] = b"praxis-grid/swim/state-broadcast/v1"; + +/// Maximum age, in milliseconds, of a pinned origin's signed broadcast +/// timestamp before it is rejected as stale. +/// +/// Set to several orders of magnitude above the default 5-second SWIM probe +/// interval (`default_probe_interval` in `operator/src/crd/grid_network.rs`), +/// comfortably covering ordinary gossip fan-out delay while still bounding a +/// captured signature's replay window to minutes rather than leaving it +/// unbounded. See [`StateBroadcast::signed_at_ms`] for what this window does +/// and does not defend against. +pub const MAX_BROADCAST_AGE_MS: u64 = 5 * 60 * 1_000; + +/// Maximum amount, in milliseconds, a pinned origin's signed broadcast +/// timestamp may be ahead of this node's own clock before it is rejected. +/// +/// Tolerates ordinary clock drift between SWIM peers without opening a +/// window for a broadcast timestamped far in the future to keep outrunning +/// [`MAX_BROADCAST_AGE_MS`] on every receiving peer. +pub const MAX_CLOCK_SKEW_AHEAD_MS: u64 = 30_000; + /// Broadcast envelope carrying one CRDT grid-state snapshot. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct StateBroadcast { @@ -63,6 +98,95 @@ pub struct StateBroadcast { /// `None` when the originating operator has no TLS certificate configured. #[serde(skip_serializing_if = "Option::is_none")] pub site_cert_pem: Option, + + /// ECDSA P-256 signature (ASN.1 DER) over [`signable_bytes`](Self::signable_bytes). + /// + /// `None` when the originating site has no signing key configured, or + /// during the rollout window before every peer signs broadcasts. A + /// receiver only requires this field once it holds a pinned identity for + /// `origin_site`; see [`StateBroadcastHandler::receive_item`]. + /// + /// [`StateBroadcastHandler::receive_item`]: foca::BroadcastHandler::receive_item + #[serde(skip_serializing_if = "Option::is_none")] + pub signature: Option>, + + /// Wall-clock time this broadcast was signed, in milliseconds since the + /// Unix epoch. + /// + /// `None` under the same conditions as [`signature`](Self::signature) + /// — no signing key configured, or the pre-rollout window. Included in + /// [`signable_bytes`](Self::signable_bytes) so a captured signature + /// cannot be re-attached to a forged, more-recent timestamp. A receiver + /// holding a pinned identity for `origin_site` rejects a signature whose + /// timestamp is more than [`MAX_BROADCAST_AGE_MS`] in the past, or more + /// than [`MAX_CLOCK_SKEW_AHEAD_MS`] in the future, bounding how long a + /// captured, validly signed broadcast can be replayed as current. + /// + /// Bounds only the *replay window*, not full replay elimination: this + /// value is never persisted, so a process restart resets every + /// receiver's notion of "now" relative to nothing durable. Full replay + /// elimination needs a persisted, monotonic revision floor surviving + /// restarts, which this in-memory crate does not provide. + #[serde(skip_serializing_if = "Option::is_none")] + pub signed_at_ms: Option, + + /// Identifier of the `GridNetwork` this broadcast's state belongs to. + /// + /// Included in [`signable_bytes`](Self::signable_bytes) so a signature + /// is scoped to one `GridNetwork`: issue [#48] documents that a single + /// cluster can run multiple `GridNetwork`s as separate tenants, + /// environments, or trust domains with independent provider + /// inventories, yet one `SwimHandle` is shared across every + /// `GridNetwork` reconcile on that cluster. Without this field, a + /// signature valid for one `GridNetwork` would also verify, bit for + /// bit, as a broadcast claiming to belong to another `GridNetwork` on + /// the same cluster — a cross-tenant replay that would defeat #48's + /// stated isolation guarantee. + /// + /// `None` for broadcasts published before a node has joined any + /// `GridNetwork` (e.g. a bare gateway-address advertisement — see + /// `publish_gateway_address_broadcast` in the operator crate), and for + /// broadcasts from operators old enough to predate this field. + /// + /// [#48]: https://github.com/praxis-proxy/grid/issues/48 + #[serde(skip_serializing_if = "Option::is_none")] + pub grid_id: Option, +} + +/// Maximum number of raw public keys retained per pinned origin. +/// +/// Mirrors `GridSiteTrustPolicy.canonical_fingerprints`'s dual-pin bound for +/// mTLS certificate rotation (`operator/src/crd/grid_site.rs`): one slot for +/// the current signing key and one for a next key during a bounded rotation +/// overlap window. +pub const MAX_PINNED_KEYS_PER_ORIGIN: usize = 2; + +/// Bounded set of accepted raw ECDSA P-256 public keys, keyed by origin site +/// name. +/// +/// Each value holds up to [`MAX_PINNED_KEYS_PER_ORIGIN`] raw uncompressed EC +/// points. A broadcast's signature verifies if it is valid under **any** key +/// in its origin's pinned set — this is what makes bounded key rotation +/// possible without an instantaneous flag-day cutover: a site publishes a +/// next key alongside its current one, callers add the next key to the pin +/// set, and once every peer has observed the rotation the old key is +/// dropped. Deliberately opaque to *how* a pinned identity was established — +/// that is an operator-level concern (see [`crate::signing`]). An origin +/// with no entry, or an empty entry, is not yet enforced against a +/// signature. Prefer [`crate::node::SwimNode::pin_origin`] over mutating +/// this map directly through [`StateBroadcastHandler::trust_store_sender`]: +/// `pin_origin` also purges any state accepted from an origin before it had +/// a pin, which a raw `watch::Sender::send`/`send_modify` call does not. +pub type TrustStore = BTreeMap>>; + +/// A caller supplied more than [`MAX_PINNED_KEYS_PER_ORIGIN`] keys for one origin. +#[derive(Debug, thiserror::Error)] +#[error("origin {origin} was pinned with {supplied} keys, exceeding the max of {MAX_PINNED_KEYS_PER_ORIGIN}")] +pub struct TooManyPinnedKeys { + /// Origin site the caller attempted to pin. + pub origin: String, + /// Number of keys the caller supplied. + pub supplied: usize, } /// Base wire-format struct. @@ -92,6 +216,66 @@ struct BroadcastExtension { gateway_address: Option, /// Optional public site certificate PEM — never a private key. site_cert_pem: Option, + /// Optional ECDSA P-256 signature over the base payload plus the other + /// extension fields. Absent on older peers and pre-rollout broadcasts. + #[serde(default)] + signature: Option>, + /// Optional wall-clock signing timestamp, milliseconds since the Unix + /// epoch. Absent on older peers and pre-rollout broadcasts. + #[serde(default)] + signed_at_ms: Option, + /// Optional owning `GridNetwork` identifier. Absent on older peers, + /// pre-rollout broadcasts, and broadcasts published before a node has + /// joined any `GridNetwork`. + #[serde(default)] + grid_id: Option, +} + +/// Extension format used by peers that predate both the `signed_at_ms` and +/// `grid_id` fields. +/// +/// bincode is not self-describing, so decoding a three-field payload as the +/// current five-field [`BroadcastExtension`] fails partway through the +/// fourth field rather than falling back to `#[serde(default)]` — `decode` +/// tries this shape before falling further back through every prior wire +/// format, so a rolling update does not silently drop or misdecode +/// `gateway_address`/`site_cert_pem`/`signature` from not-yet-upgraded peers. +#[derive(Serialize, Deserialize)] +struct PreTimestampBroadcastExtension { + /// Optional data-plane gateway address. + gateway_address: Option, + /// Optional public site certificate PEM — never a private key. + site_cert_pem: Option, + /// Optional ECDSA P-256 signature over the base payload plus the other + /// extension fields. Absent on older peers and pre-rollout broadcasts. + #[serde(default)] + signature: Option>, +} + +/// Decoded extension fields: +/// `(gateway_address, site_cert_pem, signature, signed_at_ms, grid_id)`. +type DecodedExtension = ( + Option, + Option, + Option>, + Option, + Option, +); + +/// Extension format used by peers that predate the `signature` field. +/// +/// bincode is not self-describing, so decoding a two-field payload as +/// [`PreTimestampBroadcastExtension`] fails partway through the third field +/// rather than falling back to `#[serde(default)]` — `decode` tries that +/// shape before falling further back to this one, then to the original +/// bare-`String` format, so a rolling update does not silently drop or +/// misdecode `gateway_address`/`site_cert_pem` from not-yet-upgraded peers. +#[derive(Serialize, Deserialize)] +struct PreSignatureBroadcastExtension { + /// Optional data-plane gateway address. + gateway_address: Option, + /// Optional public site certificate PEM — never a private key. + site_cert_pem: Option, } impl StateBroadcast { @@ -114,6 +298,9 @@ impl StateBroadcast { snapshot, gateway_address, site_cert_pem: None, + signature: None, + signed_at_ms: None, + grid_id: None, } } @@ -126,6 +313,62 @@ impl StateBroadcast { self } + /// Attach a signature computed over [`signable_bytes`](Self::signable_bytes). + /// + /// Callers are responsible for computing `signature` (typically via + /// [`crate::signing::sign_ecdsa_p256`] over `self.signable_bytes()`) + /// before attaching it; this method performs no verification. + #[must_use] + pub fn with_signature(mut self, signature: Option>) -> Self { + self.signature = signature; + self + } + + /// Attach the wall-clock signing timestamp (milliseconds since the Unix + /// epoch). + /// + /// Set this **before** computing [`signable_bytes`](Self::signable_bytes) + /// so the timestamp itself is covered by the signature — see + /// [`signed_at_ms`](Self::signed_at_ms)'s doc comment for why an + /// unsigned timestamp would defeat the freshness check it exists to + /// support. + #[must_use] + pub fn with_signed_at(mut self, signed_at_ms: Option) -> Self { + self.signed_at_ms = signed_at_ms; + self + } + + /// Attach the owning `GridNetwork` identifier. + /// + /// Set this **before** computing [`signable_bytes`](Self::signable_bytes) + /// so the identifier is covered by the signature — see + /// [`grid_id`](Self::grid_id)'s doc comment for the cross-`GridNetwork` + /// replay this scoping closes. + #[must_use] + pub fn with_grid_id(mut self, grid_id: Option) -> Self { + self.grid_id = grid_id; + self + } + + /// Return the canonical bytes this broadcast's signature covers. + /// + /// Prefixed with a fixed domain-separation tag followed by + /// [`encode`](Self::encode) of this broadcast with `signature` cleared, + /// so a signature can never cover itself and can never be replayed as + /// valid input to a different signing context even if the same key were + /// ever reused elsewhere. + /// + /// # Errors + /// + /// Returns a bincode encode error if the snapshot cannot be serialized. + pub fn signable_bytes(&self) -> Result, bincode::error::EncodeError> { + let mut unsigned = self.clone(); + unsigned.signature = None; + let mut signable = SIGNATURE_DOMAIN.to_vec(); + signable.extend(unsigned.encode()?); + Ok(signable) + } + /// Return this broadcast's invalidation key. #[must_use] pub fn key(&self) -> StateBroadcastKey { @@ -211,10 +454,18 @@ impl StateBroadcast { snapshot: self.snapshot.clone(), }; let mut bytes = bincode::serde::encode_to_vec(&v1, bincode::config::standard())?; - if self.gateway_address.is_some() || self.site_cert_pem.is_some() { + if self.gateway_address.is_some() + || self.site_cert_pem.is_some() + || self.signature.is_some() + || self.signed_at_ms.is_some() + || self.grid_id.is_some() + { let ext = BroadcastExtension { gateway_address: self.gateway_address.clone(), site_cert_pem: self.site_cert_pem.clone(), + signature: self.signature.clone(), + signed_at_ms: self.signed_at_ms, + grid_id: self.grid_id.clone(), }; let ext_bytes = bincode::serde::encode_to_vec(&ext, bincode::config::standard())?; bytes.extend_from_slice(&ext_bytes); @@ -225,9 +476,15 @@ impl StateBroadcast { /// Decode this broadcast from bincode bytes. /// /// Decodes the base v1 payload, then tries to decode any trailing bytes as - /// a `BroadcastExtension` struct. Falls back to the previous bare-`String` - /// format for `gateway_address` when the struct decode fails, ensuring - /// interoperability with older peers that use the first extension format. + /// a `BroadcastExtension` struct. Because bincode is not self-describing, + /// a five-field extension decode does not simply come back `Ok` with + /// `grid_id: None` when reading bytes from an older, three-field + /// peer — it fails partway through the missing fields. Falls back in + /// turn to the three-field pre-timestamp extension format, then the + /// two-field pre-signature extension format, then to the + /// original bare-`String` format for `gateway_address`, ensuring + /// interoperability with peers running any prior wire format during a + /// rolling update. /// /// Payloads without any extension decode with `gateway_address = None` and /// `site_cert_pem = None`. @@ -241,21 +498,7 @@ impl StateBroadcast { bincode::serde::decode_from_slice(bytes, bincode::config::standard())?; let remaining = bytes.get(consumed..).unwrap_or(&[]); - let (gateway_address, site_cert_pem) = if remaining.is_empty() { - (None, None) - } else { - // Try the current extension struct format. - match bincode::serde::decode_from_slice::(remaining, bincode::config::standard()) { - Ok((ext, _)) => (ext.gateway_address, ext.site_cert_pem), - Err(_) => { - // Compatibility fallback: bare String encoding for gateway_address only. - match bincode::serde::decode_from_slice::(remaining, bincode::config::standard()) { - Ok((gw, _)) => (Some(gw), None), - Err(_) => (None, None), - } - }, - } - }; + let (gateway_address, site_cert_pem, signature, signed_at_ms, grid_id) = Self::decode_extension(remaining); Ok(Self { version: v1.version, @@ -264,8 +507,48 @@ impl StateBroadcast { snapshot: v1.snapshot, gateway_address, site_cert_pem, + signature, + signed_at_ms, + grid_id, }) } + + /// Decode the trailing extension bytes, falling back across every prior + /// wire format in turn. See [`decode`](Self::decode) for why a naive + /// single-shot struct decode is not enough during a rolling update. + fn decode_extension(remaining: &[u8]) -> DecodedExtension { + if remaining.is_empty() { + return (None, None, None, None, None); + } + if let Ok((ext, _)) = + bincode::serde::decode_from_slice::(remaining, bincode::config::standard()) + { + return ( + ext.gateway_address, + ext.site_cert_pem, + ext.signature, + ext.signed_at_ms, + ext.grid_id, + ); + } + if let Ok((ext, _)) = bincode::serde::decode_from_slice::( + remaining, + bincode::config::standard(), + ) { + return (ext.gateway_address, ext.site_cert_pem, ext.signature, None, None); + } + if let Ok((ext, _)) = bincode::serde::decode_from_slice::( + remaining, + bincode::config::standard(), + ) { + return (ext.gateway_address, ext.site_cert_pem, None, None, None); + } + // Compatibility fallback: bare String encoding for gateway_address only. + match bincode::serde::decode_from_slice::(remaining, bincode::config::standard()) { + Ok((gw, _)) => (Some(gw), None, None, None, None), + Err(_) => (None, None, None, None, None), + } + } } /// Key used to replace stale queued broadcasts in foca. @@ -335,6 +618,45 @@ pub enum StateBroadcastError { /// Actual version. actual: u16, }, + + /// The origin has a pinned identity but the broadcast carries no signature. + #[error("state broadcast from pinned origin {origin_site} carries no signature")] + MissingSignature { + /// Site that originated the broadcast. + origin_site: String, + }, + + /// The broadcast's signature does not verify against the pinned identity. + #[error("state broadcast from pinned origin {origin_site} failed signature verification")] + SignatureInvalid { + /// Site that originated the broadcast. + origin_site: String, + }, + + /// The broadcast's own bytes could not be re-encoded to check its signature. + #[error("state broadcast from origin {origin_site} could not be re-encoded for signature verification: {source}")] + SignableEncode { + /// Site that originated the broadcast. + origin_site: String, + /// Underlying encode error. + source: bincode::error::EncodeError, + }, + + /// The origin has a pinned identity but the broadcast carries no signing timestamp. + #[error("state broadcast from pinned origin {origin_site} carries no signing timestamp")] + MissingTimestamp { + /// Site that originated the broadcast. + origin_site: String, + }, + + /// The broadcast's signing timestamp falls outside the accepted freshness window. + #[error( + "state broadcast from pinned origin {origin_site} has a signing timestamp outside the accepted freshness window" + )] + TimestampOutOfWindow { + /// Site that originated the broadcast. + origin_site: String, + }, } /// Per-origin metadata retained by the broadcast handler. @@ -453,6 +775,21 @@ pub struct StateBroadcastHandler { /// Watch channel for broadcasting public cert PEM updates to observers. cert_pems_tx: watch::Sender>, + /// Sender half of the pinned-identity trust store. + /// + /// Exposed via [`trust_store_sender`](Self::trust_store_sender) so a + /// caller (e.g. the operator, once it has established which certificate + /// pins an origin site's signing identity) can populate or update + /// entries at runtime, after this handler has been moved into foca. + /// Prefer [`crate::node::SwimNode::pin_origin`] over mutating this + /// sender directly: pinning through the raw sender does not purge state + /// merged from an origin before it was pinned, which is a correctness + /// gap for a signature to close, not just an inconvenience. + trust_store_tx: watch::Sender, + + /// Receiver half read synchronously by [`receive_item`](foca::BroadcastHandler::receive_item). + trust_store_rx: watch::Receiver, + /// Hard bound for per-origin revision and metadata maps. max_origins: usize, } @@ -479,6 +816,7 @@ impl StateBroadcastHandler { let (tx, _) = watch::channel(GridStateSnapshot::new(site_id)); let (gw_tx, _) = watch::channel(BTreeMap::new()); let (cert_tx, _) = watch::channel(BTreeMap::new()); + let (trust_tx, trust_rx) = watch::channel(TrustStore::new()); let max_origins = max_origins.max(1); let retained = Arc::new(Mutex::new(RetainedOrigins::default())); let control = OriginStateHandle { @@ -493,12 +831,34 @@ impl StateBroadcastHandler { retained, gateway_addrs_tx: gw_tx, cert_pems_tx: cert_tx, + trust_store_tx: trust_tx, + trust_store_rx: trust_rx, max_origins, }, control, ) } + /// Return a sender for updating the pinned-identity trust store. + /// + /// Clone and hold this to push pinned identities in after `self` has + /// been moved into foca. An origin site with no entry is not yet + /// enforced against a signature — see [`receive_item`]'s doc comment for + /// the rollout-transition rationale. + /// + /// This is the low-level primitive [`crate::node::SwimNode::pin_origin`] + /// is built on. Prefer `pin_origin` for real use: sending directly + /// through this sender skips the purge of state merged from an origin + /// before it had a pin. This accessor stays public for tests and for + /// callers that hold a bare [`StateBroadcastHandler`] outside a + /// [`SwimNode`](crate::node::SwimNode). + /// + /// [`receive_item`]: foca::BroadcastHandler::receive_item + #[must_use] + pub fn trust_store_sender(&self) -> watch::Sender { + self.trust_store_tx.clone() + } + /// Return a receiver for the live merged grid-state snapshot. /// /// Create the receiver **before** moving `self` into foca. Multiple @@ -644,20 +1004,130 @@ impl StateBroadcastHandler { }); } + /// Reject a broadcast that fails signature verification against a + /// pinned identity. + /// + /// An origin with **no** entry (or an empty entry) in the trust store + /// passes through unchecked — this is deliberate: it lets a + /// signed-broadcast rollout proceed incrementally as origins are pinned + /// one at a time, rather than requiring a synchronized flag-day + /// cutover. Once nerdalert's key-source question (grid#75) is resolved + /// and the operator starts populating pins, this becomes the + /// enforcement point; until then it is a no-op for every unpinned + /// origin. An origin pinned to more than one key (rotation overlap) + /// verifies against **any** key in its set. + #[expect( + clippy::too_many_lines, + reason = "signature-presence, timestamp-presence, signature-validity, and freshness-window checks form one \ + atomic gate with a shared rejection log message" + )] + #[expect( + clippy::cognitive_complexity, + reason = "four independent rejection branches over one broadcast, each with its own log statement, \ + read more branchily than they actually are" + )] + fn verify_signature_if_pinned(&self, broadcast: &StateBroadcast) -> Result<(), StateBroadcastError> { + /// Rejection log message shared by every failure branch below. + /// + /// Deliberately carries no payload contents, matching grid#75's + /// review request for bounded rejection metrics that never log + /// broadcast bodies; `reason` stays a small closed set of values. + const REJECTED: &str = "rejecting pinned-origin state broadcast"; + + let pinned_keys = self + .trust_store_rx + .borrow() + .get(&broadcast.origin_site) + .cloned() + .unwrap_or_default(); + if pinned_keys.is_empty() { + return Ok(()); + } + let origin_site = broadcast.origin_site.clone(); + let Some(signature) = broadcast.signature.as_ref() else { + tracing::warn!(origin_site = %origin_site, reason = "missing_signature", REJECTED); + return Err(StateBroadcastError::MissingSignature { origin_site }); + }; + let Some(signed_at_ms) = broadcast.signed_at_ms else { + tracing::warn!(origin_site = %origin_site, reason = "missing_timestamp", REJECTED); + return Err(StateBroadcastError::MissingTimestamp { origin_site }); + }; + let signable = match broadcast.signable_bytes() { + Ok(bytes) => bytes, + Err(source) => { + tracing::warn!(origin_site = %origin_site, reason = "signable_encode", REJECTED); + return Err(StateBroadcastError::SignableEncode { origin_site, source }); + }, + }; + let verified = pinned_keys + .iter() + .any(|key| crate::signing::verify_ecdsa_p256(key, &signable, signature).is_ok()); + if !verified { + tracing::warn!(origin_site = %origin_site, reason = "signature_invalid", REJECTED); + return Err(StateBroadcastError::SignatureInvalid { origin_site }); + } + if !Self::within_freshness_window(signed_at_ms) { + tracing::warn!(origin_site = %origin_site, reason = "timestamp_out_of_window", REJECTED); + return Err(StateBroadcastError::TimestampOutOfWindow { origin_site }); + } + Ok(()) + } + + /// Return true when `signed_at_ms` falls within the accepted freshness + /// window relative to this node's own clock. + /// + /// A [`SystemTime::now`] that somehow predates the Unix epoch (a + /// pathologically misconfigured clock) is treated as epoch zero rather + /// than propagating an error — every positive `signed_at_ms` then falls + /// outside the window and is rejected, failing closed rather than + /// disabling the freshness check entirely. + fn within_freshness_window(signed_at_ms: u64) -> bool { + let now_ms = u64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_millis(), + ) + .unwrap_or(u64::MAX); + let not_too_old = now_ms.saturating_sub(signed_at_ms) <= MAX_BROADCAST_AGE_MS; + let not_too_far_future = signed_at_ms.saturating_sub(now_ms) <= MAX_CLOCK_SKEW_AHEAD_MS; + not_too_old && not_too_far_future + } + /// Enforce the hard origin bound before accepting an unknown origin. + /// + /// Never evicts a pinned origin's retained state: a signature pin is an + /// authenticated fact about that origin, and evicting it would silently + /// discard the anti-replay value of the origin's tracked revision (see + /// [`verify_signature_if_pinned`](Self::verify_signature_if_pinned)) the + /// moment memory pressure forces a choice. If every retained origin is + /// pinned, the incoming origin is accepted anyway without evicting + /// anything — the map temporarily exceeds `max_origins` by at most one + /// rather than dropping an authenticated peer's state. fn make_room_for(&self, incoming_origin: &str) { let origins = self.known_origins(); if origins.contains(incoming_origin) || origins.len() < self.max_origins { return; } - if let Some(origin) = origins.into_iter().next() { + let trust_store = self.trust_store_rx.borrow(); + let Some(origin) = origins + .into_iter() + .find(|origin| trust_store.get(origin).is_none_or(Vec::is_empty)) + else { tracing::warn!( - evicted_origin = %origin, max_origins = self.max_origins, - "SWIM state origin capacity reached; evicting retained origin" + "SWIM state origin capacity reached and every retained origin is pinned; \ + accepting the new origin without evicting an authenticated peer" ); - self.remove_origin(&origin); - } + return; + }; + drop(trust_store); + tracing::warn!( + evicted_origin = %origin, + max_origins = self.max_origins, + "SWIM state origin capacity reached; evicting retained origin" + ); + self.remove_origin(&origin); } } @@ -677,6 +1147,7 @@ impl foca::BroadcastHandler for StateBroadcastHandler { actual: broadcast.version, }); } + self.verify_signature_if_pinned(&broadcast)?; self.make_room_for(&broadcast.origin_site); // Metadata-only broadcasts (gateway address or cert PEM, empty CRDT @@ -767,6 +1238,579 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + /// Encode a base v1 payload plus a two-field pre-signature extension, + /// mirroring the wire bytes a peer running the extension format that + /// predates the `signature` field would have sent. + fn encode_v1_plus_pre_signature_extension( + origin_site: &str, + revision: u64, + gateway_address: Option, + site_cert_pem: Option, + ) -> Vec { + let v1 = StateBroadcastV1 { + version: STATE_BROADCAST_VERSION, + origin_site: origin_site.to_owned(), + revision, + snapshot: snapshot(origin_site, revision, 0.4), + }; + let mut bytes = + bincode::serde::encode_to_vec(&v1, bincode::config::standard()).unwrap_or_else(|_| std::process::abort()); + let ext = PreSignatureBroadcastExtension { + gateway_address, + site_cert_pem, + }; + let ext_bytes = + bincode::serde::encode_to_vec(&ext, bincode::config::standard()).unwrap_or_else(|_| std::process::abort()); + bytes.extend_from_slice(&ext_bytes); + bytes + } + + /// Encode a base v1 payload plus a three-field pre-timestamp extension, + /// mirroring the wire bytes a peer running the extension format that + /// predates the `signed_at_ms` field would have sent. + fn encode_v1_plus_pre_timestamp_extension( + origin_site: &str, + revision: u64, + gateway_address: Option, + site_cert_pem: Option, + signature: Option>, + ) -> Vec { + let v1 = StateBroadcastV1 { + version: STATE_BROADCAST_VERSION, + origin_site: origin_site.to_owned(), + revision, + snapshot: snapshot(origin_site, revision, 0.4), + }; + let mut bytes = + bincode::serde::encode_to_vec(&v1, bincode::config::standard()).unwrap_or_else(|_| std::process::abort()); + let ext = PreTimestampBroadcastExtension { + gateway_address, + site_cert_pem, + signature, + }; + let ext_bytes = + bincode::serde::encode_to_vec(&ext, bincode::config::standard()).unwrap_or_else(|_| std::process::abort()); + bytes.extend_from_slice(&ext_bytes); + bytes + } + + /// Return the current wall-clock time in milliseconds since the Unix + /// epoch, for constructing test broadcasts with a fresh `signed_at_ms`. + fn now_ms() -> u64 { + u64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| std::process::abort()) + .as_millis(), + ) + .unwrap_or_else(|_| std::process::abort()) + } + + /// Generate an ECDSA P-256 signing key plus the raw SPKI EC point a + /// verifier needs, independent of *how* a real deployment would source + /// or pin this key material (grid#75, still open). + fn generate_signing_key_and_pubkey() -> (Vec, Vec) { + let key_pair = rcgen::KeyPair::generate().unwrap_or_else(|_| std::process::abort()); + let pkcs8_der = key_pair.serialize_der(); + let params = rcgen::CertificateParams::new(vec!["spike.grid.internal".to_owned()]) + .unwrap_or_else(|_| std::process::abort()); + let cert = params.self_signed(&key_pair).unwrap_or_else(|_| std::process::abort()); + let (_, parsed) = x509_parser::parse_x509_certificate(cert.der()).unwrap_or_else(|_| std::process::abort()); + let raw_pubkey = parsed.public_key().subject_public_key.as_ref().to_vec(); + (pkcs8_der, raw_pubkey) + } + + #[test] + fn receive_item_accepts_a_correctly_signed_broadcast_from_a_pinned_origin() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (pkcs8_der, raw_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![raw_pubkey]))); + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(now_ms())); + let signature = crate::signing::sign_ecdsa_p256( + &pkcs8_der, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let signed = unsigned.with_signature(Some(signature)); + + let key = receive(&mut handler, &signed); + + assert!( + key.is_some(), + "a correctly signed broadcast from a pinned origin must be accepted" + ); + assert!( + handler.snapshot().provider("net", "site-p", "provider").is_some(), + "the signed broadcast's provider state must be merged" + ); + } + + #[test] + fn receive_item_rejects_an_unsigned_broadcast_from_a_pinned_origin() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (_pkcs8_der, raw_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![raw_pubkey]))); + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None); + let bytes = unsigned.encode().unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::MissingSignature { origin_site }) if origin_site.as_str() == "site-p"), + "an unsigned broadcast from a pinned origin must be rejected, got {result:?}" + ); + assert!( + handler.snapshot().provider("net", "site-p", "provider").is_none(), + "a rejected broadcast must not be merged into the snapshot" + ); + } + + #[test] + fn receive_item_rejects_a_broadcast_signed_by_the_wrong_key_for_a_pinned_origin() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (_correct_key, pinned_pubkey) = generate_signing_key_and_pubkey(); + let (wrong_key, _wrong_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![pinned_pubkey]))); + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(now_ms())); + let signature = crate::signing::sign_ecdsa_p256( + &wrong_key, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let bytes = unsigned + .with_signature(Some(signature)) + .encode() + .unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::SignatureInvalid { origin_site }) if origin_site.as_str() == "site-p"), + "a broadcast signed by a key other than the pinned one must be rejected, got {result:?}" + ); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "verifies acceptance under both the 'next' and 'current' pinned keys in one proof" + )] + fn receive_item_accepts_a_broadcast_signed_by_either_key_in_a_rotated_pin_set() { + // Bounded key rotation: an origin pinned to [current, next] must + // verify against a signature made with *either* key, mirroring + // canonical_fingerprints' current+next mTLS pin overlap window. + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (current_key, current_pubkey) = generate_signing_key_and_pubkey(); + let (next_key, next_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![current_pubkey, next_pubkey]))); + + let unsigned_from_next = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(now_ms())); + let signature_from_next = crate::signing::sign_ecdsa_p256( + &next_key, + &unsigned_from_next + .signable_bytes() + .unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let signed_by_next = unsigned_from_next.with_signature(Some(signature_from_next)); + + assert!( + receive(&mut handler, &signed_by_next).is_some(), + "a broadcast signed by the 'next' key in a rotated pin set must be accepted" + ); + + let unsigned_from_current = StateBroadcast::new("site-p".to_owned(), 2, snapshot("site-p", 2, 0.2), None) + .with_signed_at(Some(now_ms())); + let signature_from_current = crate::signing::sign_ecdsa_p256( + ¤t_key, + &unsigned_from_current + .signable_bytes() + .unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let signed_by_current = unsigned_from_current.with_signature(Some(signature_from_current)); + + assert!( + receive(&mut handler, &signed_by_current).is_some(), + "a broadcast signed by the 'current' key in a rotated pin set must also be accepted" + ); + } + + #[test] + fn receive_item_rejects_a_signed_broadcast_missing_a_timestamp_from_a_pinned_origin() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (pkcs8_der, raw_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![raw_pubkey]))); + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None); + let signature = crate::signing::sign_ecdsa_p256( + &pkcs8_der, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let bytes = unsigned + .with_signature(Some(signature)) + .encode() + .unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::MissingTimestamp { origin_site }) if origin_site.as_str() == "site-p"), + "a signed broadcast with no signing timestamp from a pinned origin must be rejected, got {result:?}" + ); + } + + #[test] + fn receive_item_rejects_a_signed_broadcast_with_a_stale_timestamp_from_a_pinned_origin() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (pkcs8_der, raw_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![raw_pubkey]))); + let stale_at = now_ms().saturating_sub(MAX_BROADCAST_AGE_MS + 1_000); + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(stale_at)); + let signature = crate::signing::sign_ecdsa_p256( + &pkcs8_der, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let bytes = unsigned + .with_signature(Some(signature)) + .encode() + .unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::TimestampOutOfWindow { origin_site }) if origin_site.as_str() == "site-p"), + "a signed broadcast with a stale timestamp from a pinned origin must be rejected, got {result:?}" + ); + } + + #[test] + fn receive_item_rejects_a_signed_broadcast_with_a_far_future_timestamp_from_a_pinned_origin() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (pkcs8_der, raw_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![raw_pubkey]))); + let future_at = now_ms() + MAX_CLOCK_SKEW_AHEAD_MS + 1_000; + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(future_at)); + let signature = crate::signing::sign_ecdsa_p256( + &pkcs8_der, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let bytes = unsigned + .with_signature(Some(signature)) + .encode() + .unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::TimestampOutOfWindow { origin_site }) if origin_site.as_str() == "site-p"), + "a signed broadcast with a far-future timestamp from a pinned origin must be rejected, got {result:?}" + ); + } + + #[test] + fn within_freshness_window_accepts_a_timestamp_comfortably_within_max_age() { + let now = now_ms(); + assert!( + StateBroadcastHandler::within_freshness_window(now.saturating_sub(MAX_BROADCAST_AGE_MS - 1_000)), + "a timestamp just inside MAX_BROADCAST_AGE_MS old must be within the window" + ); + } + + #[test] + fn within_freshness_window_rejects_a_timestamp_older_than_max_age() { + let now = now_ms(); + assert!( + !StateBroadcastHandler::within_freshness_window(now.saturating_sub(MAX_BROADCAST_AGE_MS + 1_000)), + "a timestamp older than MAX_BROADCAST_AGE_MS must fall outside the window" + ); + } + + #[test] + fn within_freshness_window_accepts_a_timestamp_comfortably_ahead_within_clock_skew() { + let now = now_ms(); + assert!( + StateBroadcastHandler::within_freshness_window(now + MAX_CLOCK_SKEW_AHEAD_MS - 1_000), + "a timestamp just inside MAX_CLOCK_SKEW_AHEAD_MS ahead must be within the window" + ); + } + + #[test] + fn within_freshness_window_rejects_a_timestamp_further_ahead_than_clock_skew() { + let now = now_ms(); + assert!( + !StateBroadcastHandler::within_freshness_window(now + MAX_CLOCK_SKEW_AHEAD_MS + 1_000), + "a timestamp further ahead than MAX_CLOCK_SKEW_AHEAD_MS must fall outside the window" + ); + } + + #[test] + fn missing_timestamp_error_message_names_the_origin() { + let err = StateBroadcastError::MissingTimestamp { + origin_site: "site-p".to_owned(), + }; + assert!( + err.to_string().contains("site-p"), + "error message must name the origin site" + ); + } + + #[test] + fn timestamp_out_of_window_error_message_names_the_origin() { + let err = StateBroadcastError::TimestampOutOfWindow { + origin_site: "site-p".to_owned(), + }; + assert!( + err.to_string().contains("site-p"), + "error message must name the origin site" + ); + } + + /// Build and receive a correctly signed broadcast for a pinned origin. + fn receive_signed( + handler: &mut StateBroadcastHandler, + origin: &str, + revision: u64, + signing_key: &[u8], + ) -> Option { + let unsigned = StateBroadcast::new(origin.to_owned(), revision, snapshot(origin, revision, 0.1), None) + .with_signed_at(Some(now_ms())); + let signature = crate::signing::sign_ecdsa_p256( + signing_key, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + receive(handler, &unsigned.with_signature(Some(signature))) + } + + #[test] + fn make_room_for_never_evicts_a_pinned_origin() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 2); + let (signing_key, pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-pinned".to_owned(), vec![pubkey]))); + + drop(receive_signed(&mut handler, "site-pinned", 1, &signing_key)); + for (origin, revision) in [("site-unpinned", 2), ("site-new", 3)] { + let broadcast = StateBroadcast::new(origin.to_owned(), revision, snapshot(origin, revision, 0.1), None); + drop(receive(&mut handler, &broadcast)); + } + + let origins = handler.known_origins(); + assert!( + origins.contains("site-pinned"), + "a pinned origin must never be evicted for capacity, got {origins:?}" + ); + assert!( + !origins.contains("site-unpinned"), + "an unpinned origin must be evicted ahead of a pinned one, got {origins:?}" + ); + } + + #[test] + fn make_room_for_accepts_a_new_origin_without_evicting_when_every_retained_origin_is_pinned() { + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 2); + for origin in ["site-pinned-a", "site-pinned-b"] { + let (signing_key, pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert(origin.to_owned(), vec![pubkey]))); + drop(receive_signed(&mut handler, origin, 1, &signing_key)); + } + + let broadcast = StateBroadcast::new("site-new".to_owned(), 1, snapshot("site-new", 1, 0.1), None); + drop(receive(&mut handler, &broadcast)); + + let origins = handler.known_origins(); + assert!( + origins.contains("site-pinned-a") && origins.contains("site-pinned-b"), + "both pinned origins must survive even though capacity was reached, got {origins:?}" + ); + assert!( + origins.contains("site-new"), + "the new origin must still be accepted even though nothing could be evicted, got {origins:?}" + ); + } + + #[test] + fn signable_bytes_domain_prefix_is_load_bearing_not_decorative() { + // Proves the domain-separation prefix actually participates in what + // gets signed: a signature computed over signable_bytes() (prefix + + // payload) must not verify against the bare, unprefixed encode() of + // the same broadcast. + let (key, pubkey) = generate_signing_key_and_pubkey(); + let broadcast = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None); + let signature = crate::signing::sign_ecdsa_p256( + &key, + &broadcast.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let bare_encoded = broadcast.encode().unwrap_or_else(|_| std::process::abort()); + + let result = crate::signing::verify_ecdsa_p256(&pubkey, &bare_encoded, &signature); + + assert!( + result.is_err(), + "a signature over the domain-prefixed signable_bytes() must not verify against the bare encoded payload" + ); + } + + #[test] + fn signable_bytes_differ_for_broadcasts_that_differ_only_by_grid_id() { + // Same origin_site, revision, and snapshot -- the only difference is + // which GridNetwork the broadcast claims to belong to. If these + // produced identical signable bytes, a signature valid for one + // GridNetwork would silently double as valid for another sharing + // the same cluster's SwimHandle (issue #48's per-tenant isolation). + let base = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None); + let for_grid_a = base.clone().with_grid_id(Some("grid-a".to_owned())); + let for_grid_b = base.with_grid_id(Some("grid-b".to_owned())); + + let bytes_a = for_grid_a.signable_bytes().unwrap_or_else(|_| std::process::abort()); + let bytes_b = for_grid_b.signable_bytes().unwrap_or_else(|_| std::process::abort()); + + assert_ne!( + bytes_a, bytes_b, + "signable_bytes() must depend on grid_id, or a signature would be replayable across GridNetworks" + ); + } + + #[test] + fn receive_item_rejects_a_signature_computed_for_a_different_grid_id() { + // A signature made over a broadcast claiming grid_id="grid-a" must + // not verify once the broadcast is re-labeled grid_id="grid-b" -- + // the cross-GridNetwork replay this field exists to prevent. + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (pkcs8_der, raw_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![raw_pubkey]))); + + let signed_at_ms = Some(now_ms()); + let for_grid_a = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_grid_id(Some("grid-a".to_owned())) + .with_signed_at(signed_at_ms); + let signature = crate::signing::sign_ecdsa_p256( + &pkcs8_der, + &for_grid_a.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + + // Re-label as grid-b, reusing grid-a's signature over grid-a's bytes. + let relabeled_as_grid_b = for_grid_a.with_grid_id(Some("grid-b".to_owned())); + let bytes = relabeled_as_grid_b + .with_signature(Some(signature)) + .encode() + .unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::SignatureInvalid { origin_site }) if origin_site.as_str() == "site-p"), + "a broadcast re-labeled with a different grid_id than it was signed for must be rejected, got {result:?}" + ); + } + + #[test] + fn too_many_pinned_keys_error_message_is_descriptive() { + let err = TooManyPinnedKeys { + origin: "site-p".to_owned(), + supplied: 3, + }; + assert!(err.to_string().contains("site-p"), "error message must name the origin"); + assert!( + err.to_string().contains('3'), + "error message must name the supplied count" + ); + } + + #[test] + fn missing_signature_error_message_names_the_origin() { + let err = StateBroadcastError::MissingSignature { + origin_site: "site-p".to_owned(), + }; + assert!( + err.to_string().contains("site-p"), + "error message must name the origin site" + ); + } + + #[test] + fn signature_invalid_error_message_names_the_origin() { + let err = StateBroadcastError::SignatureInvalid { + origin_site: "site-p".to_owned(), + }; + assert!( + err.to_string().contains("site-p"), + "error message must name the origin site" + ); + } + + #[test] + fn signable_encode_error_message_names_the_origin_and_wraps_the_source() { + // A genuine bincode encode failure on `StateBroadcast`'s own fields + // (String/u64/u16/GridStateSnapshot, encoded to an in-memory Vec + // with no writer I/O) has no reachable trigger through this crate's + // public API; constructed directly to verify the message wording, + // matching this codebase's `..._error_formats_correctly` convention + // for defensive error variants (see e.g. + // `node::tests::state_broadcast_error_formats_correctly`). + let err = StateBroadcastError::SignableEncode { + origin_site: "site-p".to_owned(), + source: bincode::error::EncodeError::UnexpectedEnd, + }; + assert!( + err.to_string().contains("site-p"), + "error message must name the origin site" + ); + } + + #[test] + fn receive_item_still_merges_an_unsigned_broadcast_from_an_origin_with_no_pinned_identity() { + // Guards the incremental-rollout property documented on + // `verify_signature_if_pinned`: no synchronized flag-day cutover. + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None); + + let key = receive(&mut handler, &unsigned); + + assert!( + key.is_some(), + "an unsigned broadcast from an unpinned origin must still be accepted" + ); + assert!( + handler.snapshot().provider("net", "site-p", "provider").is_some(), + "the unsigned broadcast's provider state must be merged while the origin is unpinned" + ); + } + #[test] fn new_sets_version_origin_and_revision() { let broadcast = StateBroadcast::new("site-p".to_owned(), 7, snapshot("site-p", 7, 0.1), None); @@ -788,6 +1832,168 @@ mod tests { assert_eq!(provider.metrics.queue_depth, Some(0.1), "metric value"); } + #[test] + fn decode_recovers_gateway_and_cert_from_a_pre_signature_two_field_extension() { + // Simulates a broadcast sent by a peer running the two-field + // `BroadcastExtension` (gateway_address, site_cert_pem) that predates + // the `signature` field added in this PR. bincode is not + // self-describing: decoding those bytes as the current three-field + // struct hits `UnexpectedEnd` while reading `signature`, and + // `#[serde(default)]` never gets a chance to apply because the `?` + // inside serde's generated `visit_seq` propagates that error first. + // A rolling update must not silently drop `gateway_address`/ + // `site_cert_pem` for every broadcast from a not-yet-upgraded peer. + let bytes = encode_v1_plus_pre_signature_extension( + "site-old-peer", + 3, + Some("10.0.0.9:8443".to_owned()), + Some("-----BEGIN CERTIFICATE-----legacy-----END CERTIFICATE-----".to_owned()), + ); + + let decoded = StateBroadcast::decode(&bytes).unwrap_or_else(|_| std::process::abort()); + + assert_eq!( + decoded.gateway_address.as_deref(), + Some("10.0.0.9:8443"), + "gateway_address from a pre-signature peer must survive decode, not be silently dropped" + ); + assert_eq!( + decoded.site_cert_pem.as_deref(), + Some("-----BEGIN CERTIFICATE-----legacy-----END CERTIFICATE-----"), + "site_cert_pem from a pre-signature peer must survive decode, not be silently dropped" + ); + assert_eq!(decoded.signature, None, "a pre-signature peer never sends a signature"); + } + + #[test] + fn decode_recovers_gateway_signature_and_cert_from_a_pre_timestamp_three_field_extension() { + // Simulates a broadcast sent by a peer running the three-field + // `PreTimestampBroadcastExtension` (gateway_address, site_cert_pem, + // signature) that predates the `signed_at_ms` field added in this + // fix. Same bincode non-self-describing failure mode as the + // pre-signature case above, one field further down the chain. + let signature = vec![9_u8, 8, 7, 6]; + let bytes = encode_v1_plus_pre_timestamp_extension( + "site-old-peer", + 5, + Some("10.0.0.4:8443".to_owned()), + Some("-----BEGIN CERTIFICATE-----legacy-----END CERTIFICATE-----".to_owned()), + Some(signature.clone()), + ); + + let decoded = StateBroadcast::decode(&bytes).unwrap_or_else(|_| std::process::abort()); + + assert_eq!( + decoded.gateway_address.as_deref(), + Some("10.0.0.4:8443"), + "gateway_address from a pre-timestamp peer must survive decode, not be silently dropped" + ); + assert_eq!( + decoded.site_cert_pem.as_deref(), + Some("-----BEGIN CERTIFICATE-----legacy-----END CERTIFICATE-----"), + "site_cert_pem from a pre-timestamp peer must survive decode, not be silently dropped" + ); + assert_eq!( + decoded.signature, + Some(signature), + "signature from a pre-timestamp peer must survive decode, not be silently dropped" + ); + assert_eq!( + decoded.signed_at_ms, None, + "a pre-timestamp peer never sends a signing timestamp" + ); + assert_eq!(decoded.grid_id, None, "a pre-timestamp peer never sends a grid_id"); + } + + #[test] + fn signed_at_ms_round_trips_through_encode_and_decode() { + let broadcast = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(1_700_000_000_000)); + let bytes = broadcast.encode().unwrap_or_else(|_| std::process::abort()); + + let decoded = StateBroadcast::decode(&bytes).unwrap_or_else(|_| std::process::abort()); + + assert_eq!( + decoded.signed_at_ms, + Some(1_700_000_000_000), + "signed_at_ms must round-trip" + ); + } + + #[test] + fn grid_id_round_trips_through_encode_and_decode() { + let broadcast = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_grid_id(Some("grid-a".to_owned())); + let bytes = broadcast.encode().unwrap_or_else(|_| std::process::abort()); + + let decoded = StateBroadcast::decode(&bytes).unwrap_or_else(|_| std::process::abort()); + + assert_eq!(decoded.grid_id.as_deref(), Some("grid-a"), "grid_id must round-trip"); + } + + #[test] + fn decode_recovers_gateway_from_the_original_bare_string_extension() { + // The oldest wire format, predating even the two-field + // `PreSignatureBroadcastExtension`: a bare bincode-encoded `String` + // for `gateway_address`, with no `site_cert_pem` or `signature`. + // `decode_extension` falls all the way through to this tier only + // after both newer struct shapes fail to decode, so it must stay + // covered whenever that fallback chain is touched. + let v1 = StateBroadcastV1 { + version: STATE_BROADCAST_VERSION, + origin_site: "site-ancient-peer".to_owned(), + revision: 2, + snapshot: snapshot("site-ancient-peer", 2, 0.2), + }; + let mut bytes = + bincode::serde::encode_to_vec(&v1, bincode::config::standard()).unwrap_or_else(|_| std::process::abort()); + let gw_bytes = bincode::serde::encode_to_vec("10.0.0.5:8443".to_owned(), bincode::config::standard()) + .unwrap_or_else(|_| std::process::abort()); + bytes.extend_from_slice(&gw_bytes); + + let decoded = StateBroadcast::decode(&bytes).unwrap_or_else(|_| std::process::abort()); + + assert_eq!( + decoded.gateway_address.as_deref(), + Some("10.0.0.5:8443"), + "gateway_address from the original bare-String wire format must survive decode" + ); + assert_eq!( + decoded.site_cert_pem, None, + "the bare-String format never carries a cert" + ); + assert_eq!( + decoded.signature, None, + "the bare-String format never carries a signature" + ); + } + + #[test] + fn decode_extension_returns_no_extension_fields_for_undecodable_trailing_bytes() { + // Bytes that match none of the three known extension shapes -- the + // final fallback in `decode_extension`'s chain must degrade to no + // extension data rather than propagating a decode error, since the + // trailing bytes might belong to a future format this peer doesn't + // understand yet. + let v1 = StateBroadcastV1 { + version: STATE_BROADCAST_VERSION, + origin_site: "site-p".to_owned(), + revision: 1, + snapshot: snapshot("site-p", 1, 0.1), + }; + let mut bytes = + bincode::serde::encode_to_vec(&v1, bincode::config::standard()).unwrap_or_else(|_| std::process::abort()); + // A single 0xFF byte is not a valid bincode length prefix for a + // `String`, `PreSignatureBroadcastExtension`, or `BroadcastExtension`. + bytes.push(0xFF); + + let decoded = StateBroadcast::decode(&bytes).unwrap_or_else(|_| std::process::abort()); + + assert_eq!(decoded.gateway_address, None); + assert_eq!(decoded.site_cert_pem, None); + assert_eq!(decoded.signature, None); + } + #[test] fn grid_state_and_metadata_broadcasts_are_distinguished() { let state = StateBroadcast::new("site-p".to_owned(), 7, snapshot("site-p", 7, 0.1), None); From 7f87202c5738f31771fa1e6a2f8f7aed85379e2b Mon Sep 17 00:00:00 2001 From: Jordi Gil Date: Wed, 26 Aug 2026 10:28:05 -0400 Subject: [PATCH 6/8] test(swim): cover impersonation, tampering, and rotation for signed state broadcasts (grid#75) receive_item already rejected wrong-key signatures and accepted either key in a rotation window; add the missing tampering case -- a broadcast signed correctly, then edited before it reaches the handler (bumped revision, inflated tenant_spend). Both must fail signature verification and neither tampered payload should merge into the snapshot. These exercise the swim crate's existing sign/verify/TrustStore API only, so they hold regardless of how a real deployment ends up sourcing and pinning per-site signing keys (still open on grid#75/grid#92/grid#93). Signed-off-by: Jordi Gil --- swim/src/state_broadcast.rs | 68 +++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/swim/src/state_broadcast.rs b/swim/src/state_broadcast.rs index df712d5..4d41a91 100644 --- a/swim/src/state_broadcast.rs +++ b/swim/src/state_broadcast.rs @@ -1400,6 +1400,74 @@ mod tests { ); } + #[test] + fn receive_item_rejects_a_broadcast_whose_revision_was_tampered_after_signing() { + // A relay that forwards someone else's broadcast is a legitimate path + // (A -> B -> C); a relay that *edits* the payload in transit is not. + // Sign with the correct key, then mutate the struct before encoding + // to simulate exactly that: bytes that left the origin correctly + // signed but arrived changed. + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (signing_key, pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![pubkey]))); + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(now_ms())); + let signature = crate::signing::sign_ecdsa_p256( + &signing_key, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let mut tampered = unsigned.with_signature(Some(signature)); + tampered.revision = 99; + let bytes = tampered.encode().unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::SignatureInvalid { origin_site }) if origin_site.as_str() == "site-p"), + "a broadcast edited after signing must fail verification even though the signature itself is well-formed, got {result:?}" + ); + } + + #[test] + fn receive_item_rejects_a_broadcast_whose_tenant_spend_was_tampered_after_signing() { + // The concrete attack grid#75 is worried about: a relay (or a peer + // holding only the shared SWIM key) inflates a tenant's reported + // spend on someone else's correctly-signed broadcast. Proves that + // attack lands as a rejected broadcast, not a silently-merged one. + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (signing_key, pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![pubkey]))); + + let mut snap = snapshot("site-p", 1, 0.1); + snap.increment_tenant_spend("tenant-x", 500); + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snap, None).with_signed_at(Some(now_ms())); + let signature = crate::signing::sign_ecdsa_p256( + &signing_key, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let mut tampered = unsigned.with_signature(Some(signature)); + tampered.snapshot.increment_tenant_spend("tenant-x", 1_000_000); + let bytes = tampered.encode().unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::SignatureInvalid { origin_site }) if origin_site.as_str() == "site-p"), + "an inflated tenant_spend added after signing must be rejected, got {result:?}" + ); + assert!( + !handler.snapshot().tenant_spend.contains_key("tenant-x"), + "a rejected tampered broadcast must not merge its forged tenant_spend into the snapshot" + ); + } + #[test] #[expect( clippy::too_many_lines, From d8a192126afd91e1da7d1d2a4a211e837b7008e9 Mon Sep 17 00:00:00 2001 From: Jordi Gil Date: Wed, 26 Aug 2026 11:00:23 -0400 Subject: [PATCH 7/8] test(swim): cover rotation-completion and malformed-signature rejection Two more scenarios for signed state broadcasts: a signature from a key that was dropped at the end of a rotation (not just outnumbered by a new one), and a malformed signature that isn't valid ECDSA at all -- both must be rejected cleanly, the latter without panicking. Signed-off-by: Jordi Gil --- swim/src/state_broadcast.rs | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/swim/src/state_broadcast.rs b/swim/src/state_broadcast.rs index 4d41a91..6d60271 100644 --- a/swim/src/state_broadcast.rs +++ b/swim/src/state_broadcast.rs @@ -1468,6 +1468,68 @@ mod tests { ); } + #[test] + fn receive_item_rejects_a_broadcast_signed_by_a_key_dropped_from_a_completed_rotation() { + // Same rejection path as the wrong-key test, but exercised as a + // rotation *completion*: the old key is fully retired (not just + // outnumbered by a new one), so a straggler still signing with it + // must be rejected exactly like an impostor would be. + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (retired_key, retired_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![retired_pubkey]))); + let (_current_key, current_pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![current_pubkey]))); + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(now_ms())); + let signature = crate::signing::sign_ecdsa_p256( + &retired_key, + &unsigned.signable_bytes().unwrap_or_else(|_| std::process::abort()), + ) + .unwrap_or_else(|_| std::process::abort()); + let bytes = unsigned + .with_signature(Some(signature)) + .encode() + .unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::SignatureInvalid { origin_site }) if origin_site.as_str() == "site-p"), + "a broadcast signed by a key retired at the end of rotation must be rejected, got {result:?}" + ); + } + + #[test] + fn receive_item_rejects_a_malformed_signature_without_panicking() { + // Distinct from "signed by the wrong key": this signature isn't a + // well-formed ECDSA signature at all, proving the verifier fails + // closed on garbage input instead of panicking on it. + let (mut handler, _control) = StateBroadcastHandler::with_capacity("site-local".to_owned(), 8); + let (_signing_key, pubkey) = generate_signing_key_and_pubkey(); + handler + .trust_store_sender() + .send_modify(|store| drop(store.insert("site-p".to_owned(), vec![pubkey]))); + + let unsigned = StateBroadcast::new("site-p".to_owned(), 1, snapshot("site-p", 1, 0.1), None) + .with_signed_at(Some(now_ms())); + let bytes = unsigned + .with_signature(Some(vec![0xFF; 4])) + .encode() + .unwrap_or_else(|_| std::process::abort()); + + let result = handler.receive_item(&bytes, None); + + assert!( + matches!(&result, Err(StateBroadcastError::SignatureInvalid { origin_site }) if origin_site.as_str() == "site-p"), + "a malformed signature must be rejected cleanly, not panic, got {result:?}" + ); + } + #[test] #[expect( clippy::too_many_lines, From ad9a5e21c09c2b1a65c7d241b0721fee1fcb1b27 Mon Sep 17 00:00:00 2001 From: Jordi Gil Date: Wed, 26 Aug 2026 11:54:47 -0400 Subject: [PATCH 8/8] chore: retrigger CI (stuck queued jobs) Signed-off-by: Jordi Gil