From ba96e9444c6adfae724ee10cfcdc7b0e7bcf8c04 Mon Sep 17 00:00:00 2001 From: bjo4 <9113867+bjo4@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:59:16 +0800 Subject: [PATCH] test(kubernetes): scrape both metrics endpoints on a real cluster The chart's metrics wiring was only ever checked by rendering the templates. Rendering proves the Service ports, the headless Worker Service, and METRICS_ADDRESS exist; it does not prove anything answers. The Kubernetes suite now forwards both endpoints and runs metrics_flow, so the same properties verified under Compose are verified through the chart. Two things the script needed before that could be trusted: A port-forward whose local port is already taken exits immediately. Stderr was discarded and the PID never checked, so the suite would talk to whatever else was listening and report a platform failure that was nothing of the kind. It now fails with the ports named. The fixed sleep is replaced by waiting for each endpoint to answer, since a forward is accepted before it can carry a request and the Control Plane restarts until NATS is up. Re-running the script rotated the Secret Store encryption key and set a PostgreSQL password the retained volume never accepted, so a second run failed inside an Agent Session, far from the cause. An existing install now keeps the Secret it has. The values are read back rather than pointed at with secrets.existingSecret, which would stop the chart rendering the Secret and so delete it. Verified on kind: all eight flows pass on a fresh install, and again on a second run against the same cluster. Co-Authored-By: Claude Opus 5 (1M context) --- test/e2e/kubernetes/run.sh | 90 ++++++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/test/e2e/kubernetes/run.sh b/test/e2e/kubernetes/run.sh index 7bcf983..6049ae8 100755 --- a/test/e2e/kubernetes/run.sh +++ b/test/e2e/kubernetes/run.sh @@ -12,11 +12,13 @@ NAMESPACE="${NAMESPACE:-agent-platform}" RELEASE="${RELEASE:-agent-platform}" KIND_CLUSTER="${KIND_CLUSTER-agent-platform}" LOCAL_PORT="${LOCAL_PORT:-18080}" +LOCAL_METRICS_PORT="${LOCAL_METRICS_PORT:-19090}" +LOCAL_WORKER_METRICS_PORT="${LOCAL_WORKER_METRICS_PORT:-19091}" TAG="${TAG:-0.2.0}" # Docker-only by nature: these restart Compose services or detach container # networks. Everything else runs unchanged. -FLOWS="${FLOWS:-seed session_flow mcp_flow mcp_sse_flow mcp_stdio_flow management_flow runtime_policy_flow}" +FLOWS="${FLOWS:-seed session_flow mcp_flow mcp_sse_flow mcp_stdio_flow management_flow runtime_policy_flow metrics_flow}" cd "$REPO_ROOT" @@ -40,13 +42,39 @@ if [ -n "$KIND_CLUSTER" ]; then fi echo "==> Installing the chart" +# Generating fresh secrets on every run would rotate the Secret Store +# encryption key, leaving every credential written by an earlier run +# undecryptable, and would set a PostgreSQL password the retained volume never +# accepted. Both fail well away from their cause, so an existing install keeps +# the Secret it already has. +# +# The values are read back rather than pointed at with secrets.existingSecret, +# which would stop the chart rendering the Secret at all and so delete it. +SECRET_NAME="${RELEASE}-agent-platform-secrets" +secret_value() { + kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" -o "jsonpath={.data.$1}" | base64 -d +} +if kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" >/dev/null 2>&1; then + echo " reusing the secrets of the existing install" + SECRET_ARGS=( + --set secrets.secretEncryptionKey="$(secret_value SECRET_ENCRYPTION_KEY)" + --set secrets.workerToken="$(secret_value WORKER_TOKEN)" + --set postgresql.password="$(secret_value POSTGRES_PASSWORD)" + --set objectStore.secretKey="$(secret_value OBJECT_STORE_SECRET_KEY)" + ) +else + SECRET_ARGS=( + --set secrets.secretEncryptionKey="$(openssl rand -base64 32 | tr -d '=')" + --set secrets.workerToken="$(openssl rand -hex 24)" + --set postgresql.password="$(openssl rand -hex 16)" + --set objectStore.secretKey="$(openssl rand -hex 16)" + ) +fi + helm upgrade --install "$RELEASE" deploy/helm/agent-platform \ -n "$NAMESPACE" --create-namespace \ --set image.tag="$TAG" \ - --set secrets.secretEncryptionKey="$(openssl rand -base64 32 | tr -d '=')" \ - --set secrets.workerToken="$(openssl rand -hex 24)" \ - --set postgresql.password="$(openssl rand -hex 16)" \ - --set objectStore.secretKey="$(openssl rand -hex 16)" \ + "${SECRET_ARGS[@]}" \ --wait --timeout 10m echo "==> Deploying the fixture Provider" @@ -57,13 +85,57 @@ kubectl -n "$NAMESPACE" apply -f test/e2e/kubernetes/mock-provider.yaml kubectl -n "$NAMESPACE" rollout restart deploy/e2e-mock-provider kubectl -n "$NAMESPACE" rollout status deploy/e2e-mock-provider --timeout=180s -echo "==> Port-forwarding the console" +echo "==> Port-forwarding the console and both metrics endpoints" +FORWARD_PIDS=() +trap 'kill "${FORWARD_PIDS[@]}" 2>/dev/null || true' EXIT + kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-web" \ "${LOCAL_PORT}:8080" >/dev/null 2>&1 & -FORWARD_PID=$! -trap 'kill "$FORWARD_PID" 2>/dev/null || true' EXIT -sleep 5 +FORWARD_PIDS+=($!) +# The chart puts metrics on a port of their own on both components. Rendering +# the templates proves the wiring exists; only a scrape proves it answers, so +# the flow runs here as well as under Compose. +kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-control-plane" \ + "${LOCAL_METRICS_PORT}:9090" >/dev/null 2>&1 & +FORWARD_PIDS+=($!) +# The Worker's is headless: it exists only to give a scraper a stable target +# for a Pod that takes no inbound traffic otherwise. +kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-runtime-worker-metrics" \ + "${LOCAL_WORKER_METRICS_PORT}:9090" >/dev/null 2>&1 & +FORWARD_PIDS+=($!) + +# A port-forward whose local port is already taken dies immediately, and the +# suite then talks to whatever else is listening — which looks like a platform +# failure and is not one. Catch it here instead. +sleep 2 +for pid in "${FORWARD_PIDS[@]}"; do + if ! kill -0 "$pid" 2>/dev/null; then + echo "a port-forward exited; are ${LOCAL_PORT}, ${LOCAL_METRICS_PORT}, or" \ + "${LOCAL_WORKER_METRICS_PORT} already in use?" >&2 + exit 1 + fi +done + +# A port-forward is accepted before it can carry a request, and the Control +# Plane restarts until NATS accepts connections, so a fixed sleep turns a slow +# start into a failed first flow. Wait for each endpoint to actually answer. +wait_for() { + local url="$1" name="$2" attempt + for attempt in $(seq 60); do + if curl -fsS -o /dev/null --max-time 3 "$url"; then + return 0 + fi + sleep 2 + done + echo "$name did not answer at $url" >&2 + return 1 +} +wait_for "http://localhost:${LOCAL_PORT}/api/v1/auth/config" "the console" +wait_for "http://localhost:${LOCAL_METRICS_PORT}/metrics" "control plane metrics" +wait_for "http://localhost:${LOCAL_WORKER_METRICS_PORT}/metrics" "worker metrics" echo "==> Running flows" AGENT_PLATFORM_URL="http://localhost:${LOCAL_PORT}" \ +AGENT_PLATFORM_METRICS_URL="http://localhost:${LOCAL_METRICS_PORT}" \ +AGENT_PLATFORM_WORKER_METRICS_URL="http://localhost:${LOCAL_WORKER_METRICS_PORT}" \ python3 test/e2e/run_all.py $FLOWS