diff --git a/test/e2e/kubernetes/run.sh b/test/e2e/kubernetes/run.sh index 7bcf983..6049ae8 100755 --- a/test/e2e/kubernetes/run.sh +++ b/test/e2e/kubernetes/run.sh @@ -12,11 +12,13 @@ NAMESPACE="${NAMESPACE:-agent-platform}" RELEASE="${RELEASE:-agent-platform}" KIND_CLUSTER="${KIND_CLUSTER-agent-platform}" LOCAL_PORT="${LOCAL_PORT:-18080}" +LOCAL_METRICS_PORT="${LOCAL_METRICS_PORT:-19090}" +LOCAL_WORKER_METRICS_PORT="${LOCAL_WORKER_METRICS_PORT:-19091}" TAG="${TAG:-0.2.0}" # Docker-only by nature: these restart Compose services or detach container # networks. Everything else runs unchanged. -FLOWS="${FLOWS:-seed session_flow mcp_flow mcp_sse_flow mcp_stdio_flow management_flow runtime_policy_flow}" +FLOWS="${FLOWS:-seed session_flow mcp_flow mcp_sse_flow mcp_stdio_flow management_flow runtime_policy_flow metrics_flow}" cd "$REPO_ROOT" @@ -40,13 +42,39 @@ if [ -n "$KIND_CLUSTER" ]; then fi echo "==> Installing the chart" +# Generating fresh secrets on every run would rotate the Secret Store +# encryption key, leaving every credential written by an earlier run +# undecryptable, and would set a PostgreSQL password the retained volume never +# accepted. Both fail well away from their cause, so an existing install keeps +# the Secret it already has. +# +# The values are read back rather than pointed at with secrets.existingSecret, +# which would stop the chart rendering the Secret at all and so delete it. +SECRET_NAME="${RELEASE}-agent-platform-secrets" +secret_value() { + kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" -o "jsonpath={.data.$1}" | base64 -d +} +if kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" >/dev/null 2>&1; then + echo " reusing the secrets of the existing install" + SECRET_ARGS=( + --set secrets.secretEncryptionKey="$(secret_value SECRET_ENCRYPTION_KEY)" + --set secrets.workerToken="$(secret_value WORKER_TOKEN)" + --set postgresql.password="$(secret_value POSTGRES_PASSWORD)" + --set objectStore.secretKey="$(secret_value OBJECT_STORE_SECRET_KEY)" + ) +else + SECRET_ARGS=( + --set secrets.secretEncryptionKey="$(openssl rand -base64 32 | tr -d '=')" + --set secrets.workerToken="$(openssl rand -hex 24)" + --set postgresql.password="$(openssl rand -hex 16)" + --set objectStore.secretKey="$(openssl rand -hex 16)" + ) +fi + helm upgrade --install "$RELEASE" deploy/helm/agent-platform \ -n "$NAMESPACE" --create-namespace \ --set image.tag="$TAG" \ - --set secrets.secretEncryptionKey="$(openssl rand -base64 32 | tr -d '=')" \ - --set secrets.workerToken="$(openssl rand -hex 24)" \ - --set postgresql.password="$(openssl rand -hex 16)" \ - --set objectStore.secretKey="$(openssl rand -hex 16)" \ + "${SECRET_ARGS[@]}" \ --wait --timeout 10m echo "==> Deploying the fixture Provider" @@ -57,13 +85,57 @@ kubectl -n "$NAMESPACE" apply -f test/e2e/kubernetes/mock-provider.yaml kubectl -n "$NAMESPACE" rollout restart deploy/e2e-mock-provider kubectl -n "$NAMESPACE" rollout status deploy/e2e-mock-provider --timeout=180s -echo "==> Port-forwarding the console" +echo "==> Port-forwarding the console and both metrics endpoints" +FORWARD_PIDS=() +trap 'kill "${FORWARD_PIDS[@]}" 2>/dev/null || true' EXIT + kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-web" \ "${LOCAL_PORT}:8080" >/dev/null 2>&1 & -FORWARD_PID=$! -trap 'kill "$FORWARD_PID" 2>/dev/null || true' EXIT -sleep 5 +FORWARD_PIDS+=($!) +# The chart puts metrics on a port of their own on both components. Rendering +# the templates proves the wiring exists; only a scrape proves it answers, so +# the flow runs here as well as under Compose. +kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-control-plane" \ + "${LOCAL_METRICS_PORT}:9090" >/dev/null 2>&1 & +FORWARD_PIDS+=($!) +# The Worker's is headless: it exists only to give a scraper a stable target +# for a Pod that takes no inbound traffic otherwise. +kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}-agent-platform-runtime-worker-metrics" \ + "${LOCAL_WORKER_METRICS_PORT}:9090" >/dev/null 2>&1 & +FORWARD_PIDS+=($!) + +# A port-forward whose local port is already taken dies immediately, and the +# suite then talks to whatever else is listening — which looks like a platform +# failure and is not one. Catch it here instead. +sleep 2 +for pid in "${FORWARD_PIDS[@]}"; do + if ! kill -0 "$pid" 2>/dev/null; then + echo "a port-forward exited; are ${LOCAL_PORT}, ${LOCAL_METRICS_PORT}, or" \ + "${LOCAL_WORKER_METRICS_PORT} already in use?" >&2 + exit 1 + fi +done + +# A port-forward is accepted before it can carry a request, and the Control +# Plane restarts until NATS accepts connections, so a fixed sleep turns a slow +# start into a failed first flow. Wait for each endpoint to actually answer. +wait_for() { + local url="$1" name="$2" attempt + for attempt in $(seq 60); do + if curl -fsS -o /dev/null --max-time 3 "$url"; then + return 0 + fi + sleep 2 + done + echo "$name did not answer at $url" >&2 + return 1 +} +wait_for "http://localhost:${LOCAL_PORT}/api/v1/auth/config" "the console" +wait_for "http://localhost:${LOCAL_METRICS_PORT}/metrics" "control plane metrics" +wait_for "http://localhost:${LOCAL_WORKER_METRICS_PORT}/metrics" "worker metrics" echo "==> Running flows" AGENT_PLATFORM_URL="http://localhost:${LOCAL_PORT}" \ +AGENT_PLATFORM_METRICS_URL="http://localhost:${LOCAL_METRICS_PORT}" \ +AGENT_PLATFORM_WORKER_METRICS_URL="http://localhost:${LOCAL_WORKER_METRICS_PORT}" \ python3 test/e2e/run_all.py $FLOWS