From e909dac6b2c30dd48ceabe24f4b6799430fbf6f5 Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Wed, 16 Sep 2026 16:19:03 -0700 Subject: [PATCH 1/3] Update to the latest version of the hack scripts --- cmd/ate-setup/README.md | 12 +- cmd/ate-setup/cli-diff.md | 166 ++++++++ cmd/ate-setup/commands.md | 31 +- cmd/ate-setup/differences.md | 79 ++-- cmd/ate-setup/internal/cmd/create.go | 11 +- cmd/ate-setup/internal/cmd/deploy.go | 15 +- cmd/ate-setup/internal/cmd/root.go | 4 +- cmd/ate-setup/internal/config/config.go | 116 +++++- cmd/ate-setup/internal/config/config_test.go | 168 ++++++++ cmd/ate-setup/internal/demos/all/all.go | 5 + .../demos/countermicrovm/countermicrovm.go | 44 +++ cmd/ate-setup/internal/demos/egress/egress.go | 18 +- .../demos/egressmicrovm/egressmicrovm.go | 47 +++ .../egressmicrovmmitm/egressmicrovmmitm.go | 43 ++ .../internal/demos/egressmitm/egressmitm.go | 46 +++ .../internal/demos/jupyter/jupyter.go | 41 ++ cmd/ate-setup/internal/demos/render.go | 51 +++ cmd/ate-setup/internal/demos/render_test.go | 31 +- cmd/ate-setup/internal/demos/simple.go | 140 ------- cmd/ate-setup/internal/demos/substrate.go | 22 +- cmd/ate-setup/internal/kube/resources.go | 112 +++++- cmd/ate-setup/internal/steps/actors.go | 57 --- cmd/ate-setup/internal/steps/apienv.go | 258 ++++++++++++ cmd/ate-setup/internal/steps/apienv_test.go | 256 ++++++++++++ cmd/ate-setup/internal/steps/cloudsql.go | 262 +++++++++++++ cmd/ate-setup/internal/steps/cloudsql_test.go | 262 +++++++++++++ cmd/ate-setup/internal/steps/create.go | 57 ++- cmd/ate-setup/internal/steps/deploy.go | 34 +- cmd/ate-setup/internal/steps/overlay.go | 69 ++++ cmd/ate-setup/internal/steps/overlay_test.go | 129 ++++++ cmd/ate-setup/internal/steps/postgres.go | 56 ++- cmd/ate-setup/internal/steps/postgres_test.go | 49 ++- cmd/ate-setup/internal/steps/substrate.go | 2 +- cmd/ate-setup/main.go | 7 +- cmd/ate-setup/shim_test.go | 367 ++++++++++++++++++ internal/e2e/manifest.go | 8 +- 36 files changed, 2706 insertions(+), 369 deletions(-) create mode 100644 cmd/ate-setup/cli-diff.md create mode 100644 cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go create mode 100644 cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go create mode 100644 cmd/ate-setup/internal/demos/egressmicrovmmitm/egressmicrovmmitm.go create mode 100644 cmd/ate-setup/internal/demos/egressmitm/egressmitm.go create mode 100644 cmd/ate-setup/internal/demos/jupyter/jupyter.go create mode 100644 cmd/ate-setup/internal/demos/render.go delete mode 100644 cmd/ate-setup/internal/demos/simple.go create mode 100644 cmd/ate-setup/internal/steps/apienv.go create mode 100644 cmd/ate-setup/internal/steps/apienv_test.go create mode 100644 cmd/ate-setup/internal/steps/cloudsql.go create mode 100644 cmd/ate-setup/internal/steps/cloudsql_test.go create mode 100644 cmd/ate-setup/shim_test.go diff --git a/cmd/ate-setup/README.md b/cmd/ate-setup/README.md index 8ab2f6f8f1..d2ae95edfa 100644 --- a/cmd/ate-setup/README.md +++ b/cmd/ate-setup/README.md @@ -7,8 +7,9 @@ go run ./cmd/ate-setup [global flags] [flags] make build-ate-setup # builds bin/ate-setup ``` -`ate-setup` is a Go port of `hack/install-ate.sh` and the scripts it sources. -Both work today and can be used against the same cluster. +`ate-setup` is the installer. `hack/install-ate.sh` is a shim over it, kept so +that existing command lines and CI jobs keep working; it holds no install logic +of its own. ## Installing a release @@ -28,5 +29,8 @@ has to be readable from here as well as from the cluster. - [`commands.md`](commands.md) — every command with its `hack/install-ate.sh` equivalent. -- [`differences.md`](differences.md) — where the port deliberately behaves - differently, and what was reproduced exactly. +- [`differences.md`](differences.md) — where the Go port deliberately behaves + differently from the shell scripts it replaced, and what was reproduced + exactly. +- [`cli-diff.md`](cli-diff.md) — what a `hack/install-ate.sh` user can observe + after the switch. diff --git a/cmd/ate-setup/cli-diff.md b/cmd/ate-setup/cli-diff.md new file mode 100644 index 0000000000..f2558d76d7 --- /dev/null +++ b/cmd/ate-setup/cli-diff.md @@ -0,0 +1,166 @@ +# `hack/install-ate.sh` after the move to `ate-setup` + +`hack/install-ate.sh` no longer installs anything. It is a translation shim: it +parses the flags and environment variables the installer has always accepted +and runs `go run ./cmd/ate-setup ` for each action, in command-line +order. [`commands.md`](commands.md) has the flag-by-flag mapping; +[`differences.md`](differences.md) explains how the Go implementation differs +from the shell one it replaced. + +This document is for the person whose command line or CI job already worked. +Everything not listed here behaves as it did. What is listed here either could +not be reproduced through the shim, or was reproduced closely enough to keep +scripts working but not closely enough to keep a sharp-eyed reader from +noticing. + +## What the shim preserves + +Deliberately, and covered by `shim_test.go`: + +- Every flag, with the same spelling, the same `=value` and separated-value + forms, and the same "value flags may appear anywhere on the line" pre-scan. +- Several actions per invocation, executed in the order they were written. +- Every environment variable, including `SETUP_CSI`, `STORAGE_CLASS`, + `NO_DEV_ENV`, `ATE_INSTALL_KIND`, `KUBECTL_CONTEXT`, `PROJECT_ID`, and the + `ATE_API_POSTGRES_*` set. `ate-setup` reads them directly, so they need no + translation. +- `--help` / `-h` anywhere on the line, no arguments at all (usage, exit 1), + and an unrecognized flag (`Error: unknown option: …`, usage, exit 1) — + including after earlier actions on the same line have already run. +- `--deploy-ate-system --setup-csi=nfs` still sets the CSI driver up twice, + once inside the system deploy and once as its own action, as it always has. + +## Differences you can observe + +### One process per action + +Each action is a separate `go run ./cmd/ate-setup`. The build is cached after +the first, but a line asking for six actions pays six process starts, and each +one re-resolves the configuration — including, on GKE with `PROJECT_ID` set and +no `KUBECTL_CONTEXT`, a `gcloud container clusters get-credentials` per action +rather than one for the whole line. + +`go` is therefore required to run the installer. In exchange, `kubectl`, `jq`, +`openssl`, `sed`, `base64`, and `make` are not; see the external-binaries table +in [`differences.md`](differences.md#external-binaries) for what is still +needed. + +### Value flags are validated only when an action uses them + +The shell pre-scan validated `--atenet-dataplane`, `--podcert-workers-per-signer`, +`--rollout-timeout`, and `--benchmark-sandbox-class` on every run, whether or +not anything used them. Validation now happens inside `ate-setup`, so: + +- A line with no action at all — `./hack/install-ate.sh --rollout-timeout=zzz` + — exits 0 without complaining. It used to exit 1. +- `--benchmark-sandbox-class=zzz` is rejected only by `--deploy-benchmarks` / + `--delete-benchmarks`. Paired with any other action it is ignored, where it + used to fail the run up front. + +The four flags that apply to every action (`--atenet-dataplane`, +`--podcert-workers-per-signer`, `--rollout-timeout`, `--otlp-endpoint`) are +passed to every `ate-setup` invocation, so as long as the line has one action +they are still rejected before that action touches the cluster. + +### `--deploy-demo-autoscaled-workerpool` fails later, off Kind + +The demo registered itself only under `ATE_INSTALL_KIND=true`, so on GKE the +flag was rejected as an unknown option, before anything ran, and `--help` did +not list it. The cobra command tree is built before flags are parsed, so the +subcommand now always exists: `--help` always lists the demo, and off Kind it +fails when it runs, with `demo-autoscaled-workerpool is only supported for Kind +installations; re-run with --kind`. Earlier actions on the same line will +already have completed. `--delete-all` still skips it off Kind. + +### The demo list lives in two places + +`ATE_DEMOS` in the shim is a literal list, so that `--help` answers without a +Go build and an unknown demo is rejected here rather than reaching `ate-setup` +as an unknown subcommand. A demo added to `internal/demos/all` and not to that +list is invisible to the shim; `TestShimDemoListMatchesTheRegistry` fails when +the two drift. + +### Two step log lines changed + +`log.Step` names are otherwise unchanged, but two lost a parenthesized +qualifier that CI log scrapers may be matching on: + +| before | now | +|---|---| +| `demo-counter_deploy (with_external_volume=true)` | `demo-counter_deploy` | +| `setup_csi (nfs)` | `setup_csi` | + +### `--setup-csi` on its own does more + +The standalone action used to ensure the CRDs and then install the driver. +`ate-setup setup csi` also ensures the `ate-system` namespace, the +podcertificate CAs, and a ready podcertificate-controller with its trust +bundles first. The hostpath driver's ghostunnel sidecar projects a +podCertificate volume and cannot roll out without them, which is why +`deploy_ate_system` had always called CSI setup from that point in its +sequence; the standalone path just did not. + +### `.ate-dev-env.sh` is layered, not sourced over you + +The script sourced the file into its own shell, so an `export FOO=...` in the +file won over a `FOO` already exported in yours. `ate-setup` still runs the +file through bash — it can contain arbitrary shell — but treats the result as +the lowest-precedence layer: process environment first, then flags. If you +have been relying on the file to override your shell, unset the variable in +your shell instead. + +Sourcing is also skipped for Kind installs. `hack/install-ate-kind.sh` exports +`NO_DEV_ENV`, so that path is unchanged, but running the shim yourself with +`ATE_INSTALL_KIND=true` no longer reads the file. + +### `--rollout-timeout` reaches further + +It used to govern only the 60s workload rollout waits. It now also governs the +two waits fixed at 120s (podcertificate-controller and the CSI drivers) — +but only when it is passed, so the 60s default still cannot shorten those slow +bootstrap paths. + +### Applies are server-side + +`kubectl apply -f -` was a client-side apply. `ate-setup` uses server-side +apply with field manager `ate-setup` and `force: true`. Objects previously +installed by the shell script show both managers in `managedFields` until the +next apply reconciles them. `last-applied-configuration` stops growing on the +generated CRDs. + +### The first install after this change rolls `ate-api-server` once + +Both installers stamp `ate.dev/env-hash` on the pod template so a changed +environment starts a rollout, but they compute the digest differently. The +value is opaque, so the only consequence is one extra rollout the first time +`ate-setup` installs over a shell-installed cluster. + +### `--create-*-ca-pool-secret` is idempotent + +Against a cluster that already has the pool, these used to fail with +`AlreadyExists`. They now log `already exists; keeping it` and succeed. They +still refuse to overwrite: regenerating would rotate the root out from under +every certificate already issued from it. + +### `--setup-csi=hostpath` and `=both` off Kind are hard errors + +They used to warn and continue. Only the hostpath plugin is patched for the +single-node Kind layout; `nfs` is accepted anywhere, as before. + +### The per-demo scripts are gone + +`hack/install-demo-*.sh` and `hack/experimental-additional-egress-extproc.sh` +have been deleted — each demo is a Go package under +`cmd/ate-setup/internal/demos/` now. Nothing that goes through +`hack/install-ate.sh` notices, but anything sourcing those files directly, or +calling a `demo-…_deploy` shell function, has to move to +`go run ./cmd/ate-setup deploy demo `. + +## Not replicated, and not worth replicating + +Two behaviors of the old script were bugs that the shim does not reproduce: + +- `--delete-atenet` worked but was missing from `--help`. It is listed now. +- `--deploy-demo-jupyter` was printed twice in `--help`, and the + `demo-counter-microvm` flags were printed inside `demo-counter`'s block. + Each demo now gets exactly one block. diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 9b5889d17f..e17028e968 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -2,8 +2,10 @@ Every `ate-setup` command alongside the equivalent `hack/install-ate.sh` flag. -Both installers work today; `ate-setup` does not yet replace the shell scripts. -Use this table to translate an existing invocation. +`ate-setup` is the installer. `hack/install-ate.sh` is a shim that translates +the flags below onto these commands, so an existing invocation keeps working; +this table is how to write it directly, and +[`cli-diff.md`](cli-diff.md) covers what the translation does not cover. ``` go run ./cmd/ate-setup [global flags] [flags] @@ -12,7 +14,7 @@ go run ./cmd/ate-setup [global flags] [flags] `hack/install-ate.sh` accepts its flags in any order and runs one action per flag, in command line order. `ate-setup` runs exactly one command per invocation, so a shell line that passed several `--deploy-*` flags becomes -several `ate-setup` calls. +several `ate-setup` calls — which is what the shim does with it. ## Global flags @@ -27,6 +29,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--experimental-use-sdsmint` | `--experimental-use-sdsmint` | Mint TLS certificates on-demand via SDS in atenet egress gateway | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | +| `--otlp-endpoint URL` | `--otlp-endpoint URL`, or `ATE_OTLP_ENDPOINT=URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)) | | `--context NAME` | `KUBECTL_CONTEXT=NAME` | Kubeconfig context; still defaults to `KUBECTL_CONTEXT` | | `--kubeconfig PATH` | `KUBECONFIG=PATH` | Explicit kubeconfig path | | `--no-dev-env` | `NO_DEV_ENV=1` | Skip `.ate-dev-env.sh` at the repository root | @@ -80,7 +83,7 @@ that already names a manifest is used as written, and is not looked up. | `deploy apiserver` | `--deploy-ate-apiserver` | | `deploy ate-controller` | (no shell equivalent) | | `deploy atenet` | `--deploy-atenet` | -| `deploy postgres` | `--deploy-postgres` | +| `deploy postgres` | (no shell equivalent) | `deploy ate-system` is the whole control plane: CRDs, RBAC, the store, the apiserver, the controller, atenet, and atelet. It creates every `create` @@ -138,8 +141,12 @@ restriction, but it does need the `nfsd` kernel module loaded on the nodes. | `delete benchmarks` | `--delete-benchmarks` | | `--worker-count N` | `--benchmark-worker-count N` (default `1`) | | `--sandbox-class gvisor\|microvm` | `--benchmark-sandbox-class CLASS` (default `gvisor`) | +| `BENCHMARK_ACTOR_MEMORY=SIZE` | `--benchmark-actor-memory SIZE` (default `256Mi`) | -The two flags are per-command in `ate-setup` and global in +The memory limit has no flag: `benchmarking/workloads/deploy.sh` has always +taken it from the environment, and the shim exports it. + +The other two flags are per-command in `ate-setup` and global in `hack/install-ate.sh`, which forwards them to whichever benchmark action runs. See [`benchmarking/README.md`](../../benchmarking/README.md). @@ -157,7 +164,12 @@ See |---|---|---| | `deploy demo counter` | `--deploy-demo-counter` | A counter actor exercising snapshot, resume, and atenet ingress | | `deploy demo counter --with-external-volume [--storage-class NAME]` | `--deploy-demo-counter-with-external-volume` (`STORAGE_CLASS=NAME`) | The same, plus an external volume and a pre-seeded file to validate. Run `setup csi` first and name the class it created, e.g. `csi-nfs-sc`; defaults to `standard` | +| `deploy demo counter-microvm` | `--deploy-demo-counter-microvm` | The counter demo on micro-VM workers. Run `hack/install-microvm-deps.sh --install` first | | `deploy demo egress` | `--deploy-demo-egress` | Egress policy enforcement through atenet | +| `deploy demo egress-microvm` | `--deploy-demo-egress-microvm` | The same on micro-VM workers. Run `hack/install-microvm-deps.sh --install` first | +| `deploy demo egress-mitm` | `--deploy-demo-egress-mitm` | Egress with TLS interception. Needs an sdsmint install (`deploy atenet --experimental-use-sdsmint`) for the trust bundle | +| `deploy demo egress-microvm-mitm` | `--deploy-demo-egress-microvm-mitm` | Interception on micro-VM workers; needs both of the above | +| `deploy demo jupyter` | `--deploy-demo-jupyter` | A Jupyter notebook server per actor, reached through atenet ingress | | `deploy demo sandbox` | `--deploy-demo-sandbox` | An on-demand sandbox actor driven by the sandbox client | | `deploy demo multi-template` | `--deploy-demo-multi-template` | Two ActorTemplates sharing one WorkerPool | | `deploy demo parking` | `--deploy-demo-parking` | Actor parking and unparking on a small WorkerPool | @@ -170,9 +182,6 @@ to the deploy side only; teardown never reads them. The demo list is not hard-coded here — it is built from the registry in [`internal/demos`](internal/demos), one package per demo, so `go run ./cmd/ate-setup deploy demo --help` is authoritative for both the list -and the per-demo flags. - -The demos also each have a `hack/install-demo-*.sh`, sourced by -`hack/install-ate.sh`, which registers `--deploy-demo-NAME` / -`--delete-demo-NAME` flags on that installer. ate-setup does not use those -scripts; its demos live in [`internal/demos`](internal/demos). +and the per-demo flags. A new demo is a new package there, added to +[`internal/demos/all`](internal/demos/all) and mirrored into the `ATE_DEMOS` +list in `hack/install-ate.sh`, which a test keeps in step with the registry. diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index ff7fc16d84..d6ed406049 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -1,16 +1,19 @@ # ate-setup vs. the install shell scripts `ate-setup` is a Go port of `hack/install-ate.sh`, the seven -`hack/install-demo-*.sh` scripts it sources, and `hack/setup-csi-*-kind.sh`. +`hack/install-demo-*.sh` scripts it sourced, and `hack/setup-csi-*-kind.sh`. -The shell scripts are still present and still work; this is an additive -alternative to them, not yet a replacement. Retiring them behind compatibility -shims is left to a follow-up, so for now the two installers coexist and either -can be used against the same cluster. See [`commands.md`](commands.md) for the -flag-by-flag mapping between them. +The port has replaced them. The demo scripts are deleted, and +`hack/install-ate.sh` is a shim holding no install logic of its own: it +translates the flags and environment variables the installer has always +accepted onto `ate-setup` commands, so existing command lines keep working. See +[`commands.md`](commands.md) for the flag-by-flag mapping. -This document covers what changed *behind* that mapping. For anything not listed -here, the port is intended to be behavior-preserving. +This document covers what changed *behind* that mapping: the design decisions +of the port, against the shell it replaced. For what a user of +`hack/install-ate.sh` can observe after the switch, see +[`cli-diff.md`](cli-diff.md). For anything listed in neither, the port is +intended to be behavior-preserving. ## What is deliberately unchanged @@ -18,8 +21,8 @@ These were treated as contracts and reproduced exactly: - **Step log lines.** `log.Step` prints the same cyan `[step]: name` and the same step names (`deploy_ate_system`, `create_api_server_env_vars`, - `demo-counter_deploy (with_external_volume=true)`), so CI log scrapers keep - working. + `demo-counter_deploy`), so CI log scrapers keep working. Two lines lost a + parenthesized qualifier; see [`cli-diff.md`](cli-diff.md). - **Manifest ordering.** Applying a directory is non-recursive and lexical, as `kubectl apply -f ` was. `deploy_ate_system` depended on that ordering and the shell comments called out specific filename hazards, so @@ -35,8 +38,8 @@ These were treated as contracts and reproduced exactly: its historical value otherwise, so the 60s default cannot shorten the slow bootstrap paths. - **Rendered bytes.** `authentication.yaml` is trimmed of its trailing newline - because the shell built it inside `$(...)`, which strips them. Switching - between the two installers must not rewrite the ConfigMap. + because the shell built it inside `$(...)`, which strips them. Installing + over a shell-installed cluster must not rewrite the ConfigMap. - **ko's ldflags.** `make ldflags` emitted `-X=.Version=$(git describe --tags --always --dirty)`; `ko.Runner.ldflags` computes the identical string without depending on make. @@ -52,8 +55,8 @@ These were treated as contracts and reproduced exactly: | Repository root | `git rev-parse --show-toplevel`, then `cd` | walk up for `go.mod`; no `chdir`, all paths absolute | The one-action-per-run change is the most visible: a line that passed -`--deploy-ate-system --deploy-demo-counter` becomes two `ate-setup` calls. -`hack/install-ate.sh` still accepts the combined form. +`--deploy-ate-system --deploy-demo-counter` becomes two `ate-setup` calls. That +is what the shim does with it, so the combined form keeps working. Invalid input now fails before any cluster mutation. `--atenet-dataplane=nginx` used to be caught by a pre-scan validation pass; `--worker-count 0` was not @@ -234,8 +237,8 @@ instead of vanishing. **Actor cleanup uses the API directly.** `delete_demo_actors` required `jq`, listed actors with `kubectl-ate get actors -A -o json`, and filtered with a jq -expression. `steps.DeleteDemoActors` uses `internal/ateclient` and pages through -`ListActors`. The tolerances are preserved: no `ate-api-server`, or an apiserver +expression. `steps.DeleteSubstrateDemo` uses `internal/ateclient` and pages +through `ListActors`. The tolerances are preserved: no `ate-api-server`, or an apiserver that cannot be reached, skips cleanup rather than failing, because `delete all` runs this for every demo. @@ -259,23 +262,37 @@ stdout/stderr split matches under CI log capture. ## Known differences worth flagging -**`--setup-csi` on a non-Kind cluster.** Both installers now accept `nfs` off -Kind — only the hostpath plugin is patched for the single-node Kind layout, and -both reject `hostpath` and `both` there with a hard error rather than the -shell's old warn-and-continue. - -**Cloud SQL is shell-only.** `hack/install-ate.sh` automates Cloud SQL setup -with IAM authentication, synthesized DSNs, and the Auth Proxy sidecar -([`cloud-sql.md`](../../tools/setup-gcp/cloud-sql.md)). `ate-setup` does not yet -port this: while it honors `ATE_API_POSTGRES_CONNECTION_STRING` for generic -external databases, it ignores `ATE_API_POSTGRES_CLOUDSQL_*` and deploys the -bundled StatefulSet. Running `ate-setup` on a Cloud SQL cluster reverts the DSN -to the in-cluster database, leaving behind an orphaned proxy. Use -`hack/install-ate.sh` for Cloud SQL clusters until ported. +**`--setup-csi` on a non-Kind cluster.** `nfs` is accepted off Kind, as it was +— only the hostpath plugin is patched for the single-node Kind layout — but +`hostpath` and `both` are a hard error there rather than the shell's old +warn-and-continue. + +**The env-hash digest does not agree with the shell's.** Both stamp +`ate.dev/env-hash` on the `ate-api-server` pod template so that a changed +environment rolls the Deployment, but the shell computed it with `openssl +dgst -sha256` over jsonpath output and `ate-setup` digests the ConfigMap and +Secret contents directly. The value is opaque — only changes to it matter — so +the only consequence is that the first `ate-setup` install over a +shell-installed cluster rolls `ate-api-server` once. + +**Cloud SQL DSNs are synthesized, not adopted verbatim.** With +`ATE_API_POSTGRES_CLOUDSQL_INSTANCE` set and no explicit +`ATE_API_POSTGRES_CONNECTION_STRING`, the DSN is built passwordless +against the proxy's loopback listener from the GSA's email +([`cloud-sql.md`](../../tools/setup-gcp/cloud-sql.md)), so +`ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH=false` is rejected rather than producing a +DSN that cannot authenticate. An *unset* instance variable is distinct from an +exported empty one: unset adopts whatever the cluster already records in +`ate-api-server-envvars`, including the IP type and the GSA from the +`iam.gke.io/gcp-service-account` annotation, so a redeploy from a shell that +never exported the variables leaves a working proxy alone. Exporting it empty +removes the sidecar and the annotation and falls back to the bundled +StatefulSet. ## Testing The shell installer had no tests. `cmd/ate-setup` has unit tests for template rendering, overlay selection, config resolution, the authentication config, the -apiserver environment ConfigMap, delegated script arguments, manifest deletion, -per-demo rendering, and image reference rewriting. +apiserver environment ConfigMap and Secret, Cloud SQL resolution, the OTLP +endpoint override, delegated script arguments, manifest deletion, per-demo +rendering, and image reference rewriting. diff --git a/cmd/ate-setup/internal/cmd/create.go b/cmd/ate-setup/internal/cmd/create.go index 16ac5b4420..7dde06aa92 100644 --- a/cmd/ate-setup/internal/cmd/create.go +++ b/cmd/ate-setup/internal/cmd/create.go @@ -67,8 +67,15 @@ func init() { }, { use: "api-server-env-vars", - short: "Create the ate-api-server environment ConfigMap", - run: (*steps.Env).CreateAPIServerEnvVars, + short: "Create the ate-api-server environment ConfigMap and Secret", + run: func(e *steps.Env, ctx context.Context) error { + // The full deploy needs no guard: it updates the Deployment in + // the same run. + if err := e.EnsureEnvVarsSafeStandalone(ctx); err != nil { + return err + } + return e.CreateAPIServerEnvVars(ctx) + }, }, { use: "api-authentication-config", diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 0245d53e92..26f1560013 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -38,10 +38,8 @@ it signs, PostgreSQL, ate-api-server, ate-controller, the atenet dataplane, and the atelet DaemonSet, then waits for each to roll out. The bundled PostgreSQL StatefulSet is skipped when -ATE_API_POSTGRES_CONNECTION_STRING selects an external database. Cloud SQL is -not supported here — the ATE_API_POSTGRES_CLOUDSQL_* variables are ignored, so -use hack/install-ate.sh for a Cloud SQL install (see -cmd/ate-setup/differences.md). +ATE_API_POSTGRES_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* variables +select an external database. Shape the install with the global --atenet-dataplane flag.`, Args: cobra.NoArgs, @@ -94,12 +92,9 @@ var deployPostgresCmd = &cobra.Command{ Long: `Deploy the experimental single-replica PostgreSQL StatefulSet on its own. "deploy ate-system" already brings PostgreSQL up, unless -ATE_API_POSTGRES_CONNECTION_STRING selects an external database; this -subcommand is for bringing the StatefulSet up by itself. - -ate-setup has no Cloud SQL support: the ATE_API_POSTGRES_CLOUDSQL_* variables -are ignored here, so a Cloud SQL install needs hack/install-ate.sh (see -cmd/ate-setup/differences.md).`, +ATE_API_POSTGRES_CONNECTION_STRING or the ATE_API_POSTGRES_CLOUDSQL_* variables +select an external database; this subcommand is for bringing the StatefulSet up +by itself.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return env.DeployPostgres(cmd.Context()) diff --git a/cmd/ate-setup/internal/cmd/root.go b/cmd/ate-setup/internal/cmd/root.go index 21080865e1..e15d49b5be 100644 --- a/cmd/ate-setup/internal/cmd/root.go +++ b/cmd/ate-setup/internal/cmd/root.go @@ -85,7 +85,7 @@ func Execute() { func init() { f := rootCmd.PersistentFlags() f.BoolVar(&opts.Kind, "kind", false, - "Target a local Kind cluster: use the kind overlays, the local registry, and host-architecture builds") + "Target a local Kind cluster: use the kind overlays, the local registry, and host-architecture builds (or ATE_INSTALL_KIND=true)") f.StringVar(&opts.Kubeconfig, "kubeconfig", "", "Path to the kubeconfig file") f.StringVar(&opts.Context, "context", "", "Name of the kubeconfig context to use (defaults to KUBECTL_CONTEXT)") f.StringVar(&opts.Router, "atenet-dataplane", "", "Atenet ingress and egress dataplane: envoy or agentgateway (default envoy)") @@ -93,6 +93,8 @@ func init() { f.IntVar(&opts.PodcertWorkersPerSigner, "podcert-workers-per-signer", 0, "Number of worker goroutines per signer in podcertificate-controller") f.BoolVar(&opts.ExperimentalUseSDSMint, "experimental-use-sdsmint", false, "Deploy egress gateway with dynamic per-SNI certificate minting") f.StringVar(&opts.AdditionalEgressExtprocService, "experimental-additional-egress-extproc-service", "", "Run an additional ext_proc authorization filter served by NS/SVC:PORT (requires --experimental-use-sdsmint)") + f.StringVar(&opts.OtlpEndpoint, "otlp-endpoint", "", + "Send control plane telemetry to this OTLP collector instead of the cluster default (defaults to ATE_OTLP_ENDPOINT)") f.BoolVar(&opts.NoDevEnv, "no-dev-env", false, "Do not source .ate-dev-env.sh") f.StringVar(&opts.ImageRepo, "image-repo", "", diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 9b56eaf8ad..ffad1f540b 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -53,6 +53,14 @@ const DefaultPostgresConnectionString = "postgresql://postgres@postgres.ate-syst // ATE_API_POSTGRES_SCHEMA, the PostgreSQL schema holding the Substrate tables. const DefaultPostgresSchema = "public" +// Cloud SQL Auth Proxy IP types, the values ATE_API_POSTGRES_CLOUDSQL_IP_TYPE +// accepts. +const ( + CloudSQLIPTypePrivate = "private" + CloudSQLIPTypePublic = "public" + CloudSQLIPTypePSC = "psc" +) + // devEnvFile is the optional per-developer environment script at the repo root. const devEnvFile = ".ate-dev-env.sh" @@ -83,6 +91,12 @@ type Config struct { ClusterName string ClusterLocation string + // ExpectedJWTIssuer is the service account token issuer ate-api-server + // trusts (EXPECTED_JWT_ISSUER). It overrides both the GKE derivation from + // the coordinates above and OpenID discovery, for clusters whose issuer + // follows neither form. + ExpectedJWTIssuer string + // BucketName is the snapshot bucket demos are templated with. BucketName string @@ -103,6 +117,19 @@ type Config struct { // PostgresSchema is the PostgreSQL schema for the Substrate tables // (ATE_API_POSTGRES_SCHEMA). Empty means DefaultPostgresSchema. PostgresSchema string + // PostgresPoolMaxConns sizes the apiserver's pgxpool + // (ATE_API_POSTGRES_POOL_MAX_CONNS). It is spliced into the DSN rather + // than passed separately, because that is the only place pgxpool reads it + // from. Empty leaves the pgxpool default in place. + PostgresPoolMaxConns string + // PostgresServerCAFile is a local PEM file holding the server CA of an + // external PostgreSQL (ATE_API_POSTGRES_SERVER_CA_FILE). Its contents are + // published as the postgres-server-ca Secret, which ate-api-server mounts + // at /run/postgres-server-ca/server-ca.pem for sslmode=verify-ca DSNs. + PostgresServerCAFile string + // CloudSQL points the apiserver at a Cloud SQL instance through the Auth + // Proxy sidecar instead of a directly reachable PostgreSQL. + CloudSQL CloudSQLConfig // RolloutTimeout is the timeout duration for rollout status checks. RolloutTimeout time.Duration @@ -135,6 +162,33 @@ type Config struct { shellEnv map[string]string } +// CloudSQLConfig is the operator's Cloud SQL intent, as expressed by the +// ATE_API_POSTGRES_CLOUDSQL_* variables. +// +// Every field is empty-means-unspecified except Instance, which is three-way: +// a non-empty instance selects Cloud SQL, an explicitly empty one removes it, +// and an unset one (InstanceSet false) adopts whatever the target cluster +// already records. Without that distinction a redeploy from a shell that +// simply never exported the variable would tear the proxy sidecar out from +// under a working installation. +type CloudSQLConfig struct { + // Instance is the instance connection name, PROJECT:REGION:INSTANCE. + Instance string + // InstanceSet records whether ATE_API_POSTGRES_CLOUDSQL_INSTANCE was + // present in the environment at all, empty value included. + InstanceSet bool + + // GSA is the Google service account the proxy authenticates as, and whose + // email (minus the .gserviceaccount.com suffix) is the IAM database user. + GSA string + // IAMAuth enables automatic IAM database authentication ("true" or + // "false"). Empty defaults to enabled. + IAMAuth string + // IPType selects which instance address the proxy dials: one of the + // CloudSQLIPType constants. Empty defaults to private. + IPType string +} + // Options carries the raw flag values the root command collects, before // defaulting and validation. type Options struct { @@ -146,6 +200,7 @@ type Options struct { PodcertWorkersPerSigner int ExperimentalUseSDSMint bool AdditionalEgressExtprocService string + OtlpEndpoint string // Image source selection. ImageRepo string @@ -165,10 +220,14 @@ func Load(opts Options) (*Config, error) { env := environ() + // ATE_INSTALL_KIND is read as well as --kind: hack/install-ate-kind.sh + // selects the Kind profile by exporting it. + kind := opts.Kind || env["ATE_INSTALL_KIND"] == "true" + // Sourcing is skipped for Kind installs the same way the shell kind installer // exports NO_DEV_ENV: the GKE-shaped variables in a developer's file would // otherwise point a local install at a cloud project. - if !opts.NoDevEnv && !opts.Kind && os.Getenv("NO_DEV_ENV") == "" { + if !opts.NoDevEnv && !kind && os.Getenv("NO_DEV_ENV") == "" { path := filepath.Join(root, devEnvFile) if _, statErr := os.Stat(path); statErr == nil { sourced, srcErr := sourceShellEnv(path, root) @@ -208,32 +267,47 @@ func Load(opts Options) (*Config, error) { sdsmint := opts.ExperimentalUseSDSMint || env["ATE_EXPERIMENTAL_USE_SDSMINT"] == "true" extproc := firstNonEmpty(opts.AdditionalEgressExtprocService, env["ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE"]) + // Read with the two-value form: an exported but empty + // ATE_API_POSTGRES_CLOUDSQL_INSTANCE means "remove Cloud SQL", which an + // absent one does not. See CloudSQLConfig. + cloudsqlInstance, cloudsqlInstanceSet := env["ATE_API_POSTGRES_CLOUDSQL_INSTANCE"] + cfg := &Config{ - Root: root, - Kind: opts.Kind, - Kubeconfig: firstNonEmpty(opts.Kubeconfig, env["KUBECONFIG"]), - Context: firstNonEmpty(opts.Context, env["KUBECTL_CONTEXT"]), - ProjectID: env["PROJECT_ID"], - ClusterName: env["CLUSTER_NAME"], - ClusterLocation: env["CLUSTER_LOCATION"], - BucketName: env["BUCKET_NAME"], - KODockerRepo: env["KO_DOCKER_REPO"], - KODefaultPlatforms: env["KO_DEFAULTPLATFORMS"], - Images: loadImageSource(opts, env), - PostgresConnectionString: env["ATE_API_POSTGRES_CONNECTION_STRING"], - PostgresSchema: env["ATE_API_POSTGRES_SCHEMA"], + Root: root, + Kind: kind, + Kubeconfig: firstNonEmpty(opts.Kubeconfig, env["KUBECONFIG"]), + Context: firstNonEmpty(opts.Context, env["KUBECTL_CONTEXT"]), + ProjectID: env["PROJECT_ID"], + ClusterName: env["CLUSTER_NAME"], + ClusterLocation: env["CLUSTER_LOCATION"], + ExpectedJWTIssuer: env["EXPECTED_JWT_ISSUER"], + BucketName: env["BUCKET_NAME"], + KODockerRepo: env["KO_DOCKER_REPO"], + KODefaultPlatforms: env["KO_DEFAULTPLATFORMS"], + Images: loadImageSource(opts, env), + PostgresConnectionString: env["ATE_API_POSTGRES_CONNECTION_STRING"], + PostgresSchema: env["ATE_API_POSTGRES_SCHEMA"], + PostgresPoolMaxConns: env["ATE_API_POSTGRES_POOL_MAX_CONNS"], + PostgresServerCAFile: env["ATE_API_POSTGRES_SERVER_CA_FILE"], + CloudSQL: CloudSQLConfig{ + Instance: cloudsqlInstance, + InstanceSet: cloudsqlInstanceSet, + GSA: env["ATE_API_POSTGRES_CLOUDSQL_GSA"], + IAMAuth: env["ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH"], + IPType: env["ATE_API_POSTGRES_CLOUDSQL_IP_TYPE"], + }, RolloutTimeout: rolloutTimeout, rolloutTimeoutSet: timeoutStr != "", PodcertWorkersPerSigner: podcertWorkers, ExperimentalUseSDSMint: sdsmint, AdditionalEgressExtprocService: extproc, AnthropicAPIKey: env["ANTHROPIC_API_KEY"], - OtlpEndpoint: env["ATE_OTLP_ENDPOINT"], + OtlpEndpoint: firstNonEmpty(opts.OtlpEndpoint, env["ATE_OTLP_ENDPOINT"]), BenchmarkActorMemory: env["BENCHMARK_ACTOR_MEMORY"], shellEnv: env, } - if opts.Kind { + if kind { applyKindDefaults(cfg) } @@ -275,6 +349,15 @@ func validate(cfg *Config) error { if cfg.PodcertWorkersPerSigner < 0 { return fmt.Errorf("--podcert-workers-per-signer must be a positive integer, got %d", cfg.PodcertWorkersPerSigner) } + // Only an explicitly supplied value is checked. One adopted from the + // cluster is derived from the recorded CSQL_PROXY_* keys and so is always + // one of these by construction. + switch cfg.CloudSQL.IPType { + case "", CloudSQLIPTypePrivate, CloudSQLIPTypePublic, CloudSQLIPTypePSC: + default: + return fmt.Errorf("ATE_API_POSTGRES_CLOUDSQL_IP_TYPE must be %s, %s, or %s, got %q", + CloudSQLIPTypePrivate, CloudSQLIPTypePublic, CloudSQLIPTypePSC, cfg.CloudSQL.IPType) + } if cfg.AdditionalEgressExtprocService != "" { if err := validateExtprocService(cfg.AdditionalEgressExtprocService); err != nil { return err @@ -398,6 +481,7 @@ func (c *Config) ScriptEnv() []string { "PROJECT_ID": c.ProjectID, "CLUSTER_NAME": c.ClusterName, "CLUSTER_LOCATION": c.ClusterLocation, + "ATE_OTLP_ENDPOINT": c.OtlpEndpoint, } { if value == "" { // An empty value means "not configured". Leaving the variable set diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 6080d9e97d..5338103e88 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -40,12 +40,18 @@ func loadEnv(t *testing.T) { for _, name := range []string{ "ANTHROPIC_API_KEY", "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", + "ATE_API_POSTGRES_CLOUDSQL_GSA", + "ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH", + "ATE_API_POSTGRES_CLOUDSQL_IP_TYPE", "ATE_API_POSTGRES_CONNECTION_STRING", + "ATE_API_POSTGRES_POOL_MAX_CONNS", "ATE_API_POSTGRES_SCHEMA", + "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", "ATE_EXPERIMENTAL_USE_SDSMINT", "ATE_IMAGE_REPO", "ATE_IMAGE_TAG", + "ATE_INSTALL_KIND", "ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER", "ATE_INSTALL_ROLLOUT_TIMEOUT", "ATE_OTLP_ENDPOINT", @@ -53,6 +59,7 @@ func loadEnv(t *testing.T) { "BUCKET_NAME", "CLUSTER_LOCATION", "CLUSTER_NAME", + "EXPECTED_JWT_ISSUER", "KIND_CLUSTER_NAME", "KO_DEFAULTPLATFORMS", "KO_DOCKER_REPO", @@ -63,6 +70,19 @@ func loadEnv(t *testing.T) { } { t.Setenv(name, "") } + // Blanking this one would not read as unset: an exported but empty + // instance is the explicit "remove Cloud SQL" request. + unsetEnv(t, "ATE_API_POSTGRES_CLOUDSQL_INSTANCE") +} + +// unsetEnv removes a variable for the duration of the test. t.Setenv first, so +// that its cleanup restores whatever the caller's environment had. +func unsetEnv(t *testing.T, name string) { + t.Helper() + t.Setenv(name, "") + if err := os.Unsetenv(name); err != nil { + t.Fatalf("os.Unsetenv(%s) = %v", name, err) + } } func TestLoadDefaults(t *testing.T) { @@ -138,6 +158,154 @@ func TestLoadPostgresSchema(t *testing.T) { } } +// The apiserver reads its pool size and its server CA out of the DSN and a +// mounted file respectively, neither of which the shell installer synthesizes. +func TestLoadPostgresTuning(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_API_POSTGRES_POOL_MAX_CONNS", "50") + t.Setenv("ATE_API_POSTGRES_SERVER_CA_FILE", "/etc/ssl/server-ca.pem") + + cfg, err := Load(Options{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.PostgresPoolMaxConns != "50" { + t.Errorf("PostgresPoolMaxConns = %q, want 50", cfg.PostgresPoolMaxConns) + } + if want := "/etc/ssl/server-ca.pem"; cfg.PostgresServerCAFile != want { + t.Errorf("PostgresServerCAFile = %q, want %q", cfg.PostgresServerCAFile, want) + } +} + +// ATE_API_POSTGRES_CLOUDSQL_INSTANCE is three-way, and Load is where the +// distinction is made: everything downstream sees only Instance and +// InstanceSet. +func TestLoadCloudSQL(t *testing.T) { + t.Run("unset", func(t *testing.T) { + loadEnv(t) + cfg, err := Load(Options{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.CloudSQL.InstanceSet { + t.Errorf("CloudSQL = %+v, want InstanceSet false so the cluster's record is adopted", cfg.CloudSQL) + } + }) + + t.Run("exported but empty removes Cloud SQL", func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_API_POSTGRES_CLOUDSQL_INSTANCE", "") + cfg, err := Load(Options{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !cfg.CloudSQL.InstanceSet || cfg.CloudSQL.Instance != "" { + t.Errorf("CloudSQL = %+v, want an explicitly empty instance", cfg.CloudSQL) + } + }) + + t.Run("fully specified", func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_API_POSTGRES_CLOUDSQL_INSTANCE", "p:r:i") + t.Setenv("ATE_API_POSTGRES_CLOUDSQL_GSA", "ate@p.iam.gserviceaccount.com") + t.Setenv("ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH", "false") + t.Setenv("ATE_API_POSTGRES_CLOUDSQL_IP_TYPE", CloudSQLIPTypePSC) + + cfg, err := Load(Options{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + want := CloudSQLConfig{ + Instance: "p:r:i", + InstanceSet: true, + GSA: "ate@p.iam.gserviceaccount.com", + IAMAuth: "false", + IPType: CloudSQLIPTypePSC, + } + if cfg.CloudSQL != want { + t.Errorf("CloudSQL = %+v, want %+v", cfg.CloudSQL, want) + } + }) + + // An unrecognized IP type reaches the proxy as an unset flag, which silently + // dials the public address instead of the private one that was meant. + t.Run("rejects an unknown IP type", func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_API_POSTGRES_CLOUDSQL_IP_TYPE", "internal") + if _, err := Load(Options{}); err == nil || !strings.Contains(err.Error(), "ATE_API_POSTGRES_CLOUDSQL_IP_TYPE") { + t.Fatalf("Load() error = %v, want it to name the invalid IP type", err) + } + }) +} + +// EXPECTED_JWT_ISSUER overrides the issuer derived from the GKE coordinates, +// which is how a cluster authenticating against something other than its own +// OIDC discovery document is installed. +func TestLoadExpectedJWTIssuer(t *testing.T) { + loadEnv(t) + const issuer = "https://issuer.example.com" + t.Setenv("EXPECTED_JWT_ISSUER", issuer) + + cfg, err := Load(Options{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.ExpectedJWTIssuer != issuer { + t.Errorf("ExpectedJWTIssuer = %q, want %q", cfg.ExpectedJWTIssuer, issuer) + } +} + +// The endpoint has to reach both the Go steps and the shell scripts ate-setup +// still delegates to, or the two halves of an install export different +// collectors. +func TestLoadOtlpEndpoint(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_OTLP_ENDPOINT", "http://from-environment:4317") + + cfg, err := Load(Options{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if want := "http://from-environment:4317"; cfg.OtlpEndpoint != want { + t.Errorf("OtlpEndpoint = %q, want %q", cfg.OtlpEndpoint, want) + } + + cfg, err = Load(Options{OtlpEndpoint: "http://from-flag:4317"}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if want := "http://from-flag:4317"; cfg.OtlpEndpoint != want { + t.Errorf("OtlpEndpoint = %q, want %q", cfg.OtlpEndpoint, want) + } + if got := scriptEnvMap(t, cfg)["ATE_OTLP_ENDPOINT"]; got != "http://from-flag:4317" { + t.Errorf("ScriptEnv()[ATE_OTLP_ENDPOINT] = %q, want the flag's value", got) + } +} + +// hack/install-ate-kind.sh exports ATE_INSTALL_KIND rather than passing a flag, +// so the environment has to select the Kind profile as completely as --kind +// does; a Kind install that only half-applied would push images to the wrong +// registry. +func TestLoadKindFromEnvironment(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_INSTALL_KIND", "true") + t.Setenv("PROJECT_ID", "some-project") + + cfg, err := Load(Options{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !cfg.Kind { + t.Error("Kind = false, want true") + } + if cfg.KODockerRepo != "localhost:5001" { + t.Errorf("KODockerRepo = %q, want the Kind default", cfg.KODockerRepo) + } + if cfg.ProjectID != "" { + t.Errorf("ProjectID = %q, want it cleared by the Kind profile", cfg.ProjectID) + } +} + // ate-setup's own client resolves $KUBECONFIG through the client-go loading // rules, so the value has to reach ScriptEnv as well. Otherwise a developer who // exports KUBECONFIG without passing --kubeconfig gets an install split across diff --git a/cmd/ate-setup/internal/demos/all/all.go b/cmd/ate-setup/internal/demos/all/all.go index 68387c8b42..4b1fa244f4 100644 --- a/cmd/ate-setup/internal/demos/all/all.go +++ b/cmd/ate-setup/internal/demos/all/all.go @@ -24,7 +24,12 @@ import ( _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/autoscaledworkerpool" _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/claudemultiplex" _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/counter" + _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/countermicrovm" _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/egress" + _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/egressmicrovm" + _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/egressmicrovmmitm" + _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/egressmitm" + _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/jupyter" _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/multitemplate" _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/parking" _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/sandbox" diff --git a/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go new file mode 100644 index 0000000000..e23225370a --- /dev/null +++ b/cmd/ate-setup/internal/demos/countermicrovm/countermicrovm.go @@ -0,0 +1,44 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package countermicrovm installs the micro-VM variant of the counter demo. +// +// It needs the cluster-wide `microvm` SandboxConfig that +// hack/install-microvm-deps.sh --install creates. It has no external-volume +// option: the CSI path is covered by demo-counter. +package countermicrovm + +import ( + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "github.com/agent-substrate/substrate/internal/resources" +) + +// namespace is the pool's k8s namespace; it doubles as the atespace holding +// the demo's ActorTemplate. +const namespace = "ate-demo-counter-microvm" + +func init() { + demos.Register(&demos.Substrate{ + DemoName: "demo-counter-microvm", + Short: "The counter demo on micro-VM workers (needs hack/install-microvm-deps.sh --install)", + WorkerPoolManifest: "demos/counter/counter-microvm.yaml.tmpl", + Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "counter-microvm"}}, + Templates: []demos.SubstrateTemplate{{ + Manifest: "demos/counter/counter-microvm-template.yaml.tmpl", + Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "counter-microvm"}, + }}, + GoldenTimeout: demos.MicroVMGoldenTimeout, + }) +} diff --git a/cmd/ate-setup/internal/demos/egress/egress.go b/cmd/ate-setup/internal/demos/egress/egress.go index 768014c4c2..f681fc1a05 100644 --- a/cmd/ate-setup/internal/demos/egress/egress.go +++ b/cmd/ate-setup/internal/demos/egress/egress.go @@ -19,16 +19,22 @@ package egress import ( "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "github.com/agent-substrate/substrate/internal/resources" ) +// namespace is the pool's k8s namespace; it doubles as the atespace holding +// the demo's ActorTemplate. const namespace = "ate-demo-egress" func init() { - demos.Register(&demos.Simple{ - DemoName: "demo-egress", - Short: "Egress policy enforcement through atenet", - Template: "demos/egress/egress.yaml.tmpl", - Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress"}}, - ActorTemplates: []steps.TemplateRef{{Atespace: namespace, Name: "egress"}}, + demos.Register(&demos.Substrate{ + DemoName: "demo-egress", + Short: "Egress policy enforcement through atenet", + WorkerPoolManifest: "demos/egress/egress.yaml.tmpl", + Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress"}}, + Templates: []demos.SubstrateTemplate{{ + Manifest: "demos/egress/egress-template.yaml.tmpl", + Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "egress"}, + }}, }) } diff --git a/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go new file mode 100644 index 0000000000..67f8c4342a --- /dev/null +++ b/cmd/ate-setup/internal/demos/egressmicrovm/egressmicrovm.go @@ -0,0 +1,47 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package egressmicrovm installs the micro-VM variant of the egress demo. +// +// It needs the cluster-wide `microvm` SandboxConfig that +// hack/install-microvm-deps.sh --install creates. It is a demo of its own +// rather than a flag on demo-egress so that it appears in help and in the +// `delete all` sweep, and so that both can be installed side by side: the +// networking e2e suite runs against whichever the sandbox class under test +// selects. +package egressmicrovm + +import ( + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "github.com/agent-substrate/substrate/internal/resources" +) + +// namespace is the pool's k8s namespace; it doubles as the atespace holding +// the demo's ActorTemplate. +const namespace = "ate-demo-egress-microvm" + +func init() { + demos.Register(&demos.Substrate{ + DemoName: "demo-egress-microvm", + Short: "Egress policy enforcement on micro-VM workers (needs hack/install-microvm-deps.sh --install)", + WorkerPoolManifest: "demos/egress/egress-microvm.yaml.tmpl", + Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress-microvm"}}, + Templates: []demos.SubstrateTemplate{{ + Manifest: "demos/egress/egress-microvm-template.yaml.tmpl", + Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "egress-microvm"}, + }}, + GoldenTimeout: demos.MicroVMGoldenTimeout, + }) +} diff --git a/cmd/ate-setup/internal/demos/egressmicrovmmitm/egressmicrovmmitm.go b/cmd/ate-setup/internal/demos/egressmicrovmmitm/egressmicrovmmitm.go new file mode 100644 index 0000000000..a53c519834 --- /dev/null +++ b/cmd/ate-setup/internal/demos/egressmicrovmmitm/egressmicrovmmitm.go @@ -0,0 +1,43 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package egressmicrovmmitm installs the micro-VM MITM variant of the egress +// demo. It needs both what demo-egress-microvm needs (the cluster-wide +// `microvm` SandboxConfig from hack/install-microvm-deps.sh --install) and +// what demo-egress-mitm needs (an sdsmint install, for the trust bundle). +package egressmicrovmmitm + +import ( + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "github.com/agent-substrate/substrate/internal/resources" +) + +// namespace is the pool's k8s namespace; it doubles as the atespace holding +// the demo's ActorTemplate. +const namespace = "ate-demo-egress-microvm-mitm" + +func init() { + demos.Register(&demos.Substrate{ + DemoName: "demo-egress-microvm-mitm", + Short: "Egress MITM inspection on micro-VM workers (needs install-microvm-deps.sh and --experimental-use-sdsmint)", + WorkerPoolManifest: "demos/egress/egress-microvm-mitm.yaml.tmpl", + Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress-microvm-mitm"}}, + Templates: []demos.SubstrateTemplate{{ + Manifest: "demos/egress/egress-microvm-mitm-template.yaml.tmpl", + Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "egress-microvm-mitm"}, + }}, + GoldenTimeout: demos.MicroVMGoldenTimeout, + }) +} diff --git a/cmd/ate-setup/internal/demos/egressmitm/egressmitm.go b/cmd/ate-setup/internal/demos/egressmitm/egressmitm.go new file mode 100644 index 0000000000..af026cbd2d --- /dev/null +++ b/cmd/ate-setup/internal/demos/egressmitm/egressmitm.go @@ -0,0 +1,46 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package egressmitm installs the MITM variant of the egress demo. +// +// Its actors project the egress gateway trust bundle, which only resolves on +// an sdsmint install (deploy atenet --experimental-use-sdsmint), so it cannot +// be part of what a passthrough install deploys. A golden snapshot only exists +// once an actor starts, and an actor whose trust bundle does not resolve never +// does, so a timeout waiting for the golden is the symptom of a missing +// sdsmint install. +package egressmitm + +import ( + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "github.com/agent-substrate/substrate/internal/resources" +) + +// namespace is the pool's k8s namespace; it doubles as the atespace holding +// the demo's ActorTemplate. +const namespace = "ate-demo-egress-mitm" + +func init() { + demos.Register(&demos.Substrate{ + DemoName: "demo-egress-mitm", + Short: "Egress MITM inspection through atenet (needs an --experimental-use-sdsmint install)", + WorkerPoolManifest: "demos/egress/egress-mitm.yaml.tmpl", + Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "egress-mitm"}}, + Templates: []demos.SubstrateTemplate{{ + Manifest: "demos/egress/egress-mitm-template.yaml.tmpl", + Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "egress-mitm"}, + }}, + }) +} diff --git a/cmd/ate-setup/internal/demos/jupyter/jupyter.go b/cmd/ate-setup/internal/demos/jupyter/jupyter.go new file mode 100644 index 0000000000..b18bb4fddc --- /dev/null +++ b/cmd/ate-setup/internal/demos/jupyter/jupyter.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package jupyter installs the Jupyter demo, which runs an unmodified +// jupyter/base-notebook image as an actor that suspends when idle and resumes +// when the notebook is next opened. +package jupyter + +import ( + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" + "github.com/agent-substrate/substrate/internal/resources" +) + +// namespace is the pool's k8s namespace; it doubles as the atespace holding +// the demo's ActorTemplate. +const namespace = "ate-demo-jupyter" + +func init() { + demos.Register(&demos.Substrate{ + DemoName: "demo-jupyter", + Short: "An unmodified Jupyter notebook image as a suspending actor", + WorkerPoolManifest: "demos/jupyter/jupyter.yaml.tmpl", + Deployments: []steps.TemplateRef{{Atespace: namespace, Name: "jupyter"}}, + Templates: []demos.SubstrateTemplate{{ + Manifest: "demos/jupyter/jupyter-template.yaml.tmpl", + Ref: resources.ActorTemplateRef{Atespace: namespace, Name: "jupyter"}, + }}, + }) +} diff --git a/cmd/ate-setup/internal/demos/render.go b/cmd/ate-setup/internal/demos/render.go new file mode 100644 index 0000000000..970ecab412 --- /dev/null +++ b/cmd/ate-setup/internal/demos/render.go @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package demos + +import ( + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/render" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" +) + +// bucketNamePlaceholder is substituted into every demo template with the +// snapshot bucket for this environment. +const bucketNamePlaceholder = "BUCKET_NAME" + +// ExternalVolumePlaceholders are the optional external-volume hooks in the +// counter and autoscaled-workerpool templates. Every path except +// `deploy demo counter --with-external-volume` drops them, which removes the +// lines entirely. +var ExternalVolumePlaceholders = []string{ + "VALIDATE_EXISTING_FILE_PATH_ARG", + "EXTERNAL_VOLUME_MOUNTS", + "EXTERNAL_VOLUMES", +} + +// Render expands a demo template with the configured bucket name and the +// build version (pool templates pin worker pods to version-labeled nodes). +func Render(e *steps.Env, relPath string, extraValues map[string]string, drop []string) ([]byte, error) { + version, _, err := e.SubstrateVersion() + if err != nil { + return nil, err + } + values := map[string]string{ + bucketNamePlaceholder: e.Cfg.BucketName, + "SUBSTRATE_VERSION": version, + } + for k, v := range extraValues { + values[k] = v + } + return render.Template(e.Cfg.Path(relPath), values, drop) +} diff --git a/cmd/ate-setup/internal/demos/render_test.go b/cmd/ate-setup/internal/demos/render_test.go index 04e7222a16..144ea11e77 100644 --- a/cmd/ate-setup/internal/demos/render_test.go +++ b/cmd/ate-setup/internal/demos/render_test.go @@ -46,25 +46,20 @@ func TestDemoTemplatesRender(t *testing.T) { covered := 0 for _, demo := range demos.All() { - switch d := demo.(type) { - case interface{ SubstrateDemo() *demos.Substrate }: - covered++ - t.Run(demo.Name(), func(t *testing.T) { - s := d.SubstrateDemo() - demotest.AssertRendered(t, render(t, s.WorkerPoolManifest)) - for _, tmpl := range s.Templates { - demotest.AssertRenderedActorTemplate(t, render(t, tmpl.Manifest), tmpl.Ref) - } - }) - case interface{ TemplatePath() string }: - covered++ - t.Run(demo.Name(), func(t *testing.T) { - demotest.AssertRendered(t, render(t, d.TemplatePath())) - }) - default: - // demo-claude-code-multiplex has its own placeholders, and is - // covered by its own package's test. + // demo-claude-code-multiplex has its own placeholders, and is covered + // by its own package's test; it is the one demo this skips. + d, ok := demo.(interface{ SubstrateDemo() *demos.Substrate }) + if !ok { + continue } + covered++ + t.Run(demo.Name(), func(t *testing.T) { + s := d.SubstrateDemo() + demotest.AssertRendered(t, render(t, s.WorkerPoolManifest)) + for _, tmpl := range s.Templates { + demotest.AssertRenderedActorTemplate(t, render(t, tmpl.Manifest), tmpl.Ref) + } + }) } if want := len(demos.All()) - 1; covered != want { t.Errorf("covered %d demo templates, want %d", covered, want) diff --git a/cmd/ate-setup/internal/demos/simple.go b/cmd/ate-setup/internal/demos/simple.go deleted file mode 100644 index d4c0bc69f1..0000000000 --- a/cmd/ate-setup/internal/demos/simple.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package demos - -import ( - "context" - - "github.com/spf13/pflag" - - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/render" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/steps" -) - -// bucketNamePlaceholder is substituted into every demo template with the -// snapshot bucket for this environment. -const bucketNamePlaceholder = "BUCKET_NAME" - -// ExternalVolumePlaceholders are the optional external-volume hooks in the -// counter and autoscaled-workerpool templates. Every path except -// `deploy demo counter --with-external-volume` drops them, which removes the -// lines entirely. -var ExternalVolumePlaceholders = []string{ - "VALIDATE_EXISTING_FILE_PATH_ARG", - "EXTERNAL_VOLUME_MOUNTS", - "EXTERNAL_VOLUMES", -} - -// Render expands a demo template with the configured bucket name and the -// build version (pool templates pin worker pods to version-labeled nodes). -func Render(e *steps.Env, relPath string, extraValues map[string]string, drop []string) ([]byte, error) { - version, _, err := e.SubstrateVersion() - if err != nil { - return nil, err - } - values := map[string]string{ - bucketNamePlaceholder: e.Cfg.BucketName, - "SUBSTRATE_VERSION": version, - } - for k, v := range extraValues { - values[k] = v - } - return render.Template(e.Cfg.Path(relPath), values, drop) -} - -// Simple covers the demos that are one template plus a fixed set of readiness -// waits: render, ko apply, wait; and on delete, remove the actors then the same -// rendered manifest. -// -// Demos that need more embed it and override a method, calling back into -// DeployWorkload and WaitReady to keep the shared ordering. -type Simple struct { - // DemoName is the registry name, e.g. "demo-counter". It cannot be called - // Name: that is the accessor the Demo interface requires. - DemoName string - // Short is the one-line summary, in cobra's sense of the word. - Short string - - // Template is the *.yaml.tmpl path, relative to the repository root. - Template string - - // Deployments are the Deployments to wait for at deploy time, in order. - // The WorkerPool controller names each Deployment after its WorkerPool. - Deployments []steps.TemplateRef - - // ActorTemplates are the demo's ActorTemplates. Their actors are removed - // before the manifests at delete time. - ActorTemplates []steps.TemplateRef -} - -func (d *Simple) Name() string { return d.DemoName } -func (d *Simple) Description() string { return d.Short } - -// Flags registers nothing: most demos take no options. -func (d *Simple) Flags(*pflag.FlagSet) {} - -// TemplatePath exposes the demo's template through the Demo interface, so tests -// can check that every template renders cleanly. -func (d *Simple) TemplatePath() string { return d.Template } - -func (d *Simple) Deploy(ctx context.Context, e *steps.Env) error { - log.Step(d.DemoName + "_deploy") - if err := e.EnsureCRDs(ctx); err != nil { - return err - } - if err := d.DeployWorkload(ctx, e); err != nil { - return err - } - return d.WaitReady(ctx, e) -} - -// DeployWorkload renders the demo template and applies it through ko, without -// waiting. Demos that install add-ons alongside the workload call this directly -// so they can order the add-ons against it. -func (d *Simple) DeployWorkload(ctx context.Context, e *steps.Env) error { - manifest, err := Render(e, d.Template, nil, ExternalVolumePlaceholders) - if err != nil { - return err - } - return e.ResolveAndApplyBytes(ctx, manifest) -} - -// WaitReady blocks until the demo's Deployments are rolled out. -func (d *Simple) WaitReady(ctx context.Context, e *steps.Env) error { - if len(d.Deployments) == 0 { - return nil - } - log.Stepf("Waiting for %s to be ready...", d.DemoName) - for _, ref := range d.Deployments { - if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, ref.Atespace, ref.Name, steps.DemoTimeout); err != nil { - return err - } - } - return nil -} - -func (d *Simple) Delete(ctx context.Context, e *steps.Env) error { - log.Step(d.DemoName + "_delete") - if err := e.DeleteDemoActors(ctx, d.ActorTemplates...); err != nil { - return err - } - manifest, err := Render(e, d.Template, nil, ExternalVolumePlaceholders) - if err != nil { - return err - } - return e.Kube.DeleteBytes(ctx, manifest) -} diff --git a/cmd/ate-setup/internal/demos/substrate.go b/cmd/ate-setup/internal/demos/substrate.go index 9960d381d5..742ec854c0 100644 --- a/cmd/ate-setup/internal/demos/substrate.go +++ b/cmd/ate-setup/internal/demos/substrate.go @@ -16,6 +16,7 @@ package demos import ( "context" + "time" "github.com/spf13/pflag" @@ -25,6 +26,12 @@ import ( "github.com/agent-substrate/substrate/internal/resources" ) +// MicroVMGoldenTimeout is the golden-snapshot budget the micro-VM demos set as +// their [Substrate.GoldenTimeout]: a micro-VM golden is a cloud-hypervisor cold +// boot plus a checkpoint, on nested KVM in CI, which does not fit in +// [steps.DemoTimeout]. +const MicroVMGoldenTimeout = 600 * time.Second + // SubstrateTemplate names one protojson ActorTemplate manifest and the // resource it creates. type SubstrateTemplate struct { @@ -37,8 +44,8 @@ type SubstrateTemplate struct { // Substrate covers demos in the substrate-resource shape: one CRD manifest // for the namespace and worker pool, plus ActorTemplates created through the -// ate API. It is the Go counterpart of deploy_substrate_demo / -// delete_substrate_demo in hack/install-ate.sh. +// ate API. Most demos are one of these; a demo needing more supplies its own +// Deploy and Delete. type Substrate struct { // DemoName is the registry name, e.g. "demo-parking". DemoName string @@ -57,6 +64,11 @@ type Substrate struct { // Templates are the demo's ActorTemplates, created in order. Templates []SubstrateTemplate + // GoldenTimeout budgets the wait for each template's golden snapshot. + // Zero means [steps.DemoTimeout]. Micro-VM demos need more: their golden + // is a cloud-hypervisor cold boot plus a checkpoint, on nested KVM in CI. + GoldenTimeout time.Duration + // RenderValues optionally supplies extra placeholder values at deploy // time, for demos whose manifests need more than ${BUCKET_NAME} (e.g. a // freshly built image digest). Delete renders only the pool manifest, so @@ -162,9 +174,13 @@ func (d *Substrate) Deploy(ctx context.Context, e *steps.Env) error { // ActorTemplate pays one-time costs (downloading runsc, the first gVisor // pod start, image pulls); blocking here means callers run against an // already-warm node instead of racing that cold-start work. + golden := d.GoldenTimeout + if golden == 0 { + golden = steps.DemoTimeout + } for _, t := range d.Templates { log.Stepf("Waiting for the %s golden snapshot...", t.Ref) - if err := steps.WaitActorTemplateGolden(ctx, client, t.Ref, steps.DemoTimeout); err != nil { + if err := steps.WaitActorTemplateGolden(ctx, client, t.Ref, golden); err != nil { return err } } diff --git a/cmd/ate-setup/internal/kube/resources.go b/cmd/ate-setup/internal/kube/resources.go index 3889585702..8319b6349e 100644 --- a/cmd/ate-setup/internal/kube/resources.go +++ b/cmd/ate-setup/internal/kube/resources.go @@ -19,10 +19,13 @@ import ( "encoding/json" "fmt" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" applyconfigcorev1 "k8s.io/client-go/applyconfigurations/core/v1" + "sigs.k8s.io/yaml" ) // applyOptions are shared by the typed apply helpers below. @@ -59,6 +62,34 @@ func (c *Client) ApplyConfigMap(ctx context.Context, namespace, name string, dat return nil } +// GetConfigMap returns a ConfigMap, or nil when it does not exist. +func (c *Client) GetConfigMap(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error) { + cm, err := c.Typed.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("while getting configmap %s/%s: %w", namespace, name, err) + } + return cm, nil +} + +// MergePatchConfigMap merges keys into a ConfigMap's data, leaving the rest of +// it alone. Unlike an apply this claims no ownership of the keys it does not +// name, which is what lets the otel endpoint override amend a ConfigMap the +// bundle owns. +func (c *Client) MergePatchConfigMap(ctx context.Context, namespace, name string, data map[string]string) error { + patch, err := json.Marshal(map[string]any{"data": data}) + if err != nil { + return fmt.Errorf("while building the patch for configmap %s/%s: %w", namespace, name, err) + } + if _, err := c.Typed.CoreV1().ConfigMaps(namespace).Patch( + ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("while patching configmap %s/%s: %w", namespace, name, err) + } + return nil +} + // GetSecret returns a Secret, or nil when it does not exist. func (c *Client) GetSecret(ctx context.Context, namespace, name string) (*corev1.Secret, error) { secret, err := c.Typed.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) @@ -89,18 +120,91 @@ func (c *Client) ConfigMapExists(ctx context.Context, namespace, name string) (b return true, nil } +// GetDeployment returns a Deployment, or nil when it does not exist. +func (c *Client) GetDeployment(ctx context.Context, namespace, name string) (*appsv1.Deployment, error) { + dep, err := c.Typed.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("while getting deployment %s/%s: %w", namespace, name, err) + } + return dep, nil +} + // DeploymentExists reports whether a Deployment is present. delete_demo_actors // used this to decide whether the control plane is still up before trying to // talk to it. func (c *Client) DeploymentExists(ctx context.Context, namespace, name string) (bool, error) { - _, err := c.Typed.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + dep, err := c.GetDeployment(ctx, namespace, name) + return dep != nil, err +} + +// PatchDeployment applies a strategic merge patch to a Deployment. This is the +// `kubectl patch deployment` of the shell installer, including its +// --patch-file form: patch may be JSON or YAML. +func (c *Client) PatchDeployment(ctx context.Context, namespace, name string, patch []byte) error { + asJSON, err := yaml.YAMLToJSON(patch) if err != nil { + return fmt.Errorf("while parsing the patch for deployment %s/%s: %w", namespace, name, err) + } + if _, err := c.Typed.AppsV1().Deployments(namespace).Patch( + ctx, name, types.StrategicMergePatchType, asJSON, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("while patching deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +// DaemonSetNames lists the DaemonSets in a namespace matching a label +// selector. The atelet DaemonSet name carries a substrate version suffix, so +// the installed versions can only be found by label. +func (c *Client) DaemonSetNames(ctx context.Context, namespace, selector string) ([]string, error) { + list, err := c.Typed.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, fmt.Errorf("while listing daemonsets in %s matching %q: %w", namespace, selector, err) + } + names := make([]string, 0, len(list.Items)) + for _, ds := range list.Items { + names = append(names, ds.Name) + } + return names, nil +} + +// SetServiceAccountAnnotation adds or removes one annotation on a +// ServiceAccount; an empty value removes it. A missing ServiceAccount is not +// an error, matching the `|| true` the shell installer removed one under. +func (c *Client) SetServiceAccountAnnotation(ctx context.Context, namespace, name, key, value string) error { + var annotation any + if value != "" { + annotation = value + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": map[string]any{key: annotation}}, + }) + if err != nil { + return fmt.Errorf("while building the patch for serviceaccount %s/%s: %w", namespace, name, err) + } + if _, err := c.Typed.CoreV1().ServiceAccounts(namespace).Patch( + ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { if apierrors.IsNotFound(err) { - return false, nil + return nil } - return false, fmt.Errorf("while getting deployment %s/%s: %w", namespace, name, err) + return fmt.Errorf("while annotating serviceaccount %s/%s: %w", namespace, name, err) } - return true, nil + return nil +} + +// ServiceAccountAnnotation reads one annotation from a ServiceAccount. A +// missing ServiceAccount or annotation yields the empty string. +func (c *Client) ServiceAccountAnnotation(ctx context.Context, namespace, name, key string) (string, error) { + sa, err := c.Typed.CoreV1().ServiceAccounts(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + return "", fmt.Errorf("while getting serviceaccount %s/%s: %w", namespace, name, err) + } + return sa.Annotations[key], nil } // OIDCIssuer reads the cluster's OpenID configuration and returns its issuer. diff --git a/cmd/ate-setup/internal/steps/actors.go b/cmd/ate-setup/internal/steps/actors.go index 52437cc9e4..7c23d298d1 100644 --- a/cmd/ate-setup/internal/steps/actors.go +++ b/cmd/ate-setup/internal/steps/actors.go @@ -17,9 +17,7 @@ package steps import ( "context" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -34,61 +32,6 @@ type TemplateRef struct { Name string } -// DeleteDemoActors removes every actor created from the given ActorTemplates. -// -// Demo teardown has to do this before deleting the manifests: an ActorTemplate -// removed out from under running actors leaves them stranded. As in the shell -// version, a cluster with no ate-api-server, or an apiserver that cannot be -// reached, is not an error -- there is nothing to clean up on a cluster that -// never had the control plane, and DeleteAll runs this for every demo. -func (e *Env) DeleteDemoActors(ctx context.Context, refs ...TemplateRef) error { - if len(refs) == 0 { - return nil - } - - present, err := e.Kube.DeploymentExists(ctx, NamespaceAteSystem, "ate-api-server") - if err != nil { - return err - } - if !present { - log.Step("ate-api-server not found; skipping actor cleanup") - return nil - } - - client, err := ateclient.NewClient(ctx, e.Cfg.Kubeconfig, e.Cfg.Context, "", "", false) - if err != nil { - log.Warnf("could not connect to ate-api-server; skipping actor cleanup: %v", err) - return nil - } - defer client.Close() - - actors, err := listAllActors(ctx, client) - if err != nil { - log.Warnf("could not list actors; skipping actor cleanup: %v", err) - return nil - } - - for _, ref := range refs { - log.Stepf("Deleting actors for %s/%s", ref.Atespace, ref.Name) - for _, actor := range actors { - if actor.GetActorTemplate().GetAtespace() != ref.Atespace || actor.GetActorTemplate().GetName() != ref.Name { - continue - } - actorRef := resources.ActorRefFromActor(actor) - log.Stepf(" deleting actor %s/%s", actorRef.Atespace, actorRef.Name) - // AnyState skips the SUSPENDED precondition: teardown discards the - // actor, so there is nothing to be gained by suspending it first. - if _, err := client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ - Actor: actorRef.ToObjectRef(), - AnyState: true, - }); err != nil { - log.Warnf("while deleting actor %s/%s: %v", actorRef.Atespace, actorRef.Name, err) - } - } - } - return nil -} - // listAllActors pages through every actor in every atespace. func listAllActors(ctx context.Context, client *ateclient.Client) ([]*ateapipb.Actor, error) { var all []*ateapipb.Actor diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go new file mode 100644 index 0000000000..d47b186d65 --- /dev/null +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -0,0 +1,258 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "maps" + "os" + "regexp" + "slices" + "strings" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" +) + +// envHashAnnotation carries a digest of the apiserver's environment on the pod +// template. An envFrom source changing rolls no pods on its own, so the digest +// is what turns a new DSN into a restart. +const envHashAnnotation = "ate.dev/env-hash" + +// CreateAPIServerEnvVars reconciles how ate-api-server reaches its PostgreSQL +// store: the DSN and schema into the ate-api-server-secret-envvars Secret, the +// Cloud SQL Auth Proxy sidecar's settings into the ate-api-server-envvars +// ConfigMap, and an external server CA into postgres-server-ca. +// +// ate-api-server.yaml pulls both in through optional envFrom sources and +// resolves --postgres-connection-string=@env and --postgres-schema=@env from +// the result. It lists the secretRef last, so the Secret wins over a DSN a +// previous installer left in the ConfigMap. +func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { + log.Step("create_api_server_env_vars") + if err := e.Kube.EnsureNamespace(ctx, NamespaceAteSystem); err != nil { + return err + } + + // A DSN the operator supplied on this run, as opposed to one synthesized, + // defaulted, or adopted back from the Secret. Only the former outranks + // ATE_API_POSTGRES_POOL_MAX_CONNS below. + dsn := e.Cfg.PostgresConnectionString + dsnFromOperator := dsn != "" + + cloudsql, err := e.resolveCloudSQL(ctx) + if err != nil { + return err + } + if dsn == "" && cloudsql.Adopted { + // The instance came from the cluster, so take the DSN that goes with + // it rather than synthesizing a fresh one over the operator's edits. + if dsn, err = e.recordedDSN(ctx); err != nil { + return err + } + } + if dsn == "" { + if cloudsql.Instance != "" { + if dsn, err = cloudSQLDSN(cloudsql); err != nil { + return err + } + } else { + dsn = e.Cfg.PostgresConnString() + } + } + dsn = withPoolMaxConns(dsn, e.Cfg.PostgresPoolMaxConns, dsnFromOperator) + log.Infof("POSTGRES_CONNECTION_STRING: %s", redactDSN(dsn)) + + if err := e.Kube.ApplyConfigMap(ctx, NamespaceAteSystem, ConfigMapAPIEnvVars, cloudSQLEnvVars(cloudsql)); err != nil { + return err + } + if err := e.Kube.ApplySecret(ctx, NamespaceAteSystem, SecretAPIEnvVars, + buildAPIServerEnvVars(dsn, e.Cfg.PostgresSchemaName())); err != nil { + return err + } + if err := e.applyPostgresServerCA(ctx); err != nil { + return err + } + return e.annotateAPIServerEnvHash(ctx) +} + +// buildAPIServerEnvVars is the Secret payload. ate-api-server takes the +// connection string and the schema from it, and exits on an empty schema; an +// unrecognized key here reaches the container as a stray environment variable, +// so the set stays exactly what the shell installer's +// create_api_server_env_vars writes. +// +// The DSN can carry a password, for an external database without IAM +// authentication, which is why this is a Secret and not the ConfigMap +// alongside it. +func buildAPIServerEnvVars(connString, schema string) map[string]string { + return map[string]string{ + "ATE_API_POSTGRES_CONNECTION_STRING": connString, + "ATE_API_POSTGRES_SCHEMA": schema, + } +} + +// recordedDSN reads the connection string the cluster currently runs with. +func (e *Env) recordedDSN(ctx context.Context) (string, error) { + secret, err := e.Kube.GetSecret(ctx, NamespaceAteSystem, SecretAPIEnvVars) + if err != nil || secret == nil { + return "", err + } + return string(secret.Data["ATE_API_POSTGRES_CONNECTION_STRING"]), nil +} + +// applyPostgresServerCA publishes the server CA of an external PostgreSQL, +// which ate-api-server mounts at /run/postgres-server-ca/server-ca.pem for +// sslmode=verify-ca DSNs. For Cloud SQL: +// +// gcloud sql ssl server-ca-certs list --instance= --format="value(cert)" +func (e *Env) applyPostgresServerCA(ctx context.Context) error { + path := e.Cfg.PostgresServerCAFile + if path == "" { + return nil + } + pem, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading ATE_API_POSTGRES_SERVER_CA_FILE: %w", err) + } + return e.Kube.ApplySecret(ctx, NamespaceAteSystem, SecretPostgresServerCA, map[string]string{ + "server-ca.pem": string(pem), + }) +} + +// poolMaxConnsPattern matches the setting in either DSN format: a URI query +// parameter, delimited by &, or a keyword/value pair, delimited by a space. +var poolMaxConnsPattern = regexp.MustCompile(`pool_max_conns=[^ &]*`) + +// withPoolMaxConns splices pgxpool sizing into the DSN, the only place pgxpool +// reads it from. Without it the pool silently queues clients at its default +// size. +// +// A DSN the operator supplied on this run wins outright. The environment +// variable does however overwrite the setting in an adopted DSN, so that a +// scaling change is not silently dropped on redeploy. +func withPoolMaxConns(dsn, maxConns string, dsnFromOperator bool) string { + if maxConns == "" { + return dsn + } + if loc := poolMaxConnsPattern.FindStringIndex(dsn); loc != nil { + if dsnFromOperator { + return dsn + } + return dsn[:loc[0]] + "pool_max_conns=" + maxConns + dsn[loc[1]:] + } + switch { + case strings.Contains(dsn, "://") && strings.Contains(dsn, "?"): + return dsn + "&pool_max_conns=" + maxConns + case strings.Contains(dsn, "://"): + return dsn + "?pool_max_conns=" + maxConns + default: + return dsn + " pool_max_conns=" + maxConns + } +} + +var ( + // dsnURIPassword matches the password in a URI userinfo section. + dsnURIPassword = regexp.MustCompile(`(://[^:/@]*):[^@]*@`) + // dsnKeywordPassword matches a keyword/value or query parameter password. + dsnKeywordPassword = regexp.MustCompile(`(password=)[^ &]*`) +) + +// redactDSN masks any password before the connection string is logged. +func redactDSN(dsn string) string { + redacted := dsnURIPassword.ReplaceAllString(dsn, "$1:***@") + return dsnKeywordPassword.ReplaceAllString(redacted, "$1***") +} + +// EnsureEnvVarsSafeStandalone guards `ate-setup create api-server-env-vars` on +// a cluster installed before the DSN moved from the ConfigMap to the Secret. +// Rewriting the environment alone would prune the ConfigMap key and leave the +// running Deployment without a DSN on its next restart. A full deploy is safe +// because it updates the Deployment in the same run. +func (e *Env) EnsureEnvVarsSafeStandalone(ctx context.Context) error { + dep, err := e.Kube.GetDeployment(ctx, NamespaceAteSystem, "ate-api-server") + if err != nil { + return err + } + if dep == nil { + // Fresh install: the manifest applied later carries the secretRef. + return nil + } + containers := dep.Spec.Template.Spec.Containers + if len(containers) > 0 { + for _, source := range containers[0].EnvFrom { + if source.SecretRef != nil && source.SecretRef.Name == SecretAPIEnvVars { + return nil + } + } + } + return fmt.Errorf("the running ate-api-server Deployment does not reference the %s Secret; "+ + "rewriting the env vars alone would leave it without a DSN on its next restart. "+ + "Run `ate-setup deploy ate-apiserver` instead, which also updates the Deployment", SecretAPIEnvVars) +} + +// annotateAPIServerEnvHash stamps the pod template with a digest of the +// apiserver's environment, so that a changed DSN starts a rollout. Kubernetes +// does not restart pods when an envFrom ConfigMap or Secret changes. +func (e *Env) annotateAPIServerEnvHash(ctx context.Context) error { + dep, err := e.Kube.GetDeployment(ctx, NamespaceAteSystem, "ate-api-server") + if err != nil { + return err + } + if dep == nil { + // Fresh install: the first rollout starts with the new values. + return nil + } + + cm, err := e.Kube.GetConfigMap(ctx, NamespaceAteSystem, ConfigMapAPIEnvVars) + if err != nil { + return err + } + secret, err := e.Kube.GetSecret(ctx, NamespaceAteSystem, SecretAPIEnvVars) + if err != nil { + return err + } + var cmData map[string]string + if cm != nil { + cmData = cm.Data + } + var secretData map[string][]byte + if secret != nil { + secretData = secret.Data + } + + patch := fmt.Sprintf(`{"spec":{"template":{"metadata":{"annotations":{%q:%q}}}}}`, + envHashAnnotation, envHash(cmData, secretData)) + return e.Kube.PatchDeployment(ctx, NamespaceAteSystem, "ate-api-server", []byte(patch)) +} + +// envHash digests the apiserver's environment sources. Only changes matter, so +// the digest is an opaque value rather than a defined format; it does not +// agree with the one the shell installer computed, so the first install after +// the move to ate-setup rolls ate-api-server once. +func envHash(configMap map[string]string, secret map[string][]byte) string { + h := sha256.New() + fmt.Fprint(h, "configmap\n") + for _, k := range slices.Sorted(maps.Keys(configMap)) { + fmt.Fprintf(h, "%s=%s\n", k, configMap[k]) + } + fmt.Fprint(h, "secret\n") + for _, k := range slices.Sorted(maps.Keys(secret)) { + fmt.Fprintf(h, "%s=%s\n", k, secret[k]) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go new file mode 100644 index 0000000000..d4f4bf2e75 --- /dev/null +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -0,0 +1,256 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" +) + +// pgxpool reads its sizing out of the DSN, so an installation that gets this +// wrong queues clients silently rather than failing. +func TestWithPoolMaxConns(t *testing.T) { + for _, tc := range []struct { + name string + dsn string + maxConns string + dsnFromOperator bool + want string + }{ + { + name: "unset leaves the DSN alone", + dsn: "postgresql://p@h:5432/atepg?sslmode=disable", + want: "postgresql://p@h:5432/atepg?sslmode=disable", + }, + { + name: "URI with a query gets another parameter", + dsn: "postgresql://p@h:5432/atepg?sslmode=disable", + maxConns: "50", + want: "postgresql://p@h:5432/atepg?sslmode=disable&pool_max_conns=50", + }, + { + name: "URI without a query starts one", + dsn: "postgresql://p@h:5432/atepg", + maxConns: "50", + want: "postgresql://p@h:5432/atepg?pool_max_conns=50", + }, + { + name: "keyword/value DSN gets another pair", + dsn: "user=ate host=127.0.0.1 dbname=atepg", + maxConns: "50", + want: "user=ate host=127.0.0.1 dbname=atepg pool_max_conns=50", + }, + { + // An adopted DSN carries the previous run's value; a scaling + // change must not be silently dropped on redeploy. + name: "replaces the value in an adopted URI", + dsn: "postgresql://p@h:5432/atepg?pool_max_conns=10&sslmode=disable", + maxConns: "50", + want: "postgresql://p@h:5432/atepg?pool_max_conns=50&sslmode=disable", + }, + { + name: "replaces the value in an adopted keyword/value DSN", + dsn: "user=ate pool_max_conns=10 dbname=atepg", + maxConns: "50", + want: "user=ate pool_max_conns=50 dbname=atepg", + }, + { + // The operator spelled the whole DSN out on this run, so they + // meant the value in it. + name: "an operator-supplied value wins", + dsn: "postgresql://p@h:5432/atepg?pool_max_conns=10", + maxConns: "50", + dsnFromOperator: true, + want: "postgresql://p@h:5432/atepg?pool_max_conns=10", + }, + { + // ... but a DSN that says nothing about sizing still takes it. + name: "an operator-supplied DSN without the setting takes it", + dsn: "postgresql://p@h:5432/atepg", + maxConns: "50", + dsnFromOperator: true, + want: "postgresql://p@h:5432/atepg?pool_max_conns=50", + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := withPoolMaxConns(tc.dsn, tc.maxConns, tc.dsnFromOperator); got != tc.want { + t.Errorf("withPoolMaxConns() = %q, want %q", got, tc.want) + } + }) + } +} + +// The DSN is logged on every install, and for an external database it can +// carry a password. +func TestRedactDSN(t *testing.T) { + for _, tc := range []struct { + name string + dsn string + want string + }{ + { + name: "URI userinfo password", + dsn: "postgresql://ate:hunter2@db.example.com:5432/atepg?sslmode=require", + want: "postgresql://ate:***@db.example.com:5432/atepg?sslmode=require", + }, + { + name: "keyword/value password", + dsn: "user=ate password=hunter2 host=db.example.com", + want: "user=ate password=*** host=db.example.com", + }, + { + name: "query parameter password", + dsn: "postgresql://db.example.com/atepg?password=hunter2&sslmode=require", + want: "postgresql://db.example.com/atepg?password=***&sslmode=require", + }, + { + name: "passwordless DSN is unchanged", + dsn: "user=ate@p.iam host=127.0.0.1 port=5432 dbname=atepg sslmode=disable", + want: "user=ate@p.iam host=127.0.0.1 port=5432 dbname=atepg sslmode=disable", + }, + { + name: "the default in-cluster DSN is unchanged", + dsn: config.DefaultPostgresConnectionString, + want: config.DefaultPostgresConnectionString, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := redactDSN(tc.dsn); got != tc.want { + t.Errorf("redactDSN() = %q, want %q", got, tc.want) + } + }) + } +} + +// The digest exists to turn an envFrom change into a rollout, so what matters +// is that it moves when a value does and holds still otherwise. +func TestEnvHash(t *testing.T) { + cm := map[string]string{"A": "1", "B": "2"} + secret := map[string][]byte{"DSN": []byte("postgresql://h/atepg")} + + base := envHash(cm, secret) + if base != envHash(map[string]string{"B": "2", "A": "1"}, secret) { + t.Error("envHash() depends on map iteration order") + } + if base == envHash(cm, map[string][]byte{"DSN": []byte("postgresql://other/atepg")}) { + t.Error("envHash() did not change when the DSN did") + } + if base == envHash(map[string]string{"A": "1"}, secret) { + t.Error("envHash() did not change when a ConfigMap key was removed") + } + // The two sources are hashed into the same stream, so they need a + // separator to stay distinguishable. + if envHash(map[string]string{"X": "1"}, nil) == envHash(nil, map[string][]byte{"X": []byte("1")}) { + t.Error("envHash() does not distinguish the ConfigMap from the Secret") + } +} + +// apiServerDeployment builds an ate-api-server Deployment whose first +// container pulls in the named Secrets through envFrom. +func apiServerDeployment(secretRefs ...string) *appsv1.Deployment { + var envFrom []corev1.EnvFromSource + for _, name := range secretRefs { + envFrom = append(envFrom, corev1.EnvFromSource{ + SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: name}}, + }) + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: "ate-api-server"}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "ate-api-server", EnvFrom: envFrom}}}, + }, + }, + } +} + +// Rewriting the environment on a cluster whose Deployment predates the move of +// the DSN into a Secret would prune the ConfigMap key and leave the apiserver +// with no DSN at all on its next restart. +func TestEnsureEnvVarsSafeStandalone(t *testing.T) { + for _, tc := range []struct { + name string + dep *appsv1.Deployment + wantErr bool + }{ + { + name: "fresh install has no Deployment yet", + }, + { + name: "Deployment already reads the Secret", + dep: apiServerDeployment(SecretAPIEnvVars), + }, + { + name: "Deployment predates the Secret", + dep: apiServerDeployment(), + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var e *Env + if tc.dep == nil { + e = &Env{Cfg: &config.Config{}, Kube: fakeKube(t)} + } else { + e = &Env{Cfg: &config.Config{}, Kube: fakeKube(t, tc.dep)} + } + err := e.EnsureEnvVarsSafeStandalone(t.Context()) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), SecretAPIEnvVars) { + t.Errorf("EnsureEnvVarsSafeStandalone() error = %v, want it to name the Secret", err) + } + return + } + if err != nil { + t.Errorf("EnsureEnvVarsSafeStandalone() error = %v, want nil", err) + } + }) + } +} + +func TestAnnotateAPIServerEnvHash(t *testing.T) { + t.Run("fresh install is a no-op", func(t *testing.T) { + e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t)} + if err := e.annotateAPIServerEnvHash(t.Context()); err != nil { + t.Errorf("annotateAPIServerEnvHash() error = %v, want nil", err) + } + }) + + t.Run("stamps the pod template", func(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: SecretAPIEnvVars}, + Data: map[string][]byte{"ATE_API_POSTGRES_CONNECTION_STRING": []byte("postgresql://h/atepg")}, + } + e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t, apiServerDeployment(SecretAPIEnvVars), secret)} + + if err := e.annotateAPIServerEnvHash(t.Context()); err != nil { + t.Fatalf("annotateAPIServerEnvHash() error = %v", err) + } + + dep, err := e.Kube.GetDeployment(t.Context(), NamespaceAteSystem, "ate-api-server") + if err != nil { + t.Fatalf("GetDeployment() error = %v", err) + } + want := envHash(nil, secret.Data) + if got := dep.Spec.Template.Annotations[envHashAnnotation]; got != want { + t.Errorf("%s = %q, want %q", envHashAnnotation, got, want) + } + }) +} diff --git a/cmd/ate-setup/internal/steps/cloudsql.go b/cmd/ate-setup/internal/steps/cloudsql.go new file mode 100644 index 0000000000..20e70e330a --- /dev/null +++ b/cmd/ate-setup/internal/steps/cloudsql.go @@ -0,0 +1,262 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" +) + +// Keys the Cloud SQL Auth Proxy sidecar reads out of the ate-api-server-envvars +// ConfigMap. The proxy takes any of its flags from a CSQL_PROXY_-prefixed +// variable; the instance connection name is expanded into its args. +const ( + envCloudSQLInstance = "ATE_API_POSTGRES_CLOUDSQL_INSTANCE" + envCSQLIAMAuthn = "CSQL_PROXY_AUTO_IAM_AUTHN" + envCSQLPrivateIP = "CSQL_PROXY_PRIVATE_IP" + envCSQLPSC = "CSQL_PROXY_PSC" +) + +// workloadIdentityAnnotation links the ate-api-server KSA to the GSA whose +// ambient credentials the proxy picks up through ADC. +const workloadIdentityAnnotation = "iam.gke.io/gcp-service-account" + +// cloudSQLProxyContainer is the initContainer name in +// manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml, and the marker that +// tells the removal branch a sidecar is installed. +const cloudSQLProxyContainer = "cloud-sql-proxy" + +// gsaEmailSuffix is trimmed off the GSA email to get the Cloud SQL IAM +// database username. +const gsaEmailSuffix = ".gserviceaccount.com" + +// cloudSQLSettings is the Cloud SQL configuration for this run, after the +// operator's intent has been folded over what the cluster records. +type cloudSQLSettings struct { + // Instance is empty when Cloud SQL is not in use. + Instance string + GSA string + IAMAuth string + IPType string + // Adopted records that Instance came from the cluster rather than from + // the environment. Only then are the remaining settings inherited too. + Adopted bool +} + +// cloudSQLSettingsFrom folds the configured intent over the settings the +// cluster records in the ate-api-server-envvars ConfigMap. recorded is +// consulted only when ATE_API_POSTGRES_CLOUDSQL_INSTANCE is unset, so a +// redeploy that does not mention Cloud SQL regresses nothing to defaults. +func cloudSQLSettingsFrom(c config.CloudSQLConfig, recorded map[string]string) cloudSQLSettings { + s := cloudSQLSettings{ + Instance: c.Instance, + IAMAuth: c.IAMAuth, + IPType: c.IPType, + } + if s.IAMAuth == "" { + s.IAMAuth = "true" + } + if s.IPType == "" { + s.IPType = config.CloudSQLIPTypePrivate + } + if c.InstanceSet { + return s + } + + s.Instance = recorded[envCloudSQLInstance] + if s.Instance == "" { + return s + } + s.Adopted = true + if c.IAMAuth == "" && recorded[envCSQLIAMAuthn] != "" { + s.IAMAuth = recorded[envCSQLIAMAuthn] + } + if c.IPType == "" { + // The proxy defaults to the public address, so the absence of both + // keys means public, not the private default above. + switch { + case recorded[envCSQLPSC] != "": + s.IPType = config.CloudSQLIPTypePSC + case recorded[envCSQLPrivateIP] == "": + s.IPType = config.CloudSQLIPTypePublic + } + } + return s +} + +// recordedAPIServerEnvVars returns the ate-api-server-envvars ConfigMap data, +// or nil when the ConfigMap does not exist. +func (e *Env) recordedAPIServerEnvVars(ctx context.Context) (map[string]string, error) { + cm, err := e.Kube.GetConfigMap(ctx, NamespaceAteSystem, ConfigMapAPIEnvVars) + if err != nil || cm == nil { + return nil, err + } + return cm.Data, nil +} + +// resolveCloudSQLInstance answers only the question "is Cloud SQL in play", +// which the choice between the bundled StatefulSet and an external database +// turns on. +func (e *Env) resolveCloudSQLInstance(ctx context.Context) (string, error) { + if e.Cfg.CloudSQL.InstanceSet { + return e.Cfg.CloudSQL.Instance, nil + } + recorded, err := e.recordedAPIServerEnvVars(ctx) + if err != nil { + return "", err + } + return recorded[envCloudSQLInstance], nil +} + +// resolveCloudSQL resolves the full Cloud SQL configuration, reading the +// cluster for whatever the environment left unspecified. +func (e *Env) resolveCloudSQL(ctx context.Context) (cloudSQLSettings, error) { + var recorded map[string]string + if !e.Cfg.CloudSQL.InstanceSet { + var err error + if recorded, err = e.recordedAPIServerEnvVars(ctx); err != nil { + return cloudSQLSettings{}, err + } + } + + s := cloudSQLSettingsFrom(e.Cfg.CloudSQL, recorded) + if s.Instance == "" { + return s, nil + } + if s.Adopted { + log.Infof("Cloud SQL config adopted from cluster: %s", s.Instance) + } + + s.GSA = e.Cfg.CloudSQL.GSA + if s.GSA == "" { + gsa, err := e.Kube.ServiceAccountAnnotation(ctx, NamespaceAteSystem, "ate-api-server", workloadIdentityAnnotation) + if err != nil { + return cloudSQLSettings{}, err + } + s.GSA = gsa + } + return s, nil +} + +// cloudSQLDSN synthesizes the DSN for talking to Cloud SQL through the proxy: +// ateapi speaks plaintext to the sidecar on pod-local loopback, and the proxy +// owns TLS and IAM database authentication. +// +// The DSN is passwordless, so it only logs in when the proxy injects an IAM +// token. With IAM auth off the operator has to supply credentials themselves; +// failing here beats a PostgreSQL authentication error at pod startup. +func cloudSQLDSN(s cloudSQLSettings) (string, error) { + if s.IAMAuth == "false" { + return "", fmt.Errorf("ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH=false disables automatic IAM database " + + "authentication, so a passwordless DSN cannot be synthesized; set " + + "ATE_API_POSTGRES_CONNECTION_STRING explicitly (host=127.0.0.1 to stay on the proxy)") + } + if s.GSA == "" { + return "", fmt.Errorf("ATE_API_POSTGRES_CLOUDSQL_INSTANCE requires ATE_API_POSTGRES_CLOUDSQL_GSA " + + "(or an explicit ATE_API_POSTGRES_CONNECTION_STRING)") + } + user := strings.TrimSuffix(s.GSA, gsaEmailSuffix) + return fmt.Sprintf("user=%s host=127.0.0.1 port=5432 dbname=atepg sslmode=disable", user), nil +} + +// cloudSQLEnvVars is the ate-api-server-envvars ConfigMap payload: the proxy +// sidecar's configuration, and nothing else. It is empty without Cloud SQL, +// which prunes the keys a previous Cloud SQL install left behind. +// +// The health checks listen on 9801 because ateapi's metrics own 9090. +func cloudSQLEnvVars(s cloudSQLSettings) map[string]string { + if s.Instance == "" { + return map[string]string{} + } + data := map[string]string{ + envCloudSQLInstance: s.Instance, + envCSQLIAMAuthn: s.IAMAuth, + "CSQL_PROXY_PORT": "5432", + "CSQL_PROXY_HEALTH_CHECK": "true", + "CSQL_PROXY_HTTP_ADDRESS": "0.0.0.0", + "CSQL_PROXY_HTTP_PORT": "9801", + "CSQL_PROXY_STRUCTURED_LOGS": "true", + } + switch s.IPType { + case config.CloudSQLIPTypePrivate: + data[envCSQLPrivateIP] = "true" + case config.CloudSQLIPTypePSC: + data[envCSQLPSC] = "true" + } + return data +} + +// reconcileCloudSQLProxySidecar adds or removes the Cloud SQL Auth Proxy +// sidecar and the Workload Identity annotation on ate-api-server. It runs +// after the Deployment manifest is applied, which resets the pod template to +// the sidecar-free base. +// +// Desired state comes from resolveCloudSQL, so the removal branch fires only +// on an explicitly empty ATE_API_POSTGRES_CLOUDSQL_INSTANCE, never because a +// redeploy ran from a shell that simply did not export it. +func (e *Env) reconcileCloudSQLProxySidecar(ctx context.Context) error { + s, err := e.resolveCloudSQL(ctx) + if err != nil { + return err + } + + if s.Instance != "" { + log.Step("reconcile_cloudsql_proxy_sidecar (add)") + if s.GSA != "" { + if err := e.Kube.SetServiceAccountAnnotation(ctx, NamespaceAteSystem, "ate-api-server", + workloadIdentityAnnotation, s.GSA); err != nil { + return err + } + } + patch, err := os.ReadFile(e.Cfg.Manifest("cloudsql", "proxy-sidecar-patch.yaml")) + if err != nil { + return fmt.Errorf("reading the Cloud SQL proxy sidecar patch: %w", err) + } + return e.Kube.PatchDeployment(ctx, NamespaceAteSystem, "ate-api-server", patch) + } + + installed, err := e.cloudSQLProxyInstalled(ctx) + if err != nil || !installed { + return err + } + log.Step("reconcile_cloudsql_proxy_sidecar (remove)") + const removePatch = `{"spec":{"template":{"spec":{"initContainers":[{"name":"` + + cloudSQLProxyContainer + `","$patch":"delete"}]}}}}` + if err := e.Kube.PatchDeployment(ctx, NamespaceAteSystem, "ate-api-server", []byte(removePatch)); err != nil { + return err + } + return e.Kube.SetServiceAccountAnnotation(ctx, NamespaceAteSystem, "ate-api-server", + workloadIdentityAnnotation, "") +} + +// cloudSQLProxyInstalled reports whether ate-api-server currently runs the +// proxy sidecar. +func (e *Env) cloudSQLProxyInstalled(ctx context.Context) (bool, error) { + dep, err := e.Kube.GetDeployment(ctx, NamespaceAteSystem, "ate-api-server") + if err != nil || dep == nil { + return false, err + } + for _, c := range dep.Spec.Template.Spec.InitContainers { + if c.Name == cloudSQLProxyContainer { + return true, nil + } + } + return false, nil +} diff --git a/cmd/ate-setup/internal/steps/cloudsql_test.go b/cmd/ate-setup/internal/steps/cloudsql_test.go new file mode 100644 index 0000000000..4c1f6e2414 --- /dev/null +++ b/cmd/ate-setup/internal/steps/cloudsql_test.go @@ -0,0 +1,262 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "maps" + "os" + "slices" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/yaml" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" +) + +// fakeKube builds a kube.Client backed by the fake clientset, seeded with +// objects. A nil object is skipped, so callers can pass a helper's "absent" +// result straight through. +func fakeKube(t *testing.T, objects ...runtime.Object) *kube.Client { + t.Helper() + present := make([]runtime.Object, 0, len(objects)) + for _, o := range objects { + if o != nil { + present = append(present, o) + } + } + return &kube.Client{Typed: fake.NewSimpleClientset(present...)} +} + +// apiServerEnvVarsConfigMap seeds the ConfigMap Cloud SQL settings are adopted +// from. Nil data means the ConfigMap does not exist. +func apiServerEnvVarsConfigMap(data map[string]string) runtime.Object { + if data == nil { + return nil + } + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: ConfigMapAPIEnvVars}, + Data: data, + } +} + +// The three-way instance semantics and the adoption of the recorded proxy +// settings are the whole reason this configuration is not a plain string: a +// redeploy from a shell that never exported the variables must not regress a +// working Cloud SQL install to the defaults. +func TestCloudSQLSettingsFrom(t *testing.T) { + for _, tc := range []struct { + name string + cfg config.CloudSQLConfig + recorded map[string]string + want cloudSQLSettings + }{ + { + name: "unset and nothing recorded", + want: cloudSQLSettings{IAMAuth: "true", IPType: config.CloudSQLIPTypePrivate}, + }, + { + name: "explicit instance defaults to private IAM auth", + cfg: config.CloudSQLConfig{Instance: "p:r:i", InstanceSet: true}, + want: cloudSQLSettings{Instance: "p:r:i", IAMAuth: "true", IPType: config.CloudSQLIPTypePrivate}, + }, + { + name: "explicitly empty instance ignores the record", + cfg: config.CloudSQLConfig{InstanceSet: true}, + recorded: map[string]string{envCloudSQLInstance: "p:r:i", envCSQLPSC: "true"}, + want: cloudSQLSettings{IAMAuth: "true", IPType: config.CloudSQLIPTypePrivate}, + }, + { + name: "adopted instance inherits private IP", + recorded: map[string]string{ + envCloudSQLInstance: "p:r:i", + envCSQLPrivateIP: "true", + envCSQLIAMAuthn: "false", + }, + want: cloudSQLSettings{ + Instance: "p:r:i", IAMAuth: "false", + IPType: config.CloudSQLIPTypePrivate, Adopted: true, + }, + }, + { + name: "adopted instance inherits PSC", + recorded: map[string]string{ + envCloudSQLInstance: "p:r:i", + envCSQLPSC: "true", + envCSQLIAMAuthn: "true", + }, + want: cloudSQLSettings{ + Instance: "p:r:i", IAMAuth: "true", + IPType: config.CloudSQLIPTypePSC, Adopted: true, + }, + }, + { + // Neither key recorded: the proxy dials the public address, so + // the private default must not be reapplied. + name: "adopted instance with no IP key is public", + recorded: map[string]string{envCloudSQLInstance: "p:r:i"}, + want: cloudSQLSettings{ + Instance: "p:r:i", IAMAuth: "true", + IPType: config.CloudSQLIPTypePublic, Adopted: true, + }, + }, + { + name: "explicit settings outrank the record", + cfg: config.CloudSQLConfig{IPType: config.CloudSQLIPTypePublic, IAMAuth: "false"}, + recorded: map[string]string{envCloudSQLInstance: "p:r:i", envCSQLPSC: "true"}, + want: cloudSQLSettings{ + Instance: "p:r:i", IAMAuth: "false", + IPType: config.CloudSQLIPTypePublic, Adopted: true, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := cloudSQLSettingsFrom(tc.cfg, tc.recorded); got != tc.want { + t.Errorf("cloudSQLSettingsFrom() = %+v, want %+v", got, tc.want) + } + }) + } +} + +// The GSA is adopted from the Workload Identity annotation the previous run +// wrote, so an operator who only names the instance keeps the same database +// user. +func TestResolveCloudSQLAdoptsGSA(t *testing.T) { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: NamespaceAteSystem, + Name: "ate-api-server", + Annotations: map[string]string{workloadIdentityAnnotation: "ate@p.iam.gserviceaccount.com"}, + }, + } + e := &Env{ + Cfg: &config.Config{CloudSQL: config.CloudSQLConfig{Instance: "p:r:i", InstanceSet: true}}, + Kube: fakeKube(t, sa), + } + + got, err := e.resolveCloudSQL(t.Context()) + if err != nil { + t.Fatalf("resolveCloudSQL() error = %v", err) + } + if got.GSA != "ate@p.iam.gserviceaccount.com" { + t.Errorf("GSA = %q, want the annotated service account", got.GSA) + } +} + +func TestCloudSQLDSN(t *testing.T) { + t.Run("synthesized from the GSA", func(t *testing.T) { + got, err := cloudSQLDSN(cloudSQLSettings{ + Instance: "p:r:i", GSA: "ate@p.iam.gserviceaccount.com", IAMAuth: "true", + }) + if err != nil { + t.Fatalf("cloudSQLDSN() error = %v", err) + } + want := "user=ate@p.iam host=127.0.0.1 port=5432 dbname=atepg sslmode=disable" + if got != want { + t.Errorf("cloudSQLDSN() = %q, want %q", got, want) + } + }) + + // A passwordless DSN only logs in when the proxy injects an IAM token, so + // both of these would otherwise fail as an authentication error at pod + // startup rather than here. + t.Run("IAM auth disabled", func(t *testing.T) { + _, err := cloudSQLDSN(cloudSQLSettings{Instance: "p:r:i", GSA: "ate@p.iam.gserviceaccount.com", IAMAuth: "false"}) + if err == nil || !strings.Contains(err.Error(), "ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH=false") { + t.Errorf("cloudSQLDSN() error = %v, want it to name the disabled IAM auth", err) + } + }) + t.Run("no GSA", func(t *testing.T) { + _, err := cloudSQLDSN(cloudSQLSettings{Instance: "p:r:i", IAMAuth: "true"}) + if err == nil || !strings.Contains(err.Error(), "ATE_API_POSTGRES_CLOUDSQL_GSA") { + t.Errorf("cloudSQLDSN() error = %v, want it to name the missing GSA", err) + } + }) +} + +// The keys here are the proxy's own flag names; an unrecognized one is a flag +// the sidecar rejects at startup. +func TestCloudSQLEnvVars(t *testing.T) { + t.Run("no Cloud SQL writes nothing", func(t *testing.T) { + if got := cloudSQLEnvVars(cloudSQLSettings{}); len(got) != 0 { + t.Errorf("cloudSQLEnvVars() = %v, want empty", got) + } + }) + + for _, tc := range []struct { + ipType string + wantKey string + }{ + {ipType: config.CloudSQLIPTypePrivate, wantKey: envCSQLPrivateIP}, + {ipType: config.CloudSQLIPTypePSC, wantKey: envCSQLPSC}, + {ipType: config.CloudSQLIPTypePublic}, + } { + t.Run(tc.ipType, func(t *testing.T) { + got := cloudSQLEnvVars(cloudSQLSettings{Instance: "p:r:i", IAMAuth: "true", IPType: tc.ipType}) + + want := []string{ + envCloudSQLInstance, envCSQLIAMAuthn, + "CSQL_PROXY_HEALTH_CHECK", "CSQL_PROXY_HTTP_ADDRESS", "CSQL_PROXY_HTTP_PORT", + "CSQL_PROXY_PORT", "CSQL_PROXY_STRUCTURED_LOGS", + } + if tc.wantKey != "" { + want = append(want, tc.wantKey) + } + slices.Sort(want) + if keys := slices.Sorted(maps.Keys(got)); !slices.Equal(keys, want) { + t.Errorf("keys = %v, want %v", keys, want) + } + if got[envCloudSQLInstance] != "p:r:i" { + t.Errorf("%s = %q, want p:r:i", envCloudSQLInstance, got[envCloudSQLInstance]) + } + }) + } +} + +// The patch is applied by name after every ate-api-server apply, so a rename +// in the manifest would silently stop installing the sidecar. +func TestProxySidecarPatchNamesTheContainer(t *testing.T) { + cfg := &config.Config{Root: repoRoot(t)} + patch, err := os.ReadFile(cfg.Manifest("cloudsql", "proxy-sidecar-patch.yaml")) + if err != nil { + t.Fatalf("reading the proxy sidecar patch: %v", err) + } + + var dep appsv1.Deployment + if err := yaml.Unmarshal(patch, &dep); err != nil { + t.Fatalf("parsing the proxy sidecar patch: %v", err) + } + found := false + for _, c := range dep.Spec.Template.Spec.InitContainers { + if c.Name == cloudSQLProxyContainer { + found = true + // A plain initContainer would run to completion and block the + // pod; the proxy has to stay up alongside ateapi. + if c.RestartPolicy == nil || *c.RestartPolicy != corev1.ContainerRestartPolicyAlways { + t.Errorf("%s restartPolicy = %v, want Always so it runs as a native sidecar", c.Name, c.RestartPolicy) + } + break + } + } + if !found { + t.Errorf("the patch declares no %q initContainer; reconcileCloudSQLProxySidecar keys its removal branch on that name", cloudSQLProxyContainer) + } +} diff --git a/cmd/ate-setup/internal/steps/create.go b/cmd/ate-setup/internal/steps/create.go index cf7e3388f0..2b46e31bcc 100644 --- a/cmd/ate-setup/internal/steps/create.go +++ b/cmd/ate-setup/internal/steps/create.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" @@ -37,6 +36,8 @@ const ( SecretServiceDNSCA = "service-dns-ca-pool" SecretPodIdentityCA = "pod-identity-ca-pool" SecretEgressMITMCAPool = "egress-mitm-ca-pool" + SecretAPIEnvVars = "ate-api-server-secret-envvars" + SecretPostgresServerCA = "postgres-server-ca" ConfigMapAPIEnvVars = "ate-api-server-envvars" ConfigMapAPIAuthn = "ate-api-authentication" // poolKeyID is the identifier given to the first CA and JWT key in a new @@ -81,9 +82,11 @@ func (e *Env) CreateEgressMITMCAPoolSecret(ctx context.Context) error { return e.createPoolSecret(ctx, NamespaceAteSystem, SecretEgressMITMCAPool, poolBytes) } -// EnsureEgressMITMCAPoolSecret creates the egress MITM CA pool secret if sdsmint is enabled. +// EnsureEgressMITMCAPoolSecret creates the egress MITM CA pool secret if +// sdsmint is enabled. Both dataplanes need it: the agentgateway-egress-mitm +// overlay mounts the same Secret the envoy egress does. func (e *Env) EnsureEgressMITMCAPoolSecret(ctx context.Context) error { - if e.Cfg.Router == config.RouterAgentgateway || !e.Cfg.ExperimentalUseSDSMint { + if !e.Cfg.ExperimentalUseSDSMint { return nil } return e.ensureSecret(ctx, NamespaceAteSystem, SecretEgressMITMCAPool, e.CreateEgressMITMCAPoolSecret) @@ -119,35 +122,6 @@ func (e *Env) CreateActorIDCACertsSecret(ctx context.Context) error { }) } -// CreateAPIServerEnvVars writes the ConfigMap that tells ate-api-server how to -// reach its PostgreSQL store. ate-api-server.yaml pulls it in via an optional -// envFrom and resolves --postgres-connection-string=@env and -// --postgres-schema=@env from it. -func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { - log.Step("create_api_server_env_vars") - if err := e.Kube.EnsureNamespace(ctx, NamespaceAteSystem); err != nil { - return err - } - - connString := e.Cfg.PostgresConnString() - log.Infof("POSTGRES_CONNECTION_STRING: %s", connString) - - return e.Kube.ApplyConfigMap(ctx, NamespaceAteSystem, ConfigMapAPIEnvVars, - buildAPIServerEnvVars(connString, e.Cfg.PostgresSchemaName())) -} - -// buildAPIServerEnvVars is the ConfigMap payload. ate-api-server takes the -// connection string and the schema from it, and exits on an empty schema; an -// unrecognized key here reaches the container as a stray environment variable, -// so the set stays exactly what the shell installer's -// create_api_server_env_vars writes. -func buildAPIServerEnvVars(connString, schema string) map[string]string { - return map[string]string{ - "ATE_API_POSTGRES_CONNECTION_STRING": connString, - "ATE_API_POSTGRES_SCHEMA": schema, - } -} - // CreateAPIAuthenticationConfig writes the default ate-api-server // authentication config, pointing it at the cluster's service account issuer. func (e *Env) CreateAPIAuthenticationConfig(ctx context.Context) error { @@ -157,16 +131,29 @@ func (e *Env) CreateAPIAuthenticationConfig(ctx context.Context) error { } authnConfig := buildAuthenticationConfig(e.jwtIssuer(ctx)) + // The issuer decides which tokens the apiserver accepts at all, and a + // wrong one fails as an opaque 401 much later, so show what was written. + log.Infof("%s authentication.yaml:", ConfigMapAPIAuthn) + for _, line := range strings.Split(authnConfig, "\n") { + log.Infof(" | %s", line) + } return e.Kube.ApplyConfigMap(ctx, NamespaceAteSystem, ConfigMapAPIAuthn, map[string]string{ "authentication.yaml": authnConfig, }) } -// jwtIssuer determines the service account token issuer to trust. On GKE it is -// derived from the cluster coordinates; otherwise it comes from the cluster's -// OpenID discovery document, falling back to the in-cluster default. +// jwtIssuer determines the service account token issuer to trust. +// +// ate-api-server accepts a token only if its iss claim equals this string +// exactly. EXPECTED_JWT_ISSUER, when set, is that string; the rest is for +// clusters whose issuer follows a standard form — derived from the cluster +// coordinates on GKE, otherwise read from the cluster's OpenID discovery +// document, falling back to the in-cluster default. func (e *Env) jwtIssuer(ctx context.Context) string { cfg := e.Cfg + if cfg.ExpectedJWTIssuer != "" { + return cfg.ExpectedJWTIssuer + } if cfg.ProjectID != "" && cfg.ClusterLocation != "" && cfg.ClusterName != "" { return fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/locations/%s/clusters/%s", cfg.ProjectID, cfg.ClusterLocation, cfg.ClusterName) diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e7ea78b212..dc47018ec4 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -121,7 +121,14 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - if err := e.applyBundledPostgres(ctx); err != nil { + // Resolved before the bundle apply: adopting a Cloud SQL instance reads + // the ConfigMap that EnsureAPIServerPrerequisites has already rewritten, + // and the answer decides both the apply and the rollout wait below. + postgres, err := e.planPostgres(ctx) + if err != nil { + return err + } + if err := e.applyBundledPostgres(ctx, postgres); err != nil { return err } @@ -139,6 +146,11 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } + // After the bundle, which resets the pod template to the sidecar-free base. + if err := e.reconcileCloudSQLProxySidecar(ctx); err != nil { + return err + } + // Deploy egress gateway explicitly so kind and experimental modes are applied. if err := e.EnsureEgressMITMCAPoolSecret(ctx); err != nil { return err @@ -154,10 +166,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { log.Step("Waiting for ATE system components to be ready...") type rollout struct{ kind, name string } var waits []rollout - // Only when the bundled StatefulSet was applied above; an external - // database means it never gets deployed, and waiting on it would block - // until the timeout on an object that will never exist. - if e.useBundledPostgres() { + if postgres.bundled { waits = append(waits, rollout{kube.KindStatefulSet, "postgres"}) } waits = append(waits, @@ -172,7 +181,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } } - return nil + return e.applyOtelEndpointOverride(ctx) } // applyPodcertWorkersOverride sets WORKERS_PER_SIGNER on podcertificate-controller if configured. @@ -234,9 +243,16 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error { if err := e.applyOtelConfig(ctx); err != nil { return err } + if err := e.applyOtelEndpointOverride(ctx); err != nil { + return err + } if err := e.ResolveAndApply(ctx, e.Cfg.Manifest("ate-api-server.yaml")); err != nil { return err } + // After the manifest, which resets the pod template to the sidecar-free base. + if err := e.reconcileCloudSQLProxySidecar(ctx); err != nil { + return err + } return e.Kube.RolloutStatus(ctx, kube.KindDeployment, NamespaceAteSystem, "ate-api-server", e.Cfg.RolloutTimeout) } @@ -275,6 +291,9 @@ func (e *Env) DeployAtelet(ctx context.Context) error { if err := e.applyOtelConfig(ctx); err != nil { return err } + if err := e.applyOtelEndpointOverride(ctx); err != nil { + return err + } var manifest []byte var err error @@ -314,6 +333,9 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err := e.applyOtelConfig(ctx); err != nil { return err } + if err := e.applyOtelEndpointOverride(ctx); err != nil { + return err + } routerManifest, err := e.renderAtenetRouterManifest(ctx) if err != nil { diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index cf86d70e6d..1c1414118e 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -22,6 +22,7 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" ) // installDir is the manifest root, relative to the repository root. @@ -77,6 +78,9 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context) ([]byte, error) { if e.Cfg.AdditionalEgressExtprocService != "" { return nil, fmt.Errorf("--experimental-additional-egress-extproc-service requires --atenet-dataplane=envoy") } + if e.Cfg.ExperimentalUseSDSMint { + return e.KustomizeResolve(ctx, installDir+"/agentgateway-egress-mitm") + } return e.KustomizeResolve(ctx, installDir+"/agentgateway-egress") } @@ -270,3 +274,68 @@ func (e *Env) otelConfigPath() string { func (e *Env) applyOtelConfig(ctx context.Context) error { return e.Kube.ApplyPath(ctx, e.otelConfigPath()) } + +// otelConfigMap is the ConfigMap every control plane component reads its +// telemetry settings from through envFrom. +const otelConfigMap = "ate-otel-config" + +// otelEndpointKey is the collector address inside it. +const otelEndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT" + +// otelOverrideDeployments are the control plane Deployments that read +// ate-otel-config. ate-controller additionally copies the values onto the +// ateom worker pods it creates, so one patch reaches the whole system. +var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router"} + +// applyOtelEndpointOverride points all control plane telemetry at a different +// collector for the duration of a measurement. See +// benchmarking/telemetry/README.md. +// +// Call this AFTER every apply: the ate-system bundle carries its own copy of +// ate-otel-config, so applying it replaces an earlier patch and the endpoint +// silently returns to the cluster default. +// +// A ConfigMap change starts no rollout, because the pod template stays the +// same, so the consumers have to be restarted. Only on an actual change: a +// restart during the bundle's rollout makes the two compete, and the rollout +// wait can then exceed its timeout. An absent workload is not an error, +// because a single-component deploy has only that component. +func (e *Env) applyOtelEndpointOverride(ctx context.Context) error { + endpoint := e.Cfg.OtlpEndpoint + if endpoint == "" { + return nil + } + + cm, err := e.Kube.GetConfigMap(ctx, NamespaceAteSystem, otelConfigMap) + if err != nil { + return err + } + if cm != nil && cm.Data[otelEndpointKey] == endpoint { + return nil + } + + log.Infof("Overriding %s with %s", otelEndpointKey, endpoint) + if err := e.Kube.MergePatchConfigMap(ctx, NamespaceAteSystem, otelConfigMap, + map[string]string{otelEndpointKey: endpoint}); err != nil { + return err + } + + now := time.Now() + for _, name := range otelOverrideDeployments { + if err := e.Kube.RolloutRestartDeployment(ctx, NamespaceAteSystem, name, now); err != nil { + return err + } + } + // atelet DaemonSet names carry a version suffix; restart whichever + // versions are installed. + daemonSets, err := e.Kube.DaemonSetNames(ctx, NamespaceAteSystem, "app=atelet") + if err != nil { + return err + } + for _, name := range daemonSets { + if err := e.Kube.RolloutRestart(ctx, NamespaceAteSystem, name, now); err != nil { + return err + } + } + return nil +} diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index f2ef12a205..bd36e77d11 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -18,6 +18,9 @@ import ( "strings" "testing" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" @@ -222,3 +225,129 @@ func extProcCluster(filter map[string]any) string { name, _ := envoyGRPC["cluster_name"].(string) return name } + +// The two sdsmint switches are coupled: the MITM overlay mounts the CA pool +// Secret EnsureEgressMITMCAPoolSecret generates, so selecting one without the +// other leaves atenet-egress waiting on a Secret nobody creates. +func TestAgentgatewayEgressMITMOverlay(t *testing.T) { + cfg := &config.Config{ + Root: repoRoot(t), + Router: config.RouterAgentgateway, + ExperimentalUseSDSMint: true, + } + e := &Env{Cfg: cfg, Kube: fakeKube(t)} + + built, err := e.Kustomize(installDir + "/agentgateway-egress-mitm") + if err != nil { + t.Fatalf("Kustomize(agentgateway-egress-mitm) = %v", err) + } + if !strings.Contains(string(built), SecretEgressMITMCAPool) { + t.Errorf("the MITM overlay does not mount the %s Secret", SecretEgressMITMCAPool) + } + + if err := e.EnsureEgressMITMCAPoolSecret(t.Context()); err != nil { + t.Fatalf("EnsureEgressMITMCAPoolSecret() error = %v", err) + } + exists, err := e.Kube.SecretExists(t.Context(), NamespaceAteSystem, SecretEgressMITMCAPool) + if err != nil { + t.Fatalf("SecretExists() error = %v", err) + } + if !exists { + t.Errorf("no %s Secret was generated for the agentgateway dataplane", SecretEgressMITMCAPool) + } +} + +// otelConfig seeds the ConfigMap every component reads its collector address +// from. +func otelConfig(endpoint string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: NamespaceAteSystem, Name: otelConfigMap}, + Data: map[string]string{otelEndpointKey: endpoint}, + } +} + +// restartedAt reports whether a workload's pod template carries the restart +// annotation. +func restartedAt(t *testing.T, e *Env, kind, name string) bool { + t.Helper() + var annotations map[string]string + switch kind { + case "deployment": + dep, err := e.Kube.GetDeployment(t.Context(), NamespaceAteSystem, name) + if err != nil || dep == nil { + t.Fatalf("GetDeployment(%s) = %v, %v", name, dep, err) + } + annotations = dep.Spec.Template.Annotations + case "daemonset": + ds, err := e.Kube.Typed.AppsV1().DaemonSets(NamespaceAteSystem).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("getting daemonset %s: %v", name, err) + } + annotations = ds.Spec.Template.Annotations + } + _, ok := annotations["kubectl.kubernetes.io/restartedAt"] + return ok +} + +func TestApplyOtelEndpointOverride(t *testing.T) { + const endpoint = "http://collector.benchmark.svc:4317" + + t.Run("no endpoint configured is a no-op", func(t *testing.T) { + e := &Env{Cfg: &config.Config{}, Kube: fakeKube(t, otelConfig("http://default:4317"))} + if err := e.applyOtelEndpointOverride(t.Context()); err != nil { + t.Fatalf("applyOtelEndpointOverride() error = %v", err) + } + cm, _ := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, otelConfigMap) + if cm.Data[otelEndpointKey] != "http://default:4317" { + t.Errorf("%s = %q, want the cluster default untouched", otelEndpointKey, cm.Data[otelEndpointKey]) + } + }) + + // Restarting when nothing changed makes the restart race the rollout the + // caller is about to wait on, and `rollout status` then times out. + t.Run("already correct restarts nothing", func(t *testing.T) { + e := &Env{ + Cfg: &config.Config{OtlpEndpoint: endpoint}, + Kube: fakeKube(t, otelConfig(endpoint), apiServerDeployment()), + } + if err := e.applyOtelEndpointOverride(t.Context()); err != nil { + t.Fatalf("applyOtelEndpointOverride() error = %v", err) + } + if restartedAt(t, e, "deployment", "ate-api-server") { + t.Error("ate-api-server was restarted even though the endpoint was unchanged") + } + }) + + t.Run("patches and restarts the consumers", func(t *testing.T) { + // The atelet DaemonSet name carries a substrate version suffix, so it + // can only be found by label. + atelet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: NamespaceAteSystem, + Name: "atelet-v1-2-3", + Labels: map[string]string{"app": "atelet"}, + }, + } + e := &Env{ + Cfg: &config.Config{OtlpEndpoint: endpoint}, + Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), atelet), + } + + if err := e.applyOtelEndpointOverride(t.Context()); err != nil { + t.Fatalf("applyOtelEndpointOverride() error = %v", err) + } + + cm, _ := e.Kube.GetConfigMap(t.Context(), NamespaceAteSystem, otelConfigMap) + if cm.Data[otelEndpointKey] != endpoint { + t.Errorf("%s = %q, want %q", otelEndpointKey, cm.Data[otelEndpointKey], endpoint) + } + // ate-controller and atenet-router are absent here: a deploy of one + // component has only that component, which is not an error. + if !restartedAt(t, e, "deployment", "ate-api-server") { + t.Error("ate-api-server was not restarted") + } + if !restartedAt(t, e, "daemonset", "atelet-v1-2-3") { + t.Error("the atelet DaemonSet was not restarted") + } + }) +} diff --git a/cmd/ate-setup/internal/steps/postgres.go b/cmd/ate-setup/internal/steps/postgres.go index 5669ff3f80..07e9c16c06 100644 --- a/cmd/ate-setup/internal/steps/postgres.go +++ b/cmd/ate-setup/internal/steps/postgres.go @@ -21,20 +21,58 @@ import ( "github.com/agent-substrate/substrate/cmd/ate-setup/internal/log" ) -// useBundledPostgres reports whether ateapi uses the in-cluster database -// (when no external DSN is configured). Gates applying the bundled StatefulSet -// and waiting on its rollout in DeployAteSystem. -func (e *Env) useBundledPostgres() bool { - return e.Cfg.PostgresConnectionString == "" +// postgresPlan says where ateapi's store comes from: the bundled StatefulSet, +// or something the installer does not deploy. It gates both applying the +// StatefulSet and waiting on its rollout, since waiting on an object that will +// never exist only fails at the timeout. +type postgresPlan struct { + // bundled selects the in-cluster StatefulSet. + bundled bool + // external names what replaces it, for the log line. A DSN aimed at a + // database that was never deployed otherwise surfaces only as an + // ate-api-server rollout timeout minutes later, with nothing pointing at + // the cause. + external string +} + +// planPostgres decides between the bundled database and an external one, +// configured either as an explicit DSN or as a Cloud SQL instance — the +// latter possibly adopted from the cluster. +func (e *Env) planPostgres(ctx context.Context) (postgresPlan, error) { + if e.Cfg.PostgresConnectionString != "" { + return postgresPlan{external: "ATE_API_POSTGRES_CONNECTION_STRING"}, nil + } + instance, err := e.resolveCloudSQLInstance(ctx) + if err != nil { + return postgresPlan{}, err + } + if instance != "" { + return postgresPlan{external: "Cloud SQL instance " + instance}, nil + } + return postgresPlan{bundled: true}, nil } // applyBundledPostgres applies the bundled PostgreSQL StatefulSet, or logs that // it was skipped in favor of an external database. -func (e *Env) applyBundledPostgres(ctx context.Context) error { - if !e.useBundledPostgres() { - log.Step("Skipping bundled PostgreSQL: external database configured (ATE_API_POSTGRES_CONNECTION_STRING)") +func (e *Env) applyBundledPostgres(ctx context.Context, plan postgresPlan) error { + if !plan.bundled { + log.Stepf("Skipping bundled PostgreSQL: external database configured (%s)", plan.external) return nil } + return e.applyPostgresManifest(ctx) +} + +// applyPostgresManifest applies the bundled StatefulSet for the target +// environment. The kind overlay shrinks its CPU request and volume to what a +// local cluster can actually satisfy. +func (e *Env) applyPostgresManifest(ctx context.Context) error { + if e.Cfg.Kind { + built, err := e.Kustomize(installDir + "/kind/postgres") + if err != nil { + return err + } + return e.Kube.ApplyBytes(ctx, built) + } return e.Kube.ApplyPath(ctx, e.Cfg.Manifest("postgres", "postgres.yaml")) } @@ -66,7 +104,7 @@ func (e *Env) DeployPostgres(ctx context.Context) error { return err } - if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("postgres", "postgres.yaml")); err != nil { + if err := e.applyPostgresManifest(ctx); err != nil { return err } return e.Kube.RolloutStatus(ctx, kube.KindStatefulSet, NamespaceAteSystem, "postgres", e.Cfg.RolloutTimeout) diff --git a/cmd/ate-setup/internal/steps/postgres_test.go b/cmd/ate-setup/internal/steps/postgres_test.go index e8c1cac39c..35d774b1cc 100644 --- a/cmd/ate-setup/internal/steps/postgres_test.go +++ b/cmd/ate-setup/internal/steps/postgres_test.go @@ -22,23 +22,56 @@ import ( "github.com/agent-substrate/substrate/cmd/ate-setup/internal/config" ) -func TestUseBundledPostgres(t *testing.T) { +func TestPlanPostgres(t *testing.T) { for _, tc := range []struct { name string connString string - want bool + cloudSQL config.CloudSQLConfig + recorded map[string]string + want postgresPlan }{ - {name: "no external database", connString: "", want: true}, { - name: "external database configured", + name: "no external database", + want: postgresPlan{bundled: true}, + }, + { + name: "explicit DSN", connString: "postgresql://user@db.example.com:5432/atepg", - want: false, + want: postgresPlan{external: "ATE_API_POSTGRES_CONNECTION_STRING"}, + }, + { + name: "Cloud SQL instance from the environment", + cloudSQL: config.CloudSQLConfig{Instance: "p:r:i", InstanceSet: true}, + want: postgresPlan{external: "Cloud SQL instance p:r:i"}, + }, + { + name: "Cloud SQL instance adopted from the cluster", + recorded: map[string]string{envCloudSQLInstance: "p:r:i"}, + want: postgresPlan{external: "Cloud SQL instance p:r:i"}, + }, + { + // The removal case: the cluster still records an instance, but + // the operator asked for it to go away. + name: "explicitly empty instance ignores the cluster record", + cloudSQL: config.CloudSQLConfig{InstanceSet: true}, + recorded: map[string]string{envCloudSQLInstance: "p:r:i"}, + want: postgresPlan{bundled: true}, }, } { t.Run(tc.name, func(t *testing.T) { - e := &Env{Cfg: &config.Config{PostgresConnectionString: tc.connString}} - if got := e.useBundledPostgres(); got != tc.want { - t.Errorf("useBundledPostgres() = %v, want %v", got, tc.want) + e := &Env{ + Cfg: &config.Config{ + PostgresConnectionString: tc.connString, + CloudSQL: tc.cloudSQL, + }, + Kube: fakeKube(t, apiServerEnvVarsConfigMap(tc.recorded)), + } + got, err := e.planPostgres(t.Context()) + if err != nil { + t.Fatalf("planPostgres() error = %v", err) + } + if got != tc.want { + t.Errorf("planPostgres() = %+v, want %+v", got, tc.want) } }) } diff --git a/cmd/ate-setup/internal/steps/substrate.go b/cmd/ate-setup/internal/steps/substrate.go index db9429fddc..d985aba447 100644 --- a/cmd/ate-setup/internal/steps/substrate.go +++ b/cmd/ate-setup/internal/steps/substrate.go @@ -121,7 +121,7 @@ func WaitActorTemplateGolden(ctx context.Context, client *ateclient.Client, ref // DeleteSubstrateDemo removes a substrate demo's control-plane resources: // every actor created from the given templates, then the templates (which // server-side also removes their golden actors and snapshots), then the -// atespaces. As with DeleteDemoActors, a cluster without a reachable +// atespaces. A cluster without a reachable // ate-api-server is not an error -- there is nothing to clean up. func (e *Env) DeleteSubstrateDemo(ctx context.Context, refs []resources.ActorTemplateRef, atespaces []string) error { present, err := e.Kube.DeploymentExists(ctx, NamespaceAteSystem, "ate-api-server") diff --git a/cmd/ate-setup/main.go b/cmd/ate-setup/main.go index 5539073569..134d837aeb 100644 --- a/cmd/ate-setup/main.go +++ b/cmd/ate-setup/main.go @@ -13,9 +13,10 @@ // limitations under the License. // Command ate-setup installs and tears down Agent Substrate on a Kubernetes -// cluster. It is a Go port of hack/install-ate.sh and the scripts it sources, -// which remain in place and still work; see cmd/ate-setup/commands.md for the -// flag-by-flag mapping between the two. +// cluster. It is the installer; hack/install-ate.sh is a shim that translates +// its historical flags onto these commands. See cmd/ate-setup/commands.md for +// the flag-by-flag mapping and cmd/ate-setup/cli-diff.md for what the +// translation does not cover. package main import ( diff --git a/cmd/ate-setup/shim_test.go b/cmd/ate-setup/shim_test.go new file mode 100644 index 0000000000..378bd65f91 --- /dev/null +++ b/cmd/ate-setup/shim_test.go @@ -0,0 +1,367 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// hack/install-ate.sh translates the installer's historical flags onto this +// binary's commands. These tests pin that translation: they put a stub `go` on +// PATH, run the script, and check the command lines it would have run. Nothing +// is built and no cluster is touched, so the whole file is a unit test of the +// argument mapping -- the one part of the shim that can silently break a CI job +// or a developer's muscle memory. + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos" + // Registers every bundled demo, so demos.All() is the real list. + _ "github.com/agent-substrate/substrate/cmd/ate-setup/internal/demos/all" +) + +// stubGo is a `go` that records the arguments of every `go run ./cmd/ate-setup` +// the script makes, one invocation per line, and succeeds. +const stubGo = `#!/usr/bin/env bash +printf '%s\n' "$*" >> "${SHIM_TEST_LOG}" +` + +// runShim runs hack/install-ate.sh with args and returns the recorded +// invocations and the script's exit status. +func runShim(t *testing.T, env []string, args ...string) (invocations []string, exitCode int) { + t.Helper() + + root := repoRoot(t) + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "go"), []byte(stubGo), 0o755); err != nil { + t.Fatalf("writing the stub go: %v", err) + } + log := filepath.Join(t.TempDir(), "invocations") + + cmd := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh")) + cmd.Args = append(cmd.Args, args...) + cmd.Dir = root + // A fixed environment: the script reads SETUP_CSI and STORAGE_CLASS, so a + // developer who exports either would otherwise change the expected output. + // git needs PATH, and the stub go has to come first on it. + cmd.Env = append([]string{ + "PATH=" + bin + ":" + os.Getenv("PATH"), + "HOME=" + os.Getenv("HOME"), + "SHIM_TEST_LOG=" + log, + }, env...) + + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + exitCode = exitErr.ExitCode() + default: + t.Fatalf("running install-ate.sh %v: %v\n%s", args, err, out) + } + + recorded, err := os.ReadFile(log) + if err != nil && !os.IsNotExist(err) { + t.Fatalf("reading the invocation log: %v", err) + } + for _, line := range strings.Split(string(recorded), "\n") { + if line != "" { + invocations = append(invocations, line) + } + } + return invocations, exitCode +} + +func repoRoot(t *testing.T) string { + t.Helper() + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + t.Fatalf("git rev-parse --show-toplevel: %v", err) + } + return strings.TrimSpace(string(out)) +} + +// The script keeps its own copy of the demo list so that --help answers +// without a build and an unknown --deploy-demo-* is rejected outright. That +// copy has to track the registry: a demo added only to internal/demos would +// otherwise have no installer flag, which is exactly the regression the shim +// exists to prevent. +func TestShimDemoListMatchesTheRegistry(t *testing.T) { + script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "install-ate.sh")) + if err != nil { + t.Fatalf("reading install-ate.sh: %v", err) + } + match := regexp.MustCompile(`(?s)\nATE_DEMOS=\((.*?)\n\)`).FindSubmatch(script) + if match == nil { + t.Fatal("no ATE_DEMOS=( ... ) array in install-ate.sh") + } + listed := map[string]bool{} + for _, name := range strings.Fields(string(match[1])) { + listed[name] = true + } + + registered := map[string]bool{} + for _, d := range demos.All() { + registered[d.Name()] = true + } + + for name := range registered { + if !listed[name] { + t.Errorf("%s is registered but has no --deploy-%s flag; add it to ATE_DEMOS", name, name) + } + } + for name := range listed { + if !registered[name] { + t.Errorf("ATE_DEMOS lists %s, which no demo package registers", name) + } + } +} + +func TestShimTranslatesFlags(t *testing.T) { + const prefix = "run ./cmd/ate-setup " + + for _, tc := range []struct { + name string + env []string + args []string + want []string + }{{ + name: "deploy ate-system carries the CSI driver, and --setup-csi also acts on its own", + args: []string{"--deploy-ate-system", "--setup-csi=nfs"}, + want: []string{ + "deploy ate-system --setup-csi=nfs", + "setup csi nfs", + }, + }, { + name: "a bare --setup-csi means nfs", + args: []string{"--setup-csi"}, + want: []string{"setup csi nfs"}, + }, { + name: "--setup-csi takes a separated value", + args: []string{"--setup-csi", "hostpath"}, + want: []string{"setup csi hostpath"}, + }, { + name: "no CSI driver by default", + args: []string{"--deploy-ate-system"}, + want: []string{"deploy ate-system --setup-csi=none"}, + }, { + // The value-bearing flags were pre-scanned, so they shape every action + // regardless of where they appear. + name: "global flags apply to actions that precede them", + args: []string{"--deploy-atenet", "--atenet-dataplane", "agentgateway", "--experimental-use-sdsmint"}, + want: []string{"--atenet-dataplane=agentgateway --experimental-use-sdsmint deploy atenet"}, + }, { + name: "actions run in command line order", + args: []string{"--deploy-ate-apiserver", "--deploy-atelet", "--delete-atenet"}, + want: []string{ + "deploy apiserver", + "deploy atelet", + "delete atenet", + }, + }, { + name: "every create flag maps to a create subcommand", + args: []string{ + "--create-jwt-authority-pool-secret", + "--create-actor-id-ca-pool-secret", + "--create-actor-id-ca-certs-secret", + "--create-egress-mitm-ca-pool-secret", + "--create-podcertificate-controller-cas", + "--create-api-server-env-vars", + "--create-api-authentication-config", + }, + want: []string{ + "create jwt-authority-pool", + "create actor-id-ca-pool", + "create actor-id-ca-certs", + "create egress-mitm-ca-pool", + "create podcertificate-controller-cas", + "create api-server-env-vars", + "create api-authentication-config", + }, + }, { + name: "delete flags", + args: []string{"--delete-ate-system", "--delete-all"}, + want: []string{"delete ate-system", "delete all"}, + }, { + name: "benchmark flags are renamed and apply to both benchmark actions", + args: []string{ + "--benchmark-worker-count=4", "--deploy-benchmarks", + "--benchmark-sandbox-class", "microvm", "--delete-benchmarks", + }, + want: []string{ + "deploy benchmarks --worker-count=4 --sandbox-class=microvm", + "delete benchmarks --worker-count=4 --sandbox-class=microvm", + }, + }, { + name: "demo flags drop the demo- prefix", + args: []string{"--deploy-demo-egress-microvm-mitm", "--delete-demo-counter-microvm"}, + want: []string{ + "deploy demo egress-microvm-mitm", + "delete demo counter-microvm", + }, + }, { + name: "the counter external volume defaults to the standard storage class", + args: []string{"--deploy-demo-counter-with-external-volume"}, + want: []string{"deploy demo counter --with-external-volume --storage-class=standard"}, + }, { + name: "the counter external volume follows --setup-csi", + args: []string{"--setup-csi=hostpath", "--deploy-demo-counter-with-external-volume"}, + want: []string{ + "setup csi hostpath", + "deploy demo counter --with-external-volume --storage-class=csi-hostpath-sc", + }, + }, { + name: "STORAGE_CLASS beats the driver --setup-csi installed", + env: []string{"STORAGE_CLASS=my-class"}, + args: []string{"--setup-csi=nfs", "--deploy-demo-counter-with-external-volume"}, + want: []string{ + "setup csi nfs", + "deploy demo counter --with-external-volume --storage-class=my-class", + }, + }} { + t.Run(tc.name, func(t *testing.T) { + got, exitCode := runShim(t, tc.env, tc.args...) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + want := make([]string, len(tc.want)) + for i, w := range tc.want { + want[i] = prefix + w + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("invocations:\n got %q\n want %q", got, want) + } + }) + } +} + +// --benchmark-actor-memory has no ate-setup flag; the script exports it, which +// the stub go cannot observe through its arguments. +func TestShimExportsBenchmarkActorMemory(t *testing.T) { + root := repoRoot(t) + bin := t.TempDir() + const reportEnv = `#!/usr/bin/env bash +printf '%s\n' "BENCHMARK_ACTOR_MEMORY=${BENCHMARK_ACTOR_MEMORY:-}" +` + if err := os.WriteFile(filepath.Join(bin, "go"), []byte(reportEnv), 0o755); err != nil { + t.Fatalf("writing the stub go: %v", err) + } + + cmd := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh"), + "--benchmark-actor-memory", "512Mi", "--deploy-benchmarks") + cmd.Dir = root + cmd.Env = []string{"PATH=" + bin + ":" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} + + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("running install-ate.sh: %v\n%s", err, out) + } + if want := "BENCHMARK_ACTOR_MEMORY=512Mi\n"; string(out) != want { + t.Errorf("stub go saw %q, want %q", out, want) + } +} + +func TestShimRejectsUnknownFlags(t *testing.T) { + for _, args := range [][]string{ + {"--no-such-flag"}, + {"--deploy-demo-no-such-demo"}, + {"--delete-demo-no-such-demo"}, + } { + t.Run(strings.Join(args, " "), func(t *testing.T) { + got, exitCode := runShim(t, nil, args...) + if exitCode != 1 { + t.Errorf("exit code = %d, want 1", exitCode) + } + if len(got) != 0 { + t.Errorf("ran %q, want nothing", got) + } + }) + } +} + +// An action before an unknown flag still runs: the script dispatches as it +// walks the command line, as it always has. +func TestShimRunsActionsBeforeAnUnknownFlag(t *testing.T) { + got, exitCode := runShim(t, nil, "--deploy-atelet", "--no-such-flag") + if exitCode != 1 { + t.Errorf("exit code = %d, want 1", exitCode) + } + if want := []string{"run ./cmd/ate-setup deploy atelet"}; strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("invocations = %q, want %q", got, want) + } +} + +func TestShimUsage(t *testing.T) { + for _, tc := range []struct { + name string + args []string + exitCode int + }{ + {name: "no arguments is an error", args: nil, exitCode: 1}, + {name: "--help anywhere exits cleanly", args: []string{"--deploy-ate-system", "--help"}, exitCode: 0}, + {name: "-h anywhere exits cleanly", args: []string{"-h"}, exitCode: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + got, exitCode := runShim(t, nil, tc.args...) + if exitCode != tc.exitCode { + t.Errorf("exit code = %d, want %d", exitCode, tc.exitCode) + } + if len(got) != 0 { + t.Errorf("ran %q, want nothing: usage must not touch the cluster", got) + } + }) + } +} + +// Every flag the script dispatches on has to be documented, since --help is +// the only place a user can discover them now that the demos are Go packages. +func TestShimUsageDocumentsEveryFlag(t *testing.T) { + script, err := os.ReadFile(filepath.Join(repoRoot(t), "hack", "install-ate.sh")) + if err != nil { + t.Fatalf("reading install-ate.sh: %v", err) + } + usage := shimUsage(t) + + // The dispatch arms below `run_demo`'s definition, minus the demo + // wildcards, which usage covers one demo at a time. + for _, flag := range regexp.MustCompile(`(?m)^ (--[a-z0-9-]+)[)|]`).FindAllStringSubmatch(string(script), -1) { + if !strings.Contains(usage, flag[1]) { + t.Errorf("--help does not mention %s", flag[1]) + } + } + for _, d := range demos.All() { + for _, flag := range []string{"--deploy-" + d.Name(), "--delete-" + d.Name()} { + if !strings.Contains(usage, flag) { + t.Errorf("--help does not mention %s", flag) + } + } + } +} + +func shimUsage(t *testing.T) string { + t.Helper() + root := repoRoot(t) + cmd := exec.Command("bash", filepath.Join(root, "hack", "install-ate.sh"), "--help") + cmd.Dir = root + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} + out, err := cmd.Output() + if err != nil { + t.Fatalf("install-ate.sh --help: %v", err) + } + return string(out) +} diff --git a/internal/e2e/manifest.go b/internal/e2e/manifest.go index 19ceb7f693..5714973ba6 100644 --- a/internal/e2e/manifest.go +++ b/internal/e2e/manifest.go @@ -33,8 +33,8 @@ import ( // - inline, substituted wherever they appear (an empty value just disappears); // - block, which must be the entire content of their line. They expand to a // YAML fragment that brings its own indentation, and an empty value takes -// the whole line with it — the same trick hack/install-demo-counter.sh -// plays with `sed /.../d`. Requiring the placeholder to be the whole line +// the whole line with it — the same trick the installer's renderer plays +// with its drop list. Requiring the placeholder to be the whole line // is what lets a comment mention one without being deleted. func renderManifest(t *testing.T, relPath string, inline, blocks map[string]string) string { t.Helper() @@ -98,8 +98,8 @@ func yamlListBlock[T any](t *testing.T, key string, items []T, indent int) strin // // Through the repo's pinned ko (hack/run-tool.sh), because CI does not install // ko on PATH and every other deploy in this repo goes through that wrapper. The -// trailing `-- --context=...` mirrors run_ko in hack/install-ate.sh: ko's apply -// subcommand forwards args after `--` to kubectl. KO_CONFIG_PATH is required +// trailing `-- --context=...` is how ko's apply subcommand takes a context: it +// forwards args after `--` to kubectl. KO_CONFIG_PATH is required // because ko resolves .ko.yaml from its working directory, which is the test's // package dir rather than the repo root; without it the build silently loses // defaultPlatforms and produces images that cannot run on the cluster's nodes. From 427c8f2769dcf378805e9b39c05da0fa7930ef04 Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Wed, 16 Sep 2026 22:53:04 -0700 Subject: [PATCH 2/3] Update comments to be consistent with ate-setup --- Makefile | 4 ++-- demos/claude-code-multiplex/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1002e2f039..c2aaede94b 100644 --- a/Makefile +++ b/Makefile @@ -78,8 +78,8 @@ build-images: build-atectl: $(GO) build -ldflags "$(LDFLAGS)" -o $(ATECTL) ./cmd/kubectl-ate -# The cluster installer, a Go port of hack/install-ate.sh. Both work today; see -# cmd/ate-setup/commands.md for the flag-by-flag mapping between them. +# The cluster installer. hack/install-ate.sh is a shim over it; see +# cmd/ate-setup/commands.md for the flag-by-flag mapping between the two. .PHONY: build-ate-setup build-ate-setup: $(GO) build -ldflags "$(LDFLAGS)" -o $(ATESETUP) ./cmd/ate-setup diff --git a/demos/claude-code-multiplex/README.md b/demos/claude-code-multiplex/README.md index 0610b33b63..7bef61dbb3 100644 --- a/demos/claude-code-multiplex/README.md +++ b/demos/claude-code-multiplex/README.md @@ -44,7 +44,7 @@ This guide assumes you know Kubernetes and the general shape of agent runtimes ( |---|---| | `demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl` | Namespace and WorkerPool manifest | | `demos/claude-code-multiplex/agent-*-template.yaml.tmpl` | One protojson ActorTemplate per agent, created through the ate API with `kubectl ate create actor-template` | -| `hack/install-demo-claude-code-multiplex.sh` | Sourced by `install-ate.sh`; registers `--deploy-demo-claude-code-multiplex` and `--delete-demo-claude-code-multiplex` | +| `cmd/ate-setup/internal/demos/claudemultiplex/` | The installer's demo package, behind `--deploy-demo-claude-code-multiplex` / `--delete-demo-claude-code-multiplex` | | `demos/claude-code-multiplex/workload/` | The agent container image source (Dockerfile + entrypoint that wires Claude Code; built and pushed by the deploy step) | | `demos/claude-code-multiplex/ui/` | Static dashboard (`index.html` + `server.go`) that talks to the cluster | From af024b45afefb50c56b72a9d970f456c2cff6039 Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Wed, 16 Sep 2026 22:50:31 -0700 Subject: [PATCH 3/3] install shell now calls ate-setup --- .../experimental-additional-egress-extproc.sh | 237 --- hack/install-ate.sh | 1582 ++--------------- hack/install-demo-autoscaled-workerpool.sh | 70 - hack/install-demo-claude-code-multiplex.sh | 105 -- hack/install-demo-counter.sh | 111 -- hack/install-demo-egress.sh | 149 -- hack/install-demo-jupyter.sh | 46 - hack/install-demo-multi-template.sh | 64 - hack/install-demo-parking.sh | 42 - hack/install-demo-sandbox.sh | 42 - 10 files changed, 163 insertions(+), 2285 deletions(-) delete mode 100644 hack/experimental-additional-egress-extproc.sh delete mode 100644 hack/install-demo-autoscaled-workerpool.sh delete mode 100644 hack/install-demo-claude-code-multiplex.sh delete mode 100644 hack/install-demo-counter.sh delete mode 100644 hack/install-demo-egress.sh delete mode 100755 hack/install-demo-jupyter.sh delete mode 100644 hack/install-demo-multi-template.sh delete mode 100644 hack/install-demo-parking.sh delete mode 100644 hack/install-demo-sandbox.sh diff --git a/hack/experimental-additional-egress-extproc.sh b/hack/experimental-additional-egress-extproc.sh deleted file mode 100644 index 27bdfd5cd5..0000000000 --- a/hack/experimental-additional-egress-extproc.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. -# -# --experimental-additional-egress-extproc-service=NS/SVC:PORT runs an ext_proc -# authorization filter on the egress gateway's decrypted leg, where a request is -# a hostname, method, and path rather than the IP:port the CONNECT checkpoint -# sees. - -additional_egress_extproc_enabled() { - [[ -n "${ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE:-}" ]] -} - -# additional_egress_extproc_endpoint validates -# --experimental-additional-egress-extproc-service and echoes the DNS name to -# dial, the port, and the name to verify the server certificate against, -# space-separated. -# -# Example input: ate-system/foo:50051 -# Example output: foo.ate-system.svc.cluster.local 50051 foo.ate-system.svc -additional_egress_extproc_endpoint() { - local spec="$1" - local label='[a-z0-9]([-a-z0-9]*[a-z0-9])?' - if [[ ! "${spec}" =~ ^${label}/${label}:[0-9]+$ ]]; then - echo "Error: --experimental-additional-egress-extproc-service must be /:, got '${spec}'" >&2 - return 1 - fi - - local namespace="${spec%%/*}" - local rest="${spec#*/}" - local service="${rest%%:*}" - local port="${rest##*:}" - if (( port < 1 || port > 65535 )); then - echo "Error: --experimental-additional-egress-extproc-service port must be 1-65535, got '${port}'" >&2 - return 1 - fi - - echo "${service}.${namespace}.svc.cluster.local ${port} ${service}.${namespace}.svc" -} - -# The Envoy cluster the additional ext_proc filter dials. -readonly ADDITIONAL_EGRESS_EXTPROC_CLUSTER="additional_egress_ext_proc" - -# Note: the heredoc sections are are written at column zero; the awk in -# patch_atenet_egress_manifest will re-indent to the right column. - -emit_additional_egress_extproc_filter() { - cat <&2 - return 1 - fi - - local endpoint address port server_name - endpoint="$(additional_egress_extproc_endpoint "${ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE}")" || return 1 - read -r address port server_name <<<"${endpoint}" - - local filter_block cluster_block - filter_block="$(emit_additional_egress_extproc_filter)" || return 1 - cluster_block="$(emit_additional_egress_extproc_cluster \ - "${address}" "${port}" "${server_name}")" || return 1 - - local expected_filter_markers=2 - - # Anchored to the start of the line so that prose mentioning a marker -- the - # mitm_listener comment in the manifest names both of them -- is not itself - # replaced by a config block. - # - # Pass ATE_EXTPROC_... as env vars in ENVIRON to work around differences - # between gawk and BSD awk. - ATE_EXTPROC_FILTER_BLOCK="${filter_block}" \ - ATE_EXTPROC_CLUSTER_BLOCK="${cluster_block}" \ - awk -v want_filters="${expected_filter_markers}" ' - BEGIN { - filter = ENVIRON["ATE_EXTPROC_FILTER_BLOCK"] - cluster = ENVIRON["ATE_EXTPROC_CLUSTER_BLOCK"] - } - - /^[ \t]*#ATE_MITM_EXTPROC_FILTER/ { splice(filter); filters++; next } - /^[ \t]*#ATE_MITM_EXTPROC_CLUSTER/ { splice(cluster); clusters++; next } - { print } - - END { - if (filters != want_filters || clusters != 1) { - printf("Error: expected %d #ATE_MITM_EXTPROC_FILTER and 1 #ATE_MITM_EXTPROC_CLUSTER marker in %s, found %d and %d\n", - want_filters, FILENAME, filters, clusters) > "/dev/stderr" - exit 1 - } - } - - # Prints block at the indentation of the marker line being replaced. The - # heredocs above are written at column zero. - function splice(block, indent, lines, n, i) { - match($0, /^[ \t]*/) - indent = substr($0, 1, RLENGTH) - n = split(block, lines, "\n") - for (i = 1; i <= n; i++) { - print (lines[i] == "" ? "" : indent lines[i]) - } - } - ' "${manifest}" -} diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 5a2f411428..d218e02caf 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -13,61 +13,77 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Installs Agent Substrate. +# +# This is a translation shim: cmd/ate-setup carries out every action, and this +# script only maps the flags and environment variables the installer has always +# accepted onto its commands, so existing command lines and CI jobs keep +# working unchanged. It deliberately holds no install logic of its own. +# +# New work belongs in cmd/ate-setup. See cmd/ate-setup/commands.md for the +# equivalent invocation of each flag below, and cmd/ate-setup/cli-diff.md for +# the behaviors that differ after the move. +# +# The environment variables the installer reads -- BUCKET_NAME, KO_DOCKER_REPO, +# PROJECT_ID, CLUSTER_NAME, CLUSTER_LOCATION, KUBECTL_CONTEXT, NO_DEV_ENV, the +# ATE_API_POSTGRES_* set, and the rest -- are read by ate-setup directly, as is +# .ate-dev-env.sh. They need no translation and are not repeated here. set -o errexit -o nounset -o pipefail ROOT="$(git rev-parse --show-toplevel)" cd "${ROOT}" -# Source the environment variables if configured -# TODO: this pattern makes it difficult to switch environments. -# Developers will likely want to target both cloud and local depending on what they're working on. -if [[ -f .ate-dev-env.sh ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then - source .ate-dev-env.sh -fi - -# If the user has set KUBECTL_CONTEXT, we can assume they already have credentials. -if [[ -z "${KUBECTL_CONTEXT:-}" ]]; then - # If PROJECT_ID is set, ensure kubeconfig is configured before running any kubectl commands. - if [[ -n "${PROJECT_ID:-}" ]]; then - gcloud container clusters get-credentials "${CLUSTER_NAME}" --location "${CLUSTER_LOCATION}" --project="${PROJECT_ID}" - fi -fi -# otherwise just use the current cluster in KUBECONFIG ... - -# ATE_DEMOS is an array that registers the prefix name of the demo functions. -ATE_DEMOS=() - -# Called by a ${demo}_cmdline handler for an argument it does not own. -ate_demo_flag_unhandled() { - ATE_DEMO_FLAG_HANDLED=false -} - -# Include demos. -source "${ROOT}"/hack/install-demo-counter.sh -source "${ROOT}"/hack/install-demo-egress.sh -source "${ROOT}"/hack/install-demo-jupyter.sh -source "${ROOT}"/hack/install-demo-sandbox.sh -source "${ROOT}"/hack/install-demo-claude-code-multiplex.sh -source "${ROOT}"/hack/install-demo-multi-template.sh -source "${ROOT}"/hack/install-demo-parking.sh -source "${ROOT}"/hack/install-demo-autoscaled-workerpool.sh - -# Include the optional ext_proc filter on the egress gateway's decrypted leg, -# behind --experimental-additional-egress-extproc-service. -source "${ROOT}"/hack/experimental-additional-egress-extproc.sh - -# ANSI color codes for prettier output -COLOR_CYAN='\033[1;36m' -COLOR_RESET='\033[0m' - -function log_step() { - local step_name="$1" - echo -e "${COLOR_CYAN}[step]: ${step_name}${COLOR_RESET}" +# Demos, in the order --help lists them. Keeping the list here rather than +# asking ate-setup for it means --help answers without a build, and that a demo +# flag this script does not know is rejected outright rather than handed to +# ate-setup as an unknown subcommand. `go run ./cmd/ate-setup deploy demo +# --help` is the authoritative list; add new demos there and mirror them here. +ATE_DEMOS=( + demo-counter + demo-counter-microvm + demo-egress + demo-egress-microvm + demo-egress-mitm + demo-egress-microvm-mitm + demo-jupyter + demo-sandbox + demo-claude-code-multiplex + demo-multi-template + demo-parking + demo-autoscaled-workerpool +) + +# Prerequisites and caveats worth repeating in --help, keyed by demo. +demo_usage() { + case "$1" in + demo-counter) + echo " --deploy-demo-counter-with-external-volume Deploy demo-counter with external volume validation" + echo " (STORAGE_CLASS names the class; it otherwise follows --setup-csi)" + ;; + demo-counter-microvm|demo-egress-microvm) + echo " Needs hack/install-microvm-deps.sh --install to have run (cluster-wide microvm SandboxConfig)." + ;; + demo-egress-mitm) + echo " Needs an sdsmint install (--deploy-atenet --experimental-use-sdsmint): the actors" + echo " project the egress gateway trust bundle, which does not resolve otherwise." + ;; + demo-egress-microvm-mitm) + echo " Needs hack/install-microvm-deps.sh --install to have run (cluster-wide microvm SandboxConfig)," + echo " and an sdsmint install (--deploy-atenet --experimental-use-sdsmint) for the trust bundle." + ;; + demo-claude-code-multiplex) + echo " Required env: ANTHROPIC_API_KEY, BUCKET_NAME, KO_DOCKER_REPO" + echo " See demos/claude-code-multiplex/README.md for the walkthrough." + ;; + demo-autoscaled-workerpool) + echo " Kind only: it ships its own prometheus-adapter and a Kind-specific HPA." + ;; + esac } -# --- Helper Functions --- -function usage() { +usage() { echo "Usage: $0 [options]" echo "" echo "Overall infrastructure (all infrastructure components):" @@ -94,6 +110,7 @@ function usage() { echo " --deploy-atelet Deploy atelet only" echo " --deploy-ate-apiserver Deploy ate-api-server only" echo " --deploy-atenet Deploy atenet only" + echo " --delete-atenet Delete atenet only" echo "" echo "To create individual resources used by ate-system (Note: These are" echo "called automatically by --deploy-ate-system):" @@ -139,1267 +156,58 @@ function usage() { echo " --benchmark-actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi," echo " the smallest size microvm admits)" echo "" + local demo_name for demo_name in "${ATE_DEMOS[@]}"; do echo "Demo: ${demo_name}" echo "" echo " --deploy-${demo_name} Deploy ${demo_name}" echo " --delete-${demo_name} Delete ${demo_name}" - if declare -F "${demo_name}_usage" >/dev/null 2>&1; then - "${demo_name}_usage" - fi - done -} - -run_kubectl() { - kubectl \ - ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} \ - "$@" -} - -# run_kubectl_fatal runs kubectl and aborts the install if it fails. Demo -# handlers need this: the dispatcher below calls them from an `if` condition, -# which suppresses errexit for everything they run, so a plain run_kubectl that -# fails is silently ignored -- a broken wait then costs its whole timeout and -# lets the install "succeed" anyway. -run_kubectl_fatal() { - if ! run_kubectl "$@"; then - echo "error: kubectl $* failed" >&2 - exit 1 - fi -} - -# wait_for_pool_rollout waits for a WorkerPool's Deployment to roll out. -# ate-controller creates that Deployment, so it does not exist yet when the -# apply returns, and `kubectl rollout status` errors on a missing object rather -# than waiting for it. Gate on creation first. -wait_for_pool_rollout() { - local pool="$1" namespace="$2" timeout="${3:-300s}" - run_kubectl wait --for=create "deployment/${pool}" -n "${namespace}" \ - --timeout="${timeout}" \ - && run_kubectl rollout status "deployment/${pool}" -n "${namespace}" \ - --timeout="${timeout}" -} - -# wait_for_pool_rollout_fatal aborts the install on failure, like run_kubectl_fatal. -wait_for_pool_rollout_fatal() { - if ! wait_for_pool_rollout "$@"; then - echo "error: worker pool $1 did not roll out" >&2 - exit 1 - fi -} - -run_kubectl_ate() { - go run ./cmd/kubectl-ate \ - ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} \ - "$@" -} - -run_ko() { - # Build up a set of ldflags to pass to ko. - local ldflags=() - while IFS= read -r line || [[ -n "${line}" ]]; do - [[ -n "${line}" ]] && ldflags+=("--ldflags=${line}") - done < <(make ldflags) - - # Only ko subcommands that delegate to kubectl (apply, create, delete, run) - # accept args after `--`. ko build, resolve, deps, login etc. reject - # `--context=...` as an unknown subcommand and abort the install. - case "${1:-}" in - apply|create|delete|run) - ./hack/run-tool.sh ko "$@" \ - "${ldflags[@]}" \ - ${KUBECTL_CONTEXT:+-- --context="${KUBECTL_CONTEXT}"} - ;; - *) - ./hack/run-tool.sh ko "$@" \ - "${ldflags[@]}" - ;; - esac -} - -atenet_router() { - case "${ATE_ATENET_DATAPLANE:-envoy}" in - envoy|agentgateway) - echo "${ATE_ATENET_DATAPLANE:-envoy}" - ;; - *) - echo "Error: --atenet-dataplane must be envoy or agentgateway, got '${ATE_ATENET_DATAPLANE}'" >&2 - exit 1 - ;; - esac -} - -podcert_workers_per_signer() { - local workers="${ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER:-1}" - if ! [[ "${workers}" =~ ^[1-9][0-9]*$ ]]; then - echo "Error: --podcert-workers-per-signer must be a positive integer, got '${workers}'" >&2 - exit 1 - fi - echo "${workers}" -} - -rollout_timeout() { - local timeout="${ATE_INSTALL_ROLLOUT_TIMEOUT:-60s}" - if ! [[ "${timeout}" =~ ^(0|([0-9]+(h|m|s))+)$ ]]; then - echo "Error: --rollout-timeout must be a Go duration like 300s, 10m, or 1h30m (or 0 for no timeout), got '${timeout}'" >&2 - exit 1 - fi - echo "${timeout}" -} - -default_postgres_connection_string() { - echo "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -} - -# True if deploying the bundled in-cluster PostgreSQL. Returns false if an -# external database is configured via an explicit DSN or a Cloud SQL instance -# (whether provided in the environment or adopted from the cluster). -use_bundled_postgres() { - [[ -z "${ATE_API_POSTGRES_CONNECTION_STRING:-}" && -z "$(resolve_cloudsql_instance)" ]] -} - -# --- Versioned dataplane rendering --- -# -# The atelet DaemonSet is keyed by substrate version (name suffix + nodeSelector on -# the ate.dev/substrate-version node label) so a rolling upgrade can run two -# daemonset on disjoint node sets. - -SUBSTRATE_VERSION="" -SUBSTRATE_VERSION_SUFFIX="" - -ensure_substrate_version() { - if [[ -n "${SUBSTRATE_VERSION}" ]]; then - return 0 - fi - local version_flag="" - version_flag="$(make -s ldflags | grep 'internal/version\.Version=' | head -n 1 || true)" - local version_raw="${version_flag#*internal/version.Version=}" - if [[ -z "${version_raw}" ]]; then - echo "error: could not read the build version from 'make ldflags'" >&2 - return 1 - fi - # Verify the build label is valid format and derive the version string and object suffix. - local derived="" - derived="$(go run ./internal/versionlabel/cmd "${version_raw}")" || return 1 - SUBSTRATE_VERSION="${derived%% *}" - SUBSTRATE_VERSION_SUFFIX="${derived##* }" -} - -atelet_daemonset_name() { - echo "atelet-${SUBSTRATE_VERSION_SUFFIX}" -} - -# substitute_version fills the ${SUBSTRATE_VERSION}/${SUBSTRATE_VERSION_SUFFIX} -# placeholders in a rendered manifest stream. It runs after kustomize and ko to replace. -substitute_version() { - ensure_substrate_version - sed -e "s/: \${SUBSTRATE_VERSION}\$/: \"${SUBSTRATE_VERSION}\"/" \ - -e "s/\${SUBSTRATE_VERSION}/${SUBSTRATE_VERSION}/g" \ - -e "s/\${SUBSTRATE_VERSION_SUFFIX}/${SUBSTRATE_VERSION_SUFFIX}/g" -} - -# label_nodes_substrate_version stamps ate.dev/substrate-version on every node -# that does not carry it yet. The versioned atelet DaemonSet and worker pods -# schedule only to nodes labeled with their substrate version. -label_nodes_substrate_version() { - ensure_substrate_version - log_step "label_nodes_substrate_version (${SUBSTRATE_VERSION})" - local node="" - for node in $(run_kubectl get nodes -l '!ate.dev/substrate-version' -o name); do - run_kubectl label "${node}" "ate.dev/substrate-version=${SUBSTRATE_VERSION}" - done -} - -render_ate_system_manifests() { - local router="" - router="$(atenet_router)" - - if [[ "${router}" == "agentgateway" ]]; then - local overlay="manifests/ate-install/agentgateway" - if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - overlay="manifests/ate-install/kind-agentgateway" - fi - kubectl kustomize "${overlay}" --load-restrictor LoadRestrictionsNone | run_ko resolve -f - | substitute_version - return - fi - - if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - # Build everything resolved with Kustomize for Kind - kubectl kustomize manifests/ate-install/kind --load-restrictor LoadRestrictionsNone | run_ko resolve -f - | substitute_version - else - # Build everything resolved with base manifests for GKE - run_ko resolve -f manifests/ate-install | substitute_version - fi -} - -render_atenet_router_manifest() { - if [[ "$(atenet_router)" == "agentgateway" ]]; then - kubectl kustomize manifests/ate-install/agentgateway-router \ - --load-restrictor LoadRestrictionsNone | run_ko resolve -f - - else - run_ko resolve -f manifests/ate-install/atenet-router.yaml - fi -} - -# atenet_egress_manifest echoes the path of the egress manifest to deploy: -# the sdsmint variant under --experimental-use-sdsmint, the shipped one -# otherwise. The two are whole files rather than a Kustomize overlay because -# what differs between them is envoy.yaml, which lives as one inline string in -# the atenet-egress ConfigMap; Kustomize can replace that string but cannot -# patch into it, so an overlay would carry a full copy of it anyway. -atenet_egress_manifest() { - if [[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" == "true" ]]; then - echo "manifests/ate-install/atenet-egress-with-sdsmint.yaml" - else - echo "manifests/ate-install/atenet-egress.yaml" - fi -} - -render_atenet_egress_manifest() { - if [[ "$(atenet_router)" == "agentgateway" ]]; then - # The markers live inside Envoy's bootstrap, so there is nowhere here to - # put the filter. Refuse for the same reason patch_atenet_egress_manifest - # refuses a non-sdsmint manifest: ignoring the flag would report a - # successful install of a gateway that has no additional checkpoint on it. - if additional_egress_extproc_enabled; then - echo "Error: --experimental-additional-egress-extproc-service requires --atenet-dataplane=envoy" >&2 - return 1 - fi - local agentgateway_egress="manifests/ate-install/agentgateway-egress" - if [[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" == "true" ]]; then - agentgateway_egress="manifests/ate-install/agentgateway-egress-mitm" - fi - kubectl kustomize "${agentgateway_egress}" \ - --load-restrictor LoadRestrictionsNone | run_ko resolve -f - - elif additional_egress_extproc_enabled; then - patch_atenet_egress_manifest | run_ko resolve -f - - else - run_ko resolve -f "$(atenet_egress_manifest)" - fi -} - -# apply_atenet_egress deploys the egress gateway. -apply_atenet_egress() { - local manifests="" - manifests="$(render_atenet_egress_manifest)" - - # Whether it is already running has to be settled before the apply: a patched - # bootstrap arrives as a ConfigMap change, and an otherwise unchanged - # Deployment will not pick that up on its own. - local running=false - if run_kubectl -n ate-system get deployment/atenet-egress >/dev/null 2>&1; then - running=true - fi - - echo "${manifests}" | run_kubectl apply -f - - - if [[ "${running}" == "true" ]] && additional_egress_extproc_enabled; then - run_kubectl -n ate-system rollout restart deployment/atenet-egress - fi -} - -# Apply the ate-otel-config ConfigMap that every control plane component reads -# via envFrom. The full install gets it through render_ate_system_manifests, but -# the targeted single-component redeploys below apply raw manifests with no -# Kustomize, so they have to select the environment's copy themselves. Applying -# the base file unconditionally would overwrite a kind cluster's ConfigMap with -# the GKE endpoint and silently break telemetry for every component at once. -apply_otel_config() { - if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - run_kubectl apply -f manifests/ate-install/kind/ate-otel-config.yaml - else - run_kubectl apply -f manifests/ate-install/ate-otel-config.yaml - fi -} - -apply_postgres() { - if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - kubectl kustomize manifests/ate-install/kind/postgres \ - --load-restrictor LoadRestrictionsNone | run_kubectl apply -f - - else - run_kubectl apply -f manifests/ate-install/postgres/postgres.yaml - fi -} - -# --otlp-endpoint sends all control plane telemetry to a different collector for -# the duration of a measurement. One patch is sufficient: each component reads -# this ConfigMap through envFrom, and ate-controller copies the values to the -# ateom worker pods that it creates. See benchmarking/telemetry/README.md. -# -# Call this AFTER every apply. The ate-system bundle contains its own copy of -# ate-otel-config, thus an apply of the bundle replaces a patch that came -# before it, and the endpoint returns to the cluster default with no error -# message. -# -# A change to a ConfigMap starts no rollout, because the pod template stays the -# same. Thus restart the consumers that read it. Do the restart only when the -# value changes: a restart during the rollout of the bundle makes the two -# rollouts compete, and `kubectl rollout status` can then exceed its timeout. -# An absent workload is not an error, because a deploy of one component has -# only that component. -apply_otel_endpoint_override() { - if [[ -z "${ATE_OTLP_ENDPOINT:-}" ]]; then - return 0 - fi - - local current="" - current="$(run_kubectl -n ate-system get configmap ate-otel-config \ - -o jsonpath='{.data.OTEL_EXPORTER_OTLP_ENDPOINT}' 2>/dev/null || true)" - if [[ "${current}" == "${ATE_OTLP_ENDPOINT}" ]]; then - return 0 - fi - - echo "Overriding OTEL_EXPORTER_OTLP_ENDPOINT with ${ATE_OTLP_ENDPOINT}" - run_kubectl -n ate-system patch configmap ate-otel-config --type=merge \ - -p "{\"data\":{\"OTEL_EXPORTER_OTLP_ENDPOINT\":\"${ATE_OTLP_ENDPOINT}\"}}" - - local workload - for workload in deployment/ate-api-server deployment/ate-controller \ - deployment/atenet-router; do - if run_kubectl -n ate-system get "${workload}" >/dev/null 2>&1; then - run_kubectl -n ate-system rollout restart "${workload}" - fi - done - # atelet DaemonSet names carry a version suffix; restart whichever versions - # are installed. - local ds="" - for ds in $(run_kubectl -n ate-system get daemonset -l app=atelet -o name 2>/dev/null); do - run_kubectl -n ate-system rollout restart "${ds}" + demo_usage "${demo_name}" + echo "" done } -# Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate. -# The namespace defaults to the podcertificate controller's, where the signer -# CAs live; the actor-identity CA pool is in ate-system, so it passes its own. -ca_pool_root_pem() { - local secret="$1" - local namespace="${2:-podcertificate-controller-system}" - local pool_json="" - pool_json=$(run_kubectl get secret -n "${namespace}" "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) - local der_base64="" - der_base64=$(echo "${pool_json}" | grep -o '"RootCertificateDER":"[^"]*' | sed 's/"RootCertificateDER":"//') - echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem +# ate_setup runs one ate-setup command. GLOBAL_FLAGS holds the pre-scanned +# flags that shape every action, which is how they behaved here: a single +# command line could ask for several actions and one --atenet-dataplane. +ate_setup() { + go run ./cmd/ate-setup ${GLOBAL_FLAGS[@]+"${GLOBAL_FLAGS[@]}"} "$@" } -create_jwt_authority_pool_secret() { - log_step "create_jwt_authority_pool_secret" - run_kubectl_ate admin make-jwt-pool \ - --key-id="1" \ - --name="actor-id-jwt-pool" \ - --secret-namespace=ate-system -} - -create_actor_id_ca_pool_secret() { - log_step "create_actor_id_ca_pool_secret" - run_kubectl_ate admin make-ca-pool \ - --ca-id="1" \ - --name="actor-id-ca-pool" \ - --secret-namespace=ate-system -} - -# The egress gateway has to verify actor client certificates, which means it -# needs the actor-identity CA root. actor-id-ca-pool Secret containts both -# root and CA signing key. This derives a cert-only Secret for the signer root. -# -# TODO(liorlieberman): should this be published as ClusterTrustBundles? -create_actor_id_ca_certs_secret() { - log_step "create_actor_id_ca_certs_secret" - # Extract into its own variable first: errexit cannot see a substitution fail - # inside the create-secret argument list, which would silently produce an - # empty trust bundle and an egress gateway that rejects every actor. - local actorid_root="" - actorid_root=$(ca_pool_root_pem actor-id-ca-pool ate-system) - if [[ -z "${actorid_root}" ]]; then - echo "error: failed to extract the actor-identity CA root for actor-id-ca-certs" >&2 - return 1 - fi - - run_kubectl create secret generic actor-id-ca-certs \ - --from-literal=ca.crt="${actorid_root}" \ - -n ate-system \ - --dry-run=client -o yaml \ - | run_kubectl apply -f - -} - -# The MITM CA the egress gateway's sdsmint sidecar signs per-SNI leaves with. -# ecdsa-p256 rather than the ed25519 default: these leaves are validated by -# arbitrary clients inside actor sandboxes, where Ed25519 support cannot be -# assumed. -create_egress_mitm_ca_pool_secret() { - log_step "create_egress_mitm_ca_pool_secret" - run_kubectl_ate admin make-ca-pool \ - --ca-id="1" \ - --name="egress-mitm-ca-pool" \ - --secret-namespace=ate-system \ - --key-type=ECDSAP256 -} - -# Both egress implementations read this pool only in their opt-in MITM mode: -# AgentGateway consumes its exported TLS chain and key, while Envoy's sdsmint -# sidecar consumes the serialized pool. -ensure_egress_mitm_ca_pool_secret() { - if [[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" != "true" ]]; then - return 0 - fi - run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ - || create_egress_mitm_ca_pool_secret -} - -create_podcertificate_controller_cas() { - log_step "create_podcertificate_controller_cas" - run_kubectl create namespace podcertificate-controller-system || true - run_kubectl_ate admin make-ca-pool \ - --ca-id="1" \ - --name="service-dns-ca-pool" \ - --secret-namespace=podcertificate-controller-system - run_kubectl_ate admin make-ca-pool \ - --ca-id="1" \ - --name="pod-identity-ca-pool" \ - --secret-namespace=podcertificate-controller-system -} - -wait_for_podcertificate_trust_bundles() { - echo "Waiting for podcertificate ClusterTrustBundles to be ready..." - until run_kubectl get clustertrustbundles podidentity.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do - sleep 1 - done - until run_kubectl get clustertrustbundles servicedns.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do - sleep 1 - done -} - -# Cloud SQL intent: a SET ATE_API_POSTGRES_CLOUDSQL_INSTANCE wins, with the -# empty string meaning an explicit "remove"; an UNSET variable adopts -# whatever the cluster currently records. -resolve_cloudsql_instance() { - if [[ -n "${ATE_API_POSTGRES_CLOUDSQL_INSTANCE+x}" ]]; then - echo "${ATE_API_POSTGRES_CLOUDSQL_INSTANCE}" +# counter_storage_class names the StorageClass the counter demo's external +# volume is provisioned from. STORAGE_CLASS names it outright; without it, fall +# back to whatever --setup-csi just installed, and to ate-setup's own default +# when it installed nothing. +counter_storage_class() { + if [[ -n "${STORAGE_CLASS:-}" ]]; then + echo "${STORAGE_CLASS}" return fi - run_kubectl get configmap -n ate-system ate-api-server-envvars \ - -o jsonpath='{.data.ATE_API_POSTGRES_CLOUDSQL_INSTANCE}' 2>/dev/null || true -} - -# resolve_cloudsql_gsa mirrors resolve_cloudsql_instance for the GSA, -# adopting from the ServiceAccount's Workload Identity annotation. -resolve_cloudsql_gsa() { - if [[ -n "${ATE_API_POSTGRES_CLOUDSQL_GSA:-}" ]]; then - echo "${ATE_API_POSTGRES_CLOUDSQL_GSA}" - return - fi - run_kubectl get serviceaccount ate-api-server -n ate-system \ - -o "jsonpath={.metadata.annotations.iam\.gke\.io/gcp-service-account}" 2>/dev/null || true -} - -# recorded_envvar echoes one key from the ate-api-server-envvars ConfigMap. -recorded_envvar() { - run_kubectl get configmap -n ate-system ate-api-server-envvars \ - -o "jsonpath={.data.$1}" 2>/dev/null || true -} - -# Guards standalone --create-api-server-env-vars on older clusters. -# Since the DSN moved from a ConfigMap to a Secret, updating the env vars alone -# would prune the ConfigMap key and leave the running Deployment without a DSN. -# Full deploys are safe because they update the Deployment manifest immediately. -ensure_env_vars_safe_standalone() { - if ! run_kubectl get deployment ate-api-server -n ate-system >/dev/null 2>&1; then - return 0 # fresh install: the manifest applied later carries the secretRef - fi - local refs - refs="$(run_kubectl get deployment ate-api-server -n ate-system \ - -o jsonpath='{.spec.template.spec.containers[0].envFrom[*].secretRef.name}' 2>/dev/null || true)" - if [[ "${refs}" != *ate-api-server-secret-envvars* ]]; then - echo "Error: the running ate-api-server Deployment does not reference the" \ - "ate-api-server-secret-envvars Secret; rewriting the env vars alone would leave" \ - "it without a DSN on its next restart. Run" \ - "./hack/install-ate.sh --deploy-ate-apiserver instead, which also updates the Deployment." >&2 - exit 1 - fi -} - -# Triggers a pod rollout when the ConfigMap or Secret changes. Since envFrom -# updates don't roll pods automatically, we patch a hash of the config into -# the deployment template. -annotate_api_server_env_hash() { - if ! run_kubectl get deployment ate-api-server -n ate-system >/dev/null 2>&1; then - return 0 # fresh install: the first rollout starts with the new values - fi - local hash - hash="$({ run_kubectl get configmap -n ate-system ate-api-server-envvars \ - -o jsonpath='{.data}' 2>/dev/null || true - run_kubectl get secret -n ate-system ate-api-server-secret-envvars \ - -o jsonpath='{.data}' 2>/dev/null || true; } \ - | openssl dgst -sha256 | awk '{print $NF}')" - run_kubectl patch deployment ate-api-server -n ate-system --type=strategic -p \ - "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"ate.dev/env-hash\":\"${hash}\"}}}}}" -} - -create_api_server_env_vars() { - log_step "create_api_server_env_vars" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ - | run_kubectl apply -f - - - local postgres_connection_string="${ATE_API_POSTGRES_CONNECTION_STRING:-}" - local postgres_schema="${ATE_API_POSTGRES_SCHEMA:-public}" - # Distinguishes a DSN the operator supplied on this run from one - # synthesized, defaulted, or adopted back from the Secret: only the former - # outranks ATE_API_POSTGRES_POOL_MAX_CONNS below. - local dsn_from_operator="" - [[ -n "${postgres_connection_string}" ]] && dsn_from_operator="yes" - local cloudsql_instance cloudsql_gsa="" - local cloudsql_iam_auth="${ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH:-true}" - local cloudsql_ip_type="${ATE_API_POSTGRES_CLOUDSQL_IP_TYPE:-private}" - cloudsql_instance="$(resolve_cloudsql_instance)" - if [[ -n "${cloudsql_instance}" ]]; then - cloudsql_gsa="$(resolve_cloudsql_gsa)" - if [[ -z "${ATE_API_POSTGRES_CLOUDSQL_INSTANCE+x}" ]]; then - # Adopted from the cluster: inherit the recorded proxy settings and - # the existing DSN too, so a redeploy without the env vars regresses - # nothing to defaults. - echo "Cloud SQL config adopted from cluster: ${cloudsql_instance}" - if [[ -z "${ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH:-}" ]]; then - local recorded_iam - recorded_iam="$(recorded_envvar CSQL_PROXY_AUTO_IAM_AUTHN)" - [[ -n "${recorded_iam}" ]] && cloudsql_iam_auth="${recorded_iam}" - fi - if [[ -z "${ATE_API_POSTGRES_CLOUDSQL_IP_TYPE:-}" ]]; then - if [[ -n "$(recorded_envvar CSQL_PROXY_PSC)" ]]; then - cloudsql_ip_type="psc" - elif [[ -z "$(recorded_envvar CSQL_PROXY_PRIVATE_IP)" ]]; then - cloudsql_ip_type="public" - fi - fi - if [[ -z "${postgres_connection_string}" ]]; then - postgres_connection_string="$(run_kubectl get secret -n ate-system ate-api-server-secret-envvars \ - -o jsonpath='{.data.ATE_API_POSTGRES_CONNECTION_STRING}' 2>/dev/null | base64 --decode || true)" - fi - fi - fi - if [[ -z "${postgres_connection_string}" ]]; then - if [[ -n "${cloudsql_instance}" ]]; then - # Cloud SQL via the Auth Proxy sidecar: ateapi talks plaintext to the - # proxy on pod-local loopback; the proxy owns TLS and IAM database - # authentication. The database username is the GSA email with the - # .gserviceaccount.com suffix trimmed. - # - # The synthesized DSN is passwordless, which only logs in when the - # proxy injects an IAM token. With IAM auth disabled the operator must - # supply the credentials themselves; failing here beats a Postgres - # authentication error at pod startup. - if [[ "${cloudsql_iam_auth}" == "false" ]]; then - echo "Error: ATE_API_POSTGRES_CLOUDSQL_IAM_AUTH=false disables automatic IAM database" \ - "authentication, so a passwordless DSN cannot be synthesized; set" \ - "ATE_API_POSTGRES_CONNECTION_STRING explicitly (host=127.0.0.1 to stay on the proxy)" >&2 - exit 1 - fi - if [[ -z "${cloudsql_gsa}" ]]; then - echo "Error: ATE_API_POSTGRES_CLOUDSQL_INSTANCE requires ATE_API_POSTGRES_CLOUDSQL_GSA" \ - "(or an explicit ATE_API_POSTGRES_CONNECTION_STRING)" >&2 - exit 1 - fi - postgres_connection_string="user=${cloudsql_gsa%.gserviceaccount.com} host=127.0.0.1 port=5432 dbname=atepg sslmode=disable" - else - postgres_connection_string="$(default_postgres_connection_string)" - fi - fi - # Appends pgxpool sizing (pool_max_conns) to the DSN to prevent silent client - # queuing. Precedence: An explicitly provided DSN wins over the env var. However, - # the env var overwrites values in adopted cluster DSNs, ensuring scaling updates - # aren't silently ignored on redeploys. Handles both URI and keyword/value formats. - if [[ -n "${ATE_API_POSTGRES_POOL_MAX_CONNS:-}" ]]; then - if [[ "${postgres_connection_string}" == *pool_max_conns=* ]]; then - if [[ -z "${dsn_from_operator}" ]]; then - postgres_connection_string="$(printf '%s' "${postgres_connection_string}" \ - | sed -E "s/pool_max_conns=[^ &]*/pool_max_conns=${ATE_API_POSTGRES_POOL_MAX_CONNS}/")" - fi - elif [[ "${postgres_connection_string}" == *"://"*"?"* ]]; then - postgres_connection_string+="&pool_max_conns=${ATE_API_POSTGRES_POOL_MAX_CONNS}" - elif [[ "${postgres_connection_string}" == *"://"* ]]; then - postgres_connection_string+="?pool_max_conns=${ATE_API_POSTGRES_POOL_MAX_CONNS}" - else - postgres_connection_string+=" pool_max_conns=${ATE_API_POSTGRES_POOL_MAX_CONNS}" - fi - fi - - # Redact any password before logging (URI user:pw@host and keyword password=). - echo "POSTGRES_CONNECTION_STRING: $(printf '%s' "${postgres_connection_string}" \ - | sed -E 's#(://[^:/@]*):[^@]*@#\1:***@#; s/(password=)[^ &]*/\1***/g')" - - # Empty unless Cloud SQL is configured; expanded below with the - # ${arr[@]+...} idiom because bash 3.2's nounset rejects "${arr[@]}" on an - # empty array. - local cm_args=() - if [[ -n "${cloudsql_instance}" ]]; then - # Configuration for the Cloud SQL Auth Proxy sidecar - # (manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml). The proxy reads any - # of its flags from CSQL_PROXY_* env vars; the instance connection name - # is expanded into its args from this ConfigMap. Health checks listen on - # 9801 because ateapi's metrics own 9090. - cm_args+=( - --from-literal=ATE_API_POSTGRES_CLOUDSQL_INSTANCE="${cloudsql_instance}" - --from-literal=CSQL_PROXY_AUTO_IAM_AUTHN="${cloudsql_iam_auth}" - --from-literal=CSQL_PROXY_PORT="5432" - --from-literal=CSQL_PROXY_HEALTH_CHECK="true" - --from-literal=CSQL_PROXY_HTTP_ADDRESS="0.0.0.0" - --from-literal=CSQL_PROXY_HTTP_PORT="9801" - --from-literal=CSQL_PROXY_STRUCTURED_LOGS="true" - ) - case "${cloudsql_ip_type}" in - private) cm_args+=(--from-literal=CSQL_PROXY_PRIVATE_IP="true") ;; - psc) cm_args+=(--from-literal=CSQL_PROXY_PSC="true") ;; - public) ;; - *) - echo "Error: ATE_API_POSTGRES_CLOUDSQL_IP_TYPE must be private, public, or psc, got '${cloudsql_ip_type}'" >&2 - exit 1 - ;; - esac - fi - run_kubectl create configmap -n ate-system ate-api-server-envvars \ - ${cm_args[@]+"${cm_args[@]}"} \ - --dry-run=client -o yaml \ - | run_kubectl apply -f - - - # The Postgres DSN may carry a password (external databases without IAM - # auth), so it lives in a Secret, not the ConfigMap above. The deployment - # lists the secretRef after the configMapRef, so this value wins if both - # define the key. - run_kubectl create secret generic -n ate-system ate-api-server-secret-envvars \ - --from-literal=ATE_API_POSTGRES_CONNECTION_STRING="${postgres_connection_string}" \ - --from-literal=ATE_API_POSTGRES_SCHEMA="${postgres_schema}" \ - --dry-run=client -o yaml \ - | run_kubectl apply -f - - - # Server CA for an external Postgres, mounted by the deployment at - # /run/postgres-server-ca/server-ca.pem for sslmode=verify-ca DSNs. For - # Cloud SQL: gcloud sql ssl server-ca-certs list --instance= \ - # --format="value(cert)" > server-ca.pem - if [[ -n "${ATE_API_POSTGRES_SERVER_CA_FILE:-}" ]]; then - run_kubectl create secret generic -n ate-system postgres-server-ca \ - --from-file=server-ca.pem="${ATE_API_POSTGRES_SERVER_CA_FILE}" \ - --dry-run=client -o yaml \ - | run_kubectl apply -f - - fi - - annotate_api_server_env_hash -} - -apply_podcert_workers_override() { - if [[ -z "${ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER:-}" ]]; then - return 0 - fi - - local workers="" - workers="$(podcert_workers_per_signer)" - - local current="" - current="$(run_kubectl -n podcertificate-controller-system get deployment/podcertificate-controller \ - -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="WORKERS_PER_SIGNER")].value}' 2>/dev/null || true)" - if [[ "${current}" == "${workers}" ]]; then - return 0 - fi - - echo "Overriding WORKERS_PER_SIGNER with ${workers}" - run_kubectl -n podcertificate-controller-system set env deployment/podcertificate-controller \ - WORKERS_PER_SIGNER="${workers}" -} - -create_api_authentication_config() { - log_step "create_api_authentication_config" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ - | run_kubectl apply -f - - - # ate-api-server accepts a token only if its iss claim equals this string - # exactly. EXPECTED_JWT_ISSUER, when set, is that string; the derivation - # below is for clusters whose issuer follows the standard form. - local jwt_issuer="" - if [[ -n "${EXPECTED_JWT_ISSUER:-}" ]]; then - jwt_issuer="${EXPECTED_JWT_ISSUER}" - elif [[ -n "${PROJECT_ID:-}" && -n "${CLUSTER_LOCATION:-}" && -n "${CLUSTER_NAME:-}" ]]; then - jwt_issuer="https://container.googleapis.com/v1/projects/${PROJECT_ID}/locations/${CLUSTER_LOCATION}/clusters/${CLUSTER_NAME}" - else - jwt_issuer=$(run_kubectl get --raw /.well-known/openid-configuration 2>/dev/null | grep -o '"issuer":"[^"]*' | sed 's/"issuer":"//' || true) - if [[ -z "${jwt_issuer}" ]]; then - jwt_issuer="https://kubernetes.default.svc" - fi - fi - - local discovery_config="" - case "${jwt_issuer}" in - https://kubernetes.default.svc|https://kubernetes.default.svc.cluster.local) - discovery_config=$' certificateAuthorityFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n discoveryTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n' - ;; + case "${SETUP_CSI}" in + hostpath) echo "csi-hostpath-sc" ;; + nfs|both|true) echo "csi-nfs-sc" ;; + *) echo "standard" ;; esac - local authentication_config - authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [api.ate-system.svc]\n%s' "${jwt_issuer}" "${discovery_config}") - echo "ate-api-authentication authentication.yaml:" - echo " | ${authentication_config//$'\n'/$'\n' | }" - run_kubectl create configmap -n ate-system ate-api-authentication \ - --from-literal=authentication.yaml="${authentication_config}" \ - --dry-run=client -o yaml \ - | run_kubectl apply -f - -} - -ensure_crds() { - log_step "ensure_crds" - if run_kubectl get crd workerpools.ate.dev sandboxconfigs.ate.dev >/dev/null 2>&1; then - return - fi - - deploy_crds -} - -deploy_crds() { - log_step "deploy_crds" - run_ko apply -f manifests/ate-install/generated -} - -require_kind_for_hostpath() { - if [[ "${ATE_INSTALL_KIND:-false}" != "true" ]]; then - echo "Error: the hostpath CSI driver is only supported on Kind." >&2 - exit 1 - fi } -setup_csi() { - local driver="${SETUP_CSI:-none}" - case "${driver}" in - ""|none|false) +# run_demo maps --deploy-demo-NAME / --delete-demo-NAME onto +# `ate-setup {deploy,delete} demo NAME`. A demo this installer does not +# register is rejected the way any other unknown flag is, rather than reaching +# ate-setup as an unknown subcommand. +run_demo() { + local action="$1" flag="$2" + local demo="${flag#--"${action}"-}" + local known + for known in "${ATE_DEMOS[@]}"; do + if [[ "${demo}" == "${known}" ]]; then + ate_setup "${action}" demo "${demo#demo-}" return - ;; - esac - log_step "setup_csi (${driver})" - case "${driver}" in - nfs) - "${ROOT}/hack/setup-csi-nfs-kind.sh" - ;; - hostpath) - require_kind_for_hostpath - "${ROOT}/hack/setup-csi-hostpath-kind.sh" - ;; - both|true) - require_kind_for_hostpath - "${ROOT}/hack/setup-csi-hostpath-kind.sh" - "${ROOT}/hack/setup-csi-nfs-kind.sh" - ;; - *) - echo "Error: unknown CSI driver \"${driver}\" (valid options: nfs, hostpath, both, none)." >&2 - exit 1 - ;; - esac -} - -deploy_ate_system() { - log_step "deploy_ate_system" - # Fail fast on an unusable build version before touching the cluster. - ensure_substrate_version - - # Ensure namespace exists before applying RBAC or CRDs - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s - - # The atelet DaemonSet applied below and the demo WorkerPools' worker pods - # schedule only to version-labeled nodes. - label_nodes_substrate_version - - # Not ensure_crds: its existence check skips upgrades, stranding stale CRD - # schemas and RBAC (role.yaml has no other apply path). - deploy_crds - - # Enforce per-class SandboxConfig asset requirements (applied before any - # SandboxConfig so the configs below are validated too). - run_kubectl apply -f manifests/ate-install/sandboxconfig-validation.yaml - - # Install the cluster-wide sandbox config(s). Sandbox binaries live on - # cluster-scoped SandboxConfigs each ActorTemplate names via - # sandboxConfig.configName; gVisor templates name this one unless they - # create their own SandboxConfig. - run_kubectl apply -f manifests/ate-install/sandboxconfig-gvisor.yaml - - # Ahead of the bundle below, for the same reason as the namespace: every - # workload pulls this ConfigMap in via envFrom, and a container whose envFrom - # target is missing will not start. The bundle contains it, but a raw - # directory apply orders by filename, so ate-api-server.yaml and - # ate-controller.yaml would otherwise be created before it and sit in - # CreateContainerConfigError until it caught up. - apply_otel_config - - ensure_apiserver_prerequisites - - # Deploy podcertificate-controller first so it starts signing and creating trust bundles immediately - run_ko apply -f manifests/ate-install/pod-certificate-controller.yaml - apply_podcert_workers_override - run_kubectl rollout status deployment/podcertificate-controller -n podcertificate-controller-system --timeout=120s - - wait_for_podcertificate_trust_bundles - - # CSI setup must run after podcertificate-controller is ready and trust bundles - # exist. The ghostunnel sidecar uses projected podCertificate and clusterTrustBundle - # volumes which cannot be fulfilled until podcertcontroller is actively signing, - # otherwise rollout of csi-hostpath-socat times out. - # - # setup_csi is a no-op unless --setup-csi asked for a driver, and it is the - # one that decides which drivers need Kind, so there is no Kind gate here. - setup_csi - - if use_bundled_postgres; then - apply_postgres - else - # Say so explicitly: a DSN aimed at a database that was never deployed - # otherwise surfaces only as an ate-api-server rollout timeout minutes - # later, with nothing pointing at the cause. - local external_db="ATE_API_POSTGRES_CONNECTION_STRING" - [[ -z "${ATE_API_POSTGRES_CONNECTION_STRING:-}" ]] \ - && external_db="Cloud SQL instance $(resolve_cloudsql_instance)" - log_step "Skipping bundled PostgreSQL: external database configured (${external_db})" - fi - - local manifests="" - manifests="$(render_ate_system_manifests)" - echo "${manifests}" | run_kubectl apply -f - - - reconcile_cloudsql_proxy_sidecar - - # Applied on its own rather than through the overlay above, so - # --experimental-use-sdsmint composes with every overlay instead of needing a - # variant of each. - ensure_egress_mitm_ca_pool_secret - apply_atenet_egress - - log_step "Waiting for ATE system components to be ready..." - if use_bundled_postgres; then - run_kubectl rollout status statefulset/postgres -n ate-system --timeout="$(rollout_timeout)" - fi - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/ate-controller -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" - - # After the bundle, which carries its own copy of ate-otel-config. - apply_otel_endpoint_override -} - -# Ensure secrets and configmaps required by ate-apiserver -ensure_apiserver_prerequisites() { - log_step "ensure_apiserver_prerequisites" - run_kubectl get secret -n ate-system actor-id-jwt-pool >/dev/null 2>&1 \ - || create_jwt_authority_pool_secret - run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \ - || create_actor_id_ca_pool_secret - # Derived from actor-id-ca-pool above, so it must come after it. - run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \ - || create_actor_id_ca_certs_secret - run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \ - || create_podcertificate_controller_cas - # Always reconcile the PostgreSQL connection settings. - create_api_server_env_vars - run_kubectl get configmap -n ate-system ate-api-authentication >/dev/null 2>&1 \ - || create_api_authentication_config -} - -# Redeploy only the ate-apiserver -deploy_ate_apiserver() { - log_step "deploy_ate_apiserver" - ensure_crds - - # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s - - ensure_apiserver_prerequisites - apply_otel_config - apply_otel_endpoint_override - - run_ko apply -f manifests/ate-install/ate-api-server.yaml - reconcile_cloudsql_proxy_sidecar - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" -} - -# Reconciles the Cloud SQL Auth Proxy sidecar and Workload Identity -# annotation on ate-api-server. Runs after the deployment manifest is -# applied. Desired state comes from resolve_cloudsql_instance: a set -# ATE_API_POSTGRES_CLOUDSQL_INSTANCE wins, and an unset one adopts the -# cluster's record — so the removal branch fires only on an EXPLICITLY empty -# variable (ATE_API_POSTGRES_CLOUDSQL_INSTANCE=""), never because a redeploy -# ran from a shell that simply didn't export it. -reconcile_cloudsql_proxy_sidecar() { - local instance gsa - instance="$(resolve_cloudsql_instance)" - if [[ -n "${instance}" ]]; then - log_step "reconcile_cloudsql_proxy_sidecar (add)" - # Workload Identity: the proxy resolves the pod's ambient credentials via - # ADC, which requires the KSA to be linked to the GSA that is the Cloud - # SQL IAM database user. - gsa="$(resolve_cloudsql_gsa)" - if [[ -n "${gsa}" ]]; then - run_kubectl annotate serviceaccount ate-api-server -n ate-system \ - "iam.gke.io/gcp-service-account=${gsa}" --overwrite fi - run_kubectl patch deployment ate-api-server -n ate-system \ - --patch-file manifests/ate-install/cloudsql/proxy-sidecar-patch.yaml - elif run_kubectl get deployment ate-api-server -n ate-system \ - -o jsonpath='{.spec.template.spec.initContainers[*].name}' 2>/dev/null \ - | grep -qw cloud-sql-proxy; then - log_step "reconcile_cloudsql_proxy_sidecar (remove)" - # shellcheck disable=SC2016 - run_kubectl patch deployment ate-api-server -n ate-system --type=strategic \ - -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"cloud-sql-proxy","$patch":"delete"}]}}}}' - run_kubectl annotate serviceaccount ate-api-server -n ate-system \ - "iam.gke.io/gcp-service-account-" >/dev/null 2>&1 || true - fi -} - -deploy_atelet() { - log_step "deploy_atelet" - ensure_substrate_version - ensure_crds - - # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s - - label_nodes_substrate_version - apply_otel_config - apply_otel_endpoint_override - - local manifest="" - if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - # Use Kustomize to build and resolve the atelet DaemonSet patch - manifest=$(kubectl kustomize manifests/ate-install/kind/atelet --load-restrictor LoadRestrictionsNone | run_ko resolve -f - | substitute_version) - else - # Use base manifest for GKE - manifest=$(run_ko resolve -f manifests/ate-install/atelet.yaml | substitute_version) - fi - echo "${manifest}" | run_kubectl apply -f - - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" -} - -deploy_atenet() { - log_step "deploy_atenet" - ensure_crds - - # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s - - apply_otel_config - apply_otel_endpoint_override - - local router_manifest="" - router_manifest="$(render_atenet_router_manifest)" - echo "${router_manifest}" | run_kubectl apply -f - - - ensure_egress_mitm_ca_pool_secret - apply_atenet_egress - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" -} - -# get_actor_state echoes the actor's state enum (e.g. ACTOR_STATE_SUSPENDED). -get_actor_state() { - local actor_name="$1" - local atespace="$2" - local json - - if ! json=$(run_kubectl_ate get actor "${actor_name}" -a "${atespace}" -o json 2>/dev/null); then - return 1 - fi - jq -r '.status.state // empty' <<<"${json}" -} - -# prepare_actor_for_delete suspends (or resumes then suspends) until DeleteActor -# accepts the actor: ACTOR_STATE_SUSPENDED, ACTOR_STATE_CRASHED, or ACTOR_STATE_DELETING. -prepare_actor_for_delete() { - local actor_name="$1" - local atespace="$2" - local timeout_secs="${3:-120}" - local deadline=$((SECONDS + timeout_secs)) - local state - - while ((SECONDS < deadline)); do - if ! state=$(get_actor_state "${actor_name}" "${atespace}"); then - return 0 - fi - - case "${state}" in - ACTOR_STATE_SUSPENDED | ACTOR_STATE_CRASHED | ACTOR_STATE_DELETING) - return 0 - ;; - ACTOR_STATE_PAUSED) - run_kubectl_ate resume actor "${actor_name}" -a "${atespace}" -o json >/dev/null - ;; - ACTOR_STATE_RUNNING) - run_kubectl_ate suspend actor "${actor_name}" -a "${atespace}" -o json >/dev/null - ;; - ACTOR_STATE_RESUMING | ACTOR_STATE_SUSPENDING | ACTOR_STATE_PAUSING) - ;; - *) - echo "cannot delete actor ${actor_name}: unexpected state ${state}" >&2 - return 1 - ;; - esac - sleep 2 done - - echo "timed out waiting for actor ${actor_name} to become deletable" >&2 - return 1 -} - -# delete_demo_actors_substrate removes all actors created from a substrate -# ActorTemplate resource before the demo manifests are deleted: those -# reference their template via the actorTemplate {atespace, name} ref. -# Arguments are alternating atespace and template name. -delete_demo_actors_substrate() { - if ! command -v jq &>/dev/null; then - echo "jq is required to delete demo actors" >&2 - return 1 - fi - - if (($# == 0 || $# % 2 != 0)); then - echo "delete_demo_actors_substrate expects atespace/template pairs" >&2 - return 1 - fi - - if ! run_kubectl get deployment/ate-api-server -n ate-system >/dev/null 2>&1; then - log_step "ate-api-server not found; skipping actor cleanup" - return 0 - fi - - local actors_json - if ! actors_json=$(run_kubectl_ate get actors -A -o json 2>/dev/null); then - echo "warning: could not list actors; skipping actor cleanup" >&2 - return 0 - fi - - local template_atespace tmpl atespace actor_name - while (($# > 0)); do - template_atespace="$1" - tmpl="$2" - shift 2 - - log_step "Deleting actors for ${template_atespace}/${tmpl}" - while IFS=$'\t' read -r atespace actor_name; do - [[ -z "${actor_name}" ]] && continue - log_step " preparing actor ${atespace}/${actor_name} for delete" - prepare_actor_for_delete "${actor_name}" "${atespace}" - run_kubectl_ate delete actor "${actor_name}" -a "${atespace}" - done < <( - jq -r --arg as "${template_atespace}" --arg tmpl "${tmpl}" \ - '.actors[]? | select(.actorTemplate.atespace == $as and .actorTemplate.name == $tmpl) | "\(.metadata.atespace)\t\(.metadata.name)"' \ - <<<"${actors_json}" - ) - done -} - -# wait_actortemplate_ready polls a substrate ActorTemplate resource until its -# golden snapshot exists (the substrate counterpart of `kubectl wait -# --for=condition=Ready actortemplate/...`). Fails fast when the template -# reconciler reports an error. -wait_actortemplate_ready() { - local atespace="$1" - local template="$2" - local timeout_secs="${3:-300}" - local deadline=$((SECONDS + timeout_secs)) - local json snapshot error_message - - while ((SECONDS < deadline)); do - if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then - snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}") - if [[ -n "${snapshot}" ]]; then - return 0 - fi - error_message=$(jq -r '.status.goldenSnapshotStatus.errorMessage // empty' <<<"${json}") - if [[ -n "${error_message}" ]]; then - echo "actor template ${atespace}/${template} failed: ${error_message}" >&2 - return 1 - fi - fi - sleep 5 - done - - echo "timed out waiting for actor template ${atespace}/${template} golden snapshot" >&2 - return 1 -} - -# The helpers below deploy a demo in the substrate-resource shape: a CRD -# worker-pool manifest plus protojson ActorTemplates created through the ate -# API. The pool's k8s namespace doubles as the atespace unless a demo says -# otherwise; turns a manifest path into YAML on stdout. - -# render_demo_manifest substitutes ${BUCKET_NAME} and the version -# placeholders in a manifest. -render_demo_manifest() { - sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" "$1" | substitute_version -} - -# ensure_atespace creates the atespace if it does not already exist. The -# store enforces that a template's atespace exists at create time. -ensure_atespace() { - local atespace="$1" - if ! run_kubectl_ate create atespace "${atespace}" >/dev/null 2>&1 \ - && ! run_kubectl_ate get atespace "${atespace}" >/dev/null 2>&1; then - echo "error: failed to create atespace ${atespace}" >&2 - exit 1 - fi -} - -# create_demo_actor_template renders a protojson ActorTemplate manifest, -# creates it through the ate API, and blocks on its golden snapshot. -# Templates are immutable, so an existing one is kept in place. -create_demo_actor_template() { - local render_fn="$1" - local template_manifest="$2" - local atespace="$3" - local template="$4" - local timeout_secs="${5:-300}" - - # ko resolve replaces ko:// image references with pushed digests; - # manifests without them pass through unchanged. - if ! "${render_fn}" "${template_manifest}" \ - | run_ko resolve -f - \ - | run_kubectl_ate create actor-template -f -; then - if run_kubectl_ate get actor-template "${template}" -a "${atespace}" >/dev/null 2>&1; then - log_step "actor template ${atespace}/${template} already exists; keeping it (delete the demo to replace it)" - else - echo "error: failed to create actor template ${atespace}/${template}" >&2 - exit 1 - fi - fi - - # The substrate counterpart of `kubectl wait --for=condition=Ready - # actortemplate/...`, which does not exist for substrate resources. - log_step "Waiting for the ${atespace}/${template} golden snapshot..." - if ! wait_actortemplate_ready "${atespace}" "${template}" "${timeout_secs}"; then - exit 1 - fi -} - -# deploy_substrate_demo deploys the common substrate demo shape: one worker -# pool plus ActorTemplates given as alternating manifest / name pairs after -# the golden-snapshot timeout (micro-VM goldens pay a cloud-hypervisor cold -# boot, so those demos pass a larger budget). -deploy_substrate_demo() { - local render_fn="$1" - local pool_manifest="$2" - local atespace="$3" # also the pool's k8s namespace - local pool="$4" - local golden_timeout="$5" - shift 5 - - ensure_crds - "${render_fn}" "${pool_manifest}" | run_ko apply -f - - - log_step "Waiting for the ${pool} worker pool rollout..." - wait_for_pool_rollout_fatal "${pool}" "${atespace}" - - ensure_atespace "${atespace}" - - local template_manifest template - while (($# > 0)); do - template_manifest="$1" - template="$2" - shift 2 - create_demo_actor_template "${render_fn}" "${template_manifest}" "${atespace}" "${template}" "${golden_timeout}" - done -} - -# delete_demo_actor_template deletes a template's actors, then the template -# itself, which server-side also removes its golden actor and snapshot. -delete_demo_actor_template() { - local atespace="$1" - local template="$2" - delete_demo_actors_substrate "${atespace}" "${template}" - run_kubectl_ate delete actor-template "${template}" -a "${atespace}" 2>/dev/null \ - || log_step "actor template ${atespace}/${template} not deleted (may not exist)" -} - -# delete_substrate_demo tears down what deploy_substrate_demo deployed: -# actors, templates, the atespace, then the pool manifest. Arguments after -# the atespace are the template names in that atespace. -delete_substrate_demo() { - local render_fn="$1" - local pool_manifest="$2" - local atespace="$3" - shift 3 - - local template - for template in "$@"; do - delete_demo_actor_template "${atespace}" "${template}" - done - run_kubectl_ate delete atespace "${atespace}" 2>/dev/null \ - || log_step "atespace ${atespace} not deleted (may not exist or is not empty)" - - "${render_fn}" "${pool_manifest}" | run_kubectl delete --ignore-not-found -f - -} - -delete_ate_system() { - log_step "delete_ate_system" - if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - kubectl kustomize manifests/ate-install/kind --load-restrictor LoadRestrictionsNone \ - | run_kubectl delete --ignore-not-found -f - - else - run_kubectl delete --ignore-not-found -f manifests/ate-install - fi - - run_kubectl delete --ignore-not-found -n ate-system daemonset -l app=atelet - run_kubectl delete --ignore-not-found \ - -f manifests/ate-install/components/agentgateway/configmap.yaml - run_kubectl delete --ignore-not-found -f manifests/ate-install/postgres/postgres.yaml - run_kubectl delete --ignore-not-found -f manifests/ate-install/generated - run_kubectl label nodes -l ate.dev/substrate-version ate.dev/substrate-version- -} - -delete_atenet() { - log_step "delete_atenet" - run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-router.yaml - run_kubectl delete --ignore-not-found \ - -f manifests/ate-install/components/agentgateway/configmap.yaml - # Both egress variants, not the selected one: teardown has to clean up an - # install made with --experimental-use-sdsmint whether or not this invocation - # passes it, and either file may declare resources the other does not. - run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-egress.yaml - run_kubectl delete --ignore-not-found \ - -f manifests/ate-install/atenet-egress-with-sdsmint.yaml -} - -deploy_benchmarks() { - log_step "deploy_benchmarks (worker_count=${BENCHMARK_WORKER_COUNT}, sandbox_class=${BENCHMARK_SANDBOX_CLASS})" - # The microvm SandboxConfig lives outside --deploy-ate-system's default set - # (which only installs gvisor-default); the workloads deploy references it - # by name and would fail if we skipped this. - if [[ "${BENCHMARK_SANDBOX_CLASS}" == "microvm" ]]; then - "${ROOT}/hack/install-microvm-deps.sh" --install - fi - # Send the actor telemetry to the same place as the control plane telemetry. - local benchmark_args=(--deploy - --worker-count "${BENCHMARK_WORKER_COUNT}" - --sandbox-class "${BENCHMARK_SANDBOX_CLASS}") - if [[ -n "${ATE_OTLP_ENDPOINT:-}" ]]; then - benchmark_args+=(--otlp-endpoint "${ATE_OTLP_ENDPOINT}") - fi - if [[ -n "${BENCHMARK_ACTOR_MEMORY}" ]]; then - benchmark_args+=(--actor-memory "${BENCHMARK_ACTOR_MEMORY}") - fi - "${ROOT}/benchmarking/deploy_locust.sh" "${benchmark_args[@]}" -} - -delete_benchmarks() { - log_step "delete_benchmarks (sandbox_class=${BENCHMARK_SANDBOX_CLASS})" - "${ROOT}/benchmarking/deploy_locust.sh" --delete - # only tear down the microvm SandboxConfig if the caller opted into microvm. - if [[ "${BENCHMARK_SANDBOX_CLASS}" == "microvm" ]]; then - "${ROOT}/hack/install-microvm-deps.sh" --delete - fi -} - -delete_all() { - log_step "delete_all" - for demo_name in "${ATE_DEMOS[@]}"; do - if declare -F "${demo_name}_delete" >/dev/null 2>&1; then - "${demo_name}_delete" - fi - done - delete_ate_system + echo "Error: unknown option: ${flag}" >&2 + echo "" + usage + exit 1 } if [ "$#" -eq 0 ]; then @@ -1421,86 +229,80 @@ done # flag they configure (e.g. --benchmark-worker-count before/after # --deploy-benchmarks). The dispatch loop below also accepts these flags but # treats them as no-ops since the value is already captured here. +GLOBAL_FLAGS=() # Valid values for SETUP_CSI: nfs, hostpath, both, none. Defaults to none: a # CSI driver is an opt-in extra, and NFS needs kernel modules a plain # workstation will not have loaded. SETUP_CSI="${SETUP_CSI:-none}" -BENCHMARK_WORKER_COUNT=1 -BENCHMARK_SANDBOX_CLASS=gvisor -# Empty keeps the default in benchmarking/workloads/deploy.sh (256Mi). -BENCHMARK_ACTOR_MEMORY="" +BENCHMARK_FLAGS=() prescan_args=("$@") for ((i = 0; i < ${#prescan_args[@]}; i++)); do case "${prescan_args[i]}" in - --atenet-dataplane=*) ATE_ATENET_DATAPLANE="${prescan_args[i]#*=}" ;; + --atenet-dataplane=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; --atenet-dataplane) if (( i + 1 >= ${#prescan_args[@]} )); then echo "Error: --atenet-dataplane requires envoy or agentgateway" >&2 exit 1 fi - ATE_ATENET_DATAPLANE="${prescan_args[$((i + 1))]}" + GLOBAL_FLAGS+=("--atenet-dataplane=${prescan_args[$((i + 1))]}") ;; - --experimental-use-sdsmint) ATE_EXPERIMENTAL_USE_SDSMINT=true ;; + --experimental-use-sdsmint) GLOBAL_FLAGS+=(--experimental-use-sdsmint) ;; --experimental-additional-egress-extproc-service=*) - ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE="${prescan_args[i]#*=}" + GLOBAL_FLAGS+=("${prescan_args[i]}") ;; --experimental-additional-egress-extproc-service) if (( i + 1 >= ${#prescan_args[@]} )); then echo "Error: --experimental-additional-egress-extproc-service requires /:" >&2 exit 1 fi - ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE="${prescan_args[$((i + 1))]}" + GLOBAL_FLAGS+=("--experimental-additional-egress-extproc-service=${prescan_args[$((i + 1))]}") ;; - --podcert-workers-per-signer=*) ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER="${prescan_args[i]#*=}" ;; + --podcert-workers-per-signer=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; --podcert-workers-per-signer) if (( i + 1 >= ${#prescan_args[@]} )); then echo "Error: --podcert-workers-per-signer requires a positive integer" >&2 exit 1 fi - ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER="${prescan_args[$((i + 1))]}" + GLOBAL_FLAGS+=("--podcert-workers-per-signer=${prescan_args[$((i + 1))]}") ;; - --rollout-timeout=*) ATE_INSTALL_ROLLOUT_TIMEOUT="${prescan_args[i]#*=}" ;; + --rollout-timeout=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; --rollout-timeout) if (( i + 1 >= ${#prescan_args[@]} )); then echo "Error: --rollout-timeout requires a Go duration (e.g. 300s, 10m)" >&2 exit 1 fi - ATE_INSTALL_ROLLOUT_TIMEOUT="${prescan_args[$((i + 1))]}" + GLOBAL_FLAGS+=("--rollout-timeout=${prescan_args[$((i + 1))]}") ;; - --benchmark-worker-count) - BENCHMARK_WORKER_COUNT="${prescan_args[i+1]:-1}" + --otlp-endpoint=*) GLOBAL_FLAGS+=("${prescan_args[i]}") ;; + --otlp-endpoint) + if (( i + 1 >= ${#prescan_args[@]} )); then + echo "Error: --otlp-endpoint requires a URL" >&2 + exit 1 + fi + GLOBAL_FLAGS+=("--otlp-endpoint=${prescan_args[$((i + 1))]}") ;; - --benchmark-worker-count=*) - BENCHMARK_WORKER_COUNT="${prescan_args[i]#*=}" + --benchmark-worker-count=*) BENCHMARK_FLAGS+=("--worker-count=${prescan_args[i]#*=}") ;; + --benchmark-worker-count) + BENCHMARK_FLAGS+=("--worker-count=${prescan_args[i+1]:-1}") ;; + --benchmark-sandbox-class=*) BENCHMARK_FLAGS+=("--sandbox-class=${prescan_args[i]#*=}") ;; --benchmark-sandbox-class) if (( i + 1 >= ${#prescan_args[@]} )); then echo "Error: --benchmark-sandbox-class requires gvisor or microvm" >&2 exit 1 fi - BENCHMARK_SANDBOX_CLASS="${prescan_args[$((i + 1))]}" - ;; - --benchmark-sandbox-class=*) - BENCHMARK_SANDBOX_CLASS="${prescan_args[i]#*=}" + BENCHMARK_FLAGS+=("--sandbox-class=${prescan_args[$((i + 1))]}") ;; + # The benchmark actor memory has no ate-setup flag; it is read from the + # environment, which is also how the benchmark scripts have always taken it. + --benchmark-actor-memory=*) export BENCHMARK_ACTOR_MEMORY="${prescan_args[i]#*=}" ;; --benchmark-actor-memory) if (( i + 1 >= ${#prescan_args[@]} )); then echo "Error: --benchmark-actor-memory requires a size (e.g. 256Mi)" >&2 exit 1 fi - BENCHMARK_ACTOR_MEMORY="${prescan_args[$((i + 1))]}" - ;; - --benchmark-actor-memory=*) - BENCHMARK_ACTOR_MEMORY="${prescan_args[i]#*=}" + export BENCHMARK_ACTOR_MEMORY="${prescan_args[$((i + 1))]}" ;; - --otlp-endpoint) - if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --otlp-endpoint requires a URL" >&2 - exit 1 - fi - ATE_OTLP_ENDPOINT="${prescan_args[$((i + 1))]}" - ;; - --otlp-endpoint=*) ATE_OTLP_ENDPOINT="${prescan_args[i]#*=}" ;; --setup-csi=*) SETUP_CSI="${prescan_args[i]#*=}" ;; --setup-csi) if (( i + 1 < ${#prescan_args[@]} )) && [[ "${prescan_args[$((i + 1))]}" != --* ]]; then @@ -1511,113 +313,55 @@ for ((i = 0; i < ${#prescan_args[@]}; i++)); do ;; esac done -atenet_router >/dev/null -case "${BENCHMARK_SANDBOX_CLASS}" in - gvisor|microvm) ;; - *) - echo "Error: --benchmark-sandbox-class must be gvisor or microvm, got '${BENCHMARK_SANDBOX_CLASS}'" >&2 - exit 1 - ;; -esac -podcert_workers_per_signer >/dev/null -rollout_timeout >/dev/null +# Actions run in command line order, one ate-setup invocation each, so a single +# line can still ask for several of them. while [[ "$#" -gt 0 ]]; do - # Handlers signal an unclaimed argument via ate_demo_flag_unhandled, not exit - # status: an `if`-condition call would suppress errexit in the whole call tree. - ATE_DEMO_FLAG_HANDLED=false - for demo_name in "${ATE_DEMOS[@]}"; do - declare -F "${demo_name}_cmdline" >/dev/null 2>&1 || continue - ATE_DEMO_FLAG_HANDLED=true - "${demo_name}_cmdline" "$1" - if [[ "${ATE_DEMO_FLAG_HANDLED}" == "true" ]]; then - break - fi - done - if [[ "${ATE_DEMO_FLAG_HANDLED}" == "true" ]]; then - shift - continue - fi - case $1 in - --atenet-dataplane=*) ATE_ATENET_DATAPLANE="${1#*=}" ;; - --atenet-dataplane) - shift - if [[ "$#" -eq 0 ]]; then - echo "Error: --atenet-dataplane requires envoy or agentgateway" >&2 - exit 1 - fi - ATE_ATENET_DATAPLANE="$1" - ;; # Captured in the pre-scan above; matched here only so the `*)` branch does - # not reject it as an unknown option. - --experimental-use-sdsmint) ;; + # not reject them, and so a separated value is consumed with its flag. + --atenet-dataplane|--podcert-workers-per-signer|--rollout-timeout|--otlp-endpoint) shift ;; --experimental-additional-egress-extproc-service) shift ;; - --experimental-additional-egress-extproc-service=*) ;; - --podcert-workers-per-signer=*) ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER="${1#*=}" ;; - --podcert-workers-per-signer) - shift - if [[ "$#" -eq 0 ]]; then - echo "Error: --podcert-workers-per-signer requires a positive integer" >&2 - exit 1 - fi - ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER="$1" - ;; - --rollout-timeout=*) ATE_INSTALL_ROLLOUT_TIMEOUT="${1#*=}" ;; - --rollout-timeout) - shift - if [[ "$#" -eq 0 ]]; then - echo "Error: --rollout-timeout requires a Go duration (e.g. 300s, 10m)" >&2 - exit 1 - fi - ATE_INSTALL_ROLLOUT_TIMEOUT="$1" - ;; + --benchmark-worker-count|--benchmark-sandbox-class|--benchmark-actor-memory) shift ;; + --atenet-dataplane=*|--podcert-workers-per-signer=*|--rollout-timeout=*|--otlp-endpoint=*) ;; + --experimental-use-sdsmint|--experimental-additional-egress-extproc-service=*) ;; + --benchmark-worker-count=*|--benchmark-sandbox-class=*|--benchmark-actor-memory=*) ;; - --deploy-ate-system) deploy_ate_system ;; - --setup-csi=*) - SETUP_CSI="${1#*=}" - ensure_crds - setup_csi - ;; + --deploy-ate-system) ate_setup deploy ate-system "--setup-csi=${SETUP_CSI}" ;; + --setup-csi=*) ate_setup setup csi "${SETUP_CSI}" ;; --setup-csi) if [[ "$#" -gt 1 && "$2" != --* ]]; then shift - SETUP_CSI="$1" - else - SETUP_CSI="nfs" fi - ensure_crds - setup_csi + ate_setup setup csi "${SETUP_CSI}" ;; - --delete-ate-system) delete_ate_system ;; - --delete-all) delete_all ;; - - --deploy-atelet) deploy_atelet ;; - --deploy-ate-apiserver) deploy_ate_apiserver ;; - - --deploy-atenet) deploy_atenet ;; - --delete-atenet) delete_atenet ;; - - --deploy-benchmarks) deploy_benchmarks ;; - --delete-benchmarks) delete_benchmarks ;; - # Value captured in the pre-scan above; consume the value arg here so the - # dispatch loop's `*)` unknown-option branch doesn't reject it. - --benchmark-worker-count) shift ;; - --benchmark-worker-count=*) ;; - --benchmark-sandbox-class) shift ;; - --benchmark-sandbox-class=*) ;; - --benchmark-actor-memory) shift ;; - --benchmark-actor-memory=*) ;; - --otlp-endpoint) shift ;; - --otlp-endpoint=*) ;; - - --create-jwt-authority-pool-secret) create_jwt_authority_pool_secret ;; - --create-actor-id-ca-pool-secret) create_actor_id_ca_pool_secret ;; - --create-actor-id-ca-certs-secret) create_actor_id_ca_certs_secret ;; - --create-egress-mitm-ca-pool-secret) create_egress_mitm_ca_pool_secret ;; - --create-podcertificate-controller-cas) create_podcertificate_controller_cas ;; - --create-api-server-env-vars) ensure_env_vars_safe_standalone; create_api_server_env_vars ;; - --create-api-authentication-config) create_api_authentication_config ;; + --delete-ate-system) ate_setup delete ate-system ;; + --delete-all) ate_setup delete all ;; + + --deploy-atelet) ate_setup deploy atelet ;; + --deploy-ate-apiserver) ate_setup deploy apiserver ;; + + --deploy-atenet) ate_setup deploy atenet ;; + --delete-atenet) ate_setup delete atenet ;; + + --deploy-benchmarks) ate_setup deploy benchmarks ${BENCHMARK_FLAGS[@]+"${BENCHMARK_FLAGS[@]}"} ;; + --delete-benchmarks) ate_setup delete benchmarks ${BENCHMARK_FLAGS[@]+"${BENCHMARK_FLAGS[@]}"} ;; + + --create-jwt-authority-pool-secret) ate_setup create jwt-authority-pool ;; + --create-actor-id-ca-pool-secret) ate_setup create actor-id-ca-pool ;; + --create-actor-id-ca-certs-secret) ate_setup create actor-id-ca-certs ;; + --create-egress-mitm-ca-pool-secret) ate_setup create egress-mitm-ca-pool ;; + --create-podcertificate-controller-cas) ate_setup create podcertificate-controller-cas ;; + --create-api-server-env-vars) ate_setup create api-server-env-vars ;; + --create-api-authentication-config) ate_setup create api-authentication-config ;; + + # The one demo flag that is not just a demo name. + --deploy-demo-counter-with-external-volume) + ate_setup deploy demo counter --with-external-volume \ + "--storage-class=$(counter_storage_class)" + ;; + --deploy-demo-*) run_demo deploy "$1" ;; + --delete-demo-*) run_demo delete "$1" ;; *) # Invalid option, should usage and exit with an error. diff --git a/hack/install-demo-autoscaled-workerpool.sh b/hack/install-demo-autoscaled-workerpool.sh deleted file mode 100644 index d0c331b8f6..0000000000 --- a/hack/install-demo-autoscaled-workerpool.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. - -# This demo is kind-only: it ships its own prometheus-adapter and a kind -# specific HPA. -if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - ATE_DEMOS+=(demo-autoscaled-workerpool) # register demo-autoscaled-workerpool -fi - -demo-autoscaled-workerpool_cmdline() { - case "${1}" in - --deploy-demo-autoscaled-workerpool) demo-autoscaled-workerpool_deploy ;; - --delete-demo-autoscaled-workerpool) demo-autoscaled-workerpool_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-autoscaled-workerpool_deploy() { - log_step "demo-autoscaled-workerpool_deploy" - if [[ "${ATE_INSTALL_KIND:-false}" == "false" ]]; then - echo "Error: --deploy-demo-autoscaled-workerpool is not supported on GKE yet" >&2 - exit 1 - fi - - # Deploys the pool, then creates the actor template and waits for its - # golden snapshot. - deploy_substrate_demo render_demo_manifest \ - demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl \ - ate-demo-autoscaled-workerpool counter 300 \ - demos/autoscaled-workerpool/autoscaled-workerpool-template.yaml.tmpl counter - - log_step "Deploying prometheus-adapter and HPA for kind..." - run_kubectl apply -f demos/autoscaled-workerpool/prometheus-adapter.yaml - run_kubectl rollout status deployment/prometheus-adapter -n ate-demo-autoscaled-workerpool --timeout=120s - run_kubectl apply -f demos/autoscaled-workerpool/hpa-kind.yaml -} - -demo-autoscaled-workerpool_delete() { - log_step "demo-autoscaled-workerpool_delete" - if [[ "${ATE_INSTALL_KIND:-false}" != "true" ]]; then - echo "Error: --delete-demo-autoscaled-workerpool is not supported on GKE" >&2 - exit 1 - fi - - # The HPA goes first so it cannot scale the pool back up while the - # workload is being removed. - run_kubectl delete --ignore-not-found -f demos/autoscaled-workerpool/hpa-kind.yaml - run_kubectl delete --ignore-not-found -f demos/autoscaled-workerpool/prometheus-adapter.yaml - - delete_substrate_demo render_demo_manifest \ - demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl \ - ate-demo-autoscaled-workerpool counter -} diff --git a/hack/install-demo-claude-code-multiplex.sh b/hack/install-demo-claude-code-multiplex.sh deleted file mode 100644 index 87fbfff7bc..0000000000 --- a/hack/install-demo-claude-code-multiplex.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. - -ATE_DEMOS+=(demo-claude-code-multiplex) # register demo-claude-code-multiplex - -demo-claude-code-multiplex_cmdline() { - case "${1}" in - --deploy-demo-claude-code-multiplex) demo-claude-code-multiplex_deploy ;; - --delete-demo-claude-code-multiplex) demo-claude-code-multiplex_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -# Build the workload image, push to ${KO_DOCKER_REPO}, and echo the resolved -# digest-pinned reference (e.g. gcr.io/.../claude-multiplex-demo-workload@sha256:...). -# The workload is a Dockerfile-based Python+Claude-Code wrapper (not a Go -# binary), so it uses docker buildx rather than ko. -demo-claude-code-multiplex_build_workload() { - local repo="${KO_DOCKER_REPO}/claude-multiplex-demo-workload" - # shellcheck disable=SC2155 # safe initialization - local stage_tag="${repo}:build-$(date +%s)" - docker buildx build \ - --platform=linux/amd64 \ - --push \ - -t "${stage_tag}" \ - demos/claude-code-multiplex/workload >&2 - local digest - digest=$(docker buildx imagetools inspect "${stage_tag}" --format '{{json .}}' \ - | jq -r '.manifest.digest') - if [[ -z "${digest}" || "${digest}" == "null" ]]; then - echo "Failed to resolve workload image digest from ${stage_tag}" >&2 - return 1 - fi - echo "${repo}@${digest}" -} - -# demo-claude-code-multiplex_render substitutes the demo's placeholders in a -# manifest. DEMO_CLAUDE_WORKLOAD_IMAGE is set by the deploy function after -# building the workload image; the pool manifest uses none of these values, -# so the defaults keep delete-time rendering valid without credentials. -demo-claude-code-multiplex_render() { - sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME:-placeholder}|g" \ - -e "s|\${ANTHROPIC_API_KEY}|${ANTHROPIC_API_KEY:-placeholder}|g" \ - -e "s|\${WORKLOAD_IMAGE}|${DEMO_CLAUDE_WORKLOAD_IMAGE:-placeholder}|g" \ - "$1" -} - -demo-claude-code-multiplex_deploy() { - log_step "demo-claude-code-multiplex_deploy" - if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then - echo "ANTHROPIC_API_KEY must be set" >&2 - return 1 - fi - if [[ -z "${BUCKET_NAME:-}" ]]; then - echo "BUCKET_NAME must be set" >&2 - return 1 - fi - if [[ -z "${KO_DOCKER_REPO:-}" ]]; then - echo "KO_DOCKER_REPO must be set (see hack/ate-dev-env.sh.example)" >&2 - return 1 - fi - - DEMO_CLAUDE_WORKLOAD_IMAGE=$(demo-claude-code-multiplex_build_workload) - if [[ -z "${DEMO_CLAUDE_WORKLOAD_IMAGE}" ]]; then - return 1 - fi - log_step " workload image: ${DEMO_CLAUDE_WORKLOAD_IMAGE}" - - deploy_substrate_demo demo-claude-code-multiplex_render \ - demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl \ - claude-multiplex-demo claude-workerpool 300 \ - demos/claude-code-multiplex/agent-luna-template.yaml.tmpl agent-luna \ - demos/claude-code-multiplex/agent-mars-template.yaml.tmpl agent-mars \ - demos/claude-code-multiplex/agent-orion-template.yaml.tmpl agent-orion -} - -demo-claude-code-multiplex_delete() { - log_step "demo-claude-code-multiplex_delete" - delete_substrate_demo demo-claude-code-multiplex_render \ - demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl \ - claude-multiplex-demo agent-luna agent-mars agent-orion -} - -demo-claude-code-multiplex_usage() { - echo "" - echo " Required env: ANTHROPIC_API_KEY, BUCKET_NAME, KO_DOCKER_REPO" - echo " See demos/claude-code-multiplex/README.md for the walkthrough." -} diff --git a/hack/install-demo-counter.sh b/hack/install-demo-counter.sh deleted file mode 100644 index 8ca03887e8..0000000000 --- a/hack/install-demo-counter.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. -# -# The micro-VM variant additionally needs the cluster-wide `microvm` -# SandboxConfig from hack/install-microvm-deps.sh --install. - -ATE_DEMOS+=(demo-counter) # register demo-counter - -demo-counter_usage() { - echo " --deploy-demo-counter-with-external-volume Deploy demo-counter with external volume validation" - echo " --deploy-demo-counter-microvm Deploy demo-counter on micro-VM workers (needs install-microvm-deps.sh)" - echo " --delete-demo-counter-microvm Delete the micro-VM variant" -} - -demo-counter_cmdline() { - case "${1}" in - --deploy-demo-counter) demo-counter_deploy "false" ;; - --deploy-demo-counter-with-external-volume) demo-counter_deploy "true" ;; - --delete-demo-counter) - delete_substrate_demo demo-counter_render_plain \ - demos/counter/counter.yaml.tmpl ate-demo-counter counter - ;; - --deploy-demo-counter-microvm) - # 600s golden budget: a micro-VM golden is a cloud-hypervisor cold boot - # plus checkpoint, on nested KVM in CI. - deploy_substrate_demo render_demo_manifest \ - demos/counter/counter-microvm.yaml.tmpl ate-demo-counter-microvm counter-microvm 600 \ - demos/counter/counter-microvm-template.yaml.tmpl counter-microvm - ;; - --delete-demo-counter-microvm) - delete_substrate_demo render_demo_manifest \ - demos/counter/counter-microvm.yaml.tmpl ate-demo-counter-microvm counter-microvm - ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -# demo-counter_render substitutes the demo's placeholders in a manifest: -# demo-counter_render -# The external-volume lines are dropped unless is true. -demo-counter_render() { - local with_external_volume="$1" - local manifest="$2" - local validate_cmd=("-e" "/\${VALIDATE_EXISTING_FILE_PATH_ARG}/d") - local ext_vol_mount_cmd=("-e" "/\${EXTERNAL_VOLUME_MOUNTS}/d") - local ext_vol_spec_cmd=("-e" "/\${EXTERNAL_VOLUMES}/d") - if [[ "${with_external_volume}" == "true" ]]; then - # STORAGE_CLASS names the class outright; without it, fall back to whatever - # --setup-csi just installed, and to "standard" when it installed nothing. - local storage_class="${STORAGE_CLASS:-}" - if [[ -z "${storage_class}" ]]; then - case "${SETUP_CSI:-none}" in - hostpath) storage_class="csi-hostpath-sc" ;; - nfs|both|true) storage_class="csi-nfs-sc" ;; - *) storage_class="standard" ;; - esac - fi - - validate_cmd=("-e" "s|\${VALIDATE_EXISTING_FILE_PATH_ARG}| - --validate-existing-file-path=/external-data/test.txt|g") - ext_vol_mount_cmd=("-e" "s|\${EXTERNAL_VOLUME_MOUNTS}| - name: external-data\n mountPath: /external-data|g") - ext_vol_spec_cmd=("-e" "s|\${EXTERNAL_VOLUMES}|- name: external-data\n externalVolumeTemplate:\n capacity: 1Gi\n storageClassName: ${storage_class}|g") - fi - - sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ - "${validate_cmd[@]}" \ - "${ext_vol_mount_cmd[@]}" \ - "${ext_vol_spec_cmd[@]}" \ - "${manifest}" \ - | substitute_version -} - -# Wrappers in the helpers' one-argument shape. -demo-counter_render_plain() { demo-counter_render false "$1"; } -demo-counter_render_ext_vol() { demo-counter_render true "$1"; } - -demo-counter_deploy() { - local with_external_volume="${1:-false}" - log_step "demo-counter_deploy (with_external_volume=${with_external_volume})" - local render_fn=demo-counter_render_plain - if [[ "${with_external_volume}" == "true" ]]; then - render_fn=demo-counter_render_ext_vol - fi - deploy_substrate_demo "${render_fn}" \ - demos/counter/counter.yaml.tmpl ate-demo-counter counter 300 \ - demos/counter/counter-template.yaml.tmpl counter -} - -# demo-counter_delete tears down both variants; called by delete_all. -demo-counter_delete() { - delete_substrate_demo demo-counter_render_plain \ - demos/counter/counter.yaml.tmpl ate-demo-counter counter - delete_substrate_demo render_demo_manifest \ - demos/counter/counter-microvm.yaml.tmpl ate-demo-counter-microvm counter-microvm -} diff --git a/hack/install-demo-egress.sh b/hack/install-demo-egress.sh deleted file mode 100644 index 447ae0f03d..0000000000 --- a/hack/install-demo-egress.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. -# -# The egress demo: each variant applies a worker pool manifest, then creates -# its ActorTemplate as a substrate resource through the ate API with -# `kubectl ate create actor-template`. The micro-VM variants need the -# cluster-wide `microvm` SandboxConfig from hack/install-microvm-deps.sh -# --install; the MITM variants need an sdsmint install -# (--experimental-use-sdsmint), because their actors project the egress -# gateway trust bundle, which does not resolve otherwise. - -ATE_DEMOS+=(demo-egress) # register demo-egress -# The micro-VM variant is its own demo rather than a flag on demo-egress: that -# gets it into --help and into delete_all's teardown sweep for free, and the two -# can be installed side by side (the networking suite runs against whichever the -# sandbox class under test selects). -ATE_DEMOS+=(demo-egress-microvm) # register demo-egress-microvm -# The MITM variants are separate demos for the same reason, plus one of their -# own: they project the egress gateway trust bundle, which only resolves on an -# sdsmint install, so they cannot be part of the demos a passthrough install -# deploys. -ATE_DEMOS+=(demo-egress-mitm) # register demo-egress-mitm -ATE_DEMOS+=(demo-egress-microvm-mitm) # register demo-egress-microvm-mitm - -demo-egress_cmdline() { - case "${1}" in - --deploy-demo-egress) demo-egress_deploy ;; - --delete-demo-egress) demo-egress_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-egress-microvm_cmdline() { - case "${1}" in - --deploy-demo-egress-microvm) demo-egress-microvm_deploy ;; - --delete-demo-egress-microvm) demo-egress-microvm_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-egress-mitm_cmdline() { - case "${1}" in - --deploy-demo-egress-mitm) demo-egress-mitm_deploy ;; - --delete-demo-egress-mitm) demo-egress-mitm_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-egress-microvm-mitm_cmdline() { - case "${1}" in - --deploy-demo-egress-microvm-mitm) demo-egress-microvm-mitm_deploy ;; - --delete-demo-egress-microvm-mitm) demo-egress-microvm-mitm_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-egress_deploy() { - deploy_substrate_demo render_demo_manifest \ - demos/egress/egress.yaml.tmpl \ - ate-demo-egress egress 300 \ - demos/egress/egress-template.yaml.tmpl egress -} - -demo-egress_delete() { - delete_substrate_demo render_demo_manifest \ - demos/egress/egress.yaml.tmpl \ - ate-demo-egress egress -} - -demo-egress-microvm_usage() { - echo " Needs hack/install-microvm-deps.sh --install to have run (cluster-wide microvm SandboxConfig)." -} - -demo-egress-microvm_deploy() { - # 600s golden budget: a micro-VM golden is a cloud-hypervisor cold boot - # plus checkpoint, on nested KVM in CI. - deploy_substrate_demo render_demo_manifest \ - demos/egress/egress-microvm.yaml.tmpl \ - ate-demo-egress-microvm egress-microvm 600 \ - demos/egress/egress-microvm-template.yaml.tmpl egress-microvm -} - -demo-egress-microvm_delete() { - delete_substrate_demo render_demo_manifest \ - demos/egress/egress-microvm.yaml.tmpl \ - ate-demo-egress-microvm egress-microvm -} - -demo-egress-mitm_usage() { - echo " Needs an sdsmint install (--deploy-atenet --experimental-use-sdsmint): the actors" - echo " project the egress gateway trust bundle, which does not resolve otherwise." -} - -demo-egress-mitm_deploy() { - # The golden snapshot only exists once an actor starts, and an actor whose - # trust bundle does not resolve never does — so a timeout here is the - # symptom of a missing sdsmint install (see demo-egress-mitm_usage). - deploy_substrate_demo render_demo_manifest \ - demos/egress/egress-mitm.yaml.tmpl \ - ate-demo-egress-mitm egress-mitm 300 \ - demos/egress/egress-mitm-template.yaml.tmpl egress-mitm -} - -demo-egress-mitm_delete() { - delete_substrate_demo render_demo_manifest \ - demos/egress/egress-mitm.yaml.tmpl \ - ate-demo-egress-mitm egress-mitm -} - -demo-egress-microvm-mitm_usage() { - echo " Needs hack/install-microvm-deps.sh --install to have run (cluster-wide microvm SandboxConfig)," - echo " and an sdsmint install (--deploy-atenet --experimental-use-sdsmint) for the trust bundle." -} - -demo-egress-microvm-mitm_deploy() { - deploy_substrate_demo render_demo_manifest \ - demos/egress/egress-microvm-mitm.yaml.tmpl \ - ate-demo-egress-microvm-mitm egress-microvm-mitm 600 \ - demos/egress/egress-microvm-mitm-template.yaml.tmpl egress-microvm-mitm -} - -demo-egress-microvm-mitm_delete() { - delete_substrate_demo render_demo_manifest \ - demos/egress/egress-microvm-mitm.yaml.tmpl \ - ate-demo-egress-microvm-mitm egress-microvm-mitm -} diff --git a/hack/install-demo-jupyter.sh b/hack/install-demo-jupyter.sh deleted file mode 100755 index 2510d38487..0000000000 --- a/hack/install-demo-jupyter.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. - -ATE_DEMOS+=(demo-jupyter) # register demo-jupyter - -demo-jupyter_usage() { - echo " --deploy-demo-jupyter Deploy demo-jupyter" -} - -demo-jupyter_cmdline() { - case "${1}" in - --deploy-demo-jupyter) demo-jupyter_deploy ;; - --delete-demo-jupyter) demo-jupyter_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-jupyter_deploy() { - log_step "demo-jupyter_deploy" - deploy_substrate_demo render_demo_manifest \ - demos/jupyter/jupyter.yaml.tmpl ate-demo-jupyter jupyter 300 \ - demos/jupyter/jupyter-template.yaml.tmpl jupyter -} - -demo-jupyter_delete() { - log_step "demo-jupyter_delete" - delete_substrate_demo render_demo_manifest \ - demos/jupyter/jupyter.yaml.tmpl ate-demo-jupyter jupyter -} diff --git a/hack/install-demo-multi-template.sh b/hack/install-demo-multi-template.sh deleted file mode 100644 index 20689b4887..0000000000 --- a/hack/install-demo-multi-template.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. - -ATE_DEMOS+=(demo-multi-template) # register demo-multi-template - -demo-multi-template_cmdline() { - case "${1}" in - --deploy-demo-multi-template) demo-multi-template_deploy ;; - --delete-demo-multi-template) demo-multi-template_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -# The demo's two templates live in two different atespaces to show that pool -# selection is atespace-agnostic, so this composes the substrate helpers -# itself instead of using deploy_substrate_demo's one-atespace shape. -demo-multi-template_deploy() { - log_step "demo-multi-template_deploy" - ensure_crds - render_demo_manifest demos/multi-template/multi-template.yaml.tmpl \ - | run_ko apply -f - - - log_step "Waiting for the shared-pool worker pool rollout..." - wait_for_pool_rollout_fatal shared-pool ate-demo-multi-template-pool - - ensure_atespace ate-demo-multi-template-counter - ensure_atespace ate-demo-multi-template-fspersist - create_demo_actor_template render_demo_manifest \ - demos/multi-template/counter-template.yaml.tmpl \ - ate-demo-multi-template-counter counter - create_demo_actor_template render_demo_manifest \ - demos/multi-template/fspersist-template.yaml.tmpl \ - ate-demo-multi-template-fspersist fspersist -} - -demo-multi-template_delete() { - log_step "demo-multi-template_delete" - local atespace - delete_demo_actor_template ate-demo-multi-template-counter counter - delete_demo_actor_template ate-demo-multi-template-fspersist fspersist - for atespace in ate-demo-multi-template-counter ate-demo-multi-template-fspersist; do - run_kubectl_ate delete atespace "${atespace}" 2>/dev/null \ - || log_step "atespace ${atespace} not deleted (may not exist or is not empty)" - done - render_demo_manifest demos/multi-template/multi-template.yaml.tmpl \ - | run_kubectl delete --ignore-not-found -f - -} diff --git a/hack/install-demo-parking.sh b/hack/install-demo-parking.sh deleted file mode 100644 index d5011e5600..0000000000 --- a/hack/install-demo-parking.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. - -ATE_DEMOS+=(demo-parking) # register demo-parking - -demo-parking_cmdline() { - case "${1}" in - --deploy-demo-parking) demo-parking_deploy ;; - --delete-demo-parking) demo-parking_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-parking_deploy() { - log_step "demo-parking_deploy" - deploy_substrate_demo render_demo_manifest \ - demos/parking/parking.yaml.tmpl ate-demo-parking parking 300 \ - demos/parking/parking-template.yaml.tmpl parking -} - -demo-parking_delete() { - log_step "demo-parking_delete" - delete_substrate_demo render_demo_manifest \ - demos/parking/parking.yaml.tmpl ate-demo-parking parking -} diff --git a/hack/install-demo-sandbox.sh b/hack/install-demo-sandbox.sh deleted file mode 100644 index 207a1eb92f..0000000000 --- a/hack/install-demo-sandbox.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is sourced as part of install-ate.sh. Do not run directly. - -ATE_DEMOS+=(demo-sandbox) # register demo-sandbox - -demo-sandbox_cmdline() { - case "${1}" in - --deploy-demo-sandbox) demo-sandbox_deploy ;; - --delete-demo-sandbox) demo-sandbox_delete ;; - *) - ate_demo_flag_unhandled - ;; - esac -} - -demo-sandbox_deploy() { - log_step "demo-sandbox_deploy" - deploy_substrate_demo render_demo_manifest \ - demos/sandbox/sandbox.yaml.tmpl ate-demo-sandbox sandbox-workerpool 300 \ - demos/sandbox/sandbox-template.yaml.tmpl sandbox-template -} - -demo-sandbox_delete() { - log_step "demo-sandbox_delete" - delete_substrate_demo render_demo_manifest \ - demos/sandbox/sandbox.yaml.tmpl ate-demo-sandbox sandbox-template -}