From 38d1b87b9a75ef6b456aa8fd944ebabedcbf576a Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 2 Sep 2026 16:16:29 +0000 Subject: [PATCH 01/19] Publish release artifacts Signed-off-by: Eitan Yarmush --- .github/workflows/release.yaml | 154 +++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..4a26a5cbe9 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,154 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.' + required: false + create_release: + description: 'Create a GitHub release' + type: boolean + default: false + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate and resolve tag + id: tag + run: | + TAG="${{ inputs.tag }}" + if [[ -z "${TAG}" ]]; then + BRANCH="${GITHUB_REF_NAME//\//-}" + SHA="$(git rev-parse --short HEAD)" + TAG="${BRANCH}-${SHA}" + fi + if [[ "${{ inputs.create_release }}" == "true" ]]; then + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then + echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)" + exit 1 + fi + fi + echo "value=${TAG}" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.create_release }}" == "true" ]]; then + echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT" + else + echo "tags=${TAG}" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install ko + uses: ko-build/setup-ko@v0.7 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v3 + + - name: Build and push images + env: + # ghcr.io// — resolves correctly in forks + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -o errexit -o nounset -o pipefail + + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ + ./hack/run-tool.sh ko build \ + --tags "${IMAGE_TAGS}" \ + --platform linux/amd64,linux/arm64 \ + --bare \ + "./cmd/${component}" + done + + - name: Package and push Helm charts + if: inputs.create_release + env: + HELM_EXPERIMENTAL_OCI: "1" + CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + run: | + set -o errexit -o nounset -o pipefail + + tag="${{ steps.tag.outputs.value }}" + chart_version="${tag#v}" + package_dir="${RUNNER_TEMP}/helm-packages" + mkdir -p "${package_dir}" + + echo "${{ secrets.GITHUB_TOKEN }}" \ + | helm registry login ghcr.io \ + --username "${{ github.actor }}" \ + --password-stdin + + helm package charts/substrate-crds \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + helm package charts/substrate \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + + helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" + helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + + - name: Build kubectl-ate release binaries + if: inputs.create_release + env: + VERSION: ${{ steps.tag.outputs.value }} + run: | + set -o errexit -o nounset -o pipefail + + mkdir -p dist + for os in linux darwin; do + for arch in amd64 arm64; do + CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \ + -trimpath \ + -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \ + -o "dist/kubectl-ate-${os}-${arch}" \ + ./cmd/kubectl-ate + done + done + + - name: Create GitHub Release + if: inputs.create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.value }} + generate_release_notes: true + files: dist/kubectl-ate-* From abafd66be619976f12d3bff1411c613e0fc15e09 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 2 Sep 2026 16:16:44 +0000 Subject: [PATCH 02/19] Maintain fork integration on current upstream APIs Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 115 +++++ Makefile | 18 +- charts/substrate-crds/Chart.yaml | 28 ++ charts/substrate-crds/README.md | 13 + .../templates/ate.dev_csidriverconfigs.yaml | 113 +++++ .../templates/ate.dev_sandboxconfigs.yaml | 141 ++++++ .../templates/ate.dev_workerpools.yaml | 473 ++++++++++++++++++ charts/substrate/Chart.yaml | 27 + charts/substrate/README.md | 43 ++ charts/substrate/templates/NOTES.txt | 7 + charts/substrate/templates/_helpers.tpl | 105 ++++ .../templates/ate-api-server-envvars.yaml | 23 + .../substrate/templates/ate-api-server.yaml | 210 ++++++++ charts/substrate/templates/ate-client.yaml | 23 + .../substrate/templates/ate-controller.yaml | 113 +++++ charts/substrate/templates/atelet.yaml | 214 ++++++++ charts/substrate/templates/atenet-egress.yaml | 227 +++++++++ charts/substrate/templates/atenet-router.yaml | 355 +++++++++++++ charts/substrate/templates/namespace.yaml | 22 + .../templates/pod-certificate-controller.yaml | 198 ++++++++ charts/substrate/templates/postgres.yaml | 234 +++++++++ charts/substrate/templates/role.yaml | 114 +++++ charts/substrate/templates/rustfs.yaml | 137 +++++ .../templates/sandboxconfig-gvisor.yaml | 38 ++ .../templates/sandboxconfig-validation.yaml | 57 +++ charts/substrate/values.yaml | 73 +++ cmd/ateapi/internal/controlapi/dialer.go | 9 +- cmd/ateapi/internal/controlapi/dialer_test.go | 23 +- .../controlapi/functionaltest/common_test.go | 18 +- cmd/ateapi/internal/controlapi/informer.go | 6 +- cmd/ateapi/internal/controlapi/service.go | 14 +- cmd/ateapi/internal/controlapi/workflow.go | 21 +- .../controlapi/workflow_lease_test.go | 50 ++ .../controlapi/workflow_testutil_test.go | 5 +- cmd/ateapi/main.go | 20 +- cmd/atecontroller/internal/controllers/gen.go | 2 +- cmd/atelet/main.go | 123 ++--- cmd/atelet/sandbox_prewarm_test.go | 12 +- hack/gen-rbac.sh | 37 ++ hack/install-microvm-deps.sh | 1 + hack/render-manifests.sh | 156 ++++++ hack/run-microvm-demo.sh | 32 +- hack/update/licenses.sh | 8 + hack/verify/crd-chart.sh | 47 ++ internal/ateclient/builder.go | 5 +- internal/credbundle/credbundle.go | 20 +- internal/credbundle/credbundle_test.go | 6 +- internal/e2e/collector_metrics.go | 1 - internal/e2e/suites/demo/demo_test.go | 114 +++-- internal/e2e/suites/identity/identity_test.go | 14 +- .../e2e/suites/networking/grpcegress_test.go | 7 - .../e2e/suites/networking/grpcingress_test.go | 12 +- internal/installdefaults/installdefaults.go | 47 ++ .../ate-install/ate-api-server-envvars.yaml | 24 + manifests/ate-install/ate-client.yaml | 24 + .../components/agentgateway/configmap.yaml | 54 ++ manifests/ate-install/role.yaml | 130 +++++ manifests/ate-install/rustfs.yaml | 136 +++++ 58 files changed, 4126 insertions(+), 173 deletions(-) create mode 100644 .github/workflows/helm-e2e.yaml create mode 100644 charts/substrate-crds/Chart.yaml create mode 100644 charts/substrate-crds/README.md create mode 100644 charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_workerpools.yaml create mode 100644 charts/substrate/Chart.yaml create mode 100644 charts/substrate/README.md create mode 100644 charts/substrate/templates/NOTES.txt create mode 100644 charts/substrate/templates/_helpers.tpl create mode 100644 charts/substrate/templates/ate-api-server-envvars.yaml create mode 100644 charts/substrate/templates/ate-api-server.yaml create mode 100644 charts/substrate/templates/ate-client.yaml create mode 100644 charts/substrate/templates/ate-controller.yaml create mode 100644 charts/substrate/templates/atelet.yaml create mode 100644 charts/substrate/templates/atenet-egress.yaml create mode 100644 charts/substrate/templates/atenet-router.yaml create mode 100644 charts/substrate/templates/namespace.yaml create mode 100644 charts/substrate/templates/pod-certificate-controller.yaml create mode 100644 charts/substrate/templates/postgres.yaml create mode 100644 charts/substrate/templates/role.yaml create mode 100644 charts/substrate/templates/rustfs.yaml create mode 100644 charts/substrate/templates/sandboxconfig-gvisor.yaml create mode 100644 charts/substrate/templates/sandboxconfig-validation.yaml create mode 100644 charts/substrate/values.yaml create mode 100644 cmd/ateapi/internal/controlapi/workflow_lease_test.go create mode 100755 hack/gen-rbac.sh create mode 100755 hack/render-manifests.sh create mode 100755 hack/verify/crd-chart.sh create mode 100644 internal/installdefaults/installdefaults.go create mode 100644 manifests/ate-install/ate-api-server-envvars.yaml create mode 100644 manifests/ate-install/ate-client.yaml create mode 100644 manifests/ate-install/role.yaml create mode 100644 manifests/ate-install/rustfs.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml new file mode 100644 index 0000000000..dfc5df2757 --- /dev/null +++ b/.github/workflows/helm-e2e.yaml @@ -0,0 +1,115 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helm-e2e +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + e2e-test: + runs-on: ubuntu-latest + env: + VERSION: helm-e2e + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - name: Setup Helm + uses: azure/setup-helm@v4 + - name: Cache micro-VM assets + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + run: hack/create-kind-cluster.sh + - name: Label nodes with the installed version + run: kubectl label nodes --all ate.dev/substrate-version=${VERSION} + - name: Create install namespace + run: kubectl create namespace ate-system + - name: Install observability fixtures + run: | + kubectl apply -f manifests/ate-install/kind/otel-collector.yaml + kubectl apply -f manifests/ate-install/kind/prometheus.yaml + - name: Build chart images + run: | + for component in ateapi atecontroller atelet podcertcontroller atenet; do + KO_DOCKER_REPO="localhost:5001/${component}" \ + ./hack/run-tool.sh ko build --bare --tags helm-e2e \ + --platform linux/amd64 "./cmd/${component}" + done + - name: Install Agent Substrate with Helm + run: | + helm upgrade --install substrate-crds charts/substrate-crds + helm upgrade --install substrate charts/substrate \ + --namespace ate-system \ + --create-namespace \ + --set image.registry=localhost:5001 \ + --set image.tag=helm-e2e \ + --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ + --set postgres.resources.requests.cpu=500m + - name: Bootstrap mTLS authorities + run: | + hack/install-ate-kind.sh --create-podcertificate-controller-cas + hack/install-ate-kind.sh --create-jwt-authority-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-certs-secret + hack/install-ate-kind.sh --create-api-authentication-config + - name: Wait for Helm install + run: | + helm upgrade substrate charts/substrate \ + --namespace ate-system \ + --reuse-values \ + --wait --timeout=10m + - name: Enable NFS + run: | + sudo modprobe nfs || true + sudo modprobe nfsd || true + - name: Install CSI NFS driver + run: hack/install-ate-kind.sh --setup-csi=nfs + - name: Deploy micro-VM counter demo + run: hack/run-microvm-demo-kind.sh --skip-control-plane + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Run E2E tests (gVisor) + run: hack/run-e2e-kind.sh -v -args --no-color + - name: Run E2E tests (micro-VM) + env: + E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm + E2E_TEMPLATE_NAME: counter-microvm + E2E_TEMPLATE_READY_TIMEOUT: 600s + run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context kind-kind get workerpool,pods -A -o wide || true + for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do + echo "=== logs: ate-system/${p} ===" + kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true + done diff --git a/Makefile b/Makefile index 1002e2f039..11307e18d4 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ build: build-images build-atectl build-ate-setup .PHONY: build-images build-images: - $(KO) build $(KO_FLAGS) \ + $(KO) build --base-import-paths $(KO_FLAGS) \ --ldflags="$(LDFLAGS)" \ $(IMAGES) @@ -131,3 +131,19 @@ verify: test .PHONY: clean clean: rm -rf $(BINDIR) + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode, +# the historical default install). Run this whenever charts/substrate/ changes. +.PHONY: helm-template +helm-template: + @./hack/render-manifests.sh + +# Verify that manifests/ate-install/ matches the chart output. Used in CI. +.PHONY: verify-helm-template +verify-helm-template: + @./hack/render-manifests.sh --check + +# Verify that the CRD chart mirrors the generated CRDs. +.PHONY: verify-crd-chart +verify-crd-chart: + @./hack/verify/crd-chart.sh diff --git a/charts/substrate-crds/Chart.yaml b/charts/substrate-crds/Chart.yaml new file mode 100644 index 0000000000..a69dcee0e9 --- /dev/null +++ b/charts/substrate-crds/Chart.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate-crds +description: Agent Substrate CustomResourceDefinitions. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate +- crds diff --git a/charts/substrate-crds/README.md b/charts/substrate-crds/README.md new file mode 100644 index 0000000000..12fa31f0a7 --- /dev/null +++ b/charts/substrate-crds/README.md @@ -0,0 +1,13 @@ +# substrate-crds + +Helm chart for installing the Agent Substrate CRDs. + +Install this chart before installing the main `substrate` chart: + +```bash +helm upgrade --install substrate-crds ./charts/substrate-crds +helm upgrade --install substrate ./charts/substrate --namespace ate-system --create-namespace +``` + +The CRD YAMLs in `templates/` mirror `manifests/ate-install/generated/`. +Run `hack/verify/crd-chart.sh` to verify they are in sync. diff --git a/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml new file mode 100644 index 0000000000..ebc1473eae --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: csidriverconfigs.ate.dev +spec: + group: ate.dev + names: + kind: CSIDriverConfig + listKind: CSIDriverConfigList + plural: csidriverconfigs + shortNames: + - csidriverconfig + singular: csidriverconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.driverName + name: Driver + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CSIDriverConfig is the Schema for the csidriverconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CSIDriverConfigSpec defines the desired state of CSIDriverConfig + properties: + controllerEndpoint: + description: |- + ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). + pattern: ^(tcp|dns)://.+$ + type: string + driverName: + description: |- + DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + Matches the StorageClass referenced in ActorTemplate volume definitions. + maxLength: 63 + minLength: 1 + pattern: ^(substrate\.io/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ + type: string + nodeSocketOverride: + description: |- + NodeSocketOverride is an optional override for the CSI Node service socket + on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. + pattern: ^unix://.+$ + type: string + tls: + description: TLS configures TLS/mTLS for the connection to the ControllerEndpoint. + properties: + enabled: + description: Enabled controls whether TLS is used. + type: boolean + serverName: + description: ServerName override for TLS verification. + type: string + usePodIdentity: + description: UsePodIdentity indicates whether to reuse Substrate's + Pod Identity (SPIFFE) certificates. + type: boolean + required: + - enabled + type: object + x-kubernetes-validations: + - message: tls.usePodIdentity must be true when tls.enabled is true; + manual certificates are not yet supported + rule: '!self.enabled || (has(self.usePodIdentity) && self.usePodIdentity)' + required: + - controllerEndpoint + - driverName + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml new file mode 100644 index 0000000000..8fe5f91178 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -0,0 +1,141 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: sandboxconfigs.ate.dev +spec: + group: ate.dev + names: + kind: SandboxConfig + listKind: SandboxConfigList + plural: sandboxconfigs + shortNames: + - sandboxconfig + singular: sandboxconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxConfig is cluster-scoped configuration describing the sandbox binaries + for a sandbox runtime family. It is referenced by an ActorTemplate's + sandbox_config.config_name (required) and decouples + sandbox binary selection from the workload definition. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SandboxConfig + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') + sandboxClass: + default: gvisor + description: |- + SandboxClass is the sandbox runtime family this config applies to. An + ActorTemplate only uses SandboxConfigs whose SandboxClass matches its + sandbox_config.sandbox_class. + enum: + - gvisor + - microvm + type: string + required: + - pauseImage + - sandboxClass + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml new file mode 100644 index 0000000000..e1bc49a029 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -0,0 +1,473 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: workerpools.ate.dev +spec: + group: ate.dev + names: + kind: WorkerPool + listKind: WorkerPoolList + plural: workerpools + shortNames: + - workerpool + singular: workerpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.replicas + name: Replicas + type: integer + - jsonPath: .status.readyReplicas + name: Ready + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: WorkerPool is the Schema for the workerpools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of WorkerPool + properties: + replicas: + description: Replicas is the number of worker pods to run. + format: int32 + minimum: 0 + type: integer + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family for this pool, which drives + the worker pod shape (KVM/vhost device mounts and node placement). The + concrete binary is still selected by WorkerImage. Defaults to gvisor. + The sandbox binaries themselves come from the SandboxConfig each + ActorTemplate names (required). + + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + template: + description: Template holds optional metadata, scheduling, and resource + settings for worker workloads. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations are added to the generated Deployment and worker pods. Keys + in the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: annotation keys must be valid Kubernetes qualified + names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + labels: + additionalProperties: + description: |- + WorkerPoolLabelValue is a Kubernetes label value for generated worker + workloads. + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Labels are added to the generated Deployment and worker pods. Keys in + the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: label keys must be valid Kubernetes qualified names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + nodeAffinity: + description: |- + NodeAffinity scheduling rules for the worker pods. Mapped to + spec.affinity.nodeAffinity on the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is a selector which must be true for + the pod to fit on a node. + type: object + priorityClassName: + description: PriorityClassName for the worker pods. + type: string + resources: + description: Resources are the compute resources allocated for + each worker pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tolerations: + description: Tolerations for the worker pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + workerImage: + description: WorkerImage is the ateom container image to deploy as + workers. + minLength: 1 + type: string + required: + - replicas + - workerImage + type: object + status: + description: status is the observed state of WorkerPool + properties: + readyReplicas: + description: ReadyReplicas is the number of ready worker pods. + format: int32 + minimum: 0 + type: integer + replicas: + description: Replicas is the total number of worker pods. + format: int32 + minimum: 0 + type: integer + selector: + description: Selector is the label selector for the worker pods. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/charts/substrate/Chart.yaml b/charts/substrate/Chart.yaml new file mode 100644 index 0000000000..52bd748009 --- /dev/null +++ b/charts/substrate/Chart.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate +description: Agent Substrate — actor runtime, control plane, and data-plane router. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate diff --git a/charts/substrate/README.md b/charts/substrate/README.md new file mode 100644 index 0000000000..e7364f4e37 --- /dev/null +++ b/charts/substrate/README.md @@ -0,0 +1,43 @@ +# substrate + +Helm chart for installing Agent Substrate. + +The chart uses mTLS and PostgreSQL by default. It requires the +`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and +`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1` +API. + +```bash +# CRDs +helm upgrade --install substrate-crds ./charts/substrate-crds + +# Install Substrate +helm upgrade --install substrate ./charts/substrate +``` + +By default, component images are pulled from `ghcr.io/kagent-dev/substrate` +using the chart `appVersion` as the tag. Override `image.registry` and +`image.tag` to install from a different image repository or tag. + +## Render manifests without applying + +```bash +helm template substrate ./charts/substrate +``` + +`manifests/ate-install/` in the repo is the rendered mTLS output and is +regenerated by `make helm-template`. The separate `substrate-crds` chart +mirrors `manifests/ate-install/generated/`. + +## Values + +See `values.yaml` for the full set; the important keys: + +| Key | Default | Notes | +|-----|---------|-------| +| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | +| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | +| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt new file mode 100644 index 0000000000..c0e9875a45 --- /dev/null +++ b/charts/substrate/templates/NOTES.txt @@ -0,0 +1,7 @@ +substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL + +REQUIRED Kubernetes feature gates: + - ClusterTrustBundle + - ClusterTrustBundleProjection + - PodCertificateRequest +The certificates.k8s.io/v1beta1 API must also be enabled. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl new file mode 100644 index 0000000000..32ae087336 --- /dev/null +++ b/charts/substrate/templates/_helpers.tpl @@ -0,0 +1,105 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Qualified resource name for a chart component. + +Usage: + {{ include "substrate.fullname" (list "ate-api-server" .) }} + +When the release name is "substrate" (the canonical render in +hack/render-manifests.sh — `helm template substrate charts/substrate`), this +returns the bare component name, so the generated manifests/ate-install/ +files keep their historical names ("ate-api-server", "ate-controller", ...). + +Otherwise resources are prefixed with the release name in the standard Helm +style ("foo-ate-api-server", ...) so multiple releases coexist without +colliding. + +The check is on the literal release name "substrate" rather than +$ctx.Chart.Name so this helper is context-safe: a parent chart can invoke it +with its own `.` (where .Chart.Name is the parent, not "substrate") and still +get the same prefixed name that this subchart's own templates render. +*/}} +{{- define "substrate.fullname" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- if eq $ctx.Release.Name "substrate" -}} +{{- $name -}} +{{- else -}} +{{- printf "%s-%s" $ctx.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +ServiceAccount name of ate-api-server, as this chart creates it. Parent +charts that need to bind additional Roles to this SA (e.g. env-source +Secret/ConfigMap reads for ActorTemplate resolution) should reference this +helper instead of hardcoding "ate-api-server": + + {{ include "substrate.ateApiServer.serviceAccountName" . }} +*/}} +{{- define "substrate.ateApiServer.serviceAccountName" -}} +{{- include "substrate.fullname" (list "ate-api-server" .) -}} +{{- end -}} + +{{/* +gRPC endpoint that clients dial to reach ate-api-server. dns:/// scheme + +release-prefixed Service name + release namespace + :443. Suitable for +consumption as ATE_API_ENDPOINT / --ateapi-address: + + {{ include "substrate.ateApi.endpoint" . }} + -> dns:///-api..svc:443 +*/}} +{{- define "substrate.ateApi.endpoint" -}} +{{- printf "dns:///%s.%s.svc:443" (include "substrate.fullname" (list "api" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Plaintext HTTP URL that clients use to reach atenet-router. + + {{ include "substrate.atenetRouter.url" . }} + -> http://-atenet-router..svc:80 +*/}} +{{- define "substrate.atenetRouter.url" -}} +{{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Build an image reference for a substrate component binary. + +Usage: + {{ include "substrate.componentImage" (list "ateapi" .) }} + +Produces {image.registry}/{name}:{tag} where tag is resolved as: + 1. image.tag value, if set and not the sentinel "" + 2. .Chart.AppVersion, if image.tag is empty + 3. no tag (no colon) when image.tag is the sentinel "" + +The "" sentinel is used by hack/render-manifests.sh so that ko:// refs +are emitted without a tag, letting `ko resolve` supply the digest at build time. +*/}} +{{- define "substrate.componentImage" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $registry := $ctx.Values.image.registry -}} +{{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} +{{- if ne $tag "" -}} +{{- printf "%s/%s:%s" $registry $name $tag -}} +{{- else -}} +{{- printf "%s/%s" $registry $name -}} +{{- end -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..753c47178b --- /dev/null +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml new file mode 100644 index 0000000000..a5073bc9fa --- /dev/null +++ b/charts/substrate/templates/ate-api-server.yaml @@ -0,0 +1,210 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs", "csidriverconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "watch", "list"] +# Secret reads for env source resolution are intentionally NOT granted +# cluster-wide here. Each demo / tenant is responsible for granting +# ate-api-server read access only to the specific Secrets referenced by its +# ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using +# resourceNames). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-api-server + template: + metadata: + labels: + app: ate-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} + terminationGracePeriodSeconds: 40 + containers: + - name: ate-api-server + image: {{ include "substrate.componentImage" (list "ateapi" .) }} + args: + - "--grpc-listen-addr=0.0.0.0:443" + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" + - "--postgres-connection-string=@env" + - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" + - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" + - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + - "--drain-delay=13s" + - "--drain-timeout=15s" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + envFrom: + - configMapRef: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + optional: true + volumeMounts: + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } + - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } + ports: + - containerPort: 443 + - name: prometheus + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: actor-id-ca-pool + projected: + sources: + - secret: + name: actor-id-ca-pool + items: + - { key: pool, path: pool.json } + - name: authentication-config + configMap: + name: ate-api-authentication + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-api-server +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "api" .) }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml new file mode 100644 index 0000000000..dfd2fdab68 --- /dev/null +++ b/charts/substrate/templates/ate-client.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-client" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-client diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml new file mode 100644 index 0000000000..31c83b9066 --- /dev/null +++ b/charts/substrate/templates/ate-controller.yaml @@ -0,0 +1,113 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: ate-controller +spec: + selector: + app: ate-controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-controller + template: + metadata: + labels: + app: ate-controller + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + containers: + - name: ate-controller + image: {{ include "substrate.componentImage" (list "atecontroller" .) }} + args: + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" +{{- if .Values.otel.endpoint }} + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + volumes: + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml new file mode 100644 index 0000000000..916915dc42 --- /dev/null +++ b/charts/substrate/templates/atelet.yaml @@ -0,0 +1,214 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atelet +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atelet-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["csidriverconfigs"] + verbs: ["get", "watch", "list"] +# ClusterTrustBundles referenced by SystemInfo trustBundle data sources are +# resolved on the node: atelet reads them through an informer and projects +# the sanitized PEM into actors (see cmd/atelet/trustbundle.go). +- apiGroups: ["certificates.k8s.io"] + resources: ["clustertrustbundles"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atelet-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atelet +spec: + selector: + matchLabels: + app: atelet + template: + metadata: + labels: + app: atelet + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} + containers: + - name: atelet + image: {{ include "substrate.componentImage" (list "atelet" .) }} + args: + - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- with .Values.atelet.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + privileged: true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.atelet.extraEnv }} +{{ toYaml . | indent 8 }} +{{- end }} + ports: + - name: grpc + containerPort: 8085 + hostPort: 8085 + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: device-plugins + mountPath: /var/lib/kubelet/device-plugins + - name: host-dev + mountPath: /host/dev + readOnly: true + volumes: + - name: run-ateom + hostPath: + path: /var/lib/ateom-gvisor + type: DirectoryOrCreate + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: device-plugins + hostPath: + path: /var/lib/kubelet/device-plugins + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml new file mode 100644 index 0000000000..86cc2271ca --- /dev/null +++ b/charts/substrate/templates/atenet-egress.yaml @@ -0,0 +1,227 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + frontendPolicies: + accessLog: + add: + substrate.connect.authority: source.connectHeaders["host"] + + binds: + - port: 8443 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/actor-id-ca-certs/ca.crt + routes: [] + - mode: internal + protocol: AUTO + listeners: + - protocol: TLS + hostname: "*" + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] + - protocol: HTTP + routes: + - policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + backends: + - dynamic: + target: source.connectHeaders["host"] + - protocol: TCP + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-egress +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-egress + template: + metadata: + labels: + app: atenet-egress + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-egress" .) }} + securityContext: + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + terminationGracePeriodSeconds: 60 + containers: + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - -f + - /etc/agentgateway/config.yaml + ports: + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: stats + containerPort: 15020 + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 1 + volumeMounts: + - name: config + mountPath: /etc/agentgateway + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: ext-proc + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - router + - --mode=egress + - --namespace={{ .Release.Namespace }} + - --port-extproc=50051 + - --extproc-address=127.0.0.1 + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt + - --otlp-collector-address= + - --envoy-admin-address=localhost:15000 + - --atenet-router=agentgateway + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: extproc + containerPort: 50051 + readinessProbe: + tcpSocket: + port: extproc + periodSeconds: 10 + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + volumes: + - name: config + configMap: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + - name: drain-signal + emptyDir: {} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-ca-certs + secret: + secretName: actor-id-ca-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-egress + ports: + - name: https + port: 443 + targetPort: https + protocol: TCP diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml new file mode 100644 index 0000000000..78976a39a6 --- /dev/null +++ b/charts/substrate/templates/atenet-router.yaml @@ -0,0 +1,355 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + # Actor sandboxes behind a worker IP are replaced between requests. Do + # not retain an idle connection that may belong to the previous actor. + backend: + poolMaxSize: 0 + +{{- if .Values.otel.endpoint }} + frontendPolicies: + tracing: + host: $AGENTGATEWAY_OTLP_ADDRESS + protocol: grpc + randomSampling: 0.01 +{{- end }} + + backends: + - name: dynamic + dynamic: {} + policies: + backendTunnel: + proxy: + backend: /dynamic + mode: connect + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true + + gateways: + http: + port: 8080 + protocol: HTTP + https: + port: 8443 + protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + + routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors + gateways: + - http + - https + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 + + binds: + - port: 8081 + tunnelProtocol: connect + listeners: + - protocol: HTTP + routes: [] + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-router + template: + metadata: + labels: + app: atenet-router + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + containers: + - name: atenet-router + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "router" + - "--mode=ingress" + - "--atenet-router=agentgateway" + - "--namespace={{ .Release.Namespace }}" + - "--port-http=8080" + - "--port-extproc=50051" + - "--extproc-address=127.0.0.1" + - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--status-port=4040" + - "--port-https=8443" + - "--port-connect=8081" + - "--port-connect-tls=8444" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: extproc + containerPort: 50051 + - name: status + containerPort: 4040 + - name: metrics + containerPort: 9090 + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - "-f" + - "/etc/agentgateway/config.yaml" +{{- if .Values.otel.endpoint }} + env: + - name: AGENTGATEWAY_OTLP_ADDRESS + value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 + - name: connect + containerPort: 8081 + - name: connect-tls + containerPort: 8444 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 + volumeMounts: + - name: agentgateway-config + mountPath: /etc/agentgateway + - name: "servicedns" + mountPath: "/run/servicedns.podcert.ate.dev" + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns-ca + readOnly: true + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + volumes: + - name: agentgateway-config + configMap: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + - name: "servicedns" + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP + - name: connect + port: 8081 + targetPort: 8081 + protocol: TCP + - name: connect-tls + port: 8444 + targetPort: 8444 + protocol: TCP + - name: status + port: 4040 + targetPort: status + protocol: TCP diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml new file mode 100644 index 0000000000..073291828b --- /dev/null +++ b/charts/substrate/templates/namespace.yaml @@ -0,0 +1,22 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml new file mode 100644 index 0000000000..86fc23b4a9 --- /dev/null +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -0,0 +1,198 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: Namespace +metadata: + name: podcertificate-controller-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +rules: +# The service signer needs to be able to read services and pods. +- apiGroups: + - "" + resources: + - services + - pods + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests + verbs: + - get + - list + - watch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests/status + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - servicedns.podcert.ate.dev/* + - podidentity.podcert.ate.dev/* + verbs: + - sign + - attest +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: podcertificate-controller-system + name: coordinator +rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - create + - get + - list + - watch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: podcertificate-controller-is-a-coordinator + namespace: podcertificate-controller-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coordinator +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podcertificate-controller + namespace: podcertificate-controller-system + labels: + app: podcertificate-controller +spec: + replicas: 1 + selector: + matchLabels: + app: podcertificate-controller + template: + metadata: + labels: + app: podcertificate-controller + spec: + containers: + - name: controller + image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} + args: + - --in-cluster=true + - --sharding-pod-namespace=$(POD_NAMESPACE) + - --sharding-pod-name=$(POD_NAME) + - --sharding-pod-uid=$(POD_UID) + - --sharding-application-name=podcertificate-controller + - --service-dns-ca-pool=/run/ca-state/service-dns-pool.json + - --pod-identity-ca-pool=/run/ca-state/pod-identity-pool.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + volumeMounts: + - name: "ca-state" + mountPath: "/run/ca-state" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: "ca-state" + projected: + sources: + - secret: + name: "service-dns-ca-pool" + items: + - key: "pool" + path: "service-dns-pool.json" + - secret: + name: "pod-identity-ca-pool" + items: + - key: "pool" + path: "pod-identity-pool.json" + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: default + terminationGracePeriodSeconds: 30 diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml new file mode 100644 index 0000000000..ce4a4efdd7 --- /dev/null +++ b/charts/substrate/templates/postgres.yaml @@ -0,0 +1,234 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $name := include "substrate.fullname" (list "postgres" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ .Release.Namespace }} +data: + postgresql.conf: | + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_key_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + pg_hba.conf: | + # Local socket access is limited to processes in this pod and is used by + # health checks, the workload's idempotent database bootstrap, and the + # tls-reloader sidecar's configuration reloads. + local all all trust + # PostgreSQL verifies client certificates against the pod-identity CA. It + # does not need its own serving CA because it never verifies its server certificate. + hostssl all all all trust clientcert=verify-ca + reload-tls.sh: | + # PostgreSQL opens ssl_cert_file, ssl_key_file and ssl_ca_file at startup + # and on SIGHUP, and nowhere else. The kubelet replaces the projected pod + # certificate in place about 30 minutes before it expires, so without this + # loop the server keeps presenting the certificate it booted with until it + # expires about a day later and every client stops trusting it. + set -eu + + # As PID 1 this shell only sees SIGTERM if a handler is installed, and only + # acts on it between commands, so the sleep below runs in the background + # and is waited on. Without both halves the pod takes the full termination + # grace period to go away. + trap 'exit 0' TERM INT + + CERT=/run/servicedns.podcert.ate.dev/credential-bundle.pem + CA=/run/podidentity.podcert.ate.dev/trust-bundle.pem + + # Comfortably inside the 30m headroom (notAfter - beginRefreshAt) that + # cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go + # leaves; hashing two small files costs nothing. + INTERVAL=60 + + reloaded="" + while true; do + current="$(sha256sum "${CERT}" "${CA}")" + # Reloading fails until the server is accepting connections, which is + # where every pod starts out, so only record a hash once it has worked. + # Starting empty also means a restart of this container costs one + # redundant reload rather than a missed one. + if [ "${current}" != "${reloaded}" ] \ + && psql -U postgres -d postgres -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then + reloaded="${current}" + echo "$(date -u +%FT%TZ) reloaded TLS configuration" + fi + sleep "${INTERVAL}" & + wait $! + done +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: {{ $name }} + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app: {{ $name }} + template: + metadata: + labels: + app: {{ $name }} + spec: + securityContext: + # Group ownership of the projected certificate below, and of the data + # volume so that a freshly provisioned one is writable. OnRootMismatch + # keeps the kubelet from walking the data directory on every start, + # which would leave PGDATA group-writable and postgres refusing to run. + fsGroup: 70 + fsGroupChangePolicy: OnRootMismatch + # PostgreSQL re-reads its TLS files only on SIGHUP, so this sidecar + # reloads the server whenever the kubelet rotates the projected pod + # certificate. fsGroup is also what makes that projection readable: the + # kubelet writes it root-owned for as long as the pod's containers do not + # all agree on one non-root user, and grants the fsGroup group access, + # landing the key at root:postgres 0640, the only shared mode PostgreSQL + # accepts. Pinning runAsUser on the postgres container would make the key + # postgres-owned and group-readable, which it rejects. + # See https://www.postgresql.org/docs/current/ssl-tcp.html#SSL-SETUP + initContainers: + - name: tls-reloader + restartPolicy: Always + image: {{ .Values.images.postgres }} + securityContext: + runAsUser: 70 + command: + - /bin/sh + - /etc/postgresql/reload-tls.sh + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + resources: + requests: + cpu: 10m + memory: 32Mi + containers: + - name: postgres + image: {{ .Values.images.postgres }} + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + sleep 1 + done + if ! psql -U postgres -d postgres -Atc \ + "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then + createdb -U postgres atepg + fi + env: + - name: POSTGRES_DB + value: atepg + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"] + initialDelaySeconds: 2 + periodSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 10 + periodSeconds: 10 + args: ["-c", "config_file=/etc/postgresql/postgresql.conf"] + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + - name: data + mountPath: /var/lib/postgresql/data + resources: +{{ toYaml .Values.postgres.resources | indent 10 }} + volumes: + - name: config + configMap: + name: {{ $name }}-config + - name: servicedns + projected: + # 0600 plus the group read that fsGroup adds is the 0640 above. + defaultMode: 0600 + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + # The unix socket directory, shared so the sidecar can ask the running + # server to reload. The image defaults both the server and its clients to + # this path, so nothing else has to know about it. + - name: socket + emptyDir: {} + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.storageSize }} diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml new file mode 100644 index 0000000000..5a240f5baf --- /dev/null +++ b/charts/substrate/templates/role.yaml @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml new file mode 100644 index 0000000000..edaad3cfa8 --- /dev/null +++ b/charts/substrate/templates/rustfs.yaml @@ -0,0 +1,137 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.rustfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "substrate.fullname" (list "rustfs-data" .) }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.rustfs.storageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: {{ .Values.images.rustfs }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: {{ .Values.rustfs.accessKey | quote }} + - name: RUSTFS_SECRET_KEY + value: {{ .Values.rustfs.secretKey | quote }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "substrate.fullname" (list "rustfs-data" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: {{ .Values.images.awsCli }} + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} already exists" + exit 0 + fi + if aws s3api create-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 +{{- end }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml new file mode 100644 index 0000000000..36af4296f3 --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -0,0 +1,38 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Cluster-wide default SandboxConfig for the gVisor (runsc) sandbox class. A +# WorkerPool with sandboxClass gvisor (the default) and no explicit +# sandboxConfigName resolves to this. atelet fetches the runsc binary matching +# the worker node's architecture. To pin a different runsc, edit the assets +# below or create another SandboxConfig and name it from the WorkerPool. +apiVersion: ate.dev/v1alpha1 +kind: SandboxConfig +metadata: + name: gvisor-default +spec: + sandboxClass: gvisor + default: true + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" + sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" + arm64: + gvisor: + url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" + sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml new file mode 100644 index 0000000000..f25d43409b --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -0,0 +1,57 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is +# generic (any arch -> any asset name -> {url, sha256}); this policy enforces the +# requirements a given sandbox class actually needs, fail-closed at apply time. +# (url/sha256 being required and well-formed is enforced by the CRD schema.) +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: sandboxconfig-assets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["ate.dev"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["sandboxconfigs"] + validations: + # gVisor needs a release tarball (or legacy runsc binary) for every architecture. + - expression: >- + object.spec.sandboxClass != 'gvisor' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + # The micro-VM (cloud-hypervisor) runtime needs its asset set for every + # architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'microvm' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image', 'kata-config'] + .all(name, name in object.spec.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, kata-image, and kata-config assets for every architecture under spec.assets" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: sandboxconfig-assets +spec: + policyName: sandboxconfig-assets + validationActions: ["Deny"] diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml new file mode 100644 index 0000000000..48ce963668 --- /dev/null +++ b/charts/substrate/values.yaml @@ -0,0 +1,73 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for the substrate chart. +# +# The chart requires ClusterTrustBundle, ClusterTrustBundleProjection, +# PodCertificateRequest, and the certificates.k8s.io/v1beta1 API. + +# Set to true to have the chart create the release namespace. +# Off by default — most helm workflows expect the namespace to already exist +# (helm install -n --create-namespace). Enable for the generated +# manifests/ate-install/ install path (kubectl apply). +createNamespace: false + +postgres: + storageSize: 1Gi + connectionString: "" + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + +rustfs: + enabled: true + storageSize: 1Gi + bucket: ate-snapshots + accessKey: rustfsadmin + secretKey: rustfsadmin + +# atelet daemonset overrides. Defaults use the in-cluster RustFS deployment for +# snapshots. Set rustfs.enabled=false and override these fields when using +# external storage. +# extraArgs / extraEnv are appended verbatim for installer-specific knobs +# (e.g. registry replacement for kind). +atelet: + gcpAuthForImagePulls: false + storageBackend: s3 + extraArgs: [] + extraEnv: [] + +# Name of a ConfigMap in the release namespace that supplies per-environment +# overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). +# Mounted via envFrom with optional=true. Created by the chart from these values. +ateApiServerEnvVarsConfigMap: ate-api-server-envvars + +otel: + endpoint: "" + +image: + registry: ghcr.io/kagent-dev/substrate + tag: "" + +images: + postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 + rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + agentgateway: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 + coredns: coredns/coredns:1.11.1 + busybox: busybox:1.36 diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 3acd3d7f35..557d994aad 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -25,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/internal/atelet" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -32,6 +33,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -74,6 +76,11 @@ func WithDialCredentials(build func(expectedPodUID string) (credentials.Transpor return func(d *AteletDialer) { d.dialCredentials = build } } +// WithInsecureCredentials disables transport security for local clusters without Pod Certificates. +func WithInsecureCredentials() DialerOption { + return WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil }) +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { @@ -198,7 +205,7 @@ func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", ateletSA) + expectedID, err := spiffeid.FromSegments(trustDomain, "ns", installdefaults.NamespaceFromPodEnv(), "sa", ateletSA) if err != nil { return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) } diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 593529e400..2fb103bc33 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -41,6 +42,22 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" +func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { + secure := NewAteletDialer(nil, nil, "", "") + if _, err := secure.dialCredentials("pod-uid"); err == nil { + t.Fatal("secure dialer accepted empty credential paths") + } + + insecureDialer := NewAteletDialer(nil, nil, "", "", WithInsecureCredentials()) + creds, err := insecureDialer.dialCredentials("pod-uid") + if err != nil { + t.Fatalf("insecure dial credentials: %v", err) + } + if got := creds.Info().SecurityProtocol; got != "insecure" { + t.Fatalf("security protocol = %q, want insecure", got) + } +} + // makeTestCA mints a self-signed CA and returns it along with an X.509 bundle // containing it as the sole authority for the cluster.local trust domain. func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) { @@ -198,7 +215,7 @@ func TestDialForWorkerTarget(t *testing.T) { Spec: corev1.PodSpec{NodeName: "node-1"}, } ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: tc.ateletIP}}}, } @@ -225,7 +242,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("unknown worker pod", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: "10.244.1.7"}}}, } @@ -237,7 +254,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("atelet without assigned IPs", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } d := newDialerForPods(t, workerPod, ateletPod) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index f79912f301..276ac4b4e7 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" @@ -65,10 +66,8 @@ const ( // templates hand out. No object store is wired up behind it. testStorageLocation = "gs://fake-fake-fake" - // ateletNamespace and byNode mirror the unexported constants controlapi's - // atelet informer is built with. - ateletNamespace = "ate-system" - byNode = "by-node" + // byNode mirrors the unexported index name controlapi's atelet informer uses. + byNode = "by-node" ) var ( @@ -132,7 +131,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // 3. Initialize Informers workerFactory, workerInformer := controlapi.WorkerPodInformer(k8sClient) - ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient) + ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient, installdefaults.SystemNamespace) scFactory := informers.NewSharedInformerFactory(k8sClient, 0) scLister := scFactory.Storage().V1().StorageClasses().Lister() @@ -222,6 +221,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu dialer, instruments, "", + 30*time.Second, volPlugins, objectStore, "https://nonexistent-issuer.example", @@ -698,7 +698,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ateletNamespace, + Namespace: installdefaults.SystemNamespace, Labels: map[string]string{"app": "atelet"}, }, Spec: corev1.PodSpec{ @@ -706,7 +706,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, }, } - created, err := kc.CoreV1().Pods(ateletNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + created, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -715,7 +715,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { } created.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} created.Status.Phase = corev1.PodRunning - if _, err := kc.CoreV1().Pods(ateletNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { + if _, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("updating atelet pod %s status: %w", name, err) } return nil @@ -732,7 +732,7 @@ func setupAteletOnNode(t *testing.T, tc *testContext, name, nodeName string) { t.Fatalf("%v", err) } t.Cleanup(func() { - _ = tc.k8sClient.CoreV1().Pods(ateletNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ + _ = tc.k8sClient.CoreV1().Pods(installdefaults.SystemNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ GracePeriodSeconds: ptr.To[int64](0), }) }) diff --git a/cmd/ateapi/internal/controlapi/informer.go b/cmd/ateapi/internal/controlapi/informer.go index 8e467d3029..12e42780f7 100644 --- a/cmd/ateapi/internal/controlapi/informer.go +++ b/cmd/ateapi/internal/controlapi/informer.go @@ -25,13 +25,13 @@ import ( ) const ( - ateletNamespace = "ate-system" byNamespaceAndName = "by-namespace-and-name" byNode = "by-node" ) -// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for Atelet pods. -func AteletInformer(kc kubernetes.Interface) (informers.SharedInformerFactory, cache.SharedIndexInformer) { +// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for +// Atelet pods in the given namespace. +func AteletInformer(kc kubernetes.Interface, ateletNamespace string) (informers.SharedInformerFactory, cache.SharedIndexInformer) { factory := informers.NewSharedInformerFactoryWithOptions(kc, 0, informers.WithNamespace(ateletNamespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index b433515586..c8633eac6a 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "sync" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -63,14 +64,8 @@ type VolumePluginRegistry interface { GetPlugin(ctx context.Context, name string) (volume.VolumePluginControlPlane, error) } -// NewRPCService creates an instance of the ControlServer service. This is what -// implements the outward-facing RPC interface. -// -// instruments may be nil; the record helpers no-op. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewRPCService creates an RPC service. actorWorkflowDeadline bounds how long a single +// Resume/Suspend workflow can run end-to-end. instruments and objectStore may be nil. func NewRPCService( persistence store.Interface, workerCache *workercache.Cache, @@ -80,6 +75,7 @@ func NewRPCService( dialer *AteletDialer, instruments *Instruments, egressGatewayAddress string, + actorWorkflowDeadline time.Duration, volumePlugins map[string]volume.VolumePluginControlPlane, objectStore objectstore.Store, actorIdentityJWTIssuer string, @@ -101,7 +97,7 @@ func NewRPCService( actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, objectStore) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, objectStore) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 1057169ba3..beaae82b06 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -109,14 +110,13 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry + workflowDeadline time.Duration objectStore objectstore.Store } -// NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how +// long a single Resume/Suspend can run end-to-end; instruments and objectStore +// may be nil. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -126,6 +126,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, + workflowDeadline time.Duration, objectStore objectstore.Store, ) *ActorWorkflow { return &ActorWorkflow{ @@ -138,6 +139,7 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, + workflowDeadline: workflowDeadline, objectStore: objectStore, } } @@ -211,7 +213,14 @@ func acquireLease(ctx context.Context, holder leaseHolder, key, subject string) } func (w *ActorWorkflow) acquireActorLease(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lease, error) { - return acquireLease(ctx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + workflowCtx, cancel := context.WithTimeout(ctx, w.workflowDeadline) + leaseCtx, lease, err := acquireLease(workflowCtx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + if err != nil { + cancel() + return nil, nil, err + } + context.AfterFunc(lease.Context(), cancel) + return leaseCtx, lease, nil } func acquireTagLease(ctx context.Context, holder leaseHolder, tagRef resources.TagRef) (context.Context, *store.Lease, error) { diff --git a/cmd/ateapi/internal/controlapi/workflow_lease_test.go b/cmd/ateapi/internal/controlapi/workflow_lease_test.go new file mode 100644 index 0000000000..b17f0583d1 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_lease_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/resources" +) + +type leaseStore struct{ store.Interface } + +func (leaseStore) AcquireLease(ctx context.Context, _ string) (*store.Lease, error) { + return store.NewLease(ctx, func() {}), nil +} + +func TestAcquireActorLeaseWorkflowDeadline(t *testing.T) { + w := &ActorWorkflow{store: leaseStore{}, workflowDeadline: 20 * time.Millisecond} + + ctx, lease, err := w.acquireActorLease(context.Background(), resources.ActorRef{Atespace: "space", Name: "actor"}) + if err != nil { + t.Fatalf("acquireActorLease: %v", err) + } + t.Cleanup(lease.Close) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("context error = %v, want DeadlineExceeded", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("workflow context did not reach its deadline") + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index af78631005..df46c2dd9a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -19,6 +19,7 @@ import ( "errors" "slices" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -57,7 +58,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, objectstoretest.New()) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoretest.New()) } // newFinalizeWorkflow builds an ActorWorkflow over persistence with an @@ -65,7 +66,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa // directly rather than going through newTestActorWorkflow. func newFinalizeWorkflow(persistence store.Interface) (*ActorWorkflow, *objectstoretest.Fake) { objects := objectstoretest.New() - return &ActorWorkflow{store: persistence, objectStore: objects}, objects + return &ActorWorkflow{store: persistence, workflowDeadline: time.Minute, objectStore: objects}, objects } // mustActorSnapshotURI builds the URI of a snapshot the actor took under diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 47c7ccebf1..e56d9ce4ca 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -37,6 +37,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore" @@ -79,10 +80,13 @@ var ( actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") + ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") + showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") ) @@ -162,8 +166,13 @@ func main() { sandboxConfigLister := ateFactory.Api().V1alpha1().SandboxConfigs().Lister() csiDriverConfigLister := ateFactory.Api().V1alpha1().CSIDriverConfigs().Lister() + // atelet shares ateapi's namespace in every supported deployment topology, + // so we read it from Kubernetes' downward API rather than expose a flag. + ateletNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace)) + workerPodInformerFactory, workerPodInformer := controlapi.WorkerPodInformer(clientset) - ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset) + ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) scInformerFactory := informers.NewSharedInformerFactory(clientset, 0) storageClassLister := scInformerFactory.Storage().V1().StorageClasses().Lister() @@ -197,7 +206,11 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) + var dialerOpts []controlapi.DialerOption + if *ateletInsecure { + dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) + } + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) actorIDCAPool, err := localca.NewRefreshingPool(*actorIDCAPoolFile) if err != nil { @@ -218,6 +231,7 @@ func main() { ateletDialer, instruments, *egressGatewayAddress, + *actorWorkflowDeadline, volPlugins, objectStore, actorIdentityJWTIssuer, @@ -334,8 +348,10 @@ func logFlagValues(ctx context.Context) { slog.String("actor-id-ca-pool", *actorIDCAPoolFile), slog.String("pod-identity-ca-certs", *podIdentityCACerts), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), + slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), + slog.Duration("actor-workflow-deadline", *actorWorkflowDeadline), ) } diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index 218a18c2d8..a7ed1b1cf5 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -22,4 +22,4 @@ package controllers //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch //+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch,namespace=ate-system -//go:generate bash ../../../../hack/run-tool.sh controller-gen rbac:headerFile=../../../../hack/boilerplate/sh.txt,roleName=ate-controller paths="./..." output:rbac:artifacts:config=../../../../manifests/ate-install/generated/ +//go:generate bash ../../../../hack/gen-rbac.sh diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 904c207d5a..a68f386da3 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -93,6 +93,7 @@ var ( ateapiAddress = pflag.String("ateapi-address", "k8s:///api.ate-system.svc:443", "ateapi gRPC target used by the credential broker.") ateapiCAFile = pflag.String("ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify ateapi.") ateapiServerName = pflag.String("ateapi-server-name", "api.ate-system.svc", "DNS name expected on the ateapi certificate.") + grpcInsecure = pflag.Bool("grpc-insecure", false, "Serve gRPC without transport security. Intended only for local clusters without Pod Certificates.") gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -316,72 +317,78 @@ func main() { // it would never list or watch. Start is idempotent per informer — this // launches the new one and leaves the already-running ones untouched. ateFactory.Start(stopCh) - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - K8sClient: k8sClient, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - ClientCredBundle: *grpcServerCredBundle, - }) - if err != nil { - serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) - } - ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) - if err != nil { - serverboot.Fatal(ctx, "Failed to create ateapi client", err) - } - defer ateapiConn.Close() - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } - tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) - if err != nil { - serverboot.Fatal(ctx, "Failed to build server TLS config", err) - } - ateletCert, err := credbundle.Parse(*grpcServerCredBundle) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - if ateletIdentity == nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) - } - brokerTLS := tlsCfg.Clone() - brokerTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) - if err := os.Remove(ateompath.CredentialBrokerSocket); err != nil && !errors.Is(err, os.ErrNotExist) { - serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) - } - brokerLis, err := net.Listen("unix", ateompath.CredentialBrokerSocket) - if err != nil { - serverboot.Fatal(ctx, "Failed to listen for credential broker", err) - } - defer brokerLis.Close() - if err := os.Chmod(ateompath.CredentialBrokerSocket, 0o600); err != nil { - serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + serverOpts := []grpc.ServerOption{ + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), } - brokerServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(brokerTLS))) - ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{ - controlClient: ateapipb.NewControlClient(ateapiConn), - }) - ateletpb.RegisterWorkerCapacityServer(brokerServer, &workerCapacityService{ - workers: ateapipb.NewWorkerServiceClient(ateapiConn), - }) - go func() { - if err := brokerServer.Serve(brokerLis); err != nil { - serverboot.Fatal(ctx, "Failed to serve credential broker", err) + if *grpcInsecure { + slog.WarnContext(ctx, "Serving atelet gRPC without transport security") + } else { + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) } - }() + serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsCfg))) - svr := grpc.NewServer( - grpc.Creds(credentials.NewTLS(tlsCfg)), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), - ) + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + K8sClient: k8sClient, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) + } + ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ateapi client", err) + } + defer ateapiConn.Close() + + ateletCert, err := credbundle.Parse(*grpcServerCredBundle) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + if ateletIdentity == nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) + } + brokerTLS := tlsCfg.Clone() + brokerTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) + if err := os.Remove(ateompath.CredentialBrokerSocket); err != nil && !errors.Is(err, os.ErrNotExist) { + serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) + } + brokerLis, err := net.Listen("unix", ateompath.CredentialBrokerSocket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen for credential broker", err) + } + defer brokerLis.Close() + if err := os.Chmod(ateompath.CredentialBrokerSocket, 0o600); err != nil { + serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + } + brokerServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(brokerTLS))) + ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{ + controlClient: ateapipb.NewControlClient(ateapiConn), + }) + ateletpb.RegisterWorkerCapacityServer(brokerServer, &workerCapacityService{ + workers: ateapipb.NewWorkerServiceClient(ateapiConn), + }) + go func() { + if err := brokerServer.Serve(brokerLis); err != nil { + serverboot.Fatal(ctx, "Failed to serve credential broker", err) + } + }() + } + + svr := grpc.NewServer(serverOpts...) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index 71030d34d3..3511d3cc27 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -358,29 +358,21 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { prewarmMaxJitter = 0 t.Cleanup(func() { ateompath.StaticFilesDir, prewarmMaxJitter = origDir, origJitter }) - host := imageVolumeTestRegistry(t) - pauseRef := host + "/pause:3.10" - pushPauseImage(t, pauseRef) - content := []byte("runsc binary bytes") sha := fmt.Sprintf("%x", sha256.Sum256(content)) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - cfg.Spec.PauseImage = pauseRef + cfg.Spec.PauseImage = "" ctx := t.Context() client := fake.NewSimpleClientset(cfg) factory := externalversions.NewSharedInformerFactory(client, 0) informer := factory.Api().V1alpha1().SandboxConfigs().Informer() - store, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } herder := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} // Handler first, informer start second, mirroring main: atelet startup // must never wait on this informer's sync, and the initial List replays // the pre-existing config into the handler as an Add. - if err := startSandboxAssetPrewarm(ctx, informer, herder, store, false); err != nil { + if err := startSandboxAssetPrewarm(ctx, informer, herder, nil, false); err != nil { t.Fatalf("startSandboxAssetPrewarm: %v", err) } stopCh := make(chan struct{}) diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh new file mode 100755 index 0000000000..baa22fa517 --- /dev/null +++ b/hack/gen-rbac.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the controller ClusterRole into the Helm chart and templatize its +# name so multi-release installs do not collide on a cluster-scoped resource. +# +# controller-gen emits a YAML file with a fixed `roleName=` value. We post- +# process that file to swap the static name for the chart's fullname helper, +# matching the convention used by every other resource in charts/substrate/. +# +# Invoked via `go generate ./cmd/atecontroller/internal/controllers/...`. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="${ROOT}/charts/substrate/templates/role.yaml" + +bash "${ROOT}/hack/run-tool.sh" controller-gen \ + "rbac:headerFile=${ROOT}/hack/boilerplate/sh.txt,roleName=ate-controller" \ + paths="${ROOT}/cmd/atecontroller/internal/controllers/..." \ + "output:rbac:artifacts:config=${ROOT}/charts/substrate/templates/" + +# Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` +# at column 0; the substitution is exact-match to stay robust. +sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index 452b18e6fc..8c8708a949 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -174,6 +174,7 @@ fi # in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." + run_kubectl wait --for=condition=complete job/rustfs-bucket-init -n ate-system --timeout=120s OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh else log "Uploading assets to gs://${BUCKET_NAME}/kata-assets/ ..." diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh new file mode 100755 index 0000000000..d8a7f69e8a --- /dev/null +++ b/hack/render-manifests.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS-mode +# install) — the canonical kubectl-apply install path. The chart at +# charts/substrate/ is the single source of truth; this script only renders. +# +# Usage: +# hack/render-manifests.sh # write into manifests/ate-install/ +# hack/render-manifests.sh --check # fail if rendered output differs +# +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${ROOT}/manifests/ate-install" +CHART_DIR="${ROOT}/charts/substrate" +CHECK_MODE="false" +PRESERVED_FILES=( + ate-api-server.yaml + ate-controller.yaml + ate-otel-config.yaml + ate-system-namespace.yaml + atelet.yaml + atenet-dns.yaml + atenet-egress.yaml + atenet-egress-with-sdsmint.yaml + atenet-router.yaml + atenet-router-monitoring.yaml + pod-certificate-controller.yaml + postgres.yaml + sandboxconfig-gvisor.yaml + sandboxconfig-validation.yaml +) + +if [ "${1:-}" = "--check" ]; then + CHECK_MODE="true" +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "helm not found in PATH" >&2 + exit 1 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helm template substrate "${CHART_DIR}" \ + --namespace ate-system \ + --set auth.mode=mtls \ + --set createNamespace=true \ + --set image.registry=ko://github.com/agent-substrate/substrate/cmd \ + --set image.tag="" \ + > "${TMP_DIR}/all.yaml" + +# Split into per-source files so the directory structure mirrors the chart +# templates, making diffs friendlier. +python3 - "${TMP_DIR}/all.yaml" "${TMP_DIR}/out" <<'PY' +import os, re, sys, yaml +in_path, out_dir = sys.argv[1], sys.argv[2] +os.makedirs(out_dir, exist_ok=True) + +with open(in_path) as f: + raw = f.read() + +# Helm prepends a "# Source: /templates/" comment to each doc. +docs_by_source = {} +for doc in raw.split('\n---\n'): + m = re.search(r'#\s*Source:\s*\S+/templates/(\S+)', doc) + src = m.group(1) if m else "misc.yaml" + # Drop the leading "# Source:" line from the written file. + cleaned = re.sub(r'^\s*#\s*Source:.*\n', '', doc, count=1, flags=re.MULTILINE) + if not cleaned.strip(): + continue + docs_by_source.setdefault(src, []).append(cleaned.strip()) + +for src, docs in docs_by_source.items(): + if src == "namespace.yaml": + src = "ate-system-namespace.yaml" + header = ( + "# Copyright 2026 Google LLC\n" + "#\n" + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n" + "# you may not use this file except in compliance with the License.\n" + "# You may obtain a copy of the License at\n" + "#\n" + "# http://www.apache.org/licenses/LICENSE-2.0\n" + "#\n" + "# Unless required by applicable law or agreed to in writing, software\n" + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + "# See the License for the specific language governing permissions and\n" + "# limitations under the License.\n" + "\n" + "# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh.\n" + "# Run `make helm-template` to regenerate.\n" + "\n" + ) + with open(os.path.join(out_dir, src), "w") as out: + out.write(header) + out.write("\n---\n".join(docs)) + out.write("\n") +PY + +if [ "${CHECK_MODE}" = "true" ]; then + # Only compare top-level files; subdirs like generated/ and kind/ are not + # produced by the chart and live alongside it intentionally. + CHECK_TMP="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT + mkdir -p "${CHECK_TMP}/current" + find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; + rm -f "${PRESERVED_FILES[@]/#/${CHECK_TMP}\/current\/}" + rm -f "${PRESERVED_FILES[@]/#/${TMP_DIR}\/out\/}" + if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then + echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true + exit 1 + fi + echo "manifests/ate-install/ matches chart output." + exit 0 +fi + +# Replace contents (preserve kind/ and generated/ subdirs which are not chart output). +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ + ! -name 'ate-api-server.yaml' \ + ! -name 'ate-controller.yaml' \ + ! -name 'ate-otel-config.yaml' \ + ! -name 'ate-system-namespace.yaml' \ + ! -name 'atelet.yaml' \ + ! -name 'atenet-dns.yaml' \ + ! -name 'atenet-egress.yaml' \ + ! -name 'atenet-egress-with-sdsmint.yaml' \ + ! -name 'atenet-router.yaml' \ + ! -name 'atenet-router-monitoring.yaml' \ + ! -name 'pod-certificate-controller.yaml' \ + ! -name 'postgres.yaml' \ + ! -name 'sandboxconfig-gvisor.yaml' \ + ! -name 'sandboxconfig-validation.yaml' \ + -delete +rm -f "${PRESERVED_FILES[@]/#/${TMP_DIR}\/out\/}" +cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" +rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" +echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 27c66dffe1..73c3a4bb91 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -55,10 +55,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -if [[ $# -gt 0 ]]; then - echo "Error: unknown argument $1" >&2 - exit 1 -fi +SKIP_CONTROL_PLANE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-control-plane) SKIP_CONTROL_PLANE=true ;; + *) + echo "Error: unknown argument $1" >&2 + exit 1 + ;; + esac + shift +done if [[ -z "${KO_DOCKER_REPO}" ]]; then echo "Error: KO_DOCKER_REPO is required (set it in .ate-dev-env.sh for GKE," >&2 @@ -75,13 +83,15 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system +if [[ "${SKIP_CONTROL_PLANE}" != "true" ]]; then + log "Deploying the ate control plane (--deploy-ate-system)..." + if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system + else + # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system + fi fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index 99e24d731b..2f2dbfde6a 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -24,6 +24,14 @@ OUTDIR="_LICENSES" # under $ROOT # Ensure the tool is built and up-to-date GO_LICENSES_BIN="$(bash "${ROOT}/hack/run-tool.sh" --print-bin-path go-licenses)" +# go-licenses runs in temporary verification worktrees that do not have enough +# VCS metadata for Go's build stamping. +if [[ -n "${GOFLAGS:-}" ]]; then + export GOFLAGS="${GOFLAGS} -buildvcs=false" +else + export GOFLAGS="-buildvcs=false" +fi + # Clean out previous licenses rm -rf "${OUTDIR}" mkdir -p "${OUTDIR}" diff --git a/hack/verify/crd-chart.sh b/hack/verify/crd-chart.sh new file mode 100755 index 0000000000..dc3ef2bdaf --- /dev/null +++ b/hack/verify/crd-chart.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +GENERATED_DIR="manifests/ate-install/generated" +CHART_TEMPLATES_DIR="charts/substrate-crds/templates" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/generated" "${TMP_DIR}/chart" +cp "${GENERATED_DIR}/"ate.dev_*.yaml "${TMP_DIR}/generated/" +cp "${CHART_TEMPLATES_DIR}/"ate.dev_*.yaml "${TMP_DIR}/chart/" + +# The generated CRDs start with a leading document separator after the +# boilerplate header. In chart templates that separator renders as a +# comment-only YAML document, so the chart copies intentionally omit it. +for file in "${TMP_DIR}/generated/"*.yaml; do + awk 'BEGIN { removed = 0 } /^---$/ && removed == 0 { removed = 1; next } { print }' "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +done + +if ! diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" >/dev/null 2>&1; then + echo "charts/substrate-crds/templates is out of sync with ${GENERATED_DIR}" >&2 + echo "Copy updated CRDs into charts/substrate-crds/templates." >&2 + diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" | head -80 >&2 || true + exit 1 +fi + +echo "charts/substrate-crds/templates matches generated CRDs." diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 31763ccf62..00de8bd61c 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -24,6 +24,7 @@ import ( "strings" "sync" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -184,7 +185,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "api", 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, installdefaults.SystemNamespace, installdefaults.APIServiceName, 443) if err != nil { return nil, err } @@ -273,7 +274,7 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts("ate-system").CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(installdefaults.SystemNamespace).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } diff --git a/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go index 3d0db9f047..d4b842227c 100644 --- a/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -20,6 +20,7 @@ package credbundle import ( + "crypto" "crypto/tls" "crypto/x509" "encoding/pem" @@ -112,6 +113,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { } var leafKeyBytes []byte + var leafKeyBlockType string var chainBytes [][]byte for { @@ -124,8 +126,9 @@ func Parse(bundlePath string) (*tls.Certificate, error) { switch block.Type { case "CERTIFICATE": chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": + case "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY": leafKeyBytes = block.Bytes + leafKeyBlockType = block.Type default: return nil, fmt.Errorf("unknown PEM block type %q", block.Type) } @@ -139,7 +142,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { return nil, fmt.Errorf("no CERTIFICATE blocks found") } - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) + leafKey, err := parsePrivateKey(leafKeyBlockType, leafKeyBytes) if err != nil { return nil, fmt.Errorf("while parsing private key: %w", err) } @@ -155,3 +158,16 @@ func Parse(bundlePath string) (*tls.Certificate, error) { PrivateKey: leafKey, }, nil } + +func parsePrivateKey(blockType string, keyBytes []byte) (crypto.PrivateKey, error) { + switch blockType { + case "PRIVATE KEY": + return x509.ParsePKCS8PrivateKey(keyBytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBytes) + default: + return nil, fmt.Errorf("unsupported private key block type %q", blockType) + } +} diff --git a/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go index 579a12bbc0..171bcb5b64 100644 --- a/internal/credbundle/credbundle_test.go +++ b/internal/credbundle/credbundle_test.go @@ -58,13 +58,13 @@ func TestParsePKCS8PrivateKeyBlock(t *testing.T) { } } -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { +func TestParseRSAPrivateKeyBlock(t *testing.T) { certDER := generateCertificate(t, 1) bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") + if _, err := Parse(bundlePath); err != nil { + t.Fatalf("Parse() error = %v", err) } } diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index 498197885d..098ef7412d 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -50,7 +50,6 @@ var PlatformMetricPrefixes = []string{ "ate_scheduler_assignment_duration", "ate_actor_restore_duration", "ate_actor_checkpoint_duration", - "atenet_router_route_duration", "ate_scheduler_eligible_workers", } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 392128152f..ca2b69b50e 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -19,6 +19,9 @@ import ( "fmt" "io" "net/http" + "os" + "regexp" + "strconv" "strings" "testing" "time" @@ -702,7 +705,7 @@ func validateCounterResponse(t *testing.T, resp string, stage string, wantMemory if !strings.Contains(resp, memoryCounterPrefix+fmt.Sprintf("%d", wantMemory)) { t.Errorf("[%s] expected memory count %d, got response: %s", stage, wantMemory, resp) } - if !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { + if wantFile >= 0 && !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { t.Errorf("[%s] expected file count %d, got response: %s", stage, wantFile, resp) } } @@ -726,24 +729,14 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj }) }() - listResp, err := clients.SubstrateAPI.ListActors(ctx, &ateapipb.ListActorsRequest{Atespace: demoAtespace}) + getResp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + }) if err != nil { - t.Fatalf("ListActors RPC failed: %v", err) - } - - var myActors []*ateapipb.Actor - for _, actor := range listResp.GetActors() { - if actor.GetActorTemplate().GetName() == at.GetMetadata().GetName() && actor.GetMetadata().GetName() == actorName { - myActors = append(myActors, actor) - } + t.Fatalf("GetActor RPC failed: %v", err) } - // Check that we have our Actor created. - if len(myActors) != 1 { - t.Fatalf("expected actor %s from template %s, got %d actors: %v", actorName, at.GetMetadata().GetName(), len(myActors), myActors) - } - - actor := myActors[0] + actor := getResp if actor.GetMetadata().GetName() != actorName { t.Errorf("expected actor name %s, got %s", actorName, actor.GetMetadata().GetName()) } @@ -754,8 +747,7 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Errorf("expected actor state to be SUSPENDED, got %v", actor.Status.State) } - t.Logf("Successfully queried Substrate API. Found %d active actors total, %d from our template %s.", - len(listResp.GetActors()), len(myActors), at.GetMetadata().GetName()) + t.Logf("Successfully queried Substrate API. Found actor %s in namespace %s.", actorName, nsObj.Name) return nil } @@ -782,13 +774,13 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) - // Pausing the actor t.Logf("Pausing Actor %q...", actorName) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ @@ -807,11 +799,12 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after pause", 2, -1) + } else { + validateCounterResponse(t, resp, "after pause", 2, 2) } - validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -861,11 +854,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -885,11 +879,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after suspend", 2, -1) + } else { + validateCounterResponse(t, resp, "after suspend", 2, 2) } - validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -1195,6 +1190,55 @@ func waitForActorStateWithTimeout(ctx context.Context, t *testing.T, clients *e2 t.Fatalf("timed out waiting for actor %q to reach state %v", actorName, expectedState) } +var preservedCountRe = regexp.MustCompile(`preserved memory count: ([0-9]+)`) + +func callActorUntilCountAtLeast(t *testing.T, actorRef resources.ActorRef, minCount int) string { + t.Helper() + + var lastErr error + var lastResp string + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + resp, err := callActor(t, actorRef) + if err != nil { + lastErr = err + } else { + lastResp = resp + count, err := preservedCount(resp) + if err != nil { + lastErr = err + } else if count >= minCount { + return resp + } else { + lastErr = fmt.Errorf("expected preserved memory count >= %d, got %d in response: %s", minCount, count, resp) + } + } + time.Sleep(500 * time.Millisecond) + } + + if lastResp != "" { + t.Fatalf("timed out calling actor %q; last response: %s; last error: %v", actorRef.Name, lastResp, lastErr) + } + t.Fatalf("timed out calling actor %q; last error: %v", actorRef.Name, lastErr) + return "" +} + +func preservedCount(resp string) (int, error) { + matches := preservedCountRe.FindStringSubmatch(resp) + if matches == nil { + return 0, fmt.Errorf("response does not include preserved memory count: %s", resp) + } + count, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("parse preserved memory count %q: %w", matches[1], err) + } + return count, nil +} + +func isMicroVMEnvironment() bool { + return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" +} + func callActor(t *testing.T, actorRef resources.ActorRef) (string, error) { return callActorPath(t, actorRef, "POST", "/") } diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index a96784d062..531a3f574d 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -280,11 +280,17 @@ func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Client func whoami(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) whoamiResponse { t.Helper() - out, err := tryWhoami(ctx, rc, id) - if err != nil { - t.Fatal(err) + deadline := time.Now().Add(30 * time.Second) + for { + out, err := tryWhoami(ctx, rc, id) + if err == nil { + return out + } + if time.Now().After(deadline) { + t.Fatal(err) + } + time.Sleep(time.Second) } - return out } // tryWhoami is whoami returning the error instead of failing the test. diff --git a/internal/e2e/suites/networking/grpcegress_test.go b/internal/e2e/suites/networking/grpcegress_test.go index 9a1dbaba95..d883c0f7eb 100644 --- a/internal/e2e/suites/networking/grpcegress_test.go +++ b/internal/e2e/suites/networking/grpcegress_test.go @@ -19,11 +19,7 @@ import ( "encoding/json" "fmt" "net/http" - "strconv" "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" @@ -82,8 +78,6 @@ func TestActorEgressGRPC(t *testing.T) { // Bound the access-log scan below to lines this test could have produced. // The slack absorbs clock skew between here and the gateway's node. - since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) - const ( message = "hello over grpc" streamCount = 3 @@ -154,5 +148,4 @@ func TestActorEgressGRPC(t *testing.T) { // Everything above would also pass if the Actor's traffic had been // masqueraded straight out instead of tunneled. This is what says it went // through the gateway, on this Actor's own certificate. - assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(grpcEcho.Port)) } diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 325b9fb0d0..7bcefb6476 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -113,9 +113,15 @@ func TestIngressProtocolDowngrade(t *testing.T) { body, _ := io.ReadAll(resp.Body) resp.Body.Close() // atunnel forwards gRPC as real h2c, which the HTTP/1.1-only counter - // cannot speak — a 502 from atunnel, not a silently-downgraded 200. - if resp.StatusCode != http.StatusBadGateway { - t.Fatalf("gRPC-shaped POST = %d (body %q), want 502: gRPC must not be silently downgraded to HTTP/1.1", resp.StatusCode, body) + // cannot speak. Routers may report that as HTTP 502 or as the gRPC + // convention of HTTP 200 with a nonzero grpc-status trailer. + grpcStatus := resp.Header.Get("grpc-status") + if grpcStatus == "" { + grpcStatus = resp.Trailer.Get("grpc-status") + } + if resp.StatusCode != http.StatusBadGateway && + !(resp.StatusCode == http.StatusOK && grpcStatus != "" && grpcStatus != "0") { + t.Fatalf("gRPC-shaped POST = %d, grpc-status = %q (body %q), want an explicit upstream failure", resp.StatusCode, grpcStatus, body) } }) } diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go new file mode 100644 index 0000000000..8f47d84f0b --- /dev/null +++ b/internal/installdefaults/installdefaults.go @@ -0,0 +1,47 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package installdefaults holds the default namespace and Service names +// that match the canonical install layout in manifests/ate-install/. +// Binaries use these as flag defaults; deployments that diverge from +// the canonical layout pass actual values via the corresponding flags. +package installdefaults + +import "os" + +const ( + // SystemNamespace is the namespace where substrate's control-plane + // components and the atelet DaemonSet run. + SystemNamespace = "ate-system" + // APIServiceName is the Service name of ate-api-server. + APIServiceName = "api" + // RouterServiceName is the Service name of atenet-router. + RouterServiceName = "atenet-router" + // DNSServiceName is the Service name of substrate's CoreDNS. + DNSServiceName = "dns" + + // PodNamespaceEnv is the conventional env var name for the namespace + // a pod is running in, exposed via Kubernetes' downward API. + PodNamespaceEnv = "POD_NAMESPACE" +) + +// NamespaceFromPodEnv returns the namespace from the PodNamespaceEnv env +// var when set (typically populated via Kubernetes' downward API), and +// falls back to SystemNamespace for non-k8s invocations (tests, local dev). +func NamespaceFromPodEnv() string { + if ns := os.Getenv(PodNamespaceEnv); ns != "" { + return ns + } + return SystemNamespace +} diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..5199ab9e74 --- /dev/null +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-api-server-envvars + namespace: ate-system +data: + ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml new file mode 100644 index 0000000000..e59bd53f8f --- /dev/null +++ b/manifests/ate-install/ate-client.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-client + namespace: ate-system + labels: + apps: ate-client diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 7fee9bbc1a..a7011a504f 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -58,6 +58,32 @@ data: insecureHost: true routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + # AgentGateway only uses atunnel's CONNECT listener. + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors gateways: - http @@ -77,6 +103,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 # Terminate client CONNECT before internal HTTP routing. binds: @@ -97,6 +126,28 @@ data: listeners: - protocol: HTTP routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors-tunneled matches: - path: @@ -112,6 +163,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 --- apiVersion: v1 diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml new file mode 100644 index 0000000000..65e967c2a6 --- /dev/null +++ b/manifests/ate-install/role.yaml @@ -0,0 +1,130 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ate-controller +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ate-controller + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +--- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml new file mode 100644 index 0000000000..d6be308128 --- /dev/null +++ b/manifests/ate-install/rustfs.yaml @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rustfs-data + namespace: ate-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: rustfs + namespace: ate-system +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rustfs + namespace: ate-system +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: "rustfsadmin" + - name: RUSTFS_SECRET_KEY + value: "rustfsadmin" + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: rustfs-data +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rustfs-bucket-init + namespace: ate-system +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: + - name: AWS_ACCESS_KEY_ID + value: "rustfsadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "rustfsadmin" + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots already exists" + exit 0 + fi + if aws s3api create-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 From 5e7848d7f20d837256a057cd999bb0cebb44b00c Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 2 Sep 2026 16:16:48 +0000 Subject: [PATCH 03/19] Add fork main synchronization skill Signed-off-by: Eitan Yarmush --- .agents/skills/update-against-main/SKILL.md | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .agents/skills/update-against-main/SKILL.md diff --git a/.agents/skills/update-against-main/SKILL.md b/.agents/skills/update-against-main/SKILL.md new file mode 100644 index 0000000000..5d95417939 --- /dev/null +++ b/.agents/skills/update-against-main/SKILL.md @@ -0,0 +1,29 @@ +--- +name: update-against-main +description: Merge agent-substrate/substrate main into the kagent-dev/substrate fork's main branch, resolve conflicts, validate the result, and safely update the fork. Use only when explicitly synchronizing the fork's main branch with upstream main. Do not use for updating, rebasing, or resolving conflicts in feature branches or pull requests. +--- + +# Update Against Main + +This skill applies only to synchronizing the fork's `main` branch. Do not invoke it for a feature branch or PR merely because that branch is behind or conflicts with `main`. + +1. Confirm the worktree, current branch, tracking branch, and remotes. Do not disturb unrelated changes. +2. Fetch `origin/main` and `upstream/main`, inspect their divergence, and create a dated backup branch from `origin/main`. +3. Rebuild `main` from `upstream/main` by replaying only intentional fork feature commits in dependency order. Drop merge commits and fork commits superseded by upstream. +4. Resolve conflicts in favor of current upstream APIs while preserving the remaining fork features. Inspect the resulting diff and linear history. +5. Keep Helm charts synchronized with their corresponding manifests. When either changes, inspect and update the other while preserving intentional Helm templating and conditionals, then run `make verify-helm-template` and `make verify-crd-chart` and compare any relevant resources not covered by those checks. +6. Run `make test` and `make verify`. +7. Run the real Kind E2E matrix from `.github/workflows/pr-workflow.yaml`, but use agentgateway for all fork testing: + - Recreate the cluster with `hack/create-kind-cluster.sh`. + - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-router=agentgateway`. + - Deploy the micro-VM demo with `hack/run-microvm-demo-kind.sh --skip-control-plane` so it does not reinstall the control plane. + - Deploy the gVisor counter demo and both standard egress demos. + - The full gVisor suite: `hack/run-e2e-kind.sh -v -args --no-color` + - The full micro-VM suite with the CI environment: `E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh -v -args --no-color` + - Switch egress to agentgateway sdsmint, then run the MITM trust and targeted networking lanes for both runtimes exactly as the workflow specifies. + - Verify the live router and egress workloads use agentgateway. Never use Envoy for fork validation. +8. Treat `go test ./internal/e2e/...` without `-args --e2e` as compilation/package testing, not E2E coverage. +9. Do not push when unit, verification, or E2E checks fail or cannot run. Report the exact blocker instead. +10. After all checks pass, verify the worktree and rewritten commits, then update the fork with `git push --force-with-lease origin main`. Never use an unguarded force push. + +Use the current CI workflow as the source of truth for cluster setup, images, demos, runtime coverage, and environment variables, with the agentgateway-only override above. Never claim E2E passed unless workloads ran against the cluster. From bffc8f51bd9e13ddfe0f0b1ea5909d771a794971 Mon Sep 17 00:00:00 2001 From: Jeremy Alvis <3587901+iplay88keys@users.noreply.github.com> Date: Thu, 3 Sep 2026 04:51:09 -0700 Subject: [PATCH 04/19] Expose PostgreSQL migration settings in the Helm chart Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 2 ++ charts/substrate/templates/ate-api-server-envvars.yaml | 4 ++++ charts/substrate/templates/ate-api-server.yaml | 1 + charts/substrate/templates/postgres.yaml | 2 ++ charts/substrate/values.yaml | 2 ++ hack/render-manifests.sh | 9 ++++++--- manifests/ate-install/ate-api-server-envvars.yaml | 1 + 7 files changed, 18 insertions(+), 3 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index e7364f4e37..0640c9e9a7 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -35,7 +35,9 @@ See `values.yaml` for the full set; the important keys: | Key | Default | Notes | |-----|---------|-------| +| `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | | `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 753c47178b..ca76ae3ef8 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -14,6 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }} +{{- fail "postgres.connectionString is required when postgres.enabled=false" }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -21,3 +24,4 @@ metadata: namespace: {{ .Release.Namespace }} data: ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} + ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index a5073bc9fa..c236999e48 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -85,6 +85,7 @@ spec: - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" - "--postgres-connection-string=@env" + - "--postgres-schema=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml index ce4a4efdd7..26ddb0b1ec 100644 --- a/charts/substrate/templates/postgres.yaml +++ b/charts/substrate/templates/postgres.yaml @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.postgres.enabled }} {{- $name := include "substrate.fullname" (list "postgres" .) -}} apiVersion: v1 kind: ConfigMap @@ -232,3 +233,4 @@ spec: resources: requests: storage: {{ .Values.postgres.storageSize }} +{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 48ce963668..6bb83019ed 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -24,8 +24,10 @@ createNamespace: false postgres: + enabled: true storageSize: 1Gi connectionString: "" + schema: public resources: requests: cpu: "1" diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index d8a7f69e8a..fc6b8d6858 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -121,8 +121,9 @@ if [ "${CHECK_MODE}" = "true" ]; then trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT mkdir -p "${CHECK_TMP}/current" find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; - rm -f "${PRESERVED_FILES[@]/#/${CHECK_TMP}\/current\/}" - rm -f "${PRESERVED_FILES[@]/#/${TMP_DIR}\/out\/}" + for file in "${PRESERVED_FILES[@]}"; do + rm -f "${CHECK_TMP}/current/${file}" "${TMP_DIR}/out/${file}" + done if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true @@ -150,7 +151,9 @@ find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ ! -name 'sandboxconfig-gvisor.yaml' \ ! -name 'sandboxconfig-validation.yaml' \ -delete -rm -f "${PRESERVED_FILES[@]/#/${TMP_DIR}\/out\/}" +for file in "${PRESERVED_FILES[@]}"; do + rm -f "${TMP_DIR}/out/${file}" +done cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml index 5199ab9e74..b49cff6e1e 100644 --- a/manifests/ate-install/ate-api-server-envvars.yaml +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -22,3 +22,4 @@ metadata: namespace: ate-system data: ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + ATE_API_POSTGRES_SCHEMA: "public" From 18b0b349ebf93fada8a4af3cbe8b9bad053a10a4 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 4 Sep 2026 13:34:05 +0000 Subject: [PATCH 05/19] Fix CI regressions after upstream sync Signed-off-by: Eitan Yarmush --- charts/substrate/templates/sandboxconfig-gvisor.yaml | 9 +++------ cmd/ateapi/internal/controlapi/workflow_pause.go | 5 ++++- cmd/ateapi/internal/workercache/workercache.go | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 36af4296f3..05f851d40d 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -14,18 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# Cluster-wide default SandboxConfig for the gVisor (runsc) sandbox class. A -# WorkerPool with sandboxClass gvisor (the default) and no explicit -# sandboxConfigName resolves to this. atelet fetches the runsc binary matching -# the worker node's architecture. To pin a different runsc, edit the assets -# below or create another SandboxConfig and name it from the WorkerPool. +# Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with +# the platform so gVisor ActorTemplates have a config to name via +# sandboxConfig.configName. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default spec: sandboxClass: gvisor - default: true pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 2cb45d1835..b60afbb52c 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -238,13 +238,16 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso nodeName = worker.GetNodeName() // Drop just this actor's assignment; any other actors the worker // hosts keep theirs. - _, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) + released, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) if err != nil { if errors.Is(err, store.ErrVersionConflict) { return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") } return nil, err } + if w.workerCache != nil { + w.workerCache.Observe(released) + } } // 2. Clear the actor's assignment, now that the worker is freed diff --git a/cmd/ateapi/internal/workercache/workercache.go b/cmd/ateapi/internal/workercache/workercache.go index 6281586e63..d42be04736 100644 --- a/cmd/ateapi/internal/workercache/workercache.go +++ b/cmd/ateapi/internal/workercache/workercache.go @@ -120,6 +120,12 @@ func (c *Cache) Forget(name string) { delete(c.workers, name) } +// Observe applies a worker returned by a successful store write immediately, +// without waiting for the corresponding watch event. +func (c *Cache) Observe(worker *ateapipb.Worker) { + c.applyEvent(store.WorkerEvent{Type: store.WorkerEventUpdated, Worker: worker}) +} + func (c *Cache) sync(ctx context.Context) (*store.WorkerWatch, error) { watch, err := c.store.WatchWorkers(ctx) if err != nil { From d744133666354848717d95e392626504bf8e15a1 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 4 Sep 2026 19:42:11 +0000 Subject: [PATCH 06/19] Reconcile agentgateway egress test with upstream Signed-off-by: Eitan Yarmush --- internal/e2e/suites/networking/networking_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index ac346f8b44..2868b085ab 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -117,17 +117,12 @@ func TestActorEgress(t *testing.T) { actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} url := fmt.Sprintf("http://%s/healthz", target.Address()) - // Bound the access-log scan to lines this test produced: captured before - // the fetch, with slack for clock skew against the gateway's node. - since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) status, body := fetchThroughEgressActor(t, ctx, router, actorRef, url) if status != http.StatusOK { t.Fatalf("Actor egress fetch of %s returned HTTP %d, want 200; body: %s", url, status, body) } t.Logf("Actor egress fetch of %s succeeded; body: %s", url, body) - assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(origin.Port)) - // The Actor resolves the name itself -- DNS leaves over the UDP masquerade, // not the tunnel, and atunnel forwards the resolved address, never the // name -- so dialing by name covers a leg the by-address fetch above skips. From 962941524269d706f831851dc31bd3b418273180 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Sun, 6 Sep 2026 11:41:09 +0000 Subject: [PATCH 07/19] Configure API server object storage in Helm chart Signed-off-by: Eitan Yarmush --- charts/substrate/templates/ate-api-server.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index c236999e48..b6d1960901 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -111,6 +111,20 @@ spec: {{- if .Values.otel.endpoint }} - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} {{- end }} envFrom: - configMapRef: From 50119076905ee140462ee82c326aba3b73f26c22 Mon Sep 17 00:00:00 2001 From: Jet Chiang Date: Tue, 8 Sep 2026 11:43:13 -0400 Subject: [PATCH 08/19] Grant atelet cluster-wide read access to sandbox configs Signed-off-by: Jet Chiang Signed-off-by: Eitan Yarmush --- charts/substrate/templates/atelet.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 916915dc42..e2b4b52fb1 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -38,6 +38,11 @@ rules: - apiGroups: ["certificates.k8s.io"] resources: ["clustertrustbundles"] verbs: ["get", "watch", "list"] +# SandboxConfigs are watched to pre-download sandbox assets into the node's +# cache before the first actor needs them (see cmd/atelet/sandbox_prewarm.go). +- apiGroups: ["ate.dev"] + resources: ["sandboxconfigs"] + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From a73cc223a931aaa2e40fd23a090700b67bcb1cf9 Mon Sep 17 00:00:00 2001 From: Krisztian F <103492698+krisztianfekete@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:16:57 +0200 Subject: [PATCH 09/19] feat(chart): export the agentgateway access log over otlp (#34) Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 2 +- charts/substrate/templates/atenet-router.yaml | 7 +++++++ manifests/ate-install/kind/otel-collector.yaml | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 0640c9e9a7..d63cbf74e8 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,4 +42,4 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | -| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 78976a39a6..75f8e43a0e 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -42,6 +42,13 @@ data: host: $AGENTGATEWAY_OTLP_ADDRESS protocol: grpc randomSampling: 0.01 + # Unsampled per-request access log. It carries the ate.* actor + # attribution, which the 1%-sampled traces cannot be aggregated on. + # `fields` stays unset so the OTLP sink inherits the full field set. + accessLog: + otlp: + host: $AGENTGATEWAY_OTLP_ADDRESS + protocol: grpc {{- end }} backends: diff --git a/manifests/ate-install/kind/otel-collector.yaml b/manifests/ate-install/kind/otel-collector.yaml index 0aab804597..68b5772057 100644 --- a/manifests/ate-install/kind/otel-collector.yaml +++ b/manifests/ate-install/kind/otel-collector.yaml @@ -93,6 +93,12 @@ data: receivers: [otlp] processors: [batch] exporters: [prometheus, debug] + # The router access log arrives here. Without a logs pipeline the + # receiver rejects the export and the records are lost silently. + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug] # The count pipelines read the same receiver as the two pipelines # above, and they only count. The connector declares MutatesData From 0301654d417f2de57f30a1e8f926242ec923e794 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Thu, 10 Sep 2026 14:48:31 -0500 Subject: [PATCH 10/19] Update agentgateway implementation (#28) Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 7 ++- charts/substrate/templates/atenet-egress.yaml | 20 ++++--- charts/substrate/values.yaml | 2 +- internal/e2e/dataplane.go | 56 +++++++++++++++++++ .../e2e/suites/networking/grpcingress_test.go | 12 +--- 5 files changed, 75 insertions(+), 22 deletions(-) create mode 100644 internal/e2e/dataplane.go diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index dfc5df2757..81b3a46103 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -92,6 +92,9 @@ jobs: - name: Install CSI NFS driver run: hack/install-ate-kind.sh --setup-csi=nfs - name: Deploy micro-VM counter demo + # The deploy creates the substrate ActorTemplate and waits for its golden + # snapshot internally; the ActorTemplate CRD (and its Ready condition) + # no longer exists to wait on. run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter @@ -101,9 +104,7 @@ jobs: run: hack/run-e2e-kind.sh -v -args --no-color - name: Run E2E tests (micro-VM) env: - E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm - E2E_TEMPLATE_NAME: counter-microvm - E2E_TEMPLATE_READY_TIMEOUT: 600s + E2E_SANDBOX_CLASS: microvm run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color - name: Dump diagnostics on failure if: failure() diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 86cc2271ca..6de04ca9ee 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -32,6 +32,16 @@ data: accessLog: add: substrate.connect.authority: source.connectHeaders["host"] + # Authorize the actor identity at CONNECT-accept, before any tunnel + # (HTTP, TLS, or opaque TCP) is terminated. Fails closed when the + # control plane is unreachable. + substrateEgressActorResolution: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem binds: - port: 8443 @@ -54,15 +64,7 @@ data: target: source.connectHeaders["host"] - protocol: HTTP routes: - - policies: - substrateEgress: - host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 - policies: - backendTLS: - cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem - key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - backends: + - backends: - dynamic: target: source.connectHeaders["host"] - protocol: TCP diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 6bb83019ed..32532d7e1e 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -70,6 +70,6 @@ images: postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.988ac151 coredns: coredns/coredns:1.11.1 busybox: busybox:1.36 diff --git a/internal/e2e/dataplane.go b/internal/e2e/dataplane.go new file mode 100644 index 0000000000..39657171da --- /dev/null +++ b/internal/e2e/dataplane.go @@ -0,0 +1,56 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "sync" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + routerDataplaneOnce sync.Once + routerIsAgentgateway bool + routerDataplaneErr error +) + +// RouterIsAgentgateway reports whether the atenet-router Deployment runs the +// agentgateway dataplane. In that mode the pod has no Envoy and no +// atenet-router ext_proc process, so router-internal surfaces — the statusz +// page, atenet_router_* metrics, and Envoy's protocol mirroring to atunnel — +// do not exist. Suites gate assertions on those surfaces with this instead of +// a per-lane env knob: the deployed containers are the source of truth. +func RouterIsAgentgateway(ctx context.Context, t *testing.T) bool { + t.Helper() + routerDataplaneOnce.Do(func() { + deploy, err := GetClients().K8s.AppsV1().Deployments(routerNamespace).Get(ctx, routerService, metav1.GetOptions{}) + if err != nil { + routerDataplaneErr = err + return + } + for _, c := range deploy.Spec.Template.Spec.Containers { + if c.Name == "agentgateway" { + routerIsAgentgateway = true + return + } + } + }) + if routerDataplaneErr != nil { + t.Fatalf("detecting the router dataplane: %v", routerDataplaneErr) + } + return routerIsAgentgateway +} diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 7bcefb6476..325b9fb0d0 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -113,15 +113,9 @@ func TestIngressProtocolDowngrade(t *testing.T) { body, _ := io.ReadAll(resp.Body) resp.Body.Close() // atunnel forwards gRPC as real h2c, which the HTTP/1.1-only counter - // cannot speak. Routers may report that as HTTP 502 or as the gRPC - // convention of HTTP 200 with a nonzero grpc-status trailer. - grpcStatus := resp.Header.Get("grpc-status") - if grpcStatus == "" { - grpcStatus = resp.Trailer.Get("grpc-status") - } - if resp.StatusCode != http.StatusBadGateway && - !(resp.StatusCode == http.StatusOK && grpcStatus != "" && grpcStatus != "0") { - t.Fatalf("gRPC-shaped POST = %d, grpc-status = %q (body %q), want an explicit upstream failure", resp.StatusCode, grpcStatus, body) + // cannot speak — a 502 from atunnel, not a silently-downgraded 200. + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("gRPC-shaped POST = %d (body %q), want 502: gRPC must not be silently downgraded to HTTP/1.1", resp.StatusCode, body) } }) } From e517b5634411d81fa6714915260407e6dfc50ea6 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 10 Sep 2026 20:36:51 +0000 Subject: [PATCH 11/19] Remove obsolete actor DNS configuration Signed-off-by: Eitan Yarmush --- charts/substrate/values.yaml | 1 - hack/render-manifests.sh | 2 -- internal/installdefaults/installdefaults.go | 4 ---- 3 files changed, 7 deletions(-) diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 32532d7e1e..d67d5587b2 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -71,5 +71,4 @@ images: rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.988ac151 - coredns: coredns/coredns:1.11.1 busybox: busybox:1.36 diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index fc6b8d6858..1f6790bb73 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -34,7 +34,6 @@ PRESERVED_FILES=( ate-otel-config.yaml ate-system-namespace.yaml atelet.yaml - atenet-dns.yaml atenet-egress.yaml atenet-egress-with-sdsmint.yaml atenet-router.yaml @@ -141,7 +140,6 @@ find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ ! -name 'ate-otel-config.yaml' \ ! -name 'ate-system-namespace.yaml' \ ! -name 'atelet.yaml' \ - ! -name 'atenet-dns.yaml' \ ! -name 'atenet-egress.yaml' \ ! -name 'atenet-egress-with-sdsmint.yaml' \ ! -name 'atenet-router.yaml' \ diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index 8f47d84f0b..6d6475bc32 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -26,10 +26,6 @@ const ( SystemNamespace = "ate-system" // APIServiceName is the Service name of ate-api-server. APIServiceName = "api" - // RouterServiceName is the Service name of atenet-router. - RouterServiceName = "atenet-router" - // DNSServiceName is the Service name of substrate's CoreDNS. - DNSServiceName = "dns" // PodNamespaceEnv is the conventional env var name for the namespace // a pod is running in, exposed via Kubernetes' downward API. From 080082018d737e46d0d4fa8aed378ad9c6226fe7 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 10 Sep 2026 20:40:17 +0000 Subject: [PATCH 12/19] Pass the workflow timeout in snapshot cleanup regression test Signed-off-by: Eitan Yarmush --- cmd/ateapi/internal/controlapi/workflow_delete_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index c54c4c1098..455ef98a48 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -398,7 +399,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { }) } - actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) // Suspend the actor as far as it gets: MarkSuspending mints the // in-progress name, and the checkpoint writes under it actor, err := actorWorkflow.ensureMarkedSuspending(ctx, actorRef, actor, template) From bfe8df8ba09f46d948bf933235fb5144629c226b Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 10 Sep 2026 21:31:47 +0000 Subject: [PATCH 13/19] Update agentgateway to preserve CONNECT target ports Signed-off-by: Eitan Yarmush --- charts/substrate/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index d67d5587b2..ee420bbad9 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -70,5 +70,5 @@ images: postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.988ac151 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf busybox: busybox:1.36 From f4abeec9ab05380657ba5c842dbd580e70269da3 Mon Sep 17 00:00:00 2001 From: Krisztian F <103492698+krisztianfekete@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:22:12 +0200 Subject: [PATCH 14/19] feat(chart): configure OTLP export per signal (#37) Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 7 +++ charts/substrate/templates/_helpers.tpl | 54 +++++++++++++++++++ .../substrate/templates/ate-api-server.yaml | 5 +- .../substrate/templates/ate-controller.yaml | 5 +- charts/substrate/templates/atelet.yaml | 5 +- charts/substrate/templates/atenet-router.yaml | 36 +++++++++---- charts/substrate/values.yaml | 18 +++++++ 7 files changed, 110 insertions(+), 20 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index d63cbf74e8..516bc2b04e 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -43,3 +43,10 @@ See `values.yaml` for the full set; the important keys: | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | +| `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | +| `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` | +| `otel.traces.samplingRatio` | `0.01` | Fraction of parentless requests that start a trace, applied to the Go components and the router | +| `otel.metrics.enabled` | `true` | Sets the OTLP metrics exporter to `none`; the Go components do not honor this yet | +| `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | +| `otel.logs.enabled` | `true` | Set to `false` to export no logs; the router access log is the only OTLP log source today | +| `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 32ae087336..1179dda559 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -78,6 +78,60 @@ Plaintext HTTP URL that clients use to reach atenet-router. {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} {{- end -}} +{{/* +OTLP endpoint a signal exports to, or empty when the signal is disabled or no +endpoint resolves. The per-signal endpoint wins over the generic one, matching +the precedence the OpenTelemetry SDK gives OTEL_EXPORTER_OTLP__ENDPOINT +over OTEL_EXPORTER_OTLP_ENDPOINT. + +Usage: + {{ include "substrate.otel.signalEndpoint" (list "traces" .) }} +*/}} +{{- define "substrate.otel.signalEndpoint" -}} +{{- $signal := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $cfg := index $ctx.Values.otel $signal -}} +{{- if $cfg.enabled -}} +{{- $cfg.endpoint | default $ctx.Values.otel.endpoint -}} +{{- end -}} +{{- end -}} + +{{/* +OTEL_* env entries for a Go component, as a list of "- name/value" items. +Empty when nothing under .Values.otel is set, so callers can gate the env +key on the result. + +Usage: + {{- with include "substrate.otel.env" . }} + {{- . | trim | nindent 8 }} + {{- end }} +*/}} +{{- define "substrate.otel.env" -}} +{{- $otel := .Values.otel -}} +{{- if $otel.endpoint }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ $otel.endpoint | quote }} +{{- end }} +{{- range $signal := list "traces" "metrics" "logs" }} +{{- $cfg := index $otel $signal }} +{{- if not $cfg.enabled }} +{{- /* "none" is the SDK's own exporter name for "export nothing"; leaving the + endpoint unset would fall back to the SDK default of localhost:4317. */}} +- name: OTEL_{{ upper $signal }}_EXPORTER + value: none +{{- else if $cfg.endpoint }} +- name: OTEL_EXPORTER_OTLP_{{ upper $signal }}_ENDPOINT + value: {{ $cfg.endpoint | quote }} +{{- end }} +{{- end }} +{{- if include "substrate.otel.signalEndpoint" (list "traces" .) }} +- name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio +- name: OTEL_TRACES_SAMPLER_ARG + value: {{ $otel.traces.samplingRatio | quote }} +{{- end }} +{{- end -}} + {{/* Build an image reference for a substrate component binary. diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index b6d1960901..7718ed3827 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -108,9 +108,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} - name: ATE_STORAGE_BACKEND value: {{ .Values.atelet.storageBackend | quote }} diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 31c83b9066..d324d05af5 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -79,10 +79,9 @@ spec: - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- if .Values.otel.endpoint }} +{{- with include "substrate.otel.env" . }} env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- . | trim | nindent 8 }} {{- end }} ports: - name: metrics diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index e2b4b52fb1..2de2def545 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -133,9 +133,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} - name: ATE_STORAGE_BACKEND value: {{ .Values.atelet.storageBackend | quote }} diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 75f8e43a0e..bfa6c9e10c 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -36,19 +36,25 @@ data: backend: poolMaxSize: 0 -{{- if .Values.otel.endpoint }} +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} frontendPolicies: +{{- if $traces }} tracing: - host: $AGENTGATEWAY_OTLP_ADDRESS + host: $AGENTGATEWAY_OTLP_TRACES_ADDRESS protocol: grpc - randomSampling: 0.01 + randomSampling: {{ .Values.otel.traces.samplingRatio }} +{{- end }} +{{- if $logs }} # Unsampled per-request access log. It carries the ate.* actor - # attribution, which the 1%-sampled traces cannot be aggregated on. + # attribution, which the sampled traces cannot be aggregated on. # `fields` stays unset so the OTLP sink inherits the full field set. accessLog: otlp: - host: $AGENTGATEWAY_OTLP_ADDRESS + host: $AGENTGATEWAY_OTLP_LOGS_ADDRESS protocol: grpc +{{- end }} {{- end }} backends: @@ -238,9 +244,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} ports: - name: extproc @@ -257,10 +262,19 @@ spec: args: - "-f" - "/etc/agentgateway/config.yaml" -{{- if .Values.otel.endpoint }} +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} + # agentgateway takes host:port, not a URL, so the scheme is trimmed. env: - - name: AGENTGATEWAY_OTLP_ADDRESS - value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- if $traces }} + - name: AGENTGATEWAY_OTLP_TRACES_ADDRESS + value: {{ trimPrefix "http://" $traces | quote }} +{{- end }} +{{- if $logs }} + - name: AGENTGATEWAY_OTLP_LOGS_ADDRESS + value: {{ trimPrefix "http://" $logs | quote }} +{{- end }} {{- end }} ports: - name: http diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index ee420bbad9..7f456a20af 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -59,8 +59,26 @@ atelet: # Mounted via envFrom with optional=true. Created by the chart from these values. ateApiServerEnvVarsConfigMap: ate-api-server-envvars +# OTLP export, following the OpenTelemetry SDK environment variable spec. +# endpoint applies to every signal; a signal's own endpoint overrides it. +# A signal exports only when it is enabled and resolves to an endpoint. +# +# traces.enabled and metrics.enabled set OTEL__EXPORTER=none on the +# Go components, which do not read it yet and keep exporting; the setting +# takes effect on the router's agentgateway only. logs.enabled works in full, +# since the agentgateway access log is the only OTLP log source. otel: endpoint: "" + traces: + enabled: true + endpoint: "" + samplingRatio: 0.01 + metrics: + enabled: true + endpoint: "" + logs: + enabled: true + endpoint: "" image: registry: ghcr.io/kagent-dev/substrate From 84cac67fd9e7384c4402397d98fe443d7f9a5826 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 15 Sep 2026 14:22:24 +0000 Subject: [PATCH 15/19] Reconcile fork CRD chart and networking tests with upstream Signed-off-by: Eitan Yarmush --- charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml | 3 +-- internal/e2e/suites/networking/networking_test.go | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml index 8fe5f91178..22d46f5123 100644 --- a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -116,8 +116,7 @@ spec: - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 type: string x-kubernetes-validations: - - message: All images must be pinned (changing the image invalidates - snapshots) + - message: All images must include a digest rule: self.contains('@') sandboxClass: default: gvisor diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index 2868b085ab..ac346f8b44 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -117,12 +117,17 @@ func TestActorEgress(t *testing.T) { actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} url := fmt.Sprintf("http://%s/healthz", target.Address()) + // Bound the access-log scan to lines this test produced: captured before + // the fetch, with slack for clock skew against the gateway's node. + since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) status, body := fetchThroughEgressActor(t, ctx, router, actorRef, url) if status != http.StatusOK { t.Fatalf("Actor egress fetch of %s returned HTTP %d, want 200; body: %s", url, status, body) } t.Logf("Actor egress fetch of %s succeeded; body: %s", url, body) + assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(origin.Port)) + // The Actor resolves the name itself -- DNS leaves over the UDP masquerade, // not the tunnel, and atunnel forwards the resolved address, never the // name -- so dialing by name covers a leg the by-address fetch above skips. From 90dac89b0c56c17a00994975dd89af0ed3dd63a8 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 15 Sep 2026 14:26:39 +0000 Subject: [PATCH 16/19] Use the fork namespace default in resume wire tests Signed-off-by: Eitan Yarmush --- cmd/ateapi/internal/controlapi/workflow_resume_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 488e830a1f..0b23d36cb1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -28,6 +28,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -1270,7 +1271,7 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo Spec: corev1.PodSpec{NodeName: "node-1"}, } ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-1", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-1", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } dialer := newDialerForPods(t, workerPod, ateletPod) From 36bd285d2fb31ae69ee663b4c1b04daf4489d7f9 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 15 Sep 2026 14:31:55 +0000 Subject: [PATCH 17/19] Align egress checks with agentgateway and stabilize CA cache test Signed-off-by: Eitan Yarmush --- internal/e2e/suites/networking/networking_test.go | 10 ---------- internal/volume/csi/tls_test.go | 5 +++++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index ac346f8b44..1fa4776aae 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -152,18 +152,12 @@ func TestActorEgressHTTPS(t *testing.T) { router := mustRouterClient(t, ctx) defer router.Close() - // Bound the access-log scan below to lines this test could have produced. - // The slack absorbs clock skew between here and the gateway's node. - since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) - actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} status, body := fetchThroughEgressActor(t, ctx, router, actorRef, "https://example.com/") if status != http.StatusOK { t.Fatalf("Actor HTTPS egress fetch returned HTTP %d, want 200; body: %s", status, body) } t.Logf("Actor HTTPS egress fetch succeeded; body: %s", body) - - assertEgressGatewayConnect(t, ctx, since, actorName, "443") } // httpTarget is the origin TestActorEgressNonStandardPort dials: a plain HTTP @@ -198,8 +192,6 @@ func TestActorEgressNonStandardPort(t *testing.T) { router := mustRouterClient(t, ctx) defer router.Close() - since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) - // Address() is the ClusterIP literal, not the Service's DNS name: the // authority atunnel sends is always an address, so the name would add // nothing but a dependency on the sandbox's DNS-over-UDP masquerade path -- @@ -214,8 +206,6 @@ func TestActorEgressNonStandardPort(t *testing.T) { t.Fatalf("Actor egress fetch of %s returned HTTP %d, want 200; body: %s", url, status, body) } t.Logf("Actor egress fetch of %s succeeded", url) - - assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(httpTarget.Port)) } // fetchThroughEgressActor asks the egress demo Actor to fetch url and returns diff --git a/internal/volume/csi/tls_test.go b/internal/volume/csi/tls_test.go index ad71177441..aaa8edce05 100644 --- a/internal/volume/csi/tls_test.go +++ b/internal/volume/csi/tls_test.go @@ -391,6 +391,11 @@ func TestCAPoolCache_HitAndFileChange(t *testing.T) { // Modify the file. writeFile(t, caPath, ca.certPEM()) + // Advance mtime explicitly; consecutive writes can share a filesystem tick. + modified := cache.fi.ModTime().Add(time.Second) + if err := os.Chtimes(caPath, modified, modified); err != nil { + t.Fatal(err) + } // 3rd call should detect file change and return a newly parsed pool. pool3, err := cache.getCertPool() From cdb5d157c17d14ed5c02e48c83198d5b08d1e19b Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 15 Sep 2026 16:04:43 +0000 Subject: [PATCH 18/19] Align fork charts and validation with upstream agentgateway support Signed-off-by: Eitan Yarmush --- .agents/skills/update-against-main/SKILL.md | 2 +- .github/workflows/pr-workflow.yaml | 16 ++---- charts/substrate/templates/atenet-egress.yaml | 10 +++- charts/substrate/templates/atenet-router.yaml | 9 ++- internal/e2e/collector_metrics.go | 1 + internal/e2e/dataplane.go | 56 ------------------- .../e2e/suites/networking/grpcegress_test.go | 7 +++ .../e2e/suites/networking/networking_test.go | 10 ++++ 8 files changed, 39 insertions(+), 72 deletions(-) delete mode 100644 internal/e2e/dataplane.go diff --git a/.agents/skills/update-against-main/SKILL.md b/.agents/skills/update-against-main/SKILL.md index 5d95417939..5a62a71127 100644 --- a/.agents/skills/update-against-main/SKILL.md +++ b/.agents/skills/update-against-main/SKILL.md @@ -15,7 +15,7 @@ This skill applies only to synchronizing the fork's `main` branch. Do not invoke 6. Run `make test` and `make verify`. 7. Run the real Kind E2E matrix from `.github/workflows/pr-workflow.yaml`, but use agentgateway for all fork testing: - Recreate the cluster with `hack/create-kind-cluster.sh`. - - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-router=agentgateway`. + - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway`. - Deploy the micro-VM demo with `hack/run-microvm-demo-kind.sh --skip-control-plane` so it does not reinstall the control plane. - Deploy the gVisor counter demo and both standard egress demos. - The full gVisor suite: `hack/run-e2e-kind.sh -v -args --no-color` diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 6a0394458b..2672c916c5 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -59,15 +59,11 @@ jobs: e2e-test-matrix: name: E2E (${{ matrix.dataplane }}) runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} # TODO: Make AgentGateway required once tests show stability strategy: fail-fast: false matrix: include: - - dataplane: envoy - experimental: false - dataplane: agentgateway - experimental: true env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} steps: @@ -101,7 +97,7 @@ jobs: - name: Install Agent Substrate (${{ matrix.dataplane }}) # The dataplane selection applies to both the ingress router and egress # gateway. - run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} + run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} --rollout-timeout=300s - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | @@ -111,11 +107,8 @@ jobs: run: hack/install-ate-kind.sh --setup-csi=nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the - # counter-microvm demo onto the control plane installed above. The demo - # redeploys the control plane, so retain the selected dataplane. - env: - ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/run-microvm-demo-kind.sh + # counter-microvm demo onto the control plane installed above. + run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demos @@ -195,8 +188,7 @@ jobs: kubectl --context kind-kind get pods -A -l ate.dev/worker-pool \ -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ | while read -r ns name; do dump "$ns" "$name"; done - # Preserve the required-check name while the concrete Envoy and AgentGateway - # executions run as entries in the shared matrix above. + # Preserve the required-check name for the dataplane matrix above. e2e-test: name: e2e-test needs: e2e-test-matrix diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 6de04ca9ee..850120b65d 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -67,6 +67,14 @@ data: - backends: - dynamic: target: source.connectHeaders["host"] + policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - protocol: TCP tcpRoutes: - backends: @@ -147,7 +155,7 @@ spec: - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt - --otlp-collector-address= - --envoy-admin-address=localhost:15000 - - --atenet-router=agentgateway + - --atenet-dataplane=agentgateway env: - name: POD_NAME valueFrom: diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index bfa6c9e10c..46c244b9eb 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -31,6 +31,7 @@ data: config.yaml: | # yaml-language-server: $schema=https://agentgateway.dev/schema/config config: + statsAddr: 0.0.0.0:15020 # Actor sandboxes behind a worker IP are replaced between requests. Do # not retain an idle connection that may belong to the previous actor. backend: @@ -208,7 +209,7 @@ spec: app: atenet-router annotations: prometheus.io/scrape: "true" - prometheus.io/port: "9090" + prometheus.io/port: "15020" spec: serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} containers: @@ -217,7 +218,7 @@ spec: args: - "router" - "--mode=ingress" - - "--atenet-router=agentgateway" + - "--atenet-dataplane=agentgateway" - "--namespace={{ .Release.Namespace }}" - "--port-http=8080" - "--port-extproc=50051" @@ -374,3 +375,7 @@ spec: port: 4040 targetPort: status protocol: TCP + - name: stats + port: 15020 + targetPort: gw-metrics + protocol: TCP diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index 098ef7412d..498197885d 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -50,6 +50,7 @@ var PlatformMetricPrefixes = []string{ "ate_scheduler_assignment_duration", "ate_actor_restore_duration", "ate_actor_checkpoint_duration", + "atenet_router_route_duration", "ate_scheduler_eligible_workers", } diff --git a/internal/e2e/dataplane.go b/internal/e2e/dataplane.go deleted file mode 100644 index 39657171da..0000000000 --- a/internal/e2e/dataplane.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "context" - "sync" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var ( - routerDataplaneOnce sync.Once - routerIsAgentgateway bool - routerDataplaneErr error -) - -// RouterIsAgentgateway reports whether the atenet-router Deployment runs the -// agentgateway dataplane. In that mode the pod has no Envoy and no -// atenet-router ext_proc process, so router-internal surfaces — the statusz -// page, atenet_router_* metrics, and Envoy's protocol mirroring to atunnel — -// do not exist. Suites gate assertions on those surfaces with this instead of -// a per-lane env knob: the deployed containers are the source of truth. -func RouterIsAgentgateway(ctx context.Context, t *testing.T) bool { - t.Helper() - routerDataplaneOnce.Do(func() { - deploy, err := GetClients().K8s.AppsV1().Deployments(routerNamespace).Get(ctx, routerService, metav1.GetOptions{}) - if err != nil { - routerDataplaneErr = err - return - } - for _, c := range deploy.Spec.Template.Spec.Containers { - if c.Name == "agentgateway" { - routerIsAgentgateway = true - return - } - } - }) - if routerDataplaneErr != nil { - t.Fatalf("detecting the router dataplane: %v", routerDataplaneErr) - } - return routerIsAgentgateway -} diff --git a/internal/e2e/suites/networking/grpcegress_test.go b/internal/e2e/suites/networking/grpcegress_test.go index d883c0f7eb..9a1dbaba95 100644 --- a/internal/e2e/suites/networking/grpcegress_test.go +++ b/internal/e2e/suites/networking/grpcegress_test.go @@ -19,7 +19,11 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" @@ -78,6 +82,8 @@ func TestActorEgressGRPC(t *testing.T) { // Bound the access-log scan below to lines this test could have produced. // The slack absorbs clock skew between here and the gateway's node. + since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) + const ( message = "hello over grpc" streamCount = 3 @@ -148,4 +154,5 @@ func TestActorEgressGRPC(t *testing.T) { // Everything above would also pass if the Actor's traffic had been // masqueraded straight out instead of tunneled. This is what says it went // through the gateway, on this Actor's own certificate. + assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(grpcEcho.Port)) } diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index 1fa4776aae..ac346f8b44 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -152,12 +152,18 @@ func TestActorEgressHTTPS(t *testing.T) { router := mustRouterClient(t, ctx) defer router.Close() + // Bound the access-log scan below to lines this test could have produced. + // The slack absorbs clock skew between here and the gateway's node. + since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} status, body := fetchThroughEgressActor(t, ctx, router, actorRef, "https://example.com/") if status != http.StatusOK { t.Fatalf("Actor HTTPS egress fetch returned HTTP %d, want 200; body: %s", status, body) } t.Logf("Actor HTTPS egress fetch succeeded; body: %s", body) + + assertEgressGatewayConnect(t, ctx, since, actorName, "443") } // httpTarget is the origin TestActorEgressNonStandardPort dials: a plain HTTP @@ -192,6 +198,8 @@ func TestActorEgressNonStandardPort(t *testing.T) { router := mustRouterClient(t, ctx) defer router.Close() + since := metav1.NewTime(time.Now().Add(-1 * time.Minute)) + // Address() is the ClusterIP literal, not the Service's DNS name: the // authority atunnel sends is always an address, so the name would add // nothing but a dependency on the sandbox's DNS-over-UDP masquerade path -- @@ -206,6 +214,8 @@ func TestActorEgressNonStandardPort(t *testing.T) { t.Fatalf("Actor egress fetch of %s returned HTTP %d, want 200; body: %s", url, status, body) } t.Logf("Actor egress fetch of %s succeeded", url) + + assertEgressGatewayConnect(t, ctx, since, actorName, strconv.Itoa(httpTarget.Port)) } // fetchThroughEgressActor asks the egress demo Actor to fetch url and returns From 217b41bf6ec86443ffc2723fdbe36cdaf9efc995 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Tue, 15 Sep 2026 23:44:30 +0200 Subject: [PATCH 19/19] helm: atelet mounts /var/lib/kubelet/plugins with HostToContainer propagation atelet hostPath-mounts /var/lib/kubelet/plugins to reach the CSI driver sockets. With the default (private) propagation every atelet start copies the node's CSI globalmounts into the container's mount namespace, and kubelet's later unmount of a volume does not reach that copy: the volume's filesystem stays mounted there, the block device (and the LUKS mapper of an encrypted volume) stays open, and the volume can never be unstaged while atelet runs. Observed with Longhorn: NodeUnstageVolume failed forever with "cryptsetup luksClose: Device is still in use" after a pod moved to another node, and the pod could not attach the volume on the new node until atelet was restarted. HostToContainer (rslave) is what CSI node plugins use for the kubelet directories: mounts and unmounts made by kubelet propagate into the container, nothing propagates back. Signed-off-by: Timo Derstappen --- charts/substrate/templates/atelet.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 2de2def545..3487dd30e4 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -172,6 +172,10 @@ spec: readOnly: true - name: kubelet-plugins mountPath: /var/lib/kubelet/plugins + # HostToContainer: kubelet stages CSI volumes under this directory; a private copy + # of those mounts would keep every volume's filesystem (and LUKS mapper) open after + # kubelet unmounts it, so the volume can never be unstaged while atelet runs. + mountPropagation: HostToContainer - name: device-plugins mountPath: /var/lib/kubelet/device-plugins - name: host-dev