From 36f55e473ab565059a19d59f02296c438d7c761b Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 01/19] Publish release artifacts Build and publish versioned binaries, container images, and Helm charts from release tags. Signed-off-by: Eitan Yarmush --- .github/workflows/release.yaml | 154 +++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..4a26a5cbe9 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,154 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.' + required: false + create_release: + description: 'Create a GitHub release' + type: boolean + default: false + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate and resolve tag + id: tag + run: | + TAG="${{ inputs.tag }}" + if [[ -z "${TAG}" ]]; then + BRANCH="${GITHUB_REF_NAME//\//-}" + SHA="$(git rev-parse --short HEAD)" + TAG="${BRANCH}-${SHA}" + fi + if [[ "${{ inputs.create_release }}" == "true" ]]; then + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then + echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)" + exit 1 + fi + fi + echo "value=${TAG}" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.create_release }}" == "true" ]]; then + echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT" + else + echo "tags=${TAG}" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install ko + uses: ko-build/setup-ko@v0.7 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v3 + + - name: Build and push images + env: + # ghcr.io// — resolves correctly in forks + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -o errexit -o nounset -o pipefail + + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ + ./hack/run-tool.sh ko build \ + --tags "${IMAGE_TAGS}" \ + --platform linux/amd64,linux/arm64 \ + --bare \ + "./cmd/${component}" + done + + - name: Package and push Helm charts + if: inputs.create_release + env: + HELM_EXPERIMENTAL_OCI: "1" + CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + run: | + set -o errexit -o nounset -o pipefail + + tag="${{ steps.tag.outputs.value }}" + chart_version="${tag#v}" + package_dir="${RUNNER_TEMP}/helm-packages" + mkdir -p "${package_dir}" + + echo "${{ secrets.GITHUB_TOKEN }}" \ + | helm registry login ghcr.io \ + --username "${{ github.actor }}" \ + --password-stdin + + helm package charts/substrate-crds \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + helm package charts/substrate \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + + helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" + helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + + - name: Build kubectl-ate release binaries + if: inputs.create_release + env: + VERSION: ${{ steps.tag.outputs.value }} + run: | + set -o errexit -o nounset -o pipefail + + mkdir -p dist + for os in linux darwin; do + for arch in amd64 arm64; do + CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \ + -trimpath \ + -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \ + -o "dist/kubectl-ate-${os}-${arch}" \ + ./cmd/kubectl-ate + done + done + + - name: Create GitHub Release + if: inputs.create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.value }} + generate_release_notes: true + files: dist/kubectl-ate-* From 374faefd81f33c1f7852813a12f837ff64568bf7 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 02/19] Support deployment namespaces and opt-in local atelet transport Resolve atelet discovery and identity from the pod namespace, centralize install defaults, and allow explicitly selected local clusters to run without Pod Certificates. Keep authenticated transport as the default. Signed-off-by: Eitan Yarmush --- cmd/ateapi/internal/controlapi/dialer.go | 9 +- cmd/ateapi/internal/controlapi/dialer_test.go | 23 +++- .../controlapi/functionaltest/common_test.go | 17 ++- cmd/ateapi/internal/controlapi/informer.go | 6 +- .../controlapi/workflow_resume_test.go | 3 +- cmd/ateapi/main.go | 16 ++- cmd/atelet/main.go | 118 +++++++++--------- internal/ateclient/builder.go | 5 +- internal/installdefaults/installdefaults.go | 43 +++++++ 9 files changed, 162 insertions(+), 78 deletions(-) create mode 100644 internal/installdefaults/installdefaults.go diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 3acd3d7f35..557d994aad 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -25,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/internal/atelet" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -32,6 +33,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -74,6 +76,11 @@ func WithDialCredentials(build func(expectedPodUID string) (credentials.Transpor return func(d *AteletDialer) { d.dialCredentials = build } } +// WithInsecureCredentials disables transport security for local clusters without Pod Certificates. +func WithInsecureCredentials() DialerOption { + return WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil }) +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { @@ -198,7 +205,7 @@ func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", ateletSA) + expectedID, err := spiffeid.FromSegments(trustDomain, "ns", installdefaults.NamespaceFromPodEnv(), "sa", ateletSA) if err != nil { return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) } diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 593529e400..2fb103bc33 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -41,6 +42,22 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" +func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { + secure := NewAteletDialer(nil, nil, "", "") + if _, err := secure.dialCredentials("pod-uid"); err == nil { + t.Fatal("secure dialer accepted empty credential paths") + } + + insecureDialer := NewAteletDialer(nil, nil, "", "", WithInsecureCredentials()) + creds, err := insecureDialer.dialCredentials("pod-uid") + if err != nil { + t.Fatalf("insecure dial credentials: %v", err) + } + if got := creds.Info().SecurityProtocol; got != "insecure" { + t.Fatalf("security protocol = %q, want insecure", got) + } +} + // makeTestCA mints a self-signed CA and returns it along with an X.509 bundle // containing it as the sole authority for the cluster.local trust domain. func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) { @@ -198,7 +215,7 @@ func TestDialForWorkerTarget(t *testing.T) { Spec: corev1.PodSpec{NodeName: "node-1"}, } ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: tc.ateletIP}}}, } @@ -225,7 +242,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("unknown worker pod", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: "10.244.1.7"}}}, } @@ -237,7 +254,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("atelet without assigned IPs", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } d := newDialerForPods(t, workerPod, ateletPod) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 4a62d08a51..3564dba3c4 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" @@ -65,10 +66,8 @@ const ( // templates hand out. No object store is wired up behind it. testStorageLocation = "gs://fake-fake-fake" - // ateletNamespace and byNode mirror the unexported constants controlapi's - // atelet informer is built with. - ateletNamespace = "ate-system" - byNode = "by-node" + // byNode mirrors the unexported index name controlapi's atelet informer uses. + byNode = "by-node" ) var ( @@ -132,7 +131,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // 3. Initialize Informers workerFactory, workerInformer := controlapi.WorkerPodInformer(k8sClient) - ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient) + ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient, installdefaults.SystemNamespace) scFactory := informers.NewSharedInformerFactory(k8sClient, 0) scLister := scFactory.Storage().V1().StorageClasses().Lister() @@ -711,7 +710,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ateletNamespace, + Namespace: installdefaults.SystemNamespace, Labels: map[string]string{"app": "atelet"}, }, Spec: corev1.PodSpec{ @@ -719,7 +718,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, }, } - created, err := kc.CoreV1().Pods(ateletNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + created, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -728,7 +727,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { } created.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} created.Status.Phase = corev1.PodRunning - if _, err := kc.CoreV1().Pods(ateletNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { + if _, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("updating atelet pod %s status: %w", name, err) } return nil @@ -745,7 +744,7 @@ func setupAteletOnNode(t *testing.T, tc *testContext, name, nodeName string) { t.Fatalf("%v", err) } t.Cleanup(func() { - _ = tc.k8sClient.CoreV1().Pods(ateletNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ + _ = tc.k8sClient.CoreV1().Pods(installdefaults.SystemNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ GracePeriodSeconds: ptr.To[int64](0), }) }) diff --git a/cmd/ateapi/internal/controlapi/informer.go b/cmd/ateapi/internal/controlapi/informer.go index 8e467d3029..12e42780f7 100644 --- a/cmd/ateapi/internal/controlapi/informer.go +++ b/cmd/ateapi/internal/controlapi/informer.go @@ -25,13 +25,13 @@ import ( ) const ( - ateletNamespace = "ate-system" byNamespaceAndName = "by-namespace-and-name" byNode = "by-node" ) -// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for Atelet pods. -func AteletInformer(kc kubernetes.Interface) (informers.SharedInformerFactory, cache.SharedIndexInformer) { +// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for +// Atelet pods in the given namespace. +func AteletInformer(kc kubernetes.Interface, ateletNamespace string) (informers.SharedInformerFactory, cache.SharedIndexInformer) { factory := informers.NewSharedInformerFactoryWithOptions(kc, 0, informers.WithNamespace(ateletNamespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 17fc25159e..0d4bfa7a14 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -1268,7 +1269,7 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo Spec: corev1.PodSpec{NodeName: "node-1"}, } ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-1", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-1", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } dialer := newDialerForPods(t, workerPod, ateletPod) diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 340ea5109a..ca2277c1b9 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -37,6 +37,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore" @@ -81,6 +82,7 @@ var ( actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") + ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") @@ -169,8 +171,13 @@ func main() { sandboxConfigLister := ateFactory.Api().V1alpha1().SandboxConfigs().Lister() csiDriverConfigLister := ateFactory.Api().V1alpha1().CSIDriverConfigs().Lister() + // atelet shares ateapi's namespace in every supported deployment topology, + // so we read it from Kubernetes' downward API rather than expose a flag. + ateletNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace)) + workerPodInformerFactory, workerPodInformer := controlapi.WorkerPodInformer(clientset) - ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset) + ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) scInformerFactory := informers.NewSharedInformerFactory(clientset, 0) storageClassLister := scInformerFactory.Storage().V1().StorageClasses().Lister() @@ -204,7 +211,11 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) + var dialerOpts []controlapi.DialerOption + if *ateletInsecure { + dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) + } + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) actorIDCAPool, err := localca.NewRefreshingPool(*actorIDCAPoolFile) if err != nil { @@ -341,6 +352,7 @@ func logFlagValues(ctx context.Context) { slog.String("actor-id-ca-pool", *actorIDCAPoolFile), slog.String("pod-identity-ca-certs", *podIdentityCACerts), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), + slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), ) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 6155e49aa0..c3e740ab74 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -93,6 +93,7 @@ var ( ateapiAddress = pflag.String("ateapi-address", "k8s:///api.ate-system.svc:443", "ateapi gRPC target used by the credential broker.") ateapiCAFile = pflag.String("ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify ateapi.") ateapiServerName = pflag.String("ateapi-server-name", "api.ate-system.svc", "DNS name expected on the ateapi certificate.") + grpcInsecure = pflag.Bool("grpc-insecure", false, "Serve gRPC without transport security. Intended only for local clusters without Pod Certificates.") gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -316,73 +317,76 @@ func main() { // it would never list or watch. Start is idempotent per informer — this // launches the new one and leaves the already-running ones untouched. ateFactory.Start(stopCh) - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - K8sClient: k8sClient, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - ClientCredBundle: *grpcServerCredBundle, - }) - if err != nil { - serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) - } - ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) - if err != nil { - serverboot.Fatal(ctx, "Failed to create ateapi client", err) - } - defer ateapiConn.Close() - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } - tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) - if err != nil { - serverboot.Fatal(ctx, "Failed to build server TLS config", err) - } - ateletCert, err := credbundle.Parse(*grpcServerCredBundle) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - if ateletIdentity == nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) - } - - ateomFacingTLS := tlsCfg.Clone() - ateomFacingTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) - if err := os.Remove(ateompath.AteomSupportSocket); err != nil && !errors.Is(err, os.ErrNotExist) { - serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) - } - ateomFacingLis, err := net.Listen("unix", ateompath.AteomSupportSocket) - if err != nil { - serverboot.Fatal(ctx, "Failed to listen for credential broker", err) - } - defer ateomFacingLis.Close() - if err := os.Chmod(ateompath.AteomSupportSocket, 0o600); err != nil { - serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + serverOpts := []grpc.ServerOption{ + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), } + if *grpcInsecure { + slog.WarnContext(ctx, "Serving atelet gRPC without transport security") + } else { + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) + } + serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsCfg))) - ateomFacingSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(ateomFacingTLS))) + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + K8sClient: k8sClient, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) + } + ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ateapi client", err) + } + defer ateapiConn.Close() - ateletpb.RegisterAteomSupportServer(ateomFacingSrv, &ateomSupportServer{ - controlClient: ateapipb.NewControlClient(ateapiConn), - workers: ateapipb.NewWorkerServiceClient(ateapiConn), - }) - go func() { - if err := ateomFacingSrv.Serve(ateomFacingLis); err != nil { - serverboot.Fatal(ctx, "Failed to serve credential broker", err) + ateletCert, err := credbundle.Parse(*grpcServerCredBundle) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) } - }() + ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + if ateletIdentity == nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) + } + ateomFacingTLS := tlsCfg.Clone() + ateomFacingTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) + if err := os.Remove(ateompath.AteomSupportSocket); err != nil && !errors.Is(err, os.ErrNotExist) { + serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) + } + ateomFacingLis, err := net.Listen("unix", ateompath.AteomSupportSocket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen for credential broker", err) + } + defer ateomFacingLis.Close() + if err := os.Chmod(ateompath.AteomSupportSocket, 0o600); err != nil { + serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + } + ateomFacingSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(ateomFacingTLS))) + ateletpb.RegisterAteomSupportServer(ateomFacingSrv, &ateomSupportServer{ + controlClient: ateapipb.NewControlClient(ateapiConn), + workers: ateapipb.NewWorkerServiceClient(ateapiConn), + }) + go func() { + if err := ateomFacingSrv.Serve(ateomFacingLis); err != nil { + serverboot.Fatal(ctx, "Failed to serve credential broker", err) + } + }() + } - svr := grpc.NewServer( - grpc.Creds(credentials.NewTLS(tlsCfg)), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), - ) + svr := grpc.NewServer(serverOpts...) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 31763ccf62..00de8bd61c 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -24,6 +24,7 @@ import ( "strings" "sync" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -184,7 +185,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "api", 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, installdefaults.SystemNamespace, installdefaults.APIServiceName, 443) if err != nil { return nil, err } @@ -273,7 +274,7 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts("ate-system").CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(installdefaults.SystemNamespace).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go new file mode 100644 index 0000000000..6d6475bc32 --- /dev/null +++ b/internal/installdefaults/installdefaults.go @@ -0,0 +1,43 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package installdefaults holds the default namespace and Service names +// that match the canonical install layout in manifests/ate-install/. +// Binaries use these as flag defaults; deployments that diverge from +// the canonical layout pass actual values via the corresponding flags. +package installdefaults + +import "os" + +const ( + // SystemNamespace is the namespace where substrate's control-plane + // components and the atelet DaemonSet run. + SystemNamespace = "ate-system" + // APIServiceName is the Service name of ate-api-server. + APIServiceName = "api" + + // PodNamespaceEnv is the conventional env var name for the namespace + // a pod is running in, exposed via Kubernetes' downward API. + PodNamespaceEnv = "POD_NAMESPACE" +) + +// NamespaceFromPodEnv returns the namespace from the PodNamespaceEnv env +// var when set (typically populated via Kubernetes' downward API), and +// falls back to SystemNamespace for non-k8s invocations (tests, local dev). +func NamespaceFromPodEnv() string { + if ns := os.Getenv(PodNamespaceEnv); ns != "" { + return ns + } + return SystemNamespace +} From a93eb71ca191090865bbc279b4f5580f3cad4155 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 03/19] Bound actor workflows and refresh worker state after pause Add a configurable end-to-end workflow deadline and propagate it through lease acquisition. Apply released worker assignments to the cache immediately so subsequent scheduling sees the completed pause. Signed-off-by: Eitan Yarmush --- .../controlapi/functionaltest/common_test.go | 1 + cmd/ateapi/internal/controlapi/service.go | 14 ++---- cmd/ateapi/internal/controlapi/workflow.go | 21 +++++--- .../controlapi/workflow_delete_test.go | 5 +- .../controlapi/workflow_lease_test.go | 50 +++++++++++++++++++ .../internal/controlapi/workflow_pause.go | 5 +- .../controlapi/workflow_testutil_test.go | 5 +- .../internal/workercache/workercache.go | 6 +++ cmd/ateapi/main.go | 3 ++ 9 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/workflow_lease_test.go diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 3564dba3c4..e575e27950 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -221,6 +221,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu dialer, instruments, "", + 30*time.Second, volPlugins, objectStore, "https://nonexistent-issuer.example", diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index b433515586..c8633eac6a 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "sync" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -63,14 +64,8 @@ type VolumePluginRegistry interface { GetPlugin(ctx context.Context, name string) (volume.VolumePluginControlPlane, error) } -// NewRPCService creates an instance of the ControlServer service. This is what -// implements the outward-facing RPC interface. -// -// instruments may be nil; the record helpers no-op. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewRPCService creates an RPC service. actorWorkflowDeadline bounds how long a single +// Resume/Suspend workflow can run end-to-end. instruments and objectStore may be nil. func NewRPCService( persistence store.Interface, workerCache *workercache.Cache, @@ -80,6 +75,7 @@ func NewRPCService( dialer *AteletDialer, instruments *Instruments, egressGatewayAddress string, + actorWorkflowDeadline time.Duration, volumePlugins map[string]volume.VolumePluginControlPlane, objectStore objectstore.Store, actorIdentityJWTIssuer string, @@ -101,7 +97,7 @@ func NewRPCService( actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, objectStore) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, objectStore) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 1057169ba3..beaae82b06 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -109,14 +110,13 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry + workflowDeadline time.Duration objectStore objectstore.Store } -// NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how +// long a single Resume/Suspend can run end-to-end; instruments and objectStore +// may be nil. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -126,6 +126,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, + workflowDeadline time.Duration, objectStore objectstore.Store, ) *ActorWorkflow { return &ActorWorkflow{ @@ -138,6 +139,7 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, + workflowDeadline: workflowDeadline, objectStore: objectStore, } } @@ -211,7 +213,14 @@ func acquireLease(ctx context.Context, holder leaseHolder, key, subject string) } func (w *ActorWorkflow) acquireActorLease(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lease, error) { - return acquireLease(ctx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + workflowCtx, cancel := context.WithTimeout(ctx, w.workflowDeadline) + leaseCtx, lease, err := acquireLease(workflowCtx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + if err != nil { + cancel() + return nil, nil, err + } + context.AfterFunc(lease.Context(), cancel) + return leaseCtx, lease, nil } func acquireTagLease(ctx context.Context, holder leaseHolder, tagRef resources.TagRef) (context.Context, *store.Lease, error) { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index fa95d62fea..c93489922d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -335,7 +336,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) objects := objectstoretest.New() - w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ @@ -431,7 +432,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { }) } - actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) // Suspend the actor as far as it gets: MarkSuspending mints the // in-progress URI, and the checkpoint writes under it actor, err := actorWorkflow.ensureMarkedSuspending(ctx, actorRef, actor, template) diff --git a/cmd/ateapi/internal/controlapi/workflow_lease_test.go b/cmd/ateapi/internal/controlapi/workflow_lease_test.go new file mode 100644 index 0000000000..b17f0583d1 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_lease_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/resources" +) + +type leaseStore struct{ store.Interface } + +func (leaseStore) AcquireLease(ctx context.Context, _ string) (*store.Lease, error) { + return store.NewLease(ctx, func() {}), nil +} + +func TestAcquireActorLeaseWorkflowDeadline(t *testing.T) { + w := &ActorWorkflow{store: leaseStore{}, workflowDeadline: 20 * time.Millisecond} + + ctx, lease, err := w.acquireActorLease(context.Background(), resources.ActorRef{Atespace: "space", Name: "actor"}) + if err != nil { + t.Fatalf("acquireActorLease: %v", err) + } + t.Cleanup(lease.Close) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("context error = %v, want DeadlineExceeded", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("workflow context did not reach its deadline") + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 3fc4a77c84..f240000a71 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -238,13 +238,16 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso nodeName = worker.GetNodeName() // Drop just this actor's assignment; any other actors the worker // hosts keep theirs. - _, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) + released, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) if err != nil { if errors.Is(err, store.ErrVersionConflict) { return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") } return nil, err } + if w.workerCache != nil { + w.workerCache.Observe(released) + } } // 2. Clear the actor's assignment, now that the worker is freed diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index af78631005..df46c2dd9a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -19,6 +19,7 @@ import ( "errors" "slices" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -57,7 +58,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, objectstoretest.New()) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoretest.New()) } // newFinalizeWorkflow builds an ActorWorkflow over persistence with an @@ -65,7 +66,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa // directly rather than going through newTestActorWorkflow. func newFinalizeWorkflow(persistence store.Interface) (*ActorWorkflow, *objectstoretest.Fake) { objects := objectstoretest.New() - return &ActorWorkflow{store: persistence, objectStore: objects}, objects + return &ActorWorkflow{store: persistence, workflowDeadline: time.Minute, objectStore: objects}, objects } // mustActorSnapshotURI builds the URI of a snapshot the actor took under diff --git a/cmd/ateapi/internal/workercache/workercache.go b/cmd/ateapi/internal/workercache/workercache.go index 6281586e63..d42be04736 100644 --- a/cmd/ateapi/internal/workercache/workercache.go +++ b/cmd/ateapi/internal/workercache/workercache.go @@ -120,6 +120,12 @@ func (c *Cache) Forget(name string) { delete(c.workers, name) } +// Observe applies a worker returned by a successful store write immediately, +// without waiting for the corresponding watch event. +func (c *Cache) Observe(worker *ateapipb.Worker) { + c.applyEvent(store.WorkerEvent{Type: store.WorkerEventUpdated, Worker: worker}) +} + func (c *Cache) sync(ctx context.Context) (*store.WorkerWatch, error) { watch, err := c.store.WatchWorkers(ctx) if err != nil { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index ca2277c1b9..094429de3f 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -88,6 +88,7 @@ var ( drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") templateResyncInterval = pflag.Duration("template-resync-interval", 20*time.Second, fmt.Sprintf("Interval between actor template resyncs. Must be at least %s.", minResyncInterval)) + actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") @@ -236,6 +237,7 @@ func main() { ateletDialer, instruments, *egressGatewayAddress, + *actorWorkflowDeadline, volPlugins, objectStore, actorIdentityJWTIssuer, @@ -355,6 +357,7 @@ func logFlagValues(ctx context.Context) { slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), + slog.Duration("actor-workflow-deadline", *actorWorkflowDeadline), ) } From a67f16aeb3787cc3b5f568fc37f165148872effe Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 04/19] Accept RSA and EC private keys in credential bundles Parse PKCS1 RSA and SEC1 EC keys alongside PKCS8 keys, including regression coverage for RSA bundles. Signed-off-by: Eitan Yarmush --- internal/credbundle/credbundle.go | 20 ++++++++++++++++++-- internal/credbundle/credbundle_test.go | 6 +++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go index 3d0db9f047..d4b842227c 100644 --- a/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -20,6 +20,7 @@ package credbundle import ( + "crypto" "crypto/tls" "crypto/x509" "encoding/pem" @@ -112,6 +113,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { } var leafKeyBytes []byte + var leafKeyBlockType string var chainBytes [][]byte for { @@ -124,8 +126,9 @@ func Parse(bundlePath string) (*tls.Certificate, error) { switch block.Type { case "CERTIFICATE": chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": + case "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY": leafKeyBytes = block.Bytes + leafKeyBlockType = block.Type default: return nil, fmt.Errorf("unknown PEM block type %q", block.Type) } @@ -139,7 +142,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { return nil, fmt.Errorf("no CERTIFICATE blocks found") } - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) + leafKey, err := parsePrivateKey(leafKeyBlockType, leafKeyBytes) if err != nil { return nil, fmt.Errorf("while parsing private key: %w", err) } @@ -155,3 +158,16 @@ func Parse(bundlePath string) (*tls.Certificate, error) { PrivateKey: leafKey, }, nil } + +func parsePrivateKey(blockType string, keyBytes []byte) (crypto.PrivateKey, error) { + switch blockType { + case "PRIVATE KEY": + return x509.ParsePKCS8PrivateKey(keyBytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBytes) + default: + return nil, fmt.Errorf("unsupported private key block type %q", blockType) + } +} diff --git a/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go index 579a12bbc0..171bcb5b64 100644 --- a/internal/credbundle/credbundle_test.go +++ b/internal/credbundle/credbundle_test.go @@ -58,13 +58,13 @@ func TestParsePKCS8PrivateKeyBlock(t *testing.T) { } } -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { +func TestParseRSAPrivateKeyBlock(t *testing.T) { certDER := generateCertificate(t, 1) bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") + if _, err := Parse(bundlePath); err != nil { + t.Fatalf("Parse() error = %v", err) } } From ebe0a0768d2603f8e3be0f74ba46455cfe09c197 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 05/19] Validate agentgateway across gVisor and microVM runtimes Require the agentgateway E2E lane, reuse the installed control plane for microVM demos, wait for asset storage initialization, and accommodate runtime startup and counter persistence behavior in E2E checks. Signed-off-by: Eitan Yarmush --- .github/workflows/pr-workflow.yaml | 16 +-- hack/install-microvm-deps.sh | 1 + hack/run-microvm-demo.sh | 32 +++-- internal/e2e/suites/demo/demo_test.go | 114 ++++++++++++------ internal/e2e/suites/identity/identity_test.go | 14 ++- 5 files changed, 115 insertions(+), 62 deletions(-) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 6a0394458b..2672c916c5 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -59,15 +59,11 @@ jobs: e2e-test-matrix: name: E2E (${{ matrix.dataplane }}) runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} # TODO: Make AgentGateway required once tests show stability strategy: fail-fast: false matrix: include: - - dataplane: envoy - experimental: false - dataplane: agentgateway - experimental: true env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} steps: @@ -101,7 +97,7 @@ jobs: - name: Install Agent Substrate (${{ matrix.dataplane }}) # The dataplane selection applies to both the ingress router and egress # gateway. - run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} + run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} --rollout-timeout=300s - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | @@ -111,11 +107,8 @@ jobs: run: hack/install-ate-kind.sh --setup-csi=nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the - # counter-microvm demo onto the control plane installed above. The demo - # redeploys the control plane, so retain the selected dataplane. - env: - ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/run-microvm-demo-kind.sh + # counter-microvm demo onto the control plane installed above. + run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demos @@ -195,8 +188,7 @@ jobs: kubectl --context kind-kind get pods -A -l ate.dev/worker-pool \ -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ | while read -r ns name; do dump "$ns" "$name"; done - # Preserve the required-check name while the concrete Envoy and AgentGateway - # executions run as entries in the shared matrix above. + # Preserve the required-check name for the dataplane matrix above. e2e-test: name: e2e-test needs: e2e-test-matrix diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index 452b18e6fc..8c8708a949 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -174,6 +174,7 @@ fi # in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." + run_kubectl wait --for=condition=complete job/rustfs-bucket-init -n ate-system --timeout=120s OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh else log "Uploading assets to gs://${BUCKET_NAME}/kata-assets/ ..." diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 27c66dffe1..73c3a4bb91 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -55,10 +55,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -if [[ $# -gt 0 ]]; then - echo "Error: unknown argument $1" >&2 - exit 1 -fi +SKIP_CONTROL_PLANE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-control-plane) SKIP_CONTROL_PLANE=true ;; + *) + echo "Error: unknown argument $1" >&2 + exit 1 + ;; + esac + shift +done if [[ -z "${KO_DOCKER_REPO}" ]]; then echo "Error: KO_DOCKER_REPO is required (set it in .ate-dev-env.sh for GKE," >&2 @@ -75,13 +83,15 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system +if [[ "${SKIP_CONTROL_PLANE}" != "true" ]]; then + log "Deploying the ate control plane (--deploy-ate-system)..." + if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system + else + # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system + fi fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 392128152f..ca2b69b50e 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -19,6 +19,9 @@ import ( "fmt" "io" "net/http" + "os" + "regexp" + "strconv" "strings" "testing" "time" @@ -702,7 +705,7 @@ func validateCounterResponse(t *testing.T, resp string, stage string, wantMemory if !strings.Contains(resp, memoryCounterPrefix+fmt.Sprintf("%d", wantMemory)) { t.Errorf("[%s] expected memory count %d, got response: %s", stage, wantMemory, resp) } - if !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { + if wantFile >= 0 && !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { t.Errorf("[%s] expected file count %d, got response: %s", stage, wantFile, resp) } } @@ -726,24 +729,14 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj }) }() - listResp, err := clients.SubstrateAPI.ListActors(ctx, &ateapipb.ListActorsRequest{Atespace: demoAtespace}) + getResp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + }) if err != nil { - t.Fatalf("ListActors RPC failed: %v", err) - } - - var myActors []*ateapipb.Actor - for _, actor := range listResp.GetActors() { - if actor.GetActorTemplate().GetName() == at.GetMetadata().GetName() && actor.GetMetadata().GetName() == actorName { - myActors = append(myActors, actor) - } + t.Fatalf("GetActor RPC failed: %v", err) } - // Check that we have our Actor created. - if len(myActors) != 1 { - t.Fatalf("expected actor %s from template %s, got %d actors: %v", actorName, at.GetMetadata().GetName(), len(myActors), myActors) - } - - actor := myActors[0] + actor := getResp if actor.GetMetadata().GetName() != actorName { t.Errorf("expected actor name %s, got %s", actorName, actor.GetMetadata().GetName()) } @@ -754,8 +747,7 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Errorf("expected actor state to be SUSPENDED, got %v", actor.Status.State) } - t.Logf("Successfully queried Substrate API. Found %d active actors total, %d from our template %s.", - len(listResp.GetActors()), len(myActors), at.GetMetadata().GetName()) + t.Logf("Successfully queried Substrate API. Found actor %s in namespace %s.", actorName, nsObj.Name) return nil } @@ -782,13 +774,13 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) - // Pausing the actor t.Logf("Pausing Actor %q...", actorName) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ @@ -807,11 +799,12 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after pause", 2, -1) + } else { + validateCounterResponse(t, resp, "after pause", 2, 2) } - validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -861,11 +854,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -885,11 +879,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after suspend", 2, -1) + } else { + validateCounterResponse(t, resp, "after suspend", 2, 2) } - validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -1195,6 +1190,55 @@ func waitForActorStateWithTimeout(ctx context.Context, t *testing.T, clients *e2 t.Fatalf("timed out waiting for actor %q to reach state %v", actorName, expectedState) } +var preservedCountRe = regexp.MustCompile(`preserved memory count: ([0-9]+)`) + +func callActorUntilCountAtLeast(t *testing.T, actorRef resources.ActorRef, minCount int) string { + t.Helper() + + var lastErr error + var lastResp string + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + resp, err := callActor(t, actorRef) + if err != nil { + lastErr = err + } else { + lastResp = resp + count, err := preservedCount(resp) + if err != nil { + lastErr = err + } else if count >= minCount { + return resp + } else { + lastErr = fmt.Errorf("expected preserved memory count >= %d, got %d in response: %s", minCount, count, resp) + } + } + time.Sleep(500 * time.Millisecond) + } + + if lastResp != "" { + t.Fatalf("timed out calling actor %q; last response: %s; last error: %v", actorRef.Name, lastResp, lastErr) + } + t.Fatalf("timed out calling actor %q; last error: %v", actorRef.Name, lastErr) + return "" +} + +func preservedCount(resp string) (int, error) { + matches := preservedCountRe.FindStringSubmatch(resp) + if matches == nil { + return 0, fmt.Errorf("response does not include preserved memory count: %s", resp) + } + count, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("parse preserved memory count %q: %w", matches[1], err) + } + return count, nil +} + +func isMicroVMEnvironment() bool { + return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" +} + func callActor(t *testing.T, actorRef resources.ActorRef) (string, error) { return callActorPath(t, actorRef, "POST", "/") } diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index bf46ffa2c0..d2bae4ae18 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -278,11 +278,17 @@ func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Client func whoami(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) whoamiResponse { t.Helper() - out, err := tryWhoami(ctx, rc, id) - if err != nil { - t.Fatal(err) + deadline := time.Now().Add(30 * time.Second) + for { + out, err := tryWhoami(ctx, rc, id) + if err == nil { + return out + } + if time.Now().After(deadline) { + t.Fatal(err) + } + time.Sleep(time.Second) } - return out } // tryWhoami is whoami returning the error instead of failing the test. From 3d01e94d999dcf002330ee57c2eb081d5c4c5a4e Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 06/19] Add Helm deployment with agentgateway and CRD verification Package the control plane, workers, PostgreSQL, RustFS, and CRDs as Helm charts. Keep manifests and generated RBAC aligned, add Helm E2E checks, and include current scheduling, sandbox permissions, and agentgateway configuration. Co-authored-by: Jet Chiang Co-authored-by: Keith Mattix II Signed-off-by: Jet Chiang Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 116 +++++ Makefile | 18 +- charts/substrate-crds/Chart.yaml | 28 ++ charts/substrate-crds/README.md | 13 + .../templates/ate.dev_csidriverconfigs.yaml | 113 +++++ .../templates/ate.dev_sandboxconfigs.yaml | 140 ++++++ .../templates/ate.dev_workerpools.yaml | 473 ++++++++++++++++++ charts/substrate/Chart.yaml | 27 + charts/substrate/README.md | 43 ++ charts/substrate/templates/NOTES.txt | 7 + charts/substrate/templates/_helpers.tpl | 105 ++++ .../templates/ate-api-server-envvars.yaml | 23 + .../substrate/templates/ate-api-server.yaml | 210 ++++++++ charts/substrate/templates/ate-client.yaml | 23 + .../substrate/templates/ate-controller.yaml | 113 +++++ charts/substrate/templates/atelet.yaml | 232 +++++++++ charts/substrate/templates/atenet-egress.yaml | 237 +++++++++ charts/substrate/templates/atenet-router.yaml | 360 +++++++++++++ charts/substrate/templates/namespace.yaml | 22 + .../templates/pod-certificate-controller.yaml | 198 ++++++++ charts/substrate/templates/postgres.yaml | 234 +++++++++ charts/substrate/templates/role.yaml | 114 +++++ charts/substrate/templates/rustfs.yaml | 137 +++++ .../templates/sandboxconfig-gvisor.yaml | 35 ++ .../templates/sandboxconfig-validation.yaml | 57 +++ charts/substrate/values.yaml | 72 +++ cmd/atecontroller/internal/controllers/gen.go | 2 +- hack/gen-rbac.sh | 37 ++ hack/render-manifests.sh | 157 ++++++ hack/verify/crd-chart.sh | 47 ++ .../ate-install/ate-api-server-envvars.yaml | 24 + manifests/ate-install/ate-client.yaml | 24 + .../components/agentgateway/configmap.yaml | 54 ++ manifests/ate-install/role.yaml | 130 +++++ manifests/ate-install/rustfs.yaml | 136 +++++ 35 files changed, 3759 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/helm-e2e.yaml create mode 100644 charts/substrate-crds/Chart.yaml create mode 100644 charts/substrate-crds/README.md create mode 100644 charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_workerpools.yaml create mode 100644 charts/substrate/Chart.yaml create mode 100644 charts/substrate/README.md create mode 100644 charts/substrate/templates/NOTES.txt create mode 100644 charts/substrate/templates/_helpers.tpl create mode 100644 charts/substrate/templates/ate-api-server-envvars.yaml create mode 100644 charts/substrate/templates/ate-api-server.yaml create mode 100644 charts/substrate/templates/ate-client.yaml create mode 100644 charts/substrate/templates/ate-controller.yaml create mode 100644 charts/substrate/templates/atelet.yaml create mode 100644 charts/substrate/templates/atenet-egress.yaml create mode 100644 charts/substrate/templates/atenet-router.yaml create mode 100644 charts/substrate/templates/namespace.yaml create mode 100644 charts/substrate/templates/pod-certificate-controller.yaml create mode 100644 charts/substrate/templates/postgres.yaml create mode 100644 charts/substrate/templates/role.yaml create mode 100644 charts/substrate/templates/rustfs.yaml create mode 100644 charts/substrate/templates/sandboxconfig-gvisor.yaml create mode 100644 charts/substrate/templates/sandboxconfig-validation.yaml create mode 100644 charts/substrate/values.yaml create mode 100755 hack/gen-rbac.sh create mode 100755 hack/render-manifests.sh create mode 100755 hack/verify/crd-chart.sh create mode 100644 manifests/ate-install/ate-api-server-envvars.yaml create mode 100644 manifests/ate-install/ate-client.yaml create mode 100644 manifests/ate-install/role.yaml create mode 100644 manifests/ate-install/rustfs.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml new file mode 100644 index 0000000000..81b3a46103 --- /dev/null +++ b/.github/workflows/helm-e2e.yaml @@ -0,0 +1,116 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helm-e2e +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + e2e-test: + runs-on: ubuntu-latest + env: + VERSION: helm-e2e + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - name: Setup Helm + uses: azure/setup-helm@v4 + - name: Cache micro-VM assets + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + run: hack/create-kind-cluster.sh + - name: Label nodes with the installed version + run: kubectl label nodes --all ate.dev/substrate-version=${VERSION} + - name: Create install namespace + run: kubectl create namespace ate-system + - name: Install observability fixtures + run: | + kubectl apply -f manifests/ate-install/kind/otel-collector.yaml + kubectl apply -f manifests/ate-install/kind/prometheus.yaml + - name: Build chart images + run: | + for component in ateapi atecontroller atelet podcertcontroller atenet; do + KO_DOCKER_REPO="localhost:5001/${component}" \ + ./hack/run-tool.sh ko build --bare --tags helm-e2e \ + --platform linux/amd64 "./cmd/${component}" + done + - name: Install Agent Substrate with Helm + run: | + helm upgrade --install substrate-crds charts/substrate-crds + helm upgrade --install substrate charts/substrate \ + --namespace ate-system \ + --create-namespace \ + --set image.registry=localhost:5001 \ + --set image.tag=helm-e2e \ + --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ + --set postgres.resources.requests.cpu=500m + - name: Bootstrap mTLS authorities + run: | + hack/install-ate-kind.sh --create-podcertificate-controller-cas + hack/install-ate-kind.sh --create-jwt-authority-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-certs-secret + hack/install-ate-kind.sh --create-api-authentication-config + - name: Wait for Helm install + run: | + helm upgrade substrate charts/substrate \ + --namespace ate-system \ + --reuse-values \ + --wait --timeout=10m + - name: Enable NFS + run: | + sudo modprobe nfs || true + sudo modprobe nfsd || true + - name: Install CSI NFS driver + run: hack/install-ate-kind.sh --setup-csi=nfs + - name: Deploy micro-VM counter demo + # The deploy creates the substrate ActorTemplate and waits for its golden + # snapshot internally; the ActorTemplate CRD (and its Ready condition) + # no longer exists to wait on. + run: hack/run-microvm-demo-kind.sh --skip-control-plane + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Run E2E tests (gVisor) + run: hack/run-e2e-kind.sh -v -args --no-color + - name: Run E2E tests (micro-VM) + env: + E2E_SANDBOX_CLASS: microvm + run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context kind-kind get workerpool,pods -A -o wide || true + for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do + echo "=== logs: ate-system/${p} ===" + kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true + done diff --git a/Makefile b/Makefile index 1002e2f039..11307e18d4 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ build: build-images build-atectl build-ate-setup .PHONY: build-images build-images: - $(KO) build $(KO_FLAGS) \ + $(KO) build --base-import-paths $(KO_FLAGS) \ --ldflags="$(LDFLAGS)" \ $(IMAGES) @@ -131,3 +131,19 @@ verify: test .PHONY: clean clean: rm -rf $(BINDIR) + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode, +# the historical default install). Run this whenever charts/substrate/ changes. +.PHONY: helm-template +helm-template: + @./hack/render-manifests.sh + +# Verify that manifests/ate-install/ matches the chart output. Used in CI. +.PHONY: verify-helm-template +verify-helm-template: + @./hack/render-manifests.sh --check + +# Verify that the CRD chart mirrors the generated CRDs. +.PHONY: verify-crd-chart +verify-crd-chart: + @./hack/verify/crd-chart.sh diff --git a/charts/substrate-crds/Chart.yaml b/charts/substrate-crds/Chart.yaml new file mode 100644 index 0000000000..a69dcee0e9 --- /dev/null +++ b/charts/substrate-crds/Chart.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate-crds +description: Agent Substrate CustomResourceDefinitions. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate +- crds diff --git a/charts/substrate-crds/README.md b/charts/substrate-crds/README.md new file mode 100644 index 0000000000..12fa31f0a7 --- /dev/null +++ b/charts/substrate-crds/README.md @@ -0,0 +1,13 @@ +# substrate-crds + +Helm chart for installing the Agent Substrate CRDs. + +Install this chart before installing the main `substrate` chart: + +```bash +helm upgrade --install substrate-crds ./charts/substrate-crds +helm upgrade --install substrate ./charts/substrate --namespace ate-system --create-namespace +``` + +The CRD YAMLs in `templates/` mirror `manifests/ate-install/generated/`. +Run `hack/verify/crd-chart.sh` to verify they are in sync. diff --git a/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml new file mode 100644 index 0000000000..ebc1473eae --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: csidriverconfigs.ate.dev +spec: + group: ate.dev + names: + kind: CSIDriverConfig + listKind: CSIDriverConfigList + plural: csidriverconfigs + shortNames: + - csidriverconfig + singular: csidriverconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.driverName + name: Driver + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CSIDriverConfig is the Schema for the csidriverconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CSIDriverConfigSpec defines the desired state of CSIDriverConfig + properties: + controllerEndpoint: + description: |- + ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). + pattern: ^(tcp|dns)://.+$ + type: string + driverName: + description: |- + DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + Matches the StorageClass referenced in ActorTemplate volume definitions. + maxLength: 63 + minLength: 1 + pattern: ^(substrate\.io/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ + type: string + nodeSocketOverride: + description: |- + NodeSocketOverride is an optional override for the CSI Node service socket + on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. + pattern: ^unix://.+$ + type: string + tls: + description: TLS configures TLS/mTLS for the connection to the ControllerEndpoint. + properties: + enabled: + description: Enabled controls whether TLS is used. + type: boolean + serverName: + description: ServerName override for TLS verification. + type: string + usePodIdentity: + description: UsePodIdentity indicates whether to reuse Substrate's + Pod Identity (SPIFFE) certificates. + type: boolean + required: + - enabled + type: object + x-kubernetes-validations: + - message: tls.usePodIdentity must be true when tls.enabled is true; + manual certificates are not yet supported + rule: '!self.enabled || (has(self.usePodIdentity) && self.usePodIdentity)' + required: + - controllerEndpoint + - driverName + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml new file mode 100644 index 0000000000..22d46f5123 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -0,0 +1,140 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: sandboxconfigs.ate.dev +spec: + group: ate.dev + names: + kind: SandboxConfig + listKind: SandboxConfigList + plural: sandboxconfigs + shortNames: + - sandboxconfig + singular: sandboxconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxConfig is cluster-scoped configuration describing the sandbox binaries + for a sandbox runtime family. It is referenced by an ActorTemplate's + sandbox_config.config_name (required) and decouples + sandbox binary selection from the workload definition. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SandboxConfig + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must include a digest + rule: self.contains('@') + sandboxClass: + default: gvisor + description: |- + SandboxClass is the sandbox runtime family this config applies to. An + ActorTemplate only uses SandboxConfigs whose SandboxClass matches its + sandbox_config.sandbox_class. + enum: + - gvisor + - microvm + type: string + required: + - pauseImage + - sandboxClass + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml new file mode 100644 index 0000000000..e1bc49a029 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -0,0 +1,473 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: workerpools.ate.dev +spec: + group: ate.dev + names: + kind: WorkerPool + listKind: WorkerPoolList + plural: workerpools + shortNames: + - workerpool + singular: workerpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.replicas + name: Replicas + type: integer + - jsonPath: .status.readyReplicas + name: Ready + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: WorkerPool is the Schema for the workerpools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of WorkerPool + properties: + replicas: + description: Replicas is the number of worker pods to run. + format: int32 + minimum: 0 + type: integer + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family for this pool, which drives + the worker pod shape (KVM/vhost device mounts and node placement). The + concrete binary is still selected by WorkerImage. Defaults to gvisor. + The sandbox binaries themselves come from the SandboxConfig each + ActorTemplate names (required). + + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + template: + description: Template holds optional metadata, scheduling, and resource + settings for worker workloads. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations are added to the generated Deployment and worker pods. Keys + in the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: annotation keys must be valid Kubernetes qualified + names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + labels: + additionalProperties: + description: |- + WorkerPoolLabelValue is a Kubernetes label value for generated worker + workloads. + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Labels are added to the generated Deployment and worker pods. Keys in + the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: label keys must be valid Kubernetes qualified names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + nodeAffinity: + description: |- + NodeAffinity scheduling rules for the worker pods. Mapped to + spec.affinity.nodeAffinity on the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is a selector which must be true for + the pod to fit on a node. + type: object + priorityClassName: + description: PriorityClassName for the worker pods. + type: string + resources: + description: Resources are the compute resources allocated for + each worker pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tolerations: + description: Tolerations for the worker pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + workerImage: + description: WorkerImage is the ateom container image to deploy as + workers. + minLength: 1 + type: string + required: + - replicas + - workerImage + type: object + status: + description: status is the observed state of WorkerPool + properties: + readyReplicas: + description: ReadyReplicas is the number of ready worker pods. + format: int32 + minimum: 0 + type: integer + replicas: + description: Replicas is the total number of worker pods. + format: int32 + minimum: 0 + type: integer + selector: + description: Selector is the label selector for the worker pods. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/charts/substrate/Chart.yaml b/charts/substrate/Chart.yaml new file mode 100644 index 0000000000..52bd748009 --- /dev/null +++ b/charts/substrate/Chart.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate +description: Agent Substrate — actor runtime, control plane, and data-plane router. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate diff --git a/charts/substrate/README.md b/charts/substrate/README.md new file mode 100644 index 0000000000..e7364f4e37 --- /dev/null +++ b/charts/substrate/README.md @@ -0,0 +1,43 @@ +# substrate + +Helm chart for installing Agent Substrate. + +The chart uses mTLS and PostgreSQL by default. It requires the +`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and +`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1` +API. + +```bash +# CRDs +helm upgrade --install substrate-crds ./charts/substrate-crds + +# Install Substrate +helm upgrade --install substrate ./charts/substrate +``` + +By default, component images are pulled from `ghcr.io/kagent-dev/substrate` +using the chart `appVersion` as the tag. Override `image.registry` and +`image.tag` to install from a different image repository or tag. + +## Render manifests without applying + +```bash +helm template substrate ./charts/substrate +``` + +`manifests/ate-install/` in the repo is the rendered mTLS output and is +regenerated by `make helm-template`. The separate `substrate-crds` chart +mirrors `manifests/ate-install/generated/`. + +## Values + +See `values.yaml` for the full set; the important keys: + +| Key | Default | Notes | +|-----|---------|-------| +| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | +| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | +| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt new file mode 100644 index 0000000000..c0e9875a45 --- /dev/null +++ b/charts/substrate/templates/NOTES.txt @@ -0,0 +1,7 @@ +substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL + +REQUIRED Kubernetes feature gates: + - ClusterTrustBundle + - ClusterTrustBundleProjection + - PodCertificateRequest +The certificates.k8s.io/v1beta1 API must also be enabled. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl new file mode 100644 index 0000000000..32ae087336 --- /dev/null +++ b/charts/substrate/templates/_helpers.tpl @@ -0,0 +1,105 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Qualified resource name for a chart component. + +Usage: + {{ include "substrate.fullname" (list "ate-api-server" .) }} + +When the release name is "substrate" (the canonical render in +hack/render-manifests.sh — `helm template substrate charts/substrate`), this +returns the bare component name, so the generated manifests/ate-install/ +files keep their historical names ("ate-api-server", "ate-controller", ...). + +Otherwise resources are prefixed with the release name in the standard Helm +style ("foo-ate-api-server", ...) so multiple releases coexist without +colliding. + +The check is on the literal release name "substrate" rather than +$ctx.Chart.Name so this helper is context-safe: a parent chart can invoke it +with its own `.` (where .Chart.Name is the parent, not "substrate") and still +get the same prefixed name that this subchart's own templates render. +*/}} +{{- define "substrate.fullname" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- if eq $ctx.Release.Name "substrate" -}} +{{- $name -}} +{{- else -}} +{{- printf "%s-%s" $ctx.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +ServiceAccount name of ate-api-server, as this chart creates it. Parent +charts that need to bind additional Roles to this SA (e.g. env-source +Secret/ConfigMap reads for ActorTemplate resolution) should reference this +helper instead of hardcoding "ate-api-server": + + {{ include "substrate.ateApiServer.serviceAccountName" . }} +*/}} +{{- define "substrate.ateApiServer.serviceAccountName" -}} +{{- include "substrate.fullname" (list "ate-api-server" .) -}} +{{- end -}} + +{{/* +gRPC endpoint that clients dial to reach ate-api-server. dns:/// scheme + +release-prefixed Service name + release namespace + :443. Suitable for +consumption as ATE_API_ENDPOINT / --ateapi-address: + + {{ include "substrate.ateApi.endpoint" . }} + -> dns:///-api..svc:443 +*/}} +{{- define "substrate.ateApi.endpoint" -}} +{{- printf "dns:///%s.%s.svc:443" (include "substrate.fullname" (list "api" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Plaintext HTTP URL that clients use to reach atenet-router. + + {{ include "substrate.atenetRouter.url" . }} + -> http://-atenet-router..svc:80 +*/}} +{{- define "substrate.atenetRouter.url" -}} +{{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Build an image reference for a substrate component binary. + +Usage: + {{ include "substrate.componentImage" (list "ateapi" .) }} + +Produces {image.registry}/{name}:{tag} where tag is resolved as: + 1. image.tag value, if set and not the sentinel "" + 2. .Chart.AppVersion, if image.tag is empty + 3. no tag (no colon) when image.tag is the sentinel "" + +The "" sentinel is used by hack/render-manifests.sh so that ko:// refs +are emitted without a tag, letting `ko resolve` supply the digest at build time. +*/}} +{{- define "substrate.componentImage" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $registry := $ctx.Values.image.registry -}} +{{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} +{{- if ne $tag "" -}} +{{- printf "%s/%s:%s" $registry $name $tag -}} +{{- else -}} +{{- printf "%s/%s" $registry $name -}} +{{- end -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..753c47178b --- /dev/null +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml new file mode 100644 index 0000000000..a5073bc9fa --- /dev/null +++ b/charts/substrate/templates/ate-api-server.yaml @@ -0,0 +1,210 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs", "csidriverconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "watch", "list"] +# Secret reads for env source resolution are intentionally NOT granted +# cluster-wide here. Each demo / tenant is responsible for granting +# ate-api-server read access only to the specific Secrets referenced by its +# ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using +# resourceNames). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-api-server + template: + metadata: + labels: + app: ate-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} + terminationGracePeriodSeconds: 40 + containers: + - name: ate-api-server + image: {{ include "substrate.componentImage" (list "ateapi" .) }} + args: + - "--grpc-listen-addr=0.0.0.0:443" + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" + - "--postgres-connection-string=@env" + - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" + - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" + - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + - "--drain-delay=13s" + - "--drain-timeout=15s" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + envFrom: + - configMapRef: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + optional: true + volumeMounts: + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } + - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } + ports: + - containerPort: 443 + - name: prometheus + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: actor-id-ca-pool + projected: + sources: + - secret: + name: actor-id-ca-pool + items: + - { key: pool, path: pool.json } + - name: authentication-config + configMap: + name: ate-api-authentication + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-api-server +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "api" .) }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml new file mode 100644 index 0000000000..dfd2fdab68 --- /dev/null +++ b/charts/substrate/templates/ate-client.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-client" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-client diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml new file mode 100644 index 0000000000..31c83b9066 --- /dev/null +++ b/charts/substrate/templates/ate-controller.yaml @@ -0,0 +1,113 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: ate-controller +spec: + selector: + app: ate-controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-controller + template: + metadata: + labels: + app: ate-controller + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + containers: + - name: ate-controller + image: {{ include "substrate.componentImage" (list "atecontroller" .) }} + args: + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" +{{- if .Values.otel.endpoint }} + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + volumes: + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml new file mode 100644 index 0000000000..c026f3577f --- /dev/null +++ b/charts/substrate/templates/atelet.yaml @@ -0,0 +1,232 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atelet +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atelet-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["csidriverconfigs"] + verbs: ["get", "watch", "list"] +# ClusterTrustBundles referenced by SystemInfo trustBundle data sources are +# resolved on the node: atelet reads them through an informer and projects +# the sanitized PEM into actors (see cmd/atelet/trustbundle.go). +- apiGroups: ["certificates.k8s.io"] + resources: ["clustertrustbundles"] + verbs: ["get", "watch", "list"] +# SandboxConfigs are watched to pre-download sandbox assets into the node's +# cache before the first actor needs them (see cmd/atelet/sandbox_prewarm.go). +- apiGroups: ["ate.dev"] + resources: ["sandboxconfigs"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atelet-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ include "substrate.fullname" (list "ate-node-critical" .) }} +value: 1000000000 +globalDefault: false +description: "Node-local ate components that every actor activation depends on." +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atelet +spec: + selector: + matchLabels: + app: atelet + template: + metadata: + labels: + app: atelet + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} + priorityClassName: {{ include "substrate.fullname" (list "ate-node-critical" .) }} + containers: + - name: atelet + image: {{ include "substrate.componentImage" (list "atelet" .) }} + args: + - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- with .Values.atelet.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + privileged: true + resources: + requests: + cpu: 50m + memory: 128Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.atelet.extraEnv }} +{{ toYaml . | indent 8 }} +{{- end }} + ports: + - name: grpc + containerPort: 8085 + hostPort: 8085 + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: device-plugins + mountPath: /var/lib/kubelet/device-plugins + - name: host-dev + mountPath: /host/dev + readOnly: true + volumes: + - name: run-ateom + hostPath: + path: /var/lib/ateom-gvisor + type: DirectoryOrCreate + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: device-plugins + hostPath: + path: /var/lib/kubelet/device-plugins + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml new file mode 100644 index 0000000000..850120b65d --- /dev/null +++ b/charts/substrate/templates/atenet-egress.yaml @@ -0,0 +1,237 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + frontendPolicies: + accessLog: + add: + substrate.connect.authority: source.connectHeaders["host"] + # Authorize the actor identity at CONNECT-accept, before any tunnel + # (HTTP, TLS, or opaque TCP) is terminated. Fails closed when the + # control plane is unreachable. + substrateEgressActorResolution: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + + binds: + - port: 8443 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/actor-id-ca-certs/ca.crt + routes: [] + - mode: internal + protocol: AUTO + listeners: + - protocol: TLS + hostname: "*" + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] + - protocol: HTTP + routes: + - backends: + - dynamic: + target: source.connectHeaders["host"] + policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + - protocol: TCP + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-egress +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-egress + template: + metadata: + labels: + app: atenet-egress + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-egress" .) }} + securityContext: + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + terminationGracePeriodSeconds: 60 + containers: + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - -f + - /etc/agentgateway/config.yaml + ports: + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: stats + containerPort: 15020 + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 1 + volumeMounts: + - name: config + mountPath: /etc/agentgateway + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: ext-proc + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - router + - --mode=egress + - --namespace={{ .Release.Namespace }} + - --port-extproc=50051 + - --extproc-address=127.0.0.1 + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt + - --otlp-collector-address= + - --envoy-admin-address=localhost:15000 + - --atenet-dataplane=agentgateway + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: extproc + containerPort: 50051 + readinessProbe: + tcpSocket: + port: extproc + periodSeconds: 10 + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + volumes: + - name: config + configMap: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + - name: drain-signal + emptyDir: {} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-ca-certs + secret: + secretName: actor-id-ca-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-egress + ports: + - name: https + port: 443 + targetPort: https + protocol: TCP diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml new file mode 100644 index 0000000000..1cb84b69d5 --- /dev/null +++ b/charts/substrate/templates/atenet-router.yaml @@ -0,0 +1,360 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + statsAddr: 0.0.0.0:15020 + # Actor sandboxes behind a worker IP are replaced between requests. Do + # not retain an idle connection that may belong to the previous actor. + backend: + poolMaxSize: 0 + +{{- if .Values.otel.endpoint }} + frontendPolicies: + tracing: + host: $AGENTGATEWAY_OTLP_ADDRESS + protocol: grpc + randomSampling: 0.01 +{{- end }} + + backends: + - name: dynamic + dynamic: {} + policies: + backendTunnel: + proxy: + backend: /dynamic + mode: connect + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true + + gateways: + http: + port: 8080 + protocol: HTTP + https: + port: 8443 + protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + + routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors + gateways: + - http + - https + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 + + binds: + - port: 8081 + tunnelProtocol: connect + listeners: + - protocol: HTTP + routes: [] + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-router + template: + metadata: + labels: + app: atenet-router + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "15020" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + containers: + - name: atenet-router + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "router" + - "--mode=ingress" + - "--atenet-dataplane=agentgateway" + - "--namespace={{ .Release.Namespace }}" + - "--port-http=8080" + - "--port-extproc=50051" + - "--extproc-address=127.0.0.1" + - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--status-port=4040" + - "--port-https=8443" + - "--port-connect=8081" + - "--port-connect-tls=8444" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: extproc + containerPort: 50051 + - name: status + containerPort: 4040 + - name: metrics + containerPort: 9090 + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - "-f" + - "/etc/agentgateway/config.yaml" +{{- if .Values.otel.endpoint }} + env: + - name: AGENTGATEWAY_OTLP_ADDRESS + value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 + - name: connect + containerPort: 8081 + - name: connect-tls + containerPort: 8444 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 + volumeMounts: + - name: agentgateway-config + mountPath: /etc/agentgateway + - name: "servicedns" + mountPath: "/run/servicedns.podcert.ate.dev" + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns-ca + readOnly: true + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + volumes: + - name: agentgateway-config + configMap: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + - name: "servicedns" + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP + - name: connect + port: 8081 + targetPort: 8081 + protocol: TCP + - name: connect-tls + port: 8444 + targetPort: 8444 + protocol: TCP + - name: status + port: 4040 + targetPort: status + protocol: TCP + - name: stats + port: 15020 + targetPort: gw-metrics + protocol: TCP diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml new file mode 100644 index 0000000000..073291828b --- /dev/null +++ b/charts/substrate/templates/namespace.yaml @@ -0,0 +1,22 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml new file mode 100644 index 0000000000..86fc23b4a9 --- /dev/null +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -0,0 +1,198 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: Namespace +metadata: + name: podcertificate-controller-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +rules: +# The service signer needs to be able to read services and pods. +- apiGroups: + - "" + resources: + - services + - pods + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests + verbs: + - get + - list + - watch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests/status + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - servicedns.podcert.ate.dev/* + - podidentity.podcert.ate.dev/* + verbs: + - sign + - attest +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: podcertificate-controller-system + name: coordinator +rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - create + - get + - list + - watch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: podcertificate-controller-is-a-coordinator + namespace: podcertificate-controller-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coordinator +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podcertificate-controller + namespace: podcertificate-controller-system + labels: + app: podcertificate-controller +spec: + replicas: 1 + selector: + matchLabels: + app: podcertificate-controller + template: + metadata: + labels: + app: podcertificate-controller + spec: + containers: + - name: controller + image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} + args: + - --in-cluster=true + - --sharding-pod-namespace=$(POD_NAMESPACE) + - --sharding-pod-name=$(POD_NAME) + - --sharding-pod-uid=$(POD_UID) + - --sharding-application-name=podcertificate-controller + - --service-dns-ca-pool=/run/ca-state/service-dns-pool.json + - --pod-identity-ca-pool=/run/ca-state/pod-identity-pool.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + volumeMounts: + - name: "ca-state" + mountPath: "/run/ca-state" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: "ca-state" + projected: + sources: + - secret: + name: "service-dns-ca-pool" + items: + - key: "pool" + path: "service-dns-pool.json" + - secret: + name: "pod-identity-ca-pool" + items: + - key: "pool" + path: "pod-identity-pool.json" + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: default + terminationGracePeriodSeconds: 30 diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml new file mode 100644 index 0000000000..ce4a4efdd7 --- /dev/null +++ b/charts/substrate/templates/postgres.yaml @@ -0,0 +1,234 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $name := include "substrate.fullname" (list "postgres" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ .Release.Namespace }} +data: + postgresql.conf: | + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_key_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + pg_hba.conf: | + # Local socket access is limited to processes in this pod and is used by + # health checks, the workload's idempotent database bootstrap, and the + # tls-reloader sidecar's configuration reloads. + local all all trust + # PostgreSQL verifies client certificates against the pod-identity CA. It + # does not need its own serving CA because it never verifies its server certificate. + hostssl all all all trust clientcert=verify-ca + reload-tls.sh: | + # PostgreSQL opens ssl_cert_file, ssl_key_file and ssl_ca_file at startup + # and on SIGHUP, and nowhere else. The kubelet replaces the projected pod + # certificate in place about 30 minutes before it expires, so without this + # loop the server keeps presenting the certificate it booted with until it + # expires about a day later and every client stops trusting it. + set -eu + + # As PID 1 this shell only sees SIGTERM if a handler is installed, and only + # acts on it between commands, so the sleep below runs in the background + # and is waited on. Without both halves the pod takes the full termination + # grace period to go away. + trap 'exit 0' TERM INT + + CERT=/run/servicedns.podcert.ate.dev/credential-bundle.pem + CA=/run/podidentity.podcert.ate.dev/trust-bundle.pem + + # Comfortably inside the 30m headroom (notAfter - beginRefreshAt) that + # cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go + # leaves; hashing two small files costs nothing. + INTERVAL=60 + + reloaded="" + while true; do + current="$(sha256sum "${CERT}" "${CA}")" + # Reloading fails until the server is accepting connections, which is + # where every pod starts out, so only record a hash once it has worked. + # Starting empty also means a restart of this container costs one + # redundant reload rather than a missed one. + if [ "${current}" != "${reloaded}" ] \ + && psql -U postgres -d postgres -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then + reloaded="${current}" + echo "$(date -u +%FT%TZ) reloaded TLS configuration" + fi + sleep "${INTERVAL}" & + wait $! + done +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: {{ $name }} + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app: {{ $name }} + template: + metadata: + labels: + app: {{ $name }} + spec: + securityContext: + # Group ownership of the projected certificate below, and of the data + # volume so that a freshly provisioned one is writable. OnRootMismatch + # keeps the kubelet from walking the data directory on every start, + # which would leave PGDATA group-writable and postgres refusing to run. + fsGroup: 70 + fsGroupChangePolicy: OnRootMismatch + # PostgreSQL re-reads its TLS files only on SIGHUP, so this sidecar + # reloads the server whenever the kubelet rotates the projected pod + # certificate. fsGroup is also what makes that projection readable: the + # kubelet writes it root-owned for as long as the pod's containers do not + # all agree on one non-root user, and grants the fsGroup group access, + # landing the key at root:postgres 0640, the only shared mode PostgreSQL + # accepts. Pinning runAsUser on the postgres container would make the key + # postgres-owned and group-readable, which it rejects. + # See https://www.postgresql.org/docs/current/ssl-tcp.html#SSL-SETUP + initContainers: + - name: tls-reloader + restartPolicy: Always + image: {{ .Values.images.postgres }} + securityContext: + runAsUser: 70 + command: + - /bin/sh + - /etc/postgresql/reload-tls.sh + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + resources: + requests: + cpu: 10m + memory: 32Mi + containers: + - name: postgres + image: {{ .Values.images.postgres }} + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + sleep 1 + done + if ! psql -U postgres -d postgres -Atc \ + "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then + createdb -U postgres atepg + fi + env: + - name: POSTGRES_DB + value: atepg + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"] + initialDelaySeconds: 2 + periodSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 10 + periodSeconds: 10 + args: ["-c", "config_file=/etc/postgresql/postgresql.conf"] + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + - name: data + mountPath: /var/lib/postgresql/data + resources: +{{ toYaml .Values.postgres.resources | indent 10 }} + volumes: + - name: config + configMap: + name: {{ $name }}-config + - name: servicedns + projected: + # 0600 plus the group read that fsGroup adds is the 0640 above. + defaultMode: 0600 + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + # The unix socket directory, shared so the sidecar can ask the running + # server to reload. The image defaults both the server and its clients to + # this path, so nothing else has to know about it. + - name: socket + emptyDir: {} + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.storageSize }} diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml new file mode 100644 index 0000000000..5a240f5baf --- /dev/null +++ b/charts/substrate/templates/role.yaml @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml new file mode 100644 index 0000000000..edaad3cfa8 --- /dev/null +++ b/charts/substrate/templates/rustfs.yaml @@ -0,0 +1,137 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.rustfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "substrate.fullname" (list "rustfs-data" .) }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.rustfs.storageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: {{ .Values.images.rustfs }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: {{ .Values.rustfs.accessKey | quote }} + - name: RUSTFS_SECRET_KEY + value: {{ .Values.rustfs.secretKey | quote }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "substrate.fullname" (list "rustfs-data" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: {{ .Values.images.awsCli }} + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} already exists" + exit 0 + fi + if aws s3api create-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 +{{- end }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml new file mode 100644 index 0000000000..05f851d40d --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -0,0 +1,35 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with +# the platform so gVisor ActorTemplates have a config to name via +# sandboxConfig.configName. +apiVersion: ate.dev/v1alpha1 +kind: SandboxConfig +metadata: + name: gvisor-default +spec: + sandboxClass: gvisor + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" + sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" + arm64: + gvisor: + url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" + sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml new file mode 100644 index 0000000000..f25d43409b --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -0,0 +1,57 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is +# generic (any arch -> any asset name -> {url, sha256}); this policy enforces the +# requirements a given sandbox class actually needs, fail-closed at apply time. +# (url/sha256 being required and well-formed is enforced by the CRD schema.) +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: sandboxconfig-assets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["ate.dev"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["sandboxconfigs"] + validations: + # gVisor needs a release tarball (or legacy runsc binary) for every architecture. + - expression: >- + object.spec.sandboxClass != 'gvisor' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + # The micro-VM (cloud-hypervisor) runtime needs its asset set for every + # architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'microvm' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image', 'kata-config'] + .all(name, name in object.spec.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, kata-image, and kata-config assets for every architecture under spec.assets" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: sandboxconfig-assets +spec: + policyName: sandboxconfig-assets + validationActions: ["Deny"] diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml new file mode 100644 index 0000000000..b9c7734fe0 --- /dev/null +++ b/charts/substrate/values.yaml @@ -0,0 +1,72 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for the substrate chart. +# +# The chart requires ClusterTrustBundle, ClusterTrustBundleProjection, +# PodCertificateRequest, and the certificates.k8s.io/v1beta1 API. + +# Set to true to have the chart create the release namespace. +# Off by default — most helm workflows expect the namespace to already exist +# (helm install -n --create-namespace). Enable for the generated +# manifests/ate-install/ install path (kubectl apply). +createNamespace: false + +postgres: + storageSize: 1Gi + connectionString: "" + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + +rustfs: + enabled: true + storageSize: 1Gi + bucket: ate-snapshots + accessKey: rustfsadmin + secretKey: rustfsadmin + +# atelet daemonset overrides. Defaults use the in-cluster RustFS deployment for +# snapshots. Set rustfs.enabled=false and override these fields when using +# external storage. +# extraArgs / extraEnv are appended verbatim for installer-specific knobs +# (e.g. registry replacement for kind). +atelet: + gcpAuthForImagePulls: false + storageBackend: s3 + extraArgs: [] + extraEnv: [] + +# Name of a ConfigMap in the release namespace that supplies per-environment +# overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). +# Mounted via envFrom with optional=true. Created by the chart from these values. +ateApiServerEnvVarsConfigMap: ate-api-server-envvars + +otel: + endpoint: "" + +image: + registry: ghcr.io/kagent-dev/substrate + tag: "" + +images: + postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 + rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + busybox: busybox:1.36 diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index 218a18c2d8..a7ed1b1cf5 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -22,4 +22,4 @@ package controllers //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch //+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch,namespace=ate-system -//go:generate bash ../../../../hack/run-tool.sh controller-gen rbac:headerFile=../../../../hack/boilerplate/sh.txt,roleName=ate-controller paths="./..." output:rbac:artifacts:config=../../../../manifests/ate-install/generated/ +//go:generate bash ../../../../hack/gen-rbac.sh diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh new file mode 100755 index 0000000000..baa22fa517 --- /dev/null +++ b/hack/gen-rbac.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the controller ClusterRole into the Helm chart and templatize its +# name so multi-release installs do not collide on a cluster-scoped resource. +# +# controller-gen emits a YAML file with a fixed `roleName=` value. We post- +# process that file to swap the static name for the chart's fullname helper, +# matching the convention used by every other resource in charts/substrate/. +# +# Invoked via `go generate ./cmd/atecontroller/internal/controllers/...`. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="${ROOT}/charts/substrate/templates/role.yaml" + +bash "${ROOT}/hack/run-tool.sh" controller-gen \ + "rbac:headerFile=${ROOT}/hack/boilerplate/sh.txt,roleName=ate-controller" \ + paths="${ROOT}/cmd/atecontroller/internal/controllers/..." \ + "output:rbac:artifacts:config=${ROOT}/charts/substrate/templates/" + +# Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` +# at column 0; the substitution is exact-match to stay robust. +sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh new file mode 100755 index 0000000000..1f6790bb73 --- /dev/null +++ b/hack/render-manifests.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS-mode +# install) — the canonical kubectl-apply install path. The chart at +# charts/substrate/ is the single source of truth; this script only renders. +# +# Usage: +# hack/render-manifests.sh # write into manifests/ate-install/ +# hack/render-manifests.sh --check # fail if rendered output differs +# +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${ROOT}/manifests/ate-install" +CHART_DIR="${ROOT}/charts/substrate" +CHECK_MODE="false" +PRESERVED_FILES=( + ate-api-server.yaml + ate-controller.yaml + ate-otel-config.yaml + ate-system-namespace.yaml + atelet.yaml + atenet-egress.yaml + atenet-egress-with-sdsmint.yaml + atenet-router.yaml + atenet-router-monitoring.yaml + pod-certificate-controller.yaml + postgres.yaml + sandboxconfig-gvisor.yaml + sandboxconfig-validation.yaml +) + +if [ "${1:-}" = "--check" ]; then + CHECK_MODE="true" +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "helm not found in PATH" >&2 + exit 1 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helm template substrate "${CHART_DIR}" \ + --namespace ate-system \ + --set auth.mode=mtls \ + --set createNamespace=true \ + --set image.registry=ko://github.com/agent-substrate/substrate/cmd \ + --set image.tag="" \ + > "${TMP_DIR}/all.yaml" + +# Split into per-source files so the directory structure mirrors the chart +# templates, making diffs friendlier. +python3 - "${TMP_DIR}/all.yaml" "${TMP_DIR}/out" <<'PY' +import os, re, sys, yaml +in_path, out_dir = sys.argv[1], sys.argv[2] +os.makedirs(out_dir, exist_ok=True) + +with open(in_path) as f: + raw = f.read() + +# Helm prepends a "# Source: /templates/" comment to each doc. +docs_by_source = {} +for doc in raw.split('\n---\n'): + m = re.search(r'#\s*Source:\s*\S+/templates/(\S+)', doc) + src = m.group(1) if m else "misc.yaml" + # Drop the leading "# Source:" line from the written file. + cleaned = re.sub(r'^\s*#\s*Source:.*\n', '', doc, count=1, flags=re.MULTILINE) + if not cleaned.strip(): + continue + docs_by_source.setdefault(src, []).append(cleaned.strip()) + +for src, docs in docs_by_source.items(): + if src == "namespace.yaml": + src = "ate-system-namespace.yaml" + header = ( + "# Copyright 2026 Google LLC\n" + "#\n" + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n" + "# you may not use this file except in compliance with the License.\n" + "# You may obtain a copy of the License at\n" + "#\n" + "# http://www.apache.org/licenses/LICENSE-2.0\n" + "#\n" + "# Unless required by applicable law or agreed to in writing, software\n" + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + "# See the License for the specific language governing permissions and\n" + "# limitations under the License.\n" + "\n" + "# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh.\n" + "# Run `make helm-template` to regenerate.\n" + "\n" + ) + with open(os.path.join(out_dir, src), "w") as out: + out.write(header) + out.write("\n---\n".join(docs)) + out.write("\n") +PY + +if [ "${CHECK_MODE}" = "true" ]; then + # Only compare top-level files; subdirs like generated/ and kind/ are not + # produced by the chart and live alongside it intentionally. + CHECK_TMP="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT + mkdir -p "${CHECK_TMP}/current" + find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; + for file in "${PRESERVED_FILES[@]}"; do + rm -f "${CHECK_TMP}/current/${file}" "${TMP_DIR}/out/${file}" + done + if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then + echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true + exit 1 + fi + echo "manifests/ate-install/ matches chart output." + exit 0 +fi + +# Replace contents (preserve kind/ and generated/ subdirs which are not chart output). +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ + ! -name 'ate-api-server.yaml' \ + ! -name 'ate-controller.yaml' \ + ! -name 'ate-otel-config.yaml' \ + ! -name 'ate-system-namespace.yaml' \ + ! -name 'atelet.yaml' \ + ! -name 'atenet-egress.yaml' \ + ! -name 'atenet-egress-with-sdsmint.yaml' \ + ! -name 'atenet-router.yaml' \ + ! -name 'atenet-router-monitoring.yaml' \ + ! -name 'pod-certificate-controller.yaml' \ + ! -name 'postgres.yaml' \ + ! -name 'sandboxconfig-gvisor.yaml' \ + ! -name 'sandboxconfig-validation.yaml' \ + -delete +for file in "${PRESERVED_FILES[@]}"; do + rm -f "${TMP_DIR}/out/${file}" +done +cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" +rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" +echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/hack/verify/crd-chart.sh b/hack/verify/crd-chart.sh new file mode 100755 index 0000000000..dc3ef2bdaf --- /dev/null +++ b/hack/verify/crd-chart.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +GENERATED_DIR="manifests/ate-install/generated" +CHART_TEMPLATES_DIR="charts/substrate-crds/templates" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/generated" "${TMP_DIR}/chart" +cp "${GENERATED_DIR}/"ate.dev_*.yaml "${TMP_DIR}/generated/" +cp "${CHART_TEMPLATES_DIR}/"ate.dev_*.yaml "${TMP_DIR}/chart/" + +# The generated CRDs start with a leading document separator after the +# boilerplate header. In chart templates that separator renders as a +# comment-only YAML document, so the chart copies intentionally omit it. +for file in "${TMP_DIR}/generated/"*.yaml; do + awk 'BEGIN { removed = 0 } /^---$/ && removed == 0 { removed = 1; next } { print }' "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +done + +if ! diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" >/dev/null 2>&1; then + echo "charts/substrate-crds/templates is out of sync with ${GENERATED_DIR}" >&2 + echo "Copy updated CRDs into charts/substrate-crds/templates." >&2 + diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" | head -80 >&2 || true + exit 1 +fi + +echo "charts/substrate-crds/templates matches generated CRDs." diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..5199ab9e74 --- /dev/null +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-api-server-envvars + namespace: ate-system +data: + ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml new file mode 100644 index 0000000000..e59bd53f8f --- /dev/null +++ b/manifests/ate-install/ate-client.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-client + namespace: ate-system + labels: + apps: ate-client diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 7fee9bbc1a..a7011a504f 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -58,6 +58,32 @@ data: insecureHost: true routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + # AgentGateway only uses atunnel's CONNECT listener. + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors gateways: - http @@ -77,6 +103,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 # Terminate client CONNECT before internal HTTP routing. binds: @@ -97,6 +126,28 @@ data: listeners: - protocol: HTTP routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors-tunneled matches: - path: @@ -112,6 +163,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 --- apiVersion: v1 diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml new file mode 100644 index 0000000000..65e967c2a6 --- /dev/null +++ b/manifests/ate-install/role.yaml @@ -0,0 +1,130 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ate-controller +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ate-controller + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +--- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml new file mode 100644 index 0000000000..d6be308128 --- /dev/null +++ b/manifests/ate-install/rustfs.yaml @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rustfs-data + namespace: ate-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: rustfs + namespace: ate-system +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rustfs + namespace: ate-system +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: "rustfsadmin" + - name: RUSTFS_SECRET_KEY + value: "rustfsadmin" + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: rustfs-data +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rustfs-bucket-init + namespace: ate-system +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: + - name: AWS_ACCESS_KEY_ID + value: "rustfsadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "rustfsadmin" + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots already exists" + exit 0 + fi + if aws s3api create-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 From c4a8ce5bb49c23d5d1f8423423e06adfdb4f7013 Mon Sep 17 00:00:00 2001 From: Jeremy Alvis <3587901+iplay88keys@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 07/19] Expose PostgreSQL migration settings in the Helm chart Allow an external PostgreSQL instance and a configurable schema, validate connection settings, and pass the schema to the API server. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 2 ++ charts/substrate/templates/ate-api-server-envvars.yaml | 4 ++++ charts/substrate/templates/ate-api-server.yaml | 1 + charts/substrate/templates/postgres.yaml | 2 ++ charts/substrate/values.yaml | 2 ++ manifests/ate-install/ate-api-server-envvars.yaml | 1 + 6 files changed, 12 insertions(+) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index e7364f4e37..0640c9e9a7 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -35,7 +35,9 @@ See `values.yaml` for the full set; the important keys: | Key | Default | Notes | |-----|---------|-------| +| `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | | `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 753c47178b..ca76ae3ef8 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -14,6 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }} +{{- fail "postgres.connectionString is required when postgres.enabled=false" }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -21,3 +24,4 @@ metadata: namespace: {{ .Release.Namespace }} data: ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} + ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index a5073bc9fa..c236999e48 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -85,6 +85,7 @@ spec: - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" - "--postgres-connection-string=@env" + - "--postgres-schema=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml index ce4a4efdd7..26ddb0b1ec 100644 --- a/charts/substrate/templates/postgres.yaml +++ b/charts/substrate/templates/postgres.yaml @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.postgres.enabled }} {{- $name := include "substrate.fullname" (list "postgres" .) -}} apiVersion: v1 kind: ConfigMap @@ -232,3 +233,4 @@ spec: resources: requests: storage: {{ .Values.postgres.storageSize }} +{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index b9c7734fe0..ee420bbad9 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -24,8 +24,10 @@ createNamespace: false postgres: + enabled: true storageSize: 1Gi connectionString: "" + schema: public resources: requests: cpu: "1" diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml index 5199ab9e74..b49cff6e1e 100644 --- a/manifests/ate-install/ate-api-server-envvars.yaml +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -22,3 +22,4 @@ metadata: namespace: ate-system data: ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + ATE_API_POSTGRES_SCHEMA: "public" From c4c7f0ad9c89438ebc1d348496dfc08a86fb7476 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 08/19] Configure API server object storage in the Helm chart Wire the API server snapshot backend and S3 settings to the chart storage configuration. Signed-off-by: Eitan Yarmush --- charts/substrate/templates/ate-api-server.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index c236999e48..b6d1960901 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -111,6 +111,20 @@ spec: {{- if .Values.otel.endpoint }} - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} {{- end }} envFrom: - configMapRef: From 8da55090d57ea8ff9486d30664fc1a6bb3f10a56 Mon Sep 17 00:00:00 2001 From: Krisztian F <103492698+krisztianfekete@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 09/19] Configure per-signal OTLP export and agentgateway access logs Configure trace, metric, and log endpoints independently, expose trace sampling, and route agentgateway access logs through the collector logs pipeline. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 9 +++- charts/substrate/templates/_helpers.tpl | 54 +++++++++++++++++++ .../substrate/templates/ate-api-server.yaml | 5 +- .../substrate/templates/ate-controller.yaml | 5 +- charts/substrate/templates/atelet.yaml | 5 +- charts/substrate/templates/atenet-router.yaml | 39 ++++++++++---- charts/substrate/values.yaml | 18 +++++++ .../ate-install/kind/otel-collector.yaml | 6 +++ 8 files changed, 122 insertions(+), 19 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 0640c9e9a7..516bc2b04e 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,4 +42,11 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | -| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | +| `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | +| `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` | +| `otel.traces.samplingRatio` | `0.01` | Fraction of parentless requests that start a trace, applied to the Go components and the router | +| `otel.metrics.enabled` | `true` | Sets the OTLP metrics exporter to `none`; the Go components do not honor this yet | +| `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | +| `otel.logs.enabled` | `true` | Set to `false` to export no logs; the router access log is the only OTLP log source today | +| `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 32ae087336..1179dda559 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -78,6 +78,60 @@ Plaintext HTTP URL that clients use to reach atenet-router. {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} {{- end -}} +{{/* +OTLP endpoint a signal exports to, or empty when the signal is disabled or no +endpoint resolves. The per-signal endpoint wins over the generic one, matching +the precedence the OpenTelemetry SDK gives OTEL_EXPORTER_OTLP__ENDPOINT +over OTEL_EXPORTER_OTLP_ENDPOINT. + +Usage: + {{ include "substrate.otel.signalEndpoint" (list "traces" .) }} +*/}} +{{- define "substrate.otel.signalEndpoint" -}} +{{- $signal := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $cfg := index $ctx.Values.otel $signal -}} +{{- if $cfg.enabled -}} +{{- $cfg.endpoint | default $ctx.Values.otel.endpoint -}} +{{- end -}} +{{- end -}} + +{{/* +OTEL_* env entries for a Go component, as a list of "- name/value" items. +Empty when nothing under .Values.otel is set, so callers can gate the env +key on the result. + +Usage: + {{- with include "substrate.otel.env" . }} + {{- . | trim | nindent 8 }} + {{- end }} +*/}} +{{- define "substrate.otel.env" -}} +{{- $otel := .Values.otel -}} +{{- if $otel.endpoint }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ $otel.endpoint | quote }} +{{- end }} +{{- range $signal := list "traces" "metrics" "logs" }} +{{- $cfg := index $otel $signal }} +{{- if not $cfg.enabled }} +{{- /* "none" is the SDK's own exporter name for "export nothing"; leaving the + endpoint unset would fall back to the SDK default of localhost:4317. */}} +- name: OTEL_{{ upper $signal }}_EXPORTER + value: none +{{- else if $cfg.endpoint }} +- name: OTEL_EXPORTER_OTLP_{{ upper $signal }}_ENDPOINT + value: {{ $cfg.endpoint | quote }} +{{- end }} +{{- end }} +{{- if include "substrate.otel.signalEndpoint" (list "traces" .) }} +- name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio +- name: OTEL_TRACES_SAMPLER_ARG + value: {{ $otel.traces.samplingRatio | quote }} +{{- end }} +{{- end -}} + {{/* Build an image reference for a substrate component binary. diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index b6d1960901..7718ed3827 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -108,9 +108,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} - name: ATE_STORAGE_BACKEND value: {{ .Values.atelet.storageBackend | quote }} diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 31c83b9066..d324d05af5 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -79,10 +79,9 @@ spec: - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- if .Values.otel.endpoint }} +{{- with include "substrate.otel.env" . }} env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- . | trim | nindent 8 }} {{- end }} ports: - name: metrics diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index c026f3577f..e9db195397 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -146,9 +146,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} - name: ATE_STORAGE_BACKEND value: {{ .Values.atelet.storageBackend | quote }} diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 1cb84b69d5..46c244b9eb 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -37,12 +37,25 @@ data: backend: poolMaxSize: 0 -{{- if .Values.otel.endpoint }} +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} frontendPolicies: +{{- if $traces }} tracing: - host: $AGENTGATEWAY_OTLP_ADDRESS + host: $AGENTGATEWAY_OTLP_TRACES_ADDRESS protocol: grpc - randomSampling: 0.01 + randomSampling: {{ .Values.otel.traces.samplingRatio }} +{{- end }} +{{- if $logs }} + # Unsampled per-request access log. It carries the ate.* actor + # attribution, which the sampled traces cannot be aggregated on. + # `fields` stays unset so the OTLP sink inherits the full field set. + accessLog: + otlp: + host: $AGENTGATEWAY_OTLP_LOGS_ADDRESS + protocol: grpc +{{- end }} {{- end }} backends: @@ -232,9 +245,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} ports: - name: extproc @@ -251,10 +263,19 @@ spec: args: - "-f" - "/etc/agentgateway/config.yaml" -{{- if .Values.otel.endpoint }} +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} + # agentgateway takes host:port, not a URL, so the scheme is trimmed. env: - - name: AGENTGATEWAY_OTLP_ADDRESS - value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- if $traces }} + - name: AGENTGATEWAY_OTLP_TRACES_ADDRESS + value: {{ trimPrefix "http://" $traces | quote }} +{{- end }} +{{- if $logs }} + - name: AGENTGATEWAY_OTLP_LOGS_ADDRESS + value: {{ trimPrefix "http://" $logs | quote }} +{{- end }} {{- end }} ports: - name: http diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index ee420bbad9..7f456a20af 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -59,8 +59,26 @@ atelet: # Mounted via envFrom with optional=true. Created by the chart from these values. ateApiServerEnvVarsConfigMap: ate-api-server-envvars +# OTLP export, following the OpenTelemetry SDK environment variable spec. +# endpoint applies to every signal; a signal's own endpoint overrides it. +# A signal exports only when it is enabled and resolves to an endpoint. +# +# traces.enabled and metrics.enabled set OTEL__EXPORTER=none on the +# Go components, which do not read it yet and keep exporting; the setting +# takes effect on the router's agentgateway only. logs.enabled works in full, +# since the agentgateway access log is the only OTLP log source. otel: endpoint: "" + traces: + enabled: true + endpoint: "" + samplingRatio: 0.01 + metrics: + enabled: true + endpoint: "" + logs: + enabled: true + endpoint: "" image: registry: ghcr.io/kagent-dev/substrate diff --git a/manifests/ate-install/kind/otel-collector.yaml b/manifests/ate-install/kind/otel-collector.yaml index 0aab804597..68b5772057 100644 --- a/manifests/ate-install/kind/otel-collector.yaml +++ b/manifests/ate-install/kind/otel-collector.yaml @@ -93,6 +93,12 @@ data: receivers: [otlp] processors: [batch] exporters: [prometheus, debug] + # The router access log arrives here. Without a logs pipeline the + # receiver rejects the export and the records are lost silently. + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug] # The count pipelines read the same receiver as the two pipelines # above, and they only count. The connector declares MutatesData From 7bf05bb2a300bcd5a35ed23b5d84cb68b1999246 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:28 +0000 Subject: [PATCH 10/19] Make local verification independent of registry and filesystem timing Isolate sandbox asset download tests from pause image pulls, explicitly advance the CA file timestamp, and disable VCS stamping for license checks in temporary verification worktrees. Signed-off-by: Eitan Yarmush --- cmd/atelet/sandbox_prewarm_test.go | 12 ++---------- hack/update/licenses.sh | 8 ++++++++ internal/volume/csi/tls_test.go | 5 +++++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index 71030d34d3..3511d3cc27 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -358,29 +358,21 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { prewarmMaxJitter = 0 t.Cleanup(func() { ateompath.StaticFilesDir, prewarmMaxJitter = origDir, origJitter }) - host := imageVolumeTestRegistry(t) - pauseRef := host + "/pause:3.10" - pushPauseImage(t, pauseRef) - content := []byte("runsc binary bytes") sha := fmt.Sprintf("%x", sha256.Sum256(content)) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - cfg.Spec.PauseImage = pauseRef + cfg.Spec.PauseImage = "" ctx := t.Context() client := fake.NewSimpleClientset(cfg) factory := externalversions.NewSharedInformerFactory(client, 0) informer := factory.Api().V1alpha1().SandboxConfigs().Informer() - store, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } herder := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} // Handler first, informer start second, mirroring main: atelet startup // must never wait on this informer's sync, and the initial List replays // the pre-existing config into the handler as an Add. - if err := startSandboxAssetPrewarm(ctx, informer, herder, store, false); err != nil { + if err := startSandboxAssetPrewarm(ctx, informer, herder, nil, false); err != nil { t.Fatalf("startSandboxAssetPrewarm: %v", err) } stopCh := make(chan struct{}) diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index 99e24d731b..2f2dbfde6a 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -24,6 +24,14 @@ OUTDIR="_LICENSES" # under $ROOT # Ensure the tool is built and up-to-date GO_LICENSES_BIN="$(bash "${ROOT}/hack/run-tool.sh" --print-bin-path go-licenses)" +# go-licenses runs in temporary verification worktrees that do not have enough +# VCS metadata for Go's build stamping. +if [[ -n "${GOFLAGS:-}" ]]; then + export GOFLAGS="${GOFLAGS} -buildvcs=false" +else + export GOFLAGS="-buildvcs=false" +fi + # Clean out previous licenses rm -rf "${OUTDIR}" mkdir -p "${OUTDIR}" diff --git a/internal/volume/csi/tls_test.go b/internal/volume/csi/tls_test.go index ad71177441..aaa8edce05 100644 --- a/internal/volume/csi/tls_test.go +++ b/internal/volume/csi/tls_test.go @@ -391,6 +391,11 @@ func TestCAPoolCache_HitAndFileChange(t *testing.T) { // Modify the file. writeFile(t, caPath, ca.certPEM()) + // Advance mtime explicitly; consecutive writes can share a filesystem tick. + modified := cache.fi.ModTime().Add(time.Second) + if err := os.Chtimes(caPath, modified, modified); err != nil { + t.Fatal(err) + } // 3rd call should detect file change and return a newly parsed pool. pool3, err := cache.getCertPool() From b627ce2512412b4f4959509c684f34b8cbe3b625 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:28 +0000 Subject: [PATCH 11/19] Keep the gVisor sandbox alive until application containers are deleted Delete application containers before the pause container so their shared sandbox remains available throughout teardown. Cover the deletion order with a regression test. Signed-off-by: Eitan Yarmush --- cmd/ateom-gvisor/main.go | 4 ++-- cmd/ateom-gvisor/runsc_test.go | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 092005b556..12094f0a07 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -880,8 +880,8 @@ func (r *runsc) stopContainers(ctx context.Context, containers []*ateompb.Contai _ = r.cmdKill(ctx, ctr.GetName(), "SIGKILL") _ = r.cmdWait(ctx, ctr.GetName()) } - _ = r.cmdKill(ctx, ocispec.PauseContainer, "SIGKILL") - _ = r.cmdWait(ctx, ocispec.PauseContainer) + // Keep the sandbox alive for application deletion. cleanupContainers + // force-deletes the pause container after deleting the applications. } func (r *runsc) cleanupContainers(ctx context.Context, containers []*ateompb.Container) error { diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index c46373cd70..b772ebefc9 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -17,13 +17,51 @@ package main import ( + "context" + "os" + "path/filepath" "reflect" "testing" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/ocispec" + "github.com/agent-substrate/substrate/internal/proto/ateompb" ) +func TestCleanupKeepsSandboxAliveUntilApplicationsDeleted(t *testing.T) { + path := filepath.Join(t.TempDir(), "runsc") + // Model runsc's control socket: deleting an application fails once the + // sandbox has stopped, even when deletion is forced. + script := `#!/bin/sh +set -eu +shift 5 +case "$1" in +kill) + if [ "$2" = _pause ]; then touch "$0.stopped"; fi + ;; +delete) + if [ "$3" = _pause ]; then + touch "$0.stopped" + elif [ -e "$0.stopped" ]; then + exit 128 + fi + ;; +esac +` + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + r := &runsc{path: path, actorUID: "test-actor"} + containers := []*ateompb.Container{{Name: "counter"}} + r.stopContainers(context.Background(), containers) + if err := r.cleanupContainers(context.Background(), containers); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path + ".stopped"); err != nil { + t.Fatalf("sandbox was not stopped: %v", err) + } +} + func TestKillArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", From fe81ce868bb24c1741f2f730dad0f71aa194de9b Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:28 +0000 Subject: [PATCH 12/19] Add fork synchronization skill with temporary asset cleanup Rebuild the fork on upstream while preserving features, require agentgateway runtime validation, and use a guarded push. Delete task-owned clusters and disposable assets before finishing while preserving shared resources and recovery data. Signed-off-by: Eitan Yarmush --- .agents/skills/update-against-main/SKILL.md | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .agents/skills/update-against-main/SKILL.md diff --git a/.agents/skills/update-against-main/SKILL.md b/.agents/skills/update-against-main/SKILL.md new file mode 100644 index 0000000000..ab86694708 --- /dev/null +++ b/.agents/skills/update-against-main/SKILL.md @@ -0,0 +1,35 @@ +--- +name: update-against-main +description: Merge agent-substrate/substrate main into the kagent-dev/substrate fork's main branch, resolve conflicts, validate the result, and safely update the fork. Use only when explicitly synchronizing the fork's main branch with upstream main. Do not use for updating, rebasing, or resolving conflicts in feature branches or pull requests. +--- + +# Update Against Main + +This skill applies only to synchronizing the fork's `main` branch. Do not invoke it for a feature branch or PR merely because that branch is behind or conflicts with `main`. + +1. Confirm the worktree, current branch, tracking branch, and remotes. Do not disturb unrelated changes. +2. Fetch `origin/main` and `upstream/main`, inspect their divergence, and create a dated backup branch from `origin/main`. +3. Rebuild `main` from `upstream/main` by replaying only intentional fork feature commits in dependency order. Drop merge commits and fork commits superseded by upstream. +4. Resolve conflicts in favor of current upstream APIs while preserving the remaining fork features. Inspect the resulting diff and linear history. +5. Keep Helm charts synchronized with their corresponding manifests. When either changes, inspect and update the other while preserving intentional Helm templating and conditionals, then run `make verify-helm-template` and `make verify-crd-chart` and compare any relevant resources not covered by those checks. +6. Run `make test` and `make verify`. +7. Run the real Kind E2E matrix from `.github/workflows/pr-workflow.yaml`, but use agentgateway for all fork testing: + - Use a dedicated cluster name and kubeconfig; record the temporary assets created by this run. Before recreating with `hack/create-kind-cluster.sh`, delete any old cluster owned by this sync using `hack/kind.sh delete cluster --name "$cluster_name"` with its dedicated `KUBECONFIG`. + - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway`. + - Deploy the micro-VM demo with `hack/run-microvm-demo-kind.sh --skip-control-plane` so it does not reinstall the control plane. + - Deploy the gVisor counter demo and both standard egress demos. + - The full gVisor suite: `hack/run-e2e-kind.sh -v -args --no-color` + - The full micro-VM suite with the CI environment: `E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh -v -args --no-color` + - Switch egress to agentgateway sdsmint, then run the MITM trust and targeted networking lanes for both runtimes exactly as the workflow specifies. + - Verify the live router and egress workloads use agentgateway. Never use Envoy for fork validation. +8. Treat `go test ./internal/e2e/...` without `-args --e2e` as compilation/package testing, not E2E coverage. +9. Do not push when unit, verification, or E2E checks fail or cannot run. Report the exact blocker instead. +10. After all checks pass, verify the worktree and rewritten commits, then update the fork with `git push --force-with-lease origin main`. Never use an unguarded force push. +11. Clean up temporary assets before finishing, including on failure or cancellation: + - Stop this run's test/install processes and port-forwards. Save any diagnostics needed to explain failures before tearing down workloads. + - Delete the task-owned Kind cluster with `hack/kind.sh delete cluster --name "$cluster_name"` using its dedicated `KUBECONFIG`. Verify both the cluster and its node containers are gone before removing the kubeconfig. + - Remove this run's disposable assets: generated micro-VM disks and images, downloaded bundles, build outputs, scratch scripts, and temporary kubeconfigs. Remove task-only Docker images, containers, and volumes once no longer in use. Preserve shared assets, caches, registries, and unrelated clusters; do not use global Docker prune commands. + - After a successful push, remove clean temporary worktrees with `git worktree remove` from another checkout. Preserve backup branches, unpushed commits, uncommitted changes, and diagnostics needed for unresolved failures. + - If teardown stalls (for example, Docker reports no exit event), inspect only the task's node containers, retry scoped deletion once, and report any remaining resources and exact blocker. Do not restart the global Docker daemon or kill unrelated processes. Report cleanup separately from validation so leftover assets are not hidden by passing tests. + +Use the current CI workflow as the source of truth for cluster setup, images, demos, runtime coverage, and environment variables, with the agentgateway-only override above. Never claim E2E passed unless workloads ran against the cluster. From 11d6a03b5d106c84196a1f5db6f83e6ba6d64afe Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 10:22:49 -0400 Subject: [PATCH 13/19] Align Helm E2E with the canonical installation Select agentgateway expectations in the Helm test job, align the chart sandbox assets with the canonical manifest, and enable the CONNECT tunnel logging used by egress validation. This retains upstream gVisor checkpoint and restore fixes and closes configuration gaps between Helm and manifest installations. Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 1 + charts/substrate/templates/atenet-egress.yaml | 3 +++ charts/substrate/templates/sandboxconfig-gvisor.yaml | 8 ++++---- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 81b3a46103..7820c668a5 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -24,6 +24,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: helm-e2e + E2E_ATENET_DATAPLANE: agentgateway steps: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 850120b65d..41154a98e3 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -110,6 +110,9 @@ spec: args: - -f - /etc/agentgateway/config.yaml + env: + - name: RUST_LOG + value: debug ports: - name: https containerPort: 8443 diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 05f851d40d..286584eb01 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -27,9 +27,9 @@ spec: assets: amd64: gvisor: - url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" - sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" arm64: gvisor: - url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" - sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" From 80b9a64e1fd4371afd3bc2bc533572d1c524ae5a Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 11:17:53 -0400 Subject: [PATCH 14/19] Allow extra ateapi arguments in the Helm chart Expose ateApi.extraArgs so installations can configure API flags such as the template resync interval without editing the deployment template. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 1 + charts/substrate/templates/ate-api-server.yaml | 3 +++ charts/substrate/values.yaml | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 516bc2b04e..8a78d396fc 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,6 +42,7 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | | `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` | diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 7718ed3827..267232eb8d 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -93,6 +93,9 @@ spec: - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" - "--drain-delay=13s" - "--drain-timeout=15s" +{{- with .Values.ateApi.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} env: - name: POD_NAME valueFrom: diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 7f456a20af..b410d2b65f 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -54,6 +54,10 @@ atelet: extraArgs: [] extraEnv: [] +# Additional arguments appended to the ateapi defaults. +ateApi: + extraArgs: [] + # Name of a ConfigMap in the release namespace that supplies per-environment # overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). # Mounted via envFrom with optional=true. Created by the chart from these values. From 807dd5db9ddc252f875d156fd53ea1673d7e7b61 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 15:50:31 +0000 Subject: [PATCH 15/19] Retry layer pulls that join an eviction flight A layer pull can share a singleflight call with retirement and return without unpacking the removed layer. Distinguish pull results from retirement results and retry after retirement completes. Cover the interleaving with a deterministic concurrency test. Signed-off-by: Eitan Yarmush --- internal/imagecache/imagecache.go | 35 +++++++++++++++------------ internal/imagecache/retire_test.go | 38 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index e0a0302347..9a7d57a99c 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -652,24 +652,29 @@ func (s *Store) pull(ctx context.Context, parsedRef name.Reference, digest v1.Ha // collapsing concurrent requests for the same layer across images. func (s *Store) ensureLayer(ctx context.Context, diffID v1.Hash, layer v1.Layer) (string, error) { dir := s.layerDir(diffID) - _, err, _ := s.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { - if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { - // Refresh the dir mtime inside the flight: retireLayer re-checks - // the mtime in this same flight, so a layer reused here can - // never be renamed away between this stat and the image record - // that will re-reference it. - now := time.Now() - if err := os.Chtimes(dir, now, now); err != nil { - slog.WarnContext(ctx, "Failed to refresh layer mtime on reuse", slog.String("diffid", diffID.String()), slog.Any("err", err)) + for { + result, err, _ := s.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { + if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { + // Refresh the dir mtime inside the flight: retireLayer re-checks + // the mtime in this same flight, so a layer reused here can + // never be renamed away between this stat and the image record + // that will re-reference it. + now := time.Now() + if err := os.Chtimes(dir, now, now); err != nil { + slog.WarnContext(ctx, "Failed to refresh layer mtime on reuse", slog.String("diffid", diffID.String()), slog.Any("err", err)) + } + return dir, nil } - return nil, nil + return dir, s.unpackLayerToPool(ctx, diffID, layer) + }) + if err != nil { + return "", err + } + // A joined retirement returns nil, so the layer still needs a pull. + if result != nil { + return dir, nil } - return nil, s.unpackLayerToPool(ctx, diffID, layer) - }) - if err != nil { - return "", err } - return dir, nil } // unpackLayerToPool streams the layer (download → decompress → untar) into a diff --git a/internal/imagecache/retire_test.go b/internal/imagecache/retire_test.go index fe45c5a3cc..2abf3efa28 100644 --- a/internal/imagecache/retire_test.go +++ b/internal/imagecache/retire_test.go @@ -22,6 +22,7 @@ import ( "strings" "sync" "testing" + "testing/synctest" "time" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -201,3 +202,40 @@ func TestRetireLayerVsEnsureImageRace(t *testing.T) { t.Errorf("final layer dir missing: %v", err) } } + +func TestEnsureLayerJoinsRetirement(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + store := newTestStore(t) + layer := layerFromEntries(t, []tarEntry{ + {name: "f", typeflag: tar.TypeReg, mode: 0o644, body: "hi"}, + }) + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + + // Hold a retirement flight while the pull joins it. + release := make(chan struct{}) + go func() { + _, _, _ = store.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { + <-release + return nil, nil + }) + }() + synctest.Wait() + var dir string + go func() { + dir, err = store.ensureLayer(context.Background(), diffID, layer) + }() + synctest.Wait() + close(release) + synctest.Wait() + + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, layerFSDirName, "f")); err != nil { + t.Fatalf("layer was not unpacked after retirement: %v", err) + } + }) +} From 74056c2b61bbd0b8f7960535cece07572a8fe80a Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 10:58:09 +0000 Subject: [PATCH 16/19] Align Helm egress readiness with the metrics endpoint The egress ext_proc server listens on loopback. Probe the metrics readiness endpoint so Kubernetes can observe readiness through the pod IP, matching the upstream manifests. Signed-off-by: Eitan Yarmush --- charts/substrate/templates/atenet-egress.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 41154a98e3..74f377b4a1 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -171,10 +171,15 @@ spec: ports: - name: extproc containerPort: 50051 + - name: metrics + containerPort: 9090 readinessProbe: - tcpSocket: - port: extproc - periodSeconds: 10 + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 volumeMounts: - name: servicedns mountPath: /run/servicedns.podcert.ate.dev From 60c7821ad089ad1262ac1cdd1a76aa324c871cc4 Mon Sep 17 00:00:00 2001 From: Jeremy Alvis Date: Fri, 4 Sep 2026 11:37:07 -0700 Subject: [PATCH 17/19] Support PostgreSQL connection secrets Allow ate-api-server to read an external PostgreSQL connection string from a Secret while keeping schema configuration in the existing ConfigMap. Add Helm unit coverage for bundled, literal, Secret-backed, and invalid setups. --- .github/workflows/helm-e2e.yaml | 4 + Makefile | 4 + charts/substrate/README.md | 1 + charts/substrate/templates/_helpers.tpl | 6 + .../templates/ate-api-server-envvars.yaml | 10 +- .../substrate/templates/ate-api-server.yaml | 8 ++ charts/substrate/tests/postgres_test.yaml | 125 ++++++++++++++++++ charts/substrate/values.yaml | 6 + 8 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 charts/substrate/tests/postgres_test.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 7820c668a5..796fe4deb1 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -34,6 +34,10 @@ jobs: go-version-file: go.mod - name: Setup Helm uses: azure/setup-helm@v4 + - name: Test Helm chart + run: | + helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.3 + make helm-test - name: Cache micro-VM assets uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: diff --git a/Makefile b/Makefile index 11307e18d4..de317716f8 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,10 @@ clean: helm-template: @./hack/render-manifests.sh +.PHONY: helm-test +helm-test: + @helm unittest charts/substrate + # Verify that manifests/ate-install/ matches the chart output. Used in CI. .PHONY: verify-helm-template verify-helm-template: diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 8a78d396fc..84e4ff2ed7 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -37,6 +37,7 @@ See `values.yaml` for the full set; the important keys: |-----|---------|-------| | `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | | `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.connectionStringSecretRef` | disabled | Read an external PostgreSQL connection string from a Secret; its name defaults to `-postgres-connection` when enabled | | `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 1179dda559..efcd0d1ff6 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -78,6 +78,12 @@ Plaintext HTTP URL that clients use to reach atenet-router. {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} {{- end -}} +{{/* PostgreSQL connection Secret, when configured. */}} +{{- define "substrate.postgres.connectionStringSecretEnabled" -}} +{{- $ref := .Values.postgres.connectionStringSecretRef | default dict -}} +{{- if or (get $ref "enabled") (get $ref "name") -}}true{{- end -}} +{{- end -}} + {{/* OTLP endpoint a signal exports to, or empty when the signal is disabled or no endpoint resolves. The per-signal endpoint wins over the generic one, matching diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index ca76ae3ef8..0d18a942af 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -14,8 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }} -{{- fail "postgres.connectionString is required when postgres.enabled=false" }} +{{- $secretEnabled := include "substrate.postgres.connectionStringSecretEnabled" . -}} +{{- if and .Values.postgres.connectionString $secretEnabled }} +{{- fail "postgres.connectionString and postgres.connectionStringSecretRef are mutually exclusive" }} +{{- end }} +{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) (not $secretEnabled) }} +{{- fail "postgres.connectionString or postgres.connectionStringSecretRef is required when postgres.enabled=false" }} {{- end }} apiVersion: v1 kind: ConfigMap @@ -23,5 +27,7 @@ metadata: name: {{ .Values.ateApiServerEnvVarsConfigMap }} namespace: {{ .Release.Namespace }} data: + {{- if not $secretEnabled }} ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} + {{- end }} ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 267232eb8d..7c762f7310 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -97,6 +97,14 @@ spec: {{ toYaml . | indent 8 }} {{- end }} env: +{{- $connectionStringSecretRef := .Values.postgres.connectionStringSecretRef }} +{{- if include "substrate.postgres.connectionStringSecretEnabled" . }} + - name: ATE_API_POSTGRES_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{ get $connectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) | quote }} + key: {{ get $connectionStringSecretRef "key" | default "connectionString" | quote }} +{{- end }} - name: POD_NAME valueFrom: fieldRef: diff --git a/charts/substrate/tests/postgres_test.yaml b/charts/substrate/tests/postgres_test.yaml new file mode 100644 index 0000000000..4edb859f97 --- /dev/null +++ b/charts/substrate/tests/postgres_test.yaml @@ -0,0 +1,125 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +suite: PostgreSQL configuration +templates: +- ate-api-server-envvars.yaml +- ate-api-server.yaml +- postgres.yaml +tests: +- it: configures the bundled PostgreSQL database by default + template: ate-api-server-envvars.yaml + asserts: + - isNotNull: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + - equal: + path: data.ATE_API_POSTGRES_SCHEMA + value: public + +- it: configures an external PostgreSQL connection string and schema + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + connectionString: postgresql://user:pass@database:5432/agents + schema: substrate + asserts: + - equal: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + value: postgresql://user:pass@database:5432/agents + - equal: + path: data.ATE_API_POSTGRES_SCHEMA + value: substrate + +- it: omits bundled PostgreSQL for an external connection + template: postgres.yaml + set: + postgres: + enabled: false + connectionString: postgresql://user:pass@database:5432/agents + asserts: + - hasDocuments: + count: 0 + +- it: reads an explicitly named connection Secret + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + name: shared-database + key: url + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: shared-database + key: url + +- it: derives a release-scoped Secret name when enabled + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: RELEASE-NAME-postgres-connection + key: connectionString + +- it: keeps credentials out of the ConfigMap in Secret mode + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + schema: substrate + connectionStringSecretRef: + name: shared-database + asserts: + - notExists: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + - equal: + path: data.ATE_API_POSTGRES_SCHEMA + value: substrate + +- it: rejects disabling bundled PostgreSQL without a connection + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + asserts: + - failedTemplate: + errorMessage: postgres.connectionString or postgres.connectionStringSecretRef is required when postgres.enabled=false + +- it: rejects configuring both a literal connection and a Secret + template: ate-api-server-envvars.yaml + set: + postgres: + connectionString: postgresql://user:pass@database:5432/agents + connectionStringSecretRef: + name: shared-database + asserts: + - failedTemplate: + errorMessage: postgres.connectionString and postgres.connectionStringSecretRef are mutually exclusive diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index b410d2b65f..8526aa82ea 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -27,6 +27,12 @@ postgres: enabled: true storageSize: 1Gi connectionString: "" + # Read the connection string from a Secret instead of values/ConfigMap. + connectionStringSecretRef: + # When enabled with no name, use the release-scoped postgres-connection name. + enabled: false + name: "" + key: connectionString schema: public resources: requests: From 38af2c9e4d88fefd93524ed88785b2b5b6ae53fe Mon Sep 17 00:00:00 2001 From: Jeremy Alvis Date: Fri, 4 Sep 2026 11:40:40 -0700 Subject: [PATCH 18/19] Fix helm tests --- .github/workflows/helm-e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 796fe4deb1..53df41b600 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -36,7 +36,7 @@ jobs: uses: azure/setup-helm@v4 - name: Test Helm chart run: | - helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.3 + helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.3 --verify=false make helm-test - name: Cache micro-VM assets uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 From 612e866e719a9fe732895c0a34556acec74f48a9 Mon Sep 17 00:00:00 2001 From: Jeremy Alvis Date: Fri, 18 Sep 2026 14:36:25 -0700 Subject: [PATCH 19/19] Add in separate ddl/dml support or substrate --- charts/substrate/README.md | 6 +- charts/substrate/templates/_helpers.tpl | 6 ++ .../templates/ate-api-server-envvars.yaml | 10 +++ .../substrate/templates/ate-api-server.yaml | 9 +++ charts/substrate/tests/postgres_test.yaml | 69 +++++++++++++++++++ charts/substrate/values.yaml | 7 ++ 6 files changed, 105 insertions(+), 2 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 84e4ff2ed7..510a49522d 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -36,8 +36,10 @@ See `values.yaml` for the full set; the important keys: | Key | Default | Notes | |-----|---------|-------| | `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | -| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | -| `postgres.connectionStringSecretRef` | disabled | Read an external PostgreSQL connection string from a Secret; its name defaults to `-postgres-connection` when enabled | +| `postgres.connectionString` | `""` (in-cluster) | Runtime/DML connection for external PostgreSQL | +| `postgres.connectionStringSecretRef` | disabled | Read the runtime/DML connection from a Secret; its name defaults to `-postgres-connection` when enabled | +| `postgres.ddlConnectionString` | `""` (runtime connection) | Optional schema-owner connection for migrations and maintenance | +| `postgres.ddlConnectionStringSecretRef` | disabled | Read the optional schema-owner connection string from a Secret | | `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index efcd0d1ff6..017e2ec431 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -84,6 +84,12 @@ Plaintext HTTP URL that clients use to reach atenet-router. {{- if or (get $ref "enabled") (get $ref "name") -}}true{{- end -}} {{- end -}} +{{/* PostgreSQL DDL connection Secret, when configured. */}} +{{- define "substrate.postgres.ddlConnectionStringSecretEnabled" -}} +{{- $ref := .Values.postgres.ddlConnectionStringSecretRef | default dict -}} +{{- if or (get $ref "enabled") (get $ref "name") -}}true{{- end -}} +{{- end -}} + {{/* OTLP endpoint a signal exports to, or empty when the signal is disabled or no endpoint resolves. The per-signal endpoint wins over the generic one, matching diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 0d18a942af..3a13e254b4 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -15,12 +15,19 @@ limitations under the License. */}} {{- $secretEnabled := include "substrate.postgres.connectionStringSecretEnabled" . -}} +{{- $ddlSecretEnabled := include "substrate.postgres.ddlConnectionStringSecretEnabled" . -}} {{- if and .Values.postgres.connectionString $secretEnabled }} {{- fail "postgres.connectionString and postgres.connectionStringSecretRef are mutually exclusive" }} {{- end }} +{{- if and .Values.postgres.ddlConnectionString $ddlSecretEnabled }} +{{- fail "postgres.ddlConnectionString and postgres.ddlConnectionStringSecretRef are mutually exclusive" }} +{{- end }} {{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) (not $secretEnabled) }} {{- fail "postgres.connectionString or postgres.connectionStringSecretRef is required when postgres.enabled=false" }} {{- end }} +{{- if and (or .Values.postgres.ddlConnectionString $ddlSecretEnabled) (empty .Values.postgres.connectionString) (not $secretEnabled) }} +{{- fail "postgres.connectionString or postgres.connectionStringSecretRef is required when a DDL connection is configured" }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -30,4 +37,7 @@ data: {{- if not $secretEnabled }} ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} {{- end }} + {{- if and .Values.postgres.ddlConnectionString (not $ddlSecretEnabled) }} + ATE_API_POSTGRES_DDL_CONNECTION_STRING: {{ .Values.postgres.ddlConnectionString | quote }} + {{- end }} ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 7c762f7310..140e43cd06 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -85,6 +85,7 @@ spec: - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" - "--postgres-connection-string=@env" + - "--postgres-ddl-connection-string=@env" - "--postgres-schema=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" @@ -104,6 +105,14 @@ spec: secretKeyRef: name: {{ get $connectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) | quote }} key: {{ get $connectionStringSecretRef "key" | default "connectionString" | quote }} +{{- end }} +{{- $ddlConnectionStringSecretRef := .Values.postgres.ddlConnectionStringSecretRef }} +{{- if include "substrate.postgres.ddlConnectionStringSecretEnabled" . }} + - name: ATE_API_POSTGRES_DDL_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{ get $ddlConnectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) | quote }} + key: {{ get $ddlConnectionStringSecretRef "key" | default "ddlConnectionString" | quote }} {{- end }} - name: POD_NAME valueFrom: diff --git a/charts/substrate/tests/postgres_test.yaml b/charts/substrate/tests/postgres_test.yaml index 4edb859f97..56225fb815 100644 --- a/charts/substrate/tests/postgres_test.yaml +++ b/charts/substrate/tests/postgres_test.yaml @@ -42,6 +42,21 @@ tests: path: data.ATE_API_POSTGRES_SCHEMA value: substrate +- it: configures separate runtime and DDL connection strings + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + connectionString: postgresql://runtime@database:5432/agents + ddlConnectionString: postgresql://owner@database:5432/agents + asserts: + - equal: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + value: postgresql://runtime@database:5432/agents + - equal: + path: data.ATE_API_POSTGRES_DDL_CONNECTION_STRING + value: postgresql://owner@database:5432/agents + - it: omits bundled PostgreSQL for an external connection template: postgres.yaml set: @@ -71,6 +86,39 @@ tests: name: shared-database key: url +- it: reads separate runtime and DDL connections from a Secret + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + name: substrate-database + key: runtimeUrl + ddlConnectionStringSecretRef: + name: substrate-database + key: ddlUrl + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-ddl-connection-string=@env + - contains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: substrate-database + key: runtimeUrl + - contains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_DDL_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: substrate-database + key: ddlUrl + - it: derives a release-scoped Secret name when enabled template: ate-api-server.yaml documentIndex: 3 @@ -123,3 +171,24 @@ tests: asserts: - failedTemplate: errorMessage: postgres.connectionString and postgres.connectionStringSecretRef are mutually exclusive + +- it: rejects a DDL connection without an explicit runtime connection + template: ate-api-server-envvars.yaml + set: + postgres: + ddlConnectionString: postgresql://owner@database:5432/agents + asserts: + - failedTemplate: + errorMessage: postgres.connectionString or postgres.connectionStringSecretRef is required when a DDL connection is configured + +- it: rejects both a literal and Secret-backed DDL connection + template: ate-api-server-envvars.yaml + set: + postgres: + connectionString: postgresql://runtime@database:5432/agents + ddlConnectionString: postgresql://owner@database:5432/agents + ddlConnectionStringSecretRef: + name: substrate-database + asserts: + - failedTemplate: + errorMessage: postgres.ddlConnectionString and postgres.ddlConnectionStringSecretRef are mutually exclusive diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 8526aa82ea..6fd125bec8 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -26,6 +26,7 @@ createNamespace: false postgres: enabled: true storageSize: 1Gi + # Runtime/DML connection. Also used for DDL when no DDL connection is set. connectionString: "" # Read the connection string from a Secret instead of values/ConfigMap. connectionStringSecretRef: @@ -33,6 +34,12 @@ postgres: enabled: false name: "" key: connectionString + # Optional DDL and maintenance connection. Defaults to connectionString. + ddlConnectionString: "" + ddlConnectionStringSecretRef: + enabled: false + name: "" + key: ddlConnectionString schema: public resources: requests: