From baf9418ea6dd8b751849155fbf7d03ff2c026e4f Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 11:53:40 +0000 Subject: [PATCH 1/7] Add Kubernetes credential sidecar for agentgateway egress Resolve ate-secret URIs from Kubernetes Secrets with injector-only mTLS and default-deny atespace-to-namespace grants. Deploy the provider alongside ateapi so it can reuse the existing Pod identity, certificate, and Service. Wire the provider into agentgateway's HTTPS interception route, with optional Helm and Kustomize configuration and namespace-scoped RBAC setup. Pin the nightly containing credential-provider configuration support; its RPC and URI scheme still need alignment before end-to-end injection works. Co-authored-by: Yufan Su Signed-off-by: Eitan Yarmush --- Makefile | 1 + charts/substrate/README.md | 2 + .../substrate/templates/ate-api-server.yaml | 52 +++ charts/substrate/templates/atenet-egress.yaml | 45 +++ .../templates/credential-provider-policy.yaml | 26 ++ charts/substrate/values.yaml | 11 +- cmd/ate-setup/internal/images/images.go | 1 + cmd/k8s-credential-provider/main.go | 213 ++++++++++ cmd/k8s-credential-provider/main_test.go | 211 ++++++++++ cmd/k8s-credential-provider/manifests_test.go | 254 ++++++++++++ cmd/k8s-credential-provider/policy.go | 96 +++++ cmd/k8s-credential-provider/provider.go | 186 +++++++++ cmd/k8s-credential-provider/provider_test.go | 363 ++++++++++++++++++ docs/kubernetes-credential-provider.md | 157 ++++++++ .../kustomization.yaml | 9 + .../agentgateway/kustomization.yaml | 4 +- .../credential-provider/kustomization.yaml | 25 ++ .../credential-provider/policy.yaml | 16 + .../credential-provider/sidecar.yaml | 82 ++++ .../kubernetes-credentials/kustomization.yaml | 22 ++ 20 files changed, 1773 insertions(+), 3 deletions(-) create mode 100644 charts/substrate/templates/credential-provider-policy.yaml create mode 100644 cmd/k8s-credential-provider/main.go create mode 100644 cmd/k8s-credential-provider/main_test.go create mode 100644 cmd/k8s-credential-provider/manifests_test.go create mode 100644 cmd/k8s-credential-provider/policy.go create mode 100644 cmd/k8s-credential-provider/provider.go create mode 100644 cmd/k8s-credential-provider/provider_test.go create mode 100644 docs/kubernetes-credential-provider.md create mode 100644 manifests/ate-install/components/credential-provider/kustomization.yaml create mode 100644 manifests/ate-install/components/credential-provider/policy.yaml create mode 100644 manifests/ate-install/components/credential-provider/sidecar.yaml create mode 100644 manifests/ate-install/kubernetes-credentials/kustomization.yaml diff --git a/Makefile b/Makefile index 11307e18d4..8c24d567dc 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ CONTROL_PLANE_IMAGES := ./cmd/ateapi \ ./cmd/atecontroller \ ./cmd/atelet \ ./cmd/atenet \ + ./cmd/k8s-credential-provider \ ./cmd/podcertcontroller WORKER_IMAGES := ./cmd/ateom-gvisor \ ./cmd/ateom-microvm diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 8a78d396fc..3aea2ceb53 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,6 +42,8 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `ateApi.credentialProvider.enabled` | `false` | Add the credential-provider sidecar and AGW HTTPS injection; requires a compatible AGW image and MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | +| `ateApi.credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; Kubernetes Secret RBAC is configured separately | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 267232eb8d..23fc11b638 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -74,6 +74,9 @@ spec: annotations: prometheus.io/scrape: "true" prometheus.io/port: "9090" +{{- if .Values.ateApi.credentialProvider.enabled }} + checksum/credential-provider-policy: {{ toJson .Values.ateApi.credentialProvider.namespacePolicies | sha256sum }} +{{- end }} spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} terminationGracePeriodSeconds: 40 @@ -155,7 +158,50 @@ spec: port: 9090 initialDelaySeconds: 10 periodSeconds: 10 +{{- if .Values.ateApi.credentialProvider.enabled }} + - name: credential-provider + image: {{ include "substrate.componentImage" (list "k8s-credential-provider" .) }} + args: + - --listen-address=:50051 + - --metrics-address=:9091 + - --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem + - --client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem + - --injector-spiffe-id=spiffe://cluster.local/ns/{{ .Release.Namespace }}/sa/{{ include "substrate.fullname" (list "atenet-egress" .) }} + - --namespace-policy-file=/etc/credential-provider/policy.yaml + ports: + - name: credentials + containerPort: 50051 + - name: cred-health + containerPort: 9091 + readinessProbe: + httpGet: + path: /readyz + port: cred-health + periodSeconds: 2 + livenessProbe: + httpGet: + path: /healthz + port: cred-health + initialDelaySeconds: 10 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: credential-provider-policy, mountPath: /etc/credential-provider, readOnly: true } +{{- end }} volumes: +{{- if .Values.ateApi.credentialProvider.enabled }} + - name: credential-provider-policy + configMap: + name: {{ include "substrate.fullname" (list "credential-provider-policy" .) }} +{{- end }} - name: servicedns projected: sources: @@ -225,3 +271,9 @@ spec: protocol: TCP port: 443 targetPort: 443 +{{- if .Values.ateApi.credentialProvider.enabled }} + - name: credentials + protocol: TCP + port: 50051 + targetPort: credentials +{{- end }} diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 74f377b4a1..2aef864714 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -56,12 +56,42 @@ data: - mode: internal protocol: AUTO listeners: +{{- if .Values.ateApi.credentialProvider.enabled }} + - protocol: HTTPS + tls: + mode: dynamicCa + cert: /run/egress-mitm/tls.crt + key: /run/egress-mitm/tls.key + routes: + - backends: + - dynamic: {} + policies: + backendTLS: {} + policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + credentialProviders: + - uriAuthority: kubernetes.io + target: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- else }} - protocol: TLS hostname: "*" tcpRoutes: - backends: - dynamic: target: source.connectHeaders["host"] +{{- end }} - protocol: HTTP routes: - backends: @@ -132,6 +162,11 @@ spec: port: readiness periodSeconds: 1 volumeMounts: +{{- if .Values.ateApi.credentialProvider.enabled }} + - name: egress-mitm + mountPath: /run/egress-mitm + readOnly: true +{{- end }} - name: config mountPath: /etc/agentgateway readOnly: true @@ -193,6 +228,16 @@ spec: - name: drain-signal mountPath: /var/run/atenet volumes: +{{- if .Values.ateApi.credentialProvider.enabled }} + - name: egress-mitm + secret: + secretName: egress-mitm-ca-pool + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key +{{- end }} - name: config configMap: name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} diff --git a/charts/substrate/templates/credential-provider-policy.yaml b/charts/substrate/templates/credential-provider-policy.yaml new file mode 100644 index 0000000000..b65aa297d7 --- /dev/null +++ b/charts/substrate/templates/credential-provider-policy.yaml @@ -0,0 +1,26 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.ateApi.credentialProvider.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "credential-provider-policy" .) }} + namespace: {{ .Release.Namespace }} +data: + policy.yaml: | + policies: {{ toJson .Values.ateApi.credentialProvider.namespacePolicies }} +{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index b410d2b65f..9964f19e20 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -57,6 +57,15 @@ atelet: # Additional arguments appended to the ateapi defaults. ateApi: extraArgs: [] + # Optional Kubernetes Secret provider sharing ateapi's Pod and ServiceAccount. + # Enables AGW HTTPS interception and credential injection. Requires a compatible + # images.agentgateway build and the egress-mitm-ca-pool Secret (see docs). + # Secret read access must be granted separately with namespace-scoped RBAC. + credentialProvider: + enabled: false + namespacePolicies: [] + # - atespace: team-a + # allowedNamespaces: [team-a-secrets] # Name of a ConfigMap in the release namespace that supplies per-environment # overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). @@ -92,5 +101,5 @@ images: postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.d376b9e1@sha256:f5650ed21ab9d84a0ee5398763d5950d10880eab53f5b53e0be5e8d06767e467 busybox: busybox:1.36 diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 2b1ed1c628..93a6522a13 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -44,6 +44,7 @@ var Components = []string{ "cmd/atecontroller", "cmd/atelet", "cmd/atenet", + "cmd/k8s-credential-provider", "cmd/ateom-gvisor", "cmd/ateom-microvm", "cmd/podcertcontroller", diff --git a/cmd/k8s-credential-provider/main.go b/cmd/k8s-credential-provider/main.go new file mode 100644 index 0000000000..f82e622b0b --- /dev/null +++ b/cmd/k8s-credential-provider/main.go @@ -0,0 +1,213 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command k8s-credential-provider is the Kubernetes Secrets credential-provider +// plugin: a gRPC service that resolves ate-secret:// URIs of the kubernetes.io +// provider to Kubernetes Secret values. It is the only component in the egress +// credential-injection path with Kubernetes access; the egress gateway and its +// injector never read Secrets directly. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "log/slog" + "net" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/spf13/pflag" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/reflection" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/serverboot" + "github.com/agent-substrate/substrate/internal/version" + "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" +) + +const serviceName = "k8s-credential-provider" + +var ( + injectorSPIFFEID = pflag.String("injector-spiffe-id", "spiffe://cluster.local/ns/ate-system/sa/atenet-egress", "SPIFFE identity of the egress injector allowed to fetch credentials") + listenAddr = pflag.String("listen-address", ":50051", "gRPC listen address") + metricsAddr = pflag.String("metrics-address", ":9090", "Prometheus/health HTTP listen address") + serverBundle = pflag.String("server-cred-bundle", "", "credential bundle (PEM key+chain) presented for serving TLS (required)") + clientCAFile = pflag.String("client-ca-file", "", "CA bundle that caller (injector) client certificates must chain to (required)") + nsPolicyFile = pflag.String("namespace-policy-file", "", "path to the atespace→namespace authorization YAML (required)") + logLevel = pflag.String("log-level", "info", "one of debug, info, warn, error") + drainGrace = pflag.Duration("drain-grace", 5*time.Second, "how long to wait for in-flight RPCs on shutdown before a hard stop") +) + +func main() { + pflag.Parse() + + ctx := context.Background() + serverboot.InitLogger() + if err := serverboot.SetLogLevel(*logLevel); err != nil { + serverboot.Fatal(ctx, "invalid --log-level", err) + } + + slog.InfoContext(ctx, "starting credprovider", slog.String("version", version.String())) + + if err := run(ctx); err != nil { + serverboot.Fatal(ctx, "credprovider exited with error", err) + } +} + +func run(ctx context.Context) error { + mp, err := serverboot.InitMetrics(ctx, serviceName) + if err != nil { + return fmt.Errorf("init metrics: %w", err) + } + defer serverboot.ShutdownProvider("MeterProvider", mp.Shutdown) + + readiness := &serverboot.Readiness{} + go serverboot.StartMetricsServer(ctx, serverboot.MetricsServerOptions{ + Addr: *metricsAddr, + Readiness: readiness, + EnableHealthz: true, + }) + + client, err := newKubeClient() + if err != nil { + return fmt.Errorf("kubernetes client: %w", err) + } + + if *nsPolicyFile == "" { + return fmt.Errorf("--namespace-policy-file is required") + } + + nsAuth, err := loadNamespaceAuthorizer(*nsPolicyFile) + if err != nil { + return fmt.Errorf("namespace policy: %w", err) + } + slog.InfoContext(ctx, "loaded namespace authorization policy", slog.String("file", *nsPolicyFile)) + + creds, err := buildServerCreds(ctx) + if err != nil { + return fmt.Errorf("server credentials: %w", err) + } + + srv := grpc.NewServer( + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.Creds(creds), + ) + reflection.Register(srv) + credproviderpb.RegisterCredentialProviderServer(srv, NewServer(client, nsAuth)) + + lis, err := (&net.ListenConfig{}).Listen(ctx, "tcp", *listenAddr) + if err != nil { + return fmt.Errorf("listen on %s: %w", *listenAddr, err) + } + + shutdownCtx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + go func() { + <-shutdownCtx.Done() + slog.Info("shutting down") + readiness.MarkNotReady() + done := make(chan struct{}) + go func() { + srv.GracefulStop() + close(done) + }() + select { + case <-done: + case <-time.After(*drainGrace): + slog.Warn("graceful shutdown timed out; forcing stop", slog.Duration("grace", *drainGrace)) + srv.Stop() + } + }() + + slog.InfoContext(ctx, "credprovider listening", slog.String("address", lis.Addr().String())) + if err := srv.Serve(lis); err != nil && err != grpc.ErrServerStopped { + return fmt.Errorf("serving: %w", err) + } + return nil +} + +func newKubeClient() (kubernetes.Interface, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("in-cluster config: %w", err) + } + return kubernetes.NewForConfig(cfg) +} + +// buildServerCreds composes the mutual-TLS credentials the provider serves with: +// it presents the credential bundle to callers and requires each caller to +// present a certificate that both chains to --client-ca-file and carries the +// injector's SAN. Both --server-cred-bundle and --client-ca-file are required. +func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, error) { + if *serverBundle == "" { + return nil, fmt.Errorf("--server-cred-bundle is required") + } + if *clientCAFile == "" { + return nil, fmt.Errorf("--client-ca-file is required") + } + + id, err := url.Parse(*injectorSPIFFEID) + if err != nil || id.Scheme != "spiffe" || id.Host == "" || id.Path == "" || id.User != nil || id.RawQuery != "" || id.ForceQuery || strings.Contains(*injectorSPIFFEID, "#") { + return nil, fmt.Errorf("--injector-spiffe-id must be a SPIFFE URI") + } + + ca, err := os.ReadFile(*clientCAFile) + if err != nil { + return nil, fmt.Errorf("read --client-ca-file: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca) { + return nil, fmt.Errorf("no certificates in --client-ca-file %q", *clientCAFile) + } + + cfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + GetCertificate: credbundle.Loader(*serverBundle), + // Require a client certificate that chains to the trust bundle. + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: pool, + VerifyConnection: verifyClientSAN(*injectorSPIFFEID), + } + slog.InfoContext(ctx, "verifying caller client certificates", + slog.String("ca", *clientCAFile), slog.String("required_san", *injectorSPIFFEID)) + return credentials.NewTLS(cfg), nil +} + +// verifyClientSAN returns a TLS VerifyConnection callback that accepts a caller +// only when its certificate carries expectedSAN as a URI SAN. +func verifyClientSAN(expectedSAN string) func(tls.ConnectionState) error { + return func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 { + return fmt.Errorf("client certificate is required") + } + leaf := state.PeerCertificates[0] + for _, u := range leaf.URIs { + if u.String() == expectedSAN { + return nil + } + } + return fmt.Errorf("client certificate URI SANs %v do not include the expected injector identity %q", leaf.URIs, expectedSAN) + } +} diff --git a/cmd/k8s-credential-provider/main_test.go b/cmd/k8s-credential-provider/main_test.go new file mode 100644 index 0000000000..cabfcb9f5a --- /dev/null +++ b/cmd/k8s-credential-provider/main_test.go @@ -0,0 +1,211 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func certWithURIs(t *testing.T, uris ...string) *x509.Certificate { + t.Helper() + cert := &x509.Certificate{} + for _, u := range uris { + parsed, err := url.Parse(u) + if err != nil { + t.Fatalf("parsing SAN %q: %v", u, err) + } + cert.URIs = append(cert.URIs, parsed) + } + return cert +} + +func TestVerifyClientSAN(t *testing.T) { + injector := *injectorSPIFFEID + + tests := []struct { + name string + state tls.ConnectionState + wantErr bool + }{ + { + name: "matching SAN", + state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t, injector)}}, + }, + { + name: "matching SAN among several", + state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t, "spiffe://cluster.local/ns/other/sa/x", injector)}}, + }, + { + name: "wrong SAN", + state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t, "spiffe://cluster.local/ns/ate-system/sa/impostor")}}, + wantErr: true, + }, + { + name: "no URI SANs", + state: tls.ConnectionState{PeerCertificates: []*x509.Certificate{certWithURIs(t)}}, + wantErr: true, + }, + { + name: "no peer certificate", + state: tls.ConnectionState{}, + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := verifyClientSAN(injector)(tc.state) + if tc.wantErr && err == nil { + t.Fatal("expected an error, got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +func TestProviderMTLS(t *testing.T) { + ca, err := localca.GenerateCA("trusted", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + untrustedCA, err := localca.GenerateCA("untrusted", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + servingCert := issueCertificate(t, ca, "") + dir := t.TempDir() + oldBundle, oldCAFile, oldInjector := *serverBundle, *clientCAFile, *injectorSPIFFEID + t.Cleanup(func() { *serverBundle, *clientCAFile, *injectorSPIFFEID = oldBundle, oldCAFile, oldInjector }) + *serverBundle, *clientCAFile = filepath.Join(dir, "server.pem"), filepath.Join(dir, "ca.pem") + *injectorSPIFFEID = "spiffe://cluster.local/ns/custom/sa/release-atenet-egress" + key, err := x509.MarshalPKCS8PrivateKey(servingCert.PrivateKey) + if err != nil { + t.Fatal(err) + } + bundle := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}) + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: servingCert.Certificate[0]})...) + if err := os.WriteFile(*serverBundle, bundle, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(*clientCAFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.RootCertificate.Raw}), 0600); err != nil { + t.Fatal(err) + } + creds, err := buildServerCreds(t.Context()) + if err != nil { + t.Fatal(err) + } + client := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("credential")}, + }) + srv := grpc.NewServer(grpc.Creds(creds)) + credproviderpb.RegisterCredentialProviderServer(srv, NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}})) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + for _, tc := range []struct { + name string + certs []tls.Certificate + allowed bool + }{ + {"injector", []tls.Certificate{issueCertificate(t, ca, *injectorSPIFFEID)}, true}, + {"other workload", []tls.Certificate{issueCertificate(t, ca, "spiffe://cluster.local/ns/custom/sa/other")}, false}, + {"missing certificate", nil, false}, + {"untrusted injector", []tls.Certificate{issueCertificate(t, untrustedCA, *injectorSPIFFEID)}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + RootCAs: roots, ServerName: "api.ate-system.svc", Certificates: tc.certs, MinVersion: tls.VersionTLS13, + }))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + before := len(client.Actions()) + resp, err := credproviderpb.NewCredentialProviderClient(conn).FetchSecret(ctx, &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://kubernetes.io/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a", + }) + if tc.allowed { + if err != nil || string(resp.GetOpaqueBytes()) != "credential" { + t.Fatalf("FetchSecret: %v, %v", resp, err) + } + } else { + if err == nil { + t.Fatal("unauthorized peer received credentials") + } + if len(client.Actions()) != before { + t.Fatal("unauthorized peer reached Kubernetes") + } + } + }) + } +} + +func issueCertificate(t *testing.T, ca *localca.CA, uri string) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: serial, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), + DNSNames: []string{"api.ate-system.svc"}, KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + } + if uri != "" { + parsed, err := url.Parse(uri) + if err != nil { + t.Fatal(err) + } + template.URIs = []*url.URL{parsed} + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, &key.PublicKey, ca.SigningKey) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} diff --git a/cmd/k8s-credential-provider/manifests_test.go b/cmd/k8s-credential-provider/manifests_test.go new file mode 100644 index 0000000000..51fa0cfb9f --- /dev/null +++ b/cmd/k8s-credential-provider/manifests_test.go @@ -0,0 +1,254 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "errors" + "io" + "os/exec" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestSidecarManifests(t *testing.T) { + for _, tc := range []struct { + name, tool, namespace, prefix string + args []string + enabled bool + }{ + {name: "disabled", tool: "helm", namespace: "ate-system", args: []string{"template", "substrate", "../../charts/substrate", "-n", "ate-system"}}, + {name: "custom release", tool: "helm", namespace: "custom", prefix: "test-", enabled: true, + args: []string{"template", "test", "../../charts/substrate", "-n", "custom", "--set", "ateApi.credentialProvider.enabled=true", "--set", "ateApi.credentialProvider.namespacePolicies[0].atespace=team-a", "--set", "ateApi.credentialProvider.namespacePolicies[0].allowedNamespaces[0]=ns1"}}, + {name: "kustomize", tool: "kubectl", namespace: "ate-system", enabled: true, + args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../manifests/ate-install/kubernetes-credentials"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := exec.LookPath(tc.tool); err != nil { + t.Skipf("%s is not installed", tc.tool) + } + data, err := exec.CommandContext(t.Context(), tc.tool, tc.args...).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, data) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var sidecarFound, portFound, policyFound bool + for { + var doc struct { + Kind string + Metadata metav1.ObjectMeta + Spec struct { + Template corev1.PodTemplateSpec + Ports []corev1.ServicePort + } + Data map[string]string + Rules []rbacv1.PolicyRule + } + if err := decoder.Decode(&doc); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + switch doc.Kind { + case "Deployment": + if doc.Metadata.Name != tc.prefix+"ate-api-server" { + continue + } + pod := doc.Spec.Template.Spec + if pod.ServiceAccountName != tc.prefix+"ate-api-server" { + t.Fatalf("unexpected ServiceAccount %q", pod.ServiceAccountName) + } + for _, container := range pod.Containers { + if container.Name != "credential-provider" { + continue + } + sidecarFound = true + args := strings.Join(container.Args, " ") + for _, required := range []string{ + "--listen-address=:50051", "--metrics-address=:9091", + "--injector-spiffe-id=spiffe://cluster.local/ns/" + tc.namespace + "/sa/" + tc.prefix + "atenet-egress", + "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem", + "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem", + } { + if !strings.Contains(args, required) { + t.Errorf("sidecar missing %s", required) + } + } + if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet.Port.StrVal != "cred-health" { + t.Fatal("missing dedicated readiness probe") + } + for _, port := range container.Ports { + if port.ContainerPort == 443 || port.ContainerPort == 9090 { + t.Fatalf("sidecar conflicts with ateapi on %d", port.ContainerPort) + } + } + } + case "Service": + if doc.Metadata.Name != tc.prefix+"api" { + continue + } + for _, port := range doc.Spec.Ports { + if port.Port == 50051 && port.TargetPort.StrVal == "credentials" { + portFound = true + } + } + case "ConfigMap": + if !strings.HasPrefix(doc.Metadata.Name, tc.prefix+"credential-provider-policy") { + continue + } + policyFound = true + var policy namespacePolicyFile + if err := yaml.UnmarshalStrict([]byte(doc.Data["policy.yaml"]), &policy); err != nil { + t.Fatal(err) + } + auth, err := newNamespaceAuthorizer(policy) + if err != nil { + t.Fatal(err) + } + if auth.Allowed("team-a", "ns1") != (tc.name == "custom release") { + t.Fatal("unexpected namespace policy") + } + case "ClusterRole": + if doc.Metadata.Name != tc.prefix+"ate-api-server-role" && doc.Metadata.Name != "ate-api-server" { + continue + } + for _, rule := range doc.Rules { + for _, resource := range rule.Resources { + if resource == "secrets" || resource == "*" { + t.Fatal("sidecar grants cluster-wide Secret access") + } + } + } + } + } + if sidecarFound != tc.enabled || portFound != tc.enabled || policyFound != tc.enabled { + t.Fatalf("sidecar=%v port=%v policy=%v, enabled=%v", sidecarFound, portFound, policyFound, tc.enabled) + } + }) + } +} + +func TestAgentgatewayCredentialConfiguration(t *testing.T) { + for _, tc := range []struct { + name, tool, host, roots string + args []string + enabled bool + }{ + {name: "disabled", tool: "helm", args: []string{"template", "substrate", "../../charts/substrate", "-n", "ate-system"}}, + {name: "helm", tool: "helm", host: "test-api.custom.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", enabled: true, + args: []string{"template", "test", "../../charts/substrate", "-n", "custom", "--set", "ateApi.credentialProvider.enabled=true"}}, + {name: "kustomize", tool: "kubectl", host: "api.ate-system.svc:50051", roots: "/run/servicedns-ca/trust-bundle.pem", enabled: true, + args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../manifests/ate-install/agentgateway-egress-mitm"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := exec.LookPath(tc.tool); err != nil { + t.Skipf("%s is not installed", tc.tool) + } + data, err := exec.CommandContext(t.Context(), tc.tool, tc.args...).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, data) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + providers, mitmMounts := 0, 0 + for { + var doc struct { + Kind string + Data map[string]string + Spec struct{ Template corev1.PodTemplateSpec } + } + if err := decoder.Decode(&doc); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + if doc.Kind == "Deployment" { + for _, container := range doc.Spec.Template.Spec.Containers { + if container.Name != "agentgateway" { + continue + } + for _, mount := range container.VolumeMounts { + if mount.MountPath == "/run/egress-mitm" { + mitmMounts++ + } + } + } + } + if doc.Kind != "ConfigMap" { + continue + } + var config struct { + Binds []struct { + Listeners []struct { + Protocol string + TLS struct{ Mode, Cert, Key string } + Routes []struct { + Policies struct { + SubstrateEgress struct { + CredentialProviders []struct { + URIAuthority string `json:"uriAuthority"` + Target struct { + Host string + Policies struct { + BackendTLS struct{ Cert, Key, Root string } + } + } + } + } + } + } + } + } + } + if err := yaml.Unmarshal([]byte(doc.Data["config.yaml"]), &config); err != nil { + t.Fatal(err) + } + for _, bind := range config.Binds { + for _, listener := range bind.Listeners { + for _, route := range listener.Routes { + for _, provider := range route.Policies.SubstrateEgress.CredentialProviders { + providers++ + if listener.Protocol != "HTTPS" || listener.TLS.Mode != "dynamicCa" { + t.Fatal("credentials enabled outside TLS interception") + } + if listener.TLS.Cert != "/run/egress-mitm/tls.crt" || listener.TLS.Key != "/run/egress-mitm/tls.key" { + t.Fatal("incorrect MITM certificate paths") + } + if provider.URIAuthority != "kubernetes.io" || provider.Target.Host != tc.host { + t.Fatalf("incorrect provider: %+v", provider) + } + tls := provider.Target.Policies.BackendTLS + if tls.Root != tc.roots || tls.Cert != "/run/podidentity.podcert.ate.dev/credential-bundle.pem" || tls.Key != tls.Cert { + t.Fatalf("incorrect provider mTLS: %+v", tls) + } + } + } + } + } + } + want := 0 + if tc.enabled { + want = 1 + } + if providers != want || mitmMounts != want { + t.Fatalf("providers=%d MITM mounts=%d, want %d", providers, mitmMounts, want) + } + }) + } +} diff --git a/cmd/k8s-credential-provider/policy.go b/cmd/k8s-credential-provider/policy.go new file mode 100644 index 0000000000..8ebcc5e4a6 --- /dev/null +++ b/cmd/k8s-credential-provider/policy.go @@ -0,0 +1,96 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/agent-substrate/substrate/internal/resources" + "k8s.io/apimachinery/pkg/util/validation" + + "sigs.k8s.io/yaml" +) + +// namespacePolicyFile is the YAML the authorizer loads: a list of grants, each +// mapping one atespace to the namespaces whose Secrets it may resolve. +type namespacePolicyFile struct { + Policies []atespaceNamespacePolicy `json:"policies"` +} + +type atespaceNamespacePolicy struct { + Atespace string `json:"atespace"` + AllowedNamespaces []string `json:"allowedNamespaces"` +} + +// namespaceAuthorizer decides whether an atespace may resolve secrets in a given +// Kubernetes namespace. It is default-deny: an atespace absent from the mapping +// can resolve nothing. +type namespaceAuthorizer struct { + // allowed maps atespace -> set of permitted namespaces. + allowed map[string]map[string]struct{} +} + +// loadNamespaceAuthorizer reads the YAML policy file at path and builds an +// authorizer, so a malformed file fails startup rather than the first request. +func loadNamespaceAuthorizer(path string) (*namespaceAuthorizer, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading namespace policy file %q: %w", path, err) + } + var file namespacePolicyFile + if err := yaml.UnmarshalStrict(data, &file); err != nil { + return nil, fmt.Errorf("parsing namespace policy file %q: %w", path, err) + } + return newNamespaceAuthorizer(file) +} + +// newNamespaceAuthorizer builds an authorizer over a parsed policy file, +// validating that each grant names an atespace. +func newNamespaceAuthorizer(file namespacePolicyFile) (*namespaceAuthorizer, error) { + allowed := make(map[string]map[string]struct{}) + for i, p := range file.Policies { + if !resources.IsValidResourceName(p.Atespace) { + return nil, fmt.Errorf("namespace policy %d: valid atespace is required", i) + } + set := allowed[p.Atespace] + if set == nil { + set = make(map[string]struct{}) + allowed[p.Atespace] = set + } + for _, ns := range p.AllowedNamespaces { + if len(validation.IsDNS1123Label(ns)) != 0 { + return nil, fmt.Errorf("namespace policy %d: invalid namespace %q", i, ns) + } + set[ns] = struct{}{} + } + } + return &namespaceAuthorizer{allowed: allowed}, nil +} + +// Allowed reports whether atespace may resolve secrets in namespace. Default +// deny: an atespace absent from the mapping, or a namespace not in its list, is +// refused. +func (a *namespaceAuthorizer) Allowed(atespace, namespace string) bool { + if a == nil { + return false + } + set, ok := a.allowed[atespace] + if !ok { + return false + } + _, ok = set[namespace] + return ok +} diff --git a/cmd/k8s-credential-provider/provider.go b/cmd/k8s-credential-provider/provider.go new file mode 100644 index 0000000000..ebe9f442f5 --- /dev/null +++ b/cmd/k8s-credential-provider/provider.go @@ -0,0 +1,186 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file implements the CredentialProvider plugin API backed by Kubernetes +// Secrets. It resolves ate-secret:// URIs of the provider "kubernetes.io" to a +// Secret value read straight from the Kubernetes API — so Substrate never +// stores the secret, it only brokers a read the provider is authorized to +// perform. +package main + +import ( + "context" + "fmt" + "log/slog" + "net/url" + "strings" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" +) + +// ProviderName is the ate-secret:// URI host this backend serves. +const ProviderName = "kubernetes.io" + +// uriScheme is the only scheme a credential URI may carry. +const uriScheme = "ate-secret" + +// secretRef is a parsed ate-secret:// URI for the kubernetes.io provider. +// +// ate-secret://kubernetes.io//[/] +type secretRef struct { + Namespace string + Name string + // Key is the data key within the Secret, or "" when the URI omits it (only + // allowed when the secret contains one entry). + Key string +} + +// parseURI parses a ate-secret:// URI of the kubernetes.io provider. It +// rejects any other scheme or provider name. +func parseURI(raw string) (secretRef, error) { + u, err := url.Parse(raw) + if err != nil { + return secretRef{}, fmt.Errorf("parsing credential URI %q: %w", raw, err) + } + if u.Scheme != uriScheme { + return secretRef{}, fmt.Errorf("malformed credential URI %q: scheme is %q, want %q", raw, u.Scheme, uriScheme) + } + if u.Host != ProviderName { + return secretRef{}, fmt.Errorf("credential URI %q: provider is %q, this provider serves %q", raw, u.Host, ProviderName) + } + + if u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(raw, "#") { + return secretRef{}, fmt.Errorf("credential URI must not contain user info, a query, or a fragment") + } + + segments := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") + // / is the minimum; an optional 3rd segment is the data + // key. + if len(segments) < 2 || len(segments) > 3 { + return secretRef{}, fmt.Errorf("credential URI %q: want /[/], got %d path segments", raw, len(segments)) + } + for i, s := range segments { + if s == "" { + return secretRef{}, fmt.Errorf("credential URI %q: empty path segment %d", raw, i) + } + } + + ref := secretRef{ + Namespace: segments[0], + Name: segments[1], + } + if len(segments) == 3 { + ref.Key = segments[2] + } + if len(validation.IsDNS1123Label(ref.Namespace)) != 0 || len(validation.IsDNS1123Subdomain(ref.Name)) != 0 || (ref.Key != "" && len(validation.IsConfigMapKey(ref.Key)) != 0) { + return secretRef{}, fmt.Errorf("credential URI contains an invalid namespace, secret name, or key") + } + return ref, nil +} + +// Server implements credproviderpb.CredentialProviderServer over the Kubernetes +// API. +type Server struct { + credproviderpb.UnimplementedCredentialProviderServer + + client kubernetes.Interface + // nsAuth restricts which namespaces an atespace may resolve secrets from. + nsAuth *namespaceAuthorizer +} + +// NewServer builds a Kubernetes credential provider with a default-deny policy. +func NewServer(client kubernetes.Interface, nsAuth *namespaceAuthorizer) *Server { + return &Server{client: client, nsAuth: nsAuth} +} + +// FetchSecret resolves one ate-secret:// URI to its Secret value. +func (s *Server) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecretRequest) (*credproviderpb.FetchSecretResponse, error) { + ref, err := parseURI(req.GetUri()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + if err := s.authorize(ctx, req.GetActorSpiffeId(), ref.Namespace); err != nil { + return nil, err + } + + slog.InfoContext(ctx, "resolving credential", + slog.String("provider", ProviderName), + slog.String("namespace", ref.Namespace), + slog.String("secret", ref.Name), + slog.String("actor", req.GetActorSpiffeId()), + ) + + secret, err := s.client.CoreV1().Secrets(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, status.Errorf(codes.NotFound, "secret %s/%s not found", ref.Namespace, ref.Name) + } + if k8serrors.IsForbidden(err) { + return nil, status.Errorf(codes.PermissionDenied, "not permitted to read secret %s/%s", ref.Namespace, ref.Name) + } + return nil, status.Error(codes.Unavailable, "could not read secret from Kubernetes") + } + + value, err := selectKey(secret.Data, ref.Key) + if err != nil { + return nil, status.Errorf(codes.NotFound, "secret %s/%s: %v", ref.Namespace, ref.Name, err) + } + return &credproviderpb.FetchSecretResponse{OpaqueBytes: value}, nil +} + +// authorize enforces the atespace→namespace policy. It derives the atespace from +// the attested actor SPIFFE ID and denies unless the URI's namespace is in that +// atespace's allowed list. +func (s *Server) authorize(ctx context.Context, actorSpiffeID, namespace string) error { + actor, err := resources.ActorRefFromSPIFFEID(actorSpiffeID) + if err != nil { + slog.WarnContext(ctx, "credential request denied: unusable actor identity", slog.Any("err", err)) + return status.Error(codes.PermissionDenied, "actor identity is required and must be a valid actor SPIFFE URI") + } + if !s.nsAuth.Allowed(actor.Atespace, namespace) { + slog.WarnContext(ctx, "credential request denied: atespace not permitted for namespace", + slog.String("atespace", actor.Atespace), slog.String("namespace", namespace)) + return status.Errorf(codes.PermissionDenied, "atespace %q is not permitted to resolve secrets in namespace %q", actor.Atespace, namespace) + } + return nil +} + +// selectKey resolves which Secret data entry to return: the URI's explicit key, +// else the sole key of a single-key Secret. A URI without a key resolving a +// multi-key Secret is an error. +func selectKey(data map[string][]byte, uriKey string) ([]byte, error) { + if uriKey == "" { + if len(data) != 1 { + return nil, fmt.Errorf("no key given and the secret has %d keys; specify one in the URI", len(data)) + } + for _, v := range data { + return v, nil + } + } + v, ok := data[uriKey] + if !ok { + return nil, fmt.Errorf("key %q not present", uriKey) + } + return v, nil +} diff --git a/cmd/k8s-credential-provider/provider_test.go b/cmd/k8s-credential-provider/provider_test.go new file mode 100644 index 0000000000..3363438a12 --- /dev/null +++ b/cmd/k8s-credential-provider/provider_test.go @@ -0,0 +1,363 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" +) + +func TestParseURI(t *testing.T) { + tests := []struct { + name string + uri string + want secretRef + wantErr bool + }{ + { + name: "with key", + uri: "ate-secret://kubernetes.io/ns1/example-api/token", + want: secretRef{Namespace: "ns1", Name: "example-api", Key: "token"}, + }, + { + name: "without key", + uri: "ate-secret://kubernetes.io/ns1/example-api", + want: secretRef{Namespace: "ns1", Name: "example-api"}, + }, + {name: "wrong scheme", uri: "https://kubernetes.io/ns1/example-api", wantErr: true}, + {name: "wrong provider", uri: "ate-secret://vault.io/ns1/example-api", wantErr: true}, + {name: "too few segments", uri: "ate-secret://kubernetes.io/ns1", wantErr: true}, + {name: "too many segments", uri: "ate-secret://kubernetes.io/a/b/c/d", wantErr: true}, + {name: "user info", uri: "ate-secret://user@kubernetes.io/ns1/api/token", wantErr: true}, + {name: "query", uri: "ate-secret://kubernetes.io/ns1/api?key=token", wantErr: true}, + {name: "empty query", uri: "ate-secret://kubernetes.io/ns1/api?", wantErr: true}, + {name: "fragment", uri: "ate-secret://kubernetes.io/ns1/api#token", wantErr: true}, + {name: "empty fragment", uri: "ate-secret://kubernetes.io/ns1/api#", wantErr: true}, + {name: "trailing slash", uri: "ate-secret://kubernetes.io/ns1/api/", wantErr: true}, + {name: "empty namespace", uri: "ate-secret://kubernetes.io//api/token", wantErr: true}, + {name: "invalid namespace", uri: "ate-secret://kubernetes.io/NS/api/token", wantErr: true}, + {name: "path traversal", uri: "ate-secret://kubernetes.io/ns1/../token", wantErr: true}, + {name: "encoded slash in key", uri: "ate-secret://kubernetes.io/ns1/api/a%2Fb", wantErr: true}, + {name: "invalid key", uri: "ate-secret://kubernetes.io/ns1/api/key%20name", wantErr: true}, + {name: "unparseable", uri: "://://", wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseURI(tc.uri) + if tc.wantErr { + if err == nil { + t.Fatalf("parseURI(%q) = %+v, want error", tc.uri, got) + } + return + } + if err != nil { + t.Fatalf("parseURI(%q) unexpected error: %v", tc.uri, err) + } + if got != tc.want { + t.Errorf("parseURI(%q) = %+v, want %+v", tc.uri, got, tc.want) + } + }) + } +} + +func TestNamespaceAuthorizer(t *testing.T) { + authz, err := newNamespaceAuthorizer(namespacePolicyFile{ + Policies: []atespaceNamespacePolicy{ + {Atespace: "team-a", AllowedNamespaces: []string{"ns1", "shared"}}, + {Atespace: "team-b", AllowedNamespaces: []string{"ns2"}}, + }, + }) + if err != nil { + t.Fatalf("newNamespaceAuthorizer: %v", err) + } + tests := []struct { + atespace, namespace string + want bool + }{ + {"team-a", "ns1", true}, + {"team-a", "shared", true}, + {"team-a", "ns2", false}, // namespace not in team-a's list + {"team-b", "ns2", true}, // team-b's own namespace + {"team-c", "ns1", false}, // atespace absent -> default deny + {"team-a", "", false}, // empty namespace + } + for _, tc := range tests { + if got := authz.Allowed(tc.atespace, tc.namespace); got != tc.want { + t.Errorf("Allowed(%q, %q) = %v, want %v", tc.atespace, tc.namespace, got, tc.want) + } + } + + // An empty file denies everything. + empty, err := newNamespaceAuthorizer(namespacePolicyFile{}) + if err != nil { + t.Fatalf("newNamespaceAuthorizer(empty): %v", err) + } + if empty.Allowed("team-a", "ns1") { + t.Error("empty authorizer allowed team-a/ns1, want deny") + } + + // A policy without an atespace is rejected. + if _, err := newNamespaceAuthorizer(namespacePolicyFile{ + Policies: []atespaceNamespacePolicy{{AllowedNamespaces: []string{"ns1"}}}, + }); err == nil { + t.Error("newNamespaceAuthorizer accepted a policy with no atespace, want error") + } +} + +func TestFetchSecretAuthorization(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "example-api", Namespace: "ns1"}, + Data: map[string][]byte{"token": []byte("s3cr3t")}, + } + authz, err := newNamespaceAuthorizer(namespacePolicyFile{ + Policies: []atespaceNamespacePolicy{{Atespace: "team-a", AllowedNamespaces: []string{"ns1"}}}, + }) + if err != nil { + t.Fatalf("newNamespaceAuthorizer: %v", err) + } + const teamAURI = "spiffe://substrate-actor.local/atespace/team-a/actor/my-actor" + const teamBURI = "spiffe://substrate-actor.local/atespace/team-b/actor/my-actor" + + tests := []struct { + name string + actorSpiffeID string + uri string + wantCode codes.Code + }{ + { + name: "allowed", + actorSpiffeID: teamAURI, + uri: "ate-secret://kubernetes.io/ns1/example-api/token", + }, + { + name: "namespace not permitted", + actorSpiffeID: teamAURI, + uri: "ate-secret://kubernetes.io/ns2/example-api/token", + wantCode: codes.PermissionDenied, + }, + { + name: "unknown atespace", + actorSpiffeID: teamBURI, + uri: "ate-secret://kubernetes.io/ns1/example-api/token", + wantCode: codes.PermissionDenied, + }, + { + name: "garbage identity", + actorSpiffeID: "not-a-spiffe-uri", + uri: "ate-secret://kubernetes.io/ns1/example-api/token", + wantCode: codes.PermissionDenied, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := fake.NewSimpleClientset(secret) + srv := NewServer(client, authz) + resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: tc.actorSpiffeID}) + if tc.wantCode != codes.OK { + if len(client.Actions()) != 0 { + t.Fatal("denied request reached Kubernetes") + } + if status.Code(err) != tc.wantCode { + t.Fatalf("code = %v, want %v (err=%v)", status.Code(err), tc.wantCode, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := string(resp.GetOpaqueBytes()); got != "s3cr3t" { + t.Errorf("secret = %q, want s3cr3t", got) + } + }) + } + + // A missing authorizer must fail closed. + t.Run("nil authorizer denies", func(t *testing.T) { + srv := NewServer(fake.NewSimpleClientset(secret), nil) + if _, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://kubernetes.io/ns1/example-api/token", + ActorSpiffeId: teamAURI, + }); status.Code(err) != codes.PermissionDenied { + t.Fatalf("nil authorizer should deny, got %v", err) + } + }) +} + +func TestFetchSecret(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "example-api", Namespace: "ns1"}, + Data: map[string][]byte{ + "token": []byte("s3cr3t"), + }, + } + multiKey := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "multi", Namespace: "ns1"}, + Data: map[string][]byte{ + "a": []byte("aa"), + "b": []byte("bb"), + }, + } + + tests := []struct { + name string + uri string + want string + wantCode codes.Code + }{ + { + name: "explicit key", + uri: "ate-secret://kubernetes.io/ns1/example-api/token", + want: "s3cr3t", + }, + { + name: "single-key fallback", + uri: "ate-secret://kubernetes.io/ns1/example-api", + want: "s3cr3t", + }, + { + name: "no key, multiple keys", + uri: "ate-secret://kubernetes.io/ns1/multi", + wantCode: codes.NotFound, + }, + { + name: "missing key", + uri: "ate-secret://kubernetes.io/ns1/example-api/nope", + wantCode: codes.NotFound, + }, + { + name: "secret not found", + uri: "ate-secret://kubernetes.io/ns1/absent/token", + wantCode: codes.NotFound, + }, + { + name: "bad uri", + uri: "ate-secret://vault.io/ns1/example-api", + wantCode: codes.InvalidArgument, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := fake.NewSimpleClientset(secret, multiKey) + srv := NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/my-actor"}) + if tc.wantCode != codes.OK { + if status.Code(err) != tc.wantCode { + t.Fatalf("FetchSecret(%q) code = %v, want %v (err=%v)", tc.uri, status.Code(err), tc.wantCode, err) + } + return + } + if err != nil { + t.Fatalf("FetchSecret(%q) unexpected error: %v", tc.uri, err) + } + if got := string(resp.GetOpaqueBytes()); got != tc.want { + t.Errorf("FetchSecret(%q) = %q, want %q", tc.uri, got, tc.want) + } + }) + } +} + +func TestLoadNamespaceAuthorizer(t *testing.T) { + for _, tc := range []struct { + name, policy string + wantErr bool + }{ + {name: "valid", policy: "policies:\n- atespace: team-a\n allowedNamespaces: [ns1]\n"}, + {name: "empty", policy: "policies: []"}, + {name: "unknown field", policy: "polices: []", wantErr: true}, + {name: "duplicate field", policy: "policies: []\npolicies: []", wantErr: true}, + {name: "missing atespace", policy: "policies: [{allowedNamespaces: [ns1]}]", wantErr: true}, + {name: "invalid namespace", policy: "policies: [{atespace: team-a, allowedNamespaces: ['*']}]", wantErr: true}, + {name: "malformed", policy: "policies: [", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "policy.yaml") + if err := os.WriteFile(path, []byte(tc.policy), 0600); err != nil { + t.Fatal(err) + } + auth, err := loadNamespaceAuthorizer(path) + if (err != nil) != tc.wantErr { + t.Fatalf("loadNamespaceAuthorizer: %v", err) + } + if err == nil && auth.Allowed("team-a", "ns1") != (tc.name == "valid") { + t.Fatal("unexpected namespace grant") + } + }) + } + if _, err := loadNamespaceAuthorizer(filepath.Join(t.TempDir(), "absent")); err == nil { + t.Fatal("missing policy accepted") + } +} + +func TestFetchSecretKubernetesErrors(t *testing.T) { + for _, tc := range []struct { + name string + err error + code codes.Code + }{ + {"forbidden", k8serrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, "api", errors.New("RBAC")), codes.PermissionDenied}, + {"unavailable", errors.New("upstream response body should stay private"), codes.Unavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, tc.err }) + srv := NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + _, err := srv.FetchSecret(t.Context(), &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://kubernetes.io/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a", + }) + if status.Code(err) != tc.code { + t.Fatalf("FetchSecret: %v, want %v", err, tc.code) + } + if strings.Contains(err.Error(), "stay private") { + t.Fatal("Kubernetes response body exposed") + } + }) + } +} + +func TestFetchSecretObservesRotation(t *testing.T) { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("first")}} + client := fake.NewSimpleClientset(secret) + srv := NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + req := &credproviderpb.FetchSecretRequest{Uri: "ate-secret://kubernetes.io/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a"} + first, err := srv.FetchSecret(t.Context(), req) + if err != nil || string(first.GetOpaqueBytes()) != "first" { + t.Fatalf("first fetch: %v, %v", first, err) + } + secret.Data["token"] = []byte("rotated") + if _, err := client.CoreV1().Secrets("ns1").Update(t.Context(), secret, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + next, err := srv.FetchSecret(t.Context(), req) + if err != nil || string(next.GetOpaqueBytes()) != "rotated" { + t.Fatalf("fetch after rotation: %v, %v", next, err) + } +} diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md new file mode 100644 index 0000000000..1230912546 --- /dev/null +++ b/docs/kubernetes-credential-provider.md @@ -0,0 +1,157 @@ +# Kubernetes credential provider + +The optional `k8s-credential-provider` sidecar runs in ateapi's Pod. It serves +`CredentialProvider.FetchSecret` on `api.ate-system.svc:50051`, using ateapi's +serving certificate and ServiceAccount. ateapi's own gRPC service stays on 443. +No additional Deployment or ServiceAccount is needed. + +A URI such as `ate-secret://kubernetes.io/team-a-secrets/example-api/token` +resolves the `token` entry in that Kubernetes Secret. Omitting the key is allowed +only for a Secret with exactly one data entry. Each request reads Kubernetes, +so Secret rotation is visible on the next provider fetch. AGW currently caches +successful credentials per actor and URI for five minutes, so injection may +continue using a cached value until it expires. +Secret values are neither persisted nor logged by the provider. + +Access requires all three checks: + +- The caller presents a trusted mTLS certificate with the configured egress + injector SPIFFE identity. A different trusted workload is still rejected. +- The actor SPIFFE identity attested by that injector belongs to an atespace + explicitly granted access to the Secret's namespace. Empty policies deny all. +- ateapi's ServiceAccount has Kubernetes `get` permission on that Secret. + +The sidecar shares ateapi's Kubernetes identity and Pod failure domain. It is +process separation, not a separate Kubernetes authorization boundary. + +## Agentgateway compatibility + +This integration requires an AGW build that implements the current Substrate +credential protocol. The pinned September 17 nightly includes +`credentialProviders` configuration, but still uses the previous RPC and URI +scheme. A compatible AGW image must be pinned before enabling this feature: + +- RPC: `/credprovider.CredentialProvider/FetchSecret`. +- Request: `uri` (field 1), `actor_spiffe_id` (field 2). +- Response: `opaque_bytes` (field 1). +- URI scheme: `ate-secret://`. + +The contract is [credprovider.proto](../pkg/proto/credproviderpb/credprovider.proto). +The provider does not implement the old `RequestSecret` RPC or +`substrate-secret://` scheme. + +## Enable the sidecar + +First create the MITM CA Secret using the existing installation tooling: + +```sh +hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret +``` + +The Secret is named `egress-mitm-ca-pool` and must contain `tls.crt` and `tls.key` +in the gateway's namespace. For a non-default namespace, provision the same +Secret there. Actors must trust this CA; see the +[MITM trust bundle guide](egress-trust-bundle.md). + +For Helm, set `images.agentgateway` to the compatible image and add these values +to your existing release configuration: + +```yaml +ateApi: + credentialProvider: + enabled: true + namespacePolicies: + - atespace: team-a + allowedNamespaces: [team-a-secrets] +``` + +The chart derives the injector identity and Service name from the release name +and namespace. For example, release `demo` in namespace `platform` serves at +`demo-api.platform.svc:50051` and accepts only +`spiffe://cluster.local/ns/platform/sa/demo-atenet-egress`. +Enabling it also configures AGW's HTTPS interception route to fetch credentials +from the sidecar over mTLS. Credential providers are configured only on the HTTPS +route, so cleartext egress cannot receive injected Secrets. The feature is +disabled by default. Policy changes through Helm roll the ateapi +Pods so each sidecar loads the new policy. + +For the manifest installer, add +`manifests/ate-install/components/credential-provider` to your existing ateapi +Kustomization's `components`. Set the grants in its `policy.yaml`: + +```yaml +policies: +- atespace: team-a + allowedNamespaces: [team-a-secrets] +``` + +A ready-made overlay of the repository's ateapi defaults is also available: + +```sh +kubectl kustomize --load-restrictor=LoadRestrictionsNone \ + manifests/ate-install/kubernetes-credentials | ko apply -f - +``` + +Preserve any existing installation-specific ateapi patches in your overlay. +The component's generated ConfigMap name changes with its policy, rolling the +Pods on reapplication. Direct edits to a mounted ConfigMap require a rollout +restart: the policy and client CA bundle are loaded at startup. The serving +credential bundle follows the existing certificate loader's rotation behavior. + +## Grant Secret access + +Create the Secret in `team-a-secrets`, then grant only the required Secret reads. +For the default installation, this Role and RoleBinding allow the example above: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: egress-credentials + namespace: team-a-secrets +rules: +- apiGroups: [""] + resources: ["secrets"] + resourceNames: [example-api] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: egress-credentials + namespace: team-a-secrets +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: egress-credentials +subjects: +- kind: ServiceAccount + name: ate-api-server + namespace: ate-system +``` + +For a named Helm release, use its prefixed ateapi ServiceAccount and namespace. +Neither the chart nor the overlay grants cluster-wide Secret access. + +## Connect agentgateway + +The Helm configuration above wires `substrateEgress.credentialProviders` to the +sidecar on the HTTPS interception route. For the manifest installer, update the +AGW image in `manifests/ate-install/components/agentgateway/kustomization.yaml` +to your compatible build, install the sidecar overlay, and deploy the AGW MITM +configuration: + +```sh +hack/install-ate.sh --deploy-atenet \ + --atenet-dataplane=agentgateway \ + --experimental-use-sdsmint +``` + +The AGW MITM component points `uriAuthority: kubernetes.io` at +`api.ate-system.svc:50051`, using its existing pod identity certificate and +service DNS CA bundle. No ext_proc injector is needed. + +Set an egress policy header injection's credential URI to +`ate-secret://kubernetes.io/team-a-secrets/example-api/token`, for example with +header `authorization` and prefix `Bearer `. Namespace grants alone do not +create an egress policy. diff --git a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml index b90274819d..51b33fcef3 100644 --- a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml @@ -72,6 +72,15 @@ patches: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem + credentialProviders: + - uriAuthority: kubernetes.io + target: + host: api.ate-system.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem - protocol: HTTP routes: - backends: diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index fe062e4447..1ef20bbc79 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -42,7 +42,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.d376b9e1@sha256:f5650ed21ab9d84a0ee5398763d5950d10880eab53f5b53e0be5e8d06767e467 args: - -f - /etc/agentgateway/config.yaml @@ -118,7 +118,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.d376b9e1@sha256:f5650ed21ab9d84a0ee5398763d5950d10880eab53f5b53e0be5e8d06767e467 args: - -f - /etc/agentgateway/config.yaml diff --git a/manifests/ate-install/components/credential-provider/kustomization.yaml b/manifests/ate-install/components/credential-provider/kustomization.yaml new file mode 100644 index 0000000000..cd18534422 --- /dev/null +++ b/manifests/ate-install/components/credential-provider/kustomization.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +configMapGenerator: +- name: credential-provider-policy + namespace: ate-system + files: + - policy.yaml + +patches: +- path: sidecar.yaml diff --git a/manifests/ate-install/components/credential-provider/policy.yaml b/manifests/ate-install/components/credential-provider/policy.yaml new file mode 100644 index 0000000000..603d3898c5 --- /dev/null +++ b/manifests/ate-install/components/credential-provider/policy.yaml @@ -0,0 +1,16 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default deny. Add explicit atespace-to-namespace grants before enabling injection. +policies: [] diff --git a/manifests/ate-install/components/credential-provider/sidecar.yaml b/manifests/ate-install/components/credential-provider/sidecar.yaml new file mode 100644 index 0000000000..071cce3045 --- /dev/null +++ b/manifests/ate-install/components/credential-provider/sidecar.yaml @@ -0,0 +1,82 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-api-server + namespace: ate-system +spec: + template: + spec: + containers: + - name: credential-provider + image: ko://github.com/agent-substrate/substrate/cmd/k8s-credential-provider + args: + - --listen-address=:50051 + - --metrics-address=:9091 + - --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem + - --client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem + - --injector-spiffe-id=spiffe://cluster.local/ns/ate-system/sa/atenet-egress + - --namespace-policy-file=/etc/credential-provider/policy.yaml + ports: + - name: credentials + containerPort: 50051 + - name: cred-health + containerPort: 9091 + readinessProbe: + httpGet: + path: /readyz + port: cred-health + periodSeconds: 2 + livenessProbe: + httpGet: + path: /healthz + port: cred-health + initialDelaySeconds: 10 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: credential-provider-policy + mountPath: /etc/credential-provider + readOnly: true + volumes: + - name: credential-provider-policy + configMap: + name: credential-provider-policy +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: ate-system +spec: + ports: + - name: credentials + protocol: TCP + port: 50051 + targetPort: credentials diff --git a/manifests/ate-install/kubernetes-credentials/kustomization.yaml b/manifests/ate-install/kubernetes-credentials/kustomization.yaml new file mode 100644 index 0000000000..71da96655d --- /dev/null +++ b/manifests/ate-install/kubernetes-credentials/kustomization.yaml @@ -0,0 +1,22 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../ate-api-server.yaml + +components: +- ../components/credential-provider From 88dbebdeaaea8c62e920d7a10ad15b0f1abfb401 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 15:17:20 +0000 Subject: [PATCH 2/7] Exercise credential injection through the rendered AGW route Add an opt-in Docker integration test covering actor authentication, TLS interception, provider mTLS, namespace denial, and cleartext denial using the Helm egress configuration. Signed-off-by: Eitan Yarmush --- .../agentgateway_test.go | 260 ++++++++++++++++++ cmd/k8s-credential-provider/main_test.go | 2 +- docs/kubernetes-credential-provider.md | 18 ++ go.mod | 2 +- 4 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 cmd/k8s-credential-provider/agentgateway_test.go diff --git a/cmd/k8s-credential-provider/agentgateway_test.go b/cmd/k8s-credential-provider/agentgateway_test.go new file mode 100644 index 0000000000..06d113d5bc --- /dev/null +++ b/cmd/k8s-credential-provider/agentgateway_test.go @@ -0,0 +1,260 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes/fake" + + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" +) + +// TestAgentgatewayInjection exercises the rendered Helm route with a real AGW +// image and provider. Only Kubernetes storage and the ateapi actor lookup are +// faked. Docker must run locally on Linux for host networking. +func TestAgentgatewayInjection(t *testing.T) { + image := os.Getenv("AGENTGATEWAY_TEST_IMAGE") + if image == "" { + t.Skip("set AGENTGATEWAY_TEST_IMAGE to run the Docker integration test") + } + require.Equal(t, "linux", runtime.GOOS, "test requires Linux host networking") + dir := t.TempDir() + ca, err := localca.GenerateCA("integration", localca.KeyTypeECDSAP256, time.Hour) + require.NoError(t, err) + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + write := func(name string, data []byte) string { + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, data, 0600)) + return path + } + caPEM, err := ca.TLSCertificateChainPEM() + require.NoError(t, err) + caPath := write("ca.pem", caPEM) + caKey, err := ca.TLSPrivateKeyPEM() + require.NoError(t, err) + write("ca-key.pem", caKey) + servingCert := issueCertificate(t, ca, "") + writeBundle := func(name string, cert tls.Certificate) string { + key, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) + require.NoError(t, err) + bundle := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}) + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]})...) + return write(name, bundle) + } + oldBundle, oldCAFile, oldInjector := *serverBundle, *clientCAFile, *injectorSPIFFEID + t.Cleanup(func() { *serverBundle, *clientCAFile, *injectorSPIFFEID = oldBundle, oldCAFile, oldInjector }) + *serverBundle, *clientCAFile = writeBundle("server.pem", servingCert), caPath + *injectorSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" + writeBundle("injector.pem", issueCertificate(t, ca, *injectorSPIFFEID)) + creds, err := buildServerCreds(t.Context()) + require.NoError(t, err) + kube := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "team-a-secrets"}, + Data: map[string][]byte{"token": []byte("injected-token")}, + }) + rpc := grpc.NewServer(grpc.Creds(creds)) + credproviderpb.RegisterCredentialProviderServer(rpc, NewServer(kube, &namespaceAuthorizer{ + allowed: map[string]map[string]struct{}{"team-a": {"team-a-secrets": {}}}, + })) + ateapipb.RegisterControlServer(rpc, &credentialTestControl{}) + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = rpc.Serve(lis) }() + t.Cleanup(rpc.Stop) + + var upstreamCalls atomic.Int32 + origin := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamCalls.Add(1) + if r.Header.Get("Authorization") != "Bearer injected-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusNoContent) + })) + origin.TLS = &tls.Config{Certificates: []tls.Certificate{servingCert}, MinVersion: tls.VersionTLS13} + origin.StartTLS() + t.Cleanup(origin.Close) + _, originPort, err := net.SplitHostPort(origin.Listener.Addr().String()) + require.NoError(t, err) + target := "localhost:" + originPort + + // Keep the chart's route and TLS policies; substitute only local endpoints + // and test certificates, including the TLS origin's private CA. + rendered, err := exec.CommandContext(t.Context(), "helm", "template", "substrate", "../../charts/substrate", + "-n", "ate-system", "--set", "ateApi.credentialProvider.enabled=true").CombinedOutput() + require.NoError(t, err, "%s", rendered) + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) + var config string + for { + var doc struct { + Kind string + Data map[string]string + } + err := decoder.Decode(&doc) + if err == io.EOF { + break + } + require.NoError(t, err) + if doc.Kind == "ConfigMap" && strings.Contains(doc.Data["config.yaml"], "credentialProviders:") { + config = doc.Data["config.yaml"] + } + } + require.NotEmpty(t, config) + portReservation, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + gatewayAddr := portReservation.Addr().String() + _, gatewayPort, err := net.SplitHostPort(gatewayAddr) + require.NoError(t, err) + require.NoError(t, portReservation.Close()) + config = strings.NewReplacer( + "api.ate-system.svc:443", lis.Addr().String(), + "api.ate-system.svc:50051", lis.Addr().String(), + "port: 8443", "port: "+gatewayPort, + "/run/servicedns.podcert.ate.dev/credential-bundle.pem", "/config/server.pem", + "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "/config/injector.pem", + "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "/config/ca.pem", + "/run/actor-id-ca-certs/ca.crt", "/config/ca.pem", + "/run/egress-mitm/tls.crt", "/config/ca.pem", + "/run/egress-mitm/tls.key", "/config/ca-key.pem", + "backendTLS: {}", "backendTLS: {root: /config/ca.pem}", + ).Replace(config) + write("config.yaml", []byte("config:\n adminAddr: 127.0.0.1:0\n statsAddr: 127.0.0.1:0\n readinessAddr: 127.0.0.1:0\n"+config)) + container, err := exec.CommandContext(t.Context(), "docker", "run", "--detach", "--rm", "--network=host", + "--user=0:0", "--volume", dir+":/config:ro", image, "-f", "/config/config.yaml").CombinedOutput() + require.NoError(t, err, "%s", container) + id := strings.TrimSpace(string(container)) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if t.Failed() { + logs, _ := exec.CommandContext(ctx, "docker", "logs", id).CombinedOutput() + t.Logf("AGW logs:\n%s", logs) + } + out, err := exec.CommandContext(ctx, "docker", "stop", "--time=1", id).CombinedOutput() + if err != nil { + t.Errorf("stop AGW: %v: %s", err, out) + } + }) + require.Eventually(t, func() bool { + conn, err := net.DialTimeout("tcp", gatewayAddr, 100*time.Millisecond) + if err != nil { + return false + } + _ = conn.Close() + return true + }, 20*time.Second, 100*time.Millisecond, "AGW did not start") + + for _, tc := range []struct { + name, atespace string + useTLS bool + wantStatus int + }{ + {"allowed", "team-a", true, http.StatusNoContent}, + {"different atespace cannot use cached secret", "team-b", true, http.StatusForbidden}, + {"cleartext cannot receive secrets", "team-a", false, http.StatusForbidden}, + } { + t.Run(tc.name, func(t *testing.T) { + beforeKube, beforeOrigin := len(kube.Actions()), upstreamCalls.Load() + actorCert := issueCertificate(t, ca, "") + template, err := x509.ParseCertificate(actorCert.Certificate[0]) + require.NoError(t, err) + require.NoError(t, substratex509.AddActorIdentityToCertificate(&substratex509.ActorIdentity{ + Atespace: tc.atespace, ActorName: "actor", ActorUid: "uid-1", Purpose: substratex509.ActorIdentityPurposeAtunnel, + }, template)) + der, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, + actorCert.PrivateKey.(crypto.Signer).Public(), ca.SigningKey) + require.NoError(t, err) + actorCert.Certificate = [][]byte{der} + outer, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", gatewayAddr, &tls.Config{ + RootCAs: roots, Certificates: []tls.Certificate{actorCert}, MinVersion: tls.VersionTLS13, + }) + require.NoError(t, err) + defer outer.Close() + require.NoError(t, outer.SetDeadline(time.Now().Add(10*time.Second))) + _, err = fmt.Fprintf(outer, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", target, target) + require.NoError(t, err) + response, err := http.ReadResponse(bufio.NewReader(outer), &http.Request{Method: http.MethodConnect}) + require.NoError(t, err) + require.Equal(t, http.StatusOK, response.StatusCode, "actor CONNECT authorization") + var tunnel net.Conn = outer + if tc.useTLS { + tunnel = tls.Client(outer, &tls.Config{RootCAs: roots, ServerName: "localhost", MinVersion: tls.VersionTLS13}) + } + _, err = fmt.Fprintf(tunnel, "GET / HTTP/1.1\r\nHost: %s\r\nAuthorization: actor-supplied\r\nConnection: close\r\n\r\n", target) + require.NoError(t, err) + response, err = http.ReadResponse(bufio.NewReader(tunnel), &http.Request{Method: http.MethodGet}) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, tc.wantStatus, response.StatusCode) + if tc.wantStatus == http.StatusNoContent { + require.Len(t, kube.Actions(), beforeKube+1) + require.Equal(t, beforeOrigin+1, upstreamCalls.Load()) + } else { + require.Len(t, kube.Actions(), beforeKube, "denied request must not read Kubernetes") + require.Equal(t, beforeOrigin, upstreamCalls.Load(), "denied request must not reach upstream") + } + }) + } +} + +type credentialTestControl struct { + ateapipb.UnimplementedControlServer +} + +func (*credentialTestControl) GetActor(_ context.Context, _ *ateapipb.GetActorRequest) (*ateapipb.Actor, error) { + return &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Uid: "uid-1"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, + }, nil +} + +func (*credentialTestControl) GetActorEgressPolicy(_ context.Context, _ *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { + return &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Hostnames: &ateapipb.HostnameRule{ + Patterns: []string{"localhost"}, + Effects: &ateapipb.EgressRuleEffects{InjectStaticHeaders: []*ateapipb.CredentialHeaderInjection{{ + Header: "authorization", Prefix: "Bearer ", CredentialUri: "ate-secret://kubernetes.io/team-a-secrets/api/token", + }}}, + }}}}, nil +} diff --git a/cmd/k8s-credential-provider/main_test.go b/cmd/k8s-credential-provider/main_test.go index cabfcb9f5a..03542b263b 100644 --- a/cmd/k8s-credential-provider/main_test.go +++ b/cmd/k8s-credential-provider/main_test.go @@ -193,7 +193,7 @@ func issueCertificate(t *testing.T, ca *localca.CA, uri string) tls.Certificate } template := &x509.Certificate{ SerialNumber: serial, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), - DNSNames: []string{"api.ate-system.svc"}, KeyUsage: x509.KeyUsageDigitalSignature, + DNSNames: []string{"api.ate-system.svc", "localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, } if uri != "" { diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md index 1230912546..4e1b8c7882 100644 --- a/docs/kubernetes-credential-provider.md +++ b/docs/kubernetes-credential-provider.md @@ -155,3 +155,21 @@ Set an egress policy header injection's credential URI to `ate-secret://kubernetes.io/team-a-secrets/example-api/token`, for example with header `authorization` and prefix `Bearer `. Namespace grants alone do not create an egress policy. + +## Validate an AGW image locally + +On Linux with a local Docker daemon and Helm installed, set +`AGENTGATEWAY_TEST_IMAGE` to the image reference from `images.agentgateway` in +`charts/substrate/values.yaml`, then run: + +```sh +go test -race ./cmd/k8s-credential-provider -run '^TestAgentgatewayInjection$' -count=1 -v +``` + +The test skips unless that environment variable is exported. It uses the +rendered Helm egress configuration, an authenticated actor CONNECT tunnel, TLS +interception, the real credential-provider server over mTLS, and a TLS upstream. +It verifies header injection, denial for an ungranted atespace after a successful +fetch, and denial of credential injection on cleartext egress. Kubernetes Secret +storage and ateapi's actor/policy RPCs are faked; it does not validate cluster +RBAC, certificate provisioning, or Pod deployment. diff --git a/go.mod b/go.mod index 4edf49121d..2e692a6a65 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spiffe/go-spiffe/v2 v2.7.0 + github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 github.com/vishvananda/netlink v1.3.1 github.com/vishvananda/netns v0.0.5 @@ -194,7 +195,6 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shirou/gopsutil/v4 v4.26.6 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/stretchr/testify v1.12.1 // indirect github.com/testcontainers/testcontainers-go v0.44.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect From eec5e542bb707e4abc2c861b8d900faca8c8b8ce Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 15:42:49 +0000 Subject: [PATCH 3/7] Pin AGW with the current credential-provider protocol Use the published multi-architecture build supporting FetchSecret and ate-secret URIs. Document the tested integration and keep Docker pull diagnostics out of the integration test container ID. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 2 +- charts/substrate/values.yaml | 6 +++--- .../agentgateway_test.go | 11 +++++++---- docs/kubernetes-credential-provider.md | 17 ++++++++--------- .../components/agentgateway/kustomization.yaml | 4 ++-- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 3aea2ceb53..106d174dd6 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,7 +42,7 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | -| `ateApi.credentialProvider.enabled` | `false` | Add the credential-provider sidecar and AGW HTTPS injection; requires a compatible AGW image and MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | +| `ateApi.credentialProvider.enabled` | `false` | Add the credential-provider sidecar and AGW HTTPS injection; requires a MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | | `ateApi.credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; Kubernetes Secret RBAC is configured separately | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 9964f19e20..37ed45af23 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -58,8 +58,8 @@ atelet: ateApi: extraArgs: [] # Optional Kubernetes Secret provider sharing ateapi's Pod and ServiceAccount. - # Enables AGW HTTPS interception and credential injection. Requires a compatible - # images.agentgateway build and the egress-mitm-ca-pool Secret (see docs). + # Enables AGW HTTPS interception and credential injection with the pinned image. + # Requires the egress-mitm-ca-pool Secret (see docs). # Secret read access must be granted separately with namespace-scoped RBAC. credentialProvider: enabled: false @@ -101,5 +101,5 @@ images: postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.d376b9e1@sha256:f5650ed21ab9d84a0ee5398763d5950d10880eab53f5b53e0be5e8d06767e467 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e busybox: busybox:1.36 diff --git a/cmd/k8s-credential-provider/agentgateway_test.go b/cmd/k8s-credential-provider/agentgateway_test.go index 06d113d5bc..4ae6c8cffb 100644 --- a/cmd/k8s-credential-provider/agentgateway_test.go +++ b/cmd/k8s-credential-provider/agentgateway_test.go @@ -161,9 +161,12 @@ func TestAgentgatewayInjection(t *testing.T) { "backendTLS: {}", "backendTLS: {root: /config/ca.pem}", ).Replace(config) write("config.yaml", []byte("config:\n adminAddr: 127.0.0.1:0\n statsAddr: 127.0.0.1:0\n readinessAddr: 127.0.0.1:0\n"+config)) - container, err := exec.CommandContext(t.Context(), "docker", "run", "--detach", "--rm", "--network=host", - "--user=0:0", "--volume", dir+":/config:ro", image, "-f", "/config/config.yaml").CombinedOutput() - require.NoError(t, err, "%s", container) + command := exec.CommandContext(t.Context(), "docker", "run", "--detach", "--rm", "--network=host", + "--user=0:0", "--volume", dir+":/config:ro", image, "-f", "/config/config.yaml") + var stderr bytes.Buffer + command.Stderr = &stderr + container, err := command.Output() + require.NoError(t, err, "%s", stderr.String()) id := strings.TrimSpace(string(container)) t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) @@ -172,7 +175,7 @@ func TestAgentgatewayInjection(t *testing.T) { logs, _ := exec.CommandContext(ctx, "docker", "logs", id).CombinedOutput() t.Logf("AGW logs:\n%s", logs) } - out, err := exec.CommandContext(ctx, "docker", "stop", "--time=1", id).CombinedOutput() + out, err := exec.CommandContext(ctx, "docker", "stop", "--timeout=1", id).CombinedOutput() if err != nil { t.Errorf("stop AGW: %v: %s", err, out) } diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md index 4e1b8c7882..ea4fdaf562 100644 --- a/docs/kubernetes-credential-provider.md +++ b/docs/kubernetes-credential-provider.md @@ -26,10 +26,10 @@ process separation, not a separate Kubernetes authorization boundary. ## Agentgateway compatibility -This integration requires an AGW build that implements the current Substrate -credential protocol. The pinned September 17 nightly includes -`credentialProviders` configuration, but still uses the previous RPC and URI -scheme. A compatible AGW image must be pinned before enabling this feature: +The pinned AGW nightly includes the +[credential protocol update](https://github.com/agentgateway/agentgateway/pull/3524) +from [this build](https://github.com/agentgateway/agentgateway/actions/runs/35238449333). +It implements the current Substrate credential protocol: - RPC: `/credprovider.CredentialProvider/FetchSecret`. - Request: `uri` (field 1), `actor_spiffe_id` (field 2). @@ -53,8 +53,8 @@ in the gateway's namespace. For a non-default namespace, provision the same Secret there. Actors must trust this CA; see the [MITM trust bundle guide](egress-trust-bundle.md). -For Helm, set `images.agentgateway` to the compatible image and add these values -to your existing release configuration: +For Helm, keep the pinned `images.agentgateway` image and add these values to +your existing release configuration: ```yaml ateApi: @@ -136,9 +136,8 @@ Neither the chart nor the overlay grants cluster-wide Secret access. ## Connect agentgateway The Helm configuration above wires `substrateEgress.credentialProviders` to the -sidecar on the HTTPS interception route. For the manifest installer, update the -AGW image in `manifests/ate-install/components/agentgateway/kustomization.yaml` -to your compatible build, install the sidecar overlay, and deploy the AGW MITM +sidecar on the HTTPS interception route. For the manifest installer, keep the +pinned AGW image, install the sidecar overlay, and deploy the AGW MITM configuration: ```sh diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 1ef20bbc79..cdff87553f 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -42,7 +42,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.d376b9e1@sha256:f5650ed21ab9d84a0ee5398763d5950d10880eab53f5b53e0be5e8d06767e467 + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e args: - -f - /etc/agentgateway/config.yaml @@ -118,7 +118,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.d376b9e1@sha256:f5650ed21ab9d84a0ee5398763d5950d10880eab53f5b53e0be5e8d06767e467 + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e args: - -f - /etc/agentgateway/config.yaml From e845397657d25684bd8f2d23e160338421fe9cd5 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 19:04:47 +0000 Subject: [PATCH 4/7] Exercise Kubernetes credential injection through real actors in Helm E2E Validate exact header injection and namespace, RBAC, and cleartext denials against a local HTTPS origin. Remove fixture workers before their namespace to avoid delayed namespace finalization. Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 13 +- docs/kubernetes-credential-provider.md | 27 +++ internal/e2e/fixture.go | 10 + internal/e2e/fixtures/testserver/http.go | 37 ++- internal/e2e/fixtures/testserver/http_test.go | 51 +++++ internal/e2e/serverpod.go | 8 +- internal/e2e/serverpod_test.go | 14 ++ .../suites/credentials/credentials_test.go | 210 ++++++++++++++++++ .../e2e/suites/credentials/testmain_test.go | 24 ++ internal/e2e/suites/credentials/values.yaml | 20 ++ 10 files changed, 406 insertions(+), 8 deletions(-) create mode 100644 internal/e2e/fixtures/testserver/http_test.go create mode 100644 internal/e2e/suites/credentials/credentials_test.go create mode 100644 internal/e2e/suites/credentials/testmain_test.go create mode 100644 internal/e2e/suites/credentials/values.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 7820c668a5..0943951a47 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -57,7 +57,7 @@ jobs: kubectl apply -f manifests/ate-install/kind/prometheus.yaml - name: Build chart images run: | - for component in ateapi atecontroller atelet podcertcontroller atenet; do + for component in ateapi atecontroller atelet podcertcontroller atenet k8s-credential-provider; do KO_DOCKER_REPO="localhost:5001/${component}" \ ./hack/run-tool.sh ko build --bare --tags helm-e2e \ --platform linux/amd64 "./cmd/${component}" @@ -107,6 +107,17 @@ jobs: env: E2E_SANDBOX_CLASS: microvm run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Enable Kubernetes credential injection + # Interception changes egress TLS, so enable it after the standard lanes. + run: | + hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret + helm upgrade substrate charts/substrate --namespace ate-system \ + --reuse-values -f internal/e2e/suites/credentials/values.yaml \ + --wait --timeout=5m + - name: Run E2E tests (Kubernetes credentials) + env: + E2E_CREDENTIAL_PROVIDER: "1" + run: hack/run-e2e-kind.sh ./internal/e2e/suites/credentials -v -args --no-color - name: Dump diagnostics on failure if: failure() run: | diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md index ea4fdaf562..75c18a36e8 100644 --- a/docs/kubernetes-credential-provider.md +++ b/docs/kubernetes-credential-provider.md @@ -172,3 +172,30 @@ It verifies header injection, denial for an ungranted atespace after a successfu fetch, and denial of credential injection on cleartext egress. Kubernetes Secret storage and ateapi's actor/policy RPCs are faked; it does not validate cluster RBAC, certificate provisioning, or Pod deployment. + +## Cluster E2E test + +The Helm CI workflow builds the sidecar image and runs +`internal/e2e/suites/credentials` after enabling the feature. The test deploys a +real actor and an in-cluster HTTPS origin, creates Secrets and namespace-scoped +RBAC, and exercises the actual ateapi, AGW, and provider. It checks the exact +injected token, an unauthenticated-origin control, namespace-policy denial, +Kubernetes RBAC denial, and cleartext denial. SubjectAccessReviews verify the +permission assumptions behind each denial. + +To run it on a dedicated Helm-installed Kind cluster with this branch's images +(including `k8s-credential-provider`) available: + +```sh +hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret +helm upgrade substrate charts/substrate --namespace ate-system \ + --reuse-values -f internal/e2e/suites/credentials/values.yaml \ + --wait --timeout=5m +E2E_ATENET_DATAPLANE=agentgateway E2E_CREDENTIAL_PROVIDER=1 \ + hack/run-e2e-kind.sh ./internal/e2e/suites/credentials -v -args --no-color +``` + +This configuration enables TLS interception cluster-wide, so run it after tests +that require passthrough egress. The test temporarily trusts the cluster's serving +CA for the local HTTPS origin, restoring that gateway configuration on cleanup. +It does not modify the actor or credential-provider authentication settings. diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index ee3d1e6829..9f67b18986 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -147,6 +147,16 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, t.Fatalf("fixture %s declares templates in different atespaces (%q and %q)", manifests.Template, atespace, got) } } + t.Cleanup(func() { + // Remove workers before the namespace so its controller does not wait + // on their one-hour termination grace estimate after the Pods exit. + delArgs := []string{"delete", "workerpools", "--all", "--namespace=" + atespace, + "--ignore-not-found", "--cascade=foreground", "--timeout=2m"} + if KubeContext != "" { + delArgs = append([]string{"--context=" + KubeContext}, delArgs...) + } + RunCmd(t, "kubectl", delArgs...) + }) if _, err := clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: atespace}}}); err != nil && status.Code(err) != codes.AlreadyExists { t.Fatalf("failed to create atespace %q: %v", atespace, err) diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index 0ae303baa8..a4a3d06436 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -15,21 +15,22 @@ package main import ( + "crypto/tls" "log" "net/http" + "os" "time" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/spf13/cobra" ) // newHTTPCmd is a plain HTTP/1.1 origin an Actor's egress lands on. It exists so -// a test can assert the destination port is recovered from SO_ORIGINAL_DST -// rather than defaulted from the URL scheme: the actor fetches its /healthz on a -// non-standard port, and the gateway's access log is expected to carry that -// port. There is nothing to serve beyond readiness, so /healthz is all it -// answers. +// a test can assert the destination port is recovered from SO_ORIGINAL_DST. +// It can also serve TLS and verify an injected Authorization header against a +// mounted token for the credential-provider E2E test. func newHTTPCmd() *cobra.Command { - var listenAddress string + var listenAddress, tlsBundle, authorizationFile string cmd := &cobra.Command{ Use: "http", Short: "Serve a plain HTTP/1.1 origin answering /healthz.", @@ -39,6 +40,9 @@ func newHTTPCmd() *cobra.Command { mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + if authorizationFile != "" { + mux.HandleFunc("/credential", credentialHandler(authorizationFile)) + } server := &http.Server{ Addr: listenAddress, @@ -47,9 +51,30 @@ func newHTTPCmd() *cobra.Command { WriteTimeout: 2 * time.Minute, } log.Printf("testserver http: listening on %s", listenAddress) + if tlsBundle != "" { + server.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS13, GetCertificate: credbundle.Loader(tlsBundle)} + return server.ListenAndServeTLS("", "") + } return server.ListenAndServe() }, } cmd.Flags().StringVar(&listenAddress, "listen", ":8080", "Address the HTTP origin listens on.") + cmd.Flags().StringVar(&tlsBundle, "tls-bundle", "", "Serve HTTPS using this certificate and key bundle.") + cmd.Flags().StringVar(&authorizationFile, "authorization-file", "", "Enable /credential, requiring a Bearer token matching this file.") return cmd } + +func credentialHandler(path string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, err := os.ReadFile(path) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + if len(token) == 0 || r.Header.Get("Authorization") != "Bearer "+string(token) { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusNoContent) + } +} diff --git a/internal/e2e/fixtures/testserver/http_test.go b/internal/e2e/fixtures/testserver/http_test.go new file mode 100644 index 0000000000..b42c292677 --- /dev/null +++ b/internal/e2e/fixtures/testserver/http_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestCredentialHandler(t *testing.T) { + path := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(path, []byte("expected-token"), 0600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + header string + status int + }{ + {"", http.StatusUnauthorized}, + {"Bearer wrong-token", http.StatusUnauthorized}, + {"Bearer expected-token", http.StatusNoContent}, + } { + req := httptest.NewRequest(http.MethodGet, "/credential", nil) + req.Header.Set("Authorization", tc.header) + resp := httptest.NewRecorder() + credentialHandler(path)(resp, req) + if resp.Code != tc.status || resp.Body.Len() != 0 { + t.Errorf("header %q: status=%d body=%q, want status=%d and no body", tc.header, resp.Code, resp.Body.String(), tc.status) + } + } + resp := httptest.NewRecorder() + credentialHandler(path+"-missing")(resp, httptest.NewRequest(http.MethodGet, "/credential", nil)) + if resp.Code != http.StatusInternalServerError { + t.Fatalf("unreadable credential file: status=%d, want 500", resp.Code) + } +} diff --git a/internal/e2e/serverpod.go b/internal/e2e/serverpod.go index ea4d186337..a9727de00d 100644 --- a/internal/e2e/serverpod.go +++ b/internal/e2e/serverpod.go @@ -68,6 +68,8 @@ type ServerPod struct { // an HTTP GET. A gRPC server answers an HTTP request with a protocol error, // so a server speaking grpc must set this and register the health service. GRPCProbe bool + // HTTPSProbe uses HTTPS for readiness. Ignored when GRPCProbe is set. + HTTPSProbe bool // HealthPath is the HTTP readiness path, defaulting to /healthz. Ignored // when GRPCProbe is set. HealthPath string @@ -179,5 +181,9 @@ func serverReadinessProbe(spec ServerPod, targetPort string) string { if path == "" { path = "/healthz" } - return fmt.Sprintf(" httpGet:\n path: %s\n port: %s", path, targetPort) + probe := fmt.Sprintf(" httpGet:\n path: %s\n port: %s", path, targetPort) + if spec.HTTPSProbe { + probe += "\n scheme: HTTPS" + } + return probe } diff --git a/internal/e2e/serverpod_test.go b/internal/e2e/serverpod_test.go index 53848d371e..429ef95678 100644 --- a/internal/e2e/serverpod_test.go +++ b/internal/e2e/serverpod_test.go @@ -156,6 +156,20 @@ func TestRenderServerPod_HTTPProbe(t *testing.T) { } } +func TestRenderServerPod_HTTPSProbe(t *testing.T) { + pod, service := renderServerPodDocs(t, ServerPod{ + Name: "tlsorigin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http"}, Port: 443, TargetPort: 8443, HTTPSProbe: true, + }) + probe := pod.Spec.Containers[0].ReadinessProbe.HTTPGet + if probe == nil || probe.Scheme != corev1.URISchemeHTTPS || probe.Port.IntValue() != 8443 { + t.Fatalf("HTTPS readiness probe = %+v, want HTTPS on container port 8443", probe) + } + if service.Spec.Ports[0].Port != 443 { + t.Fatal("origin Service must expose port 443") + } +} + // TestRenderServerPod covers where each port lands: every field kubelet or // the binary reaches follows the listener, while the Service alone keeps the // published port. diff --git a/internal/e2e/suites/credentials/credentials_test.go b/internal/e2e/suites/credentials/credentials_test.go new file mode 100644 index 0000000000..03fe439f73 --- /dev/null +++ b/internal/e2e/suites/credentials/credentials_test.go @@ -0,0 +1,210 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credentials + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// TestKubernetesCredentialInjection requires the Helm values in values.yaml and +// an egress MITM CA. It uses real actors, Kubernetes RBAC, and the deployed +// provider. The origin uses the cluster's serving CA to keep all traffic local. +func TestKubernetesCredentialInjection(t *testing.T) { + if os.Getenv("E2E_CREDENTIAL_PROVIDER") == "" { + t.Skip("enable the credential-provider Helm E2E values and set E2E_CREDENTIAL_PROVIDER=1") + } + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + require.NoError(t, err) + ctx := t.Context() + clients := e2e.GetClients() + namespace, template := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials", e2e.WithTrustBundle()) + otherNamespace := e2e.CreateNamespace(t).Name + api, err := clients.K8s.AppsV1().Deployments("ate-system").Get(ctx, "ate-api-server", metav1.GetOptions{}) + require.NoError(t, err) + serviceAccount := api.Spec.Template.Spec.ServiceAccountName + require.NotEmpty(t, serviceAccount) + + for _, secret := range []struct{ namespace, name string }{ + {namespace, "allowed"}, {namespace, "no-rbac"}, {otherNamespace, "allowed"}, + } { + _, err := clients.K8s.CoreV1().Secrets(secret.namespace).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secret.name}, + Data: map[string][]byte{"token": []byte("e2e-credential-token")}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + for _, ns := range []string{namespace, otherNamespace} { + _, err := clients.K8s.RbacV1().Roles(ns).Create(ctx, &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "credential-reader"}, + Rules: []rbacv1.PolicyRule{{APIGroups: []string{""}, Resources: []string{"secrets"}, + ResourceNames: []string{"allowed"}, Verbs: []string{"get"}}}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + _, err = clients.K8s.RbacV1().RoleBindings(ns).Create(ctx, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "credential-reader"}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "credential-reader"}, + Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Namespace: "ate-system", Name: serviceAccount}}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + // Prove the negative controls isolate different boundaries: Kubernetes + // permits the other namespace, but the provider policy does not; within + // the granted namespace, Kubernetes itself refuses the second Secret. + for _, tc := range []struct { + namespace, secret string + allowed bool + }{{namespace, "allowed", true}, {otherNamespace, "allowed", true}, {namespace, "no-rbac", false}} { + require.Eventually(t, func() bool { + review, err := clients.K8s.AuthorizationV1().SubjectAccessReviews().Create(ctx, &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + User: "system:serviceaccount:ate-system:" + serviceAccount, + Groups: []string{"system:serviceaccounts", "system:serviceaccounts:ate-system", "system:authenticated"}, + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Namespace: tc.namespace, Verb: "get", Resource: "secrets", Name: tc.secret, + }, + }, + }, metav1.CreateOptions{}) + return err == nil && review.Status.Allowed == tc.allowed + }, 30*time.Second, time.Second, "unexpected Secret RBAC for %s/%s", tc.namespace, tc.secret) + } + + trustOriginCA(t, clients) + e2e.DeployServerPod(t, ctx, e2e.ServerPod{ + Name: "credential-origin", Namespace: namespace, + ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--tls-bundle=/run/tls/bundle.pem", "--authorization-file=/run/token/token"}, + Port: 443, TargetPort: 8443, HTTPSProbe: true, + Volumes: []corev1.Volume{ + {Name: "token", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "allowed"}}}, + {Name: "tls", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{PodCertificate: &corev1.PodCertificateProjection{ + SignerName: "servicedns.podcert.ate.dev/identity", KeyType: "ECDSAP256", CredentialBundlePath: "bundle.pem", + }}}, + }}}, + }, + VolumeMounts: []corev1.VolumeMount{{Name: "tls", MountPath: "/run/tls", ReadOnly: true}, {Name: "token", MountPath: "/run/token", ReadOnly: true}}, + }) + host := "credential-origin." + namespace + ".svc" + router, err := e2e.NewRouterClient(ctx) + require.NoError(t, err) + t.Cleanup(router.Close) + for _, tc := range []struct { + name, secretNamespace, secret, scheme string + want string + }{ + {"without-injection", "", "", "https", "401"}, + {"allowed", namespace, "allowed", "https", "204"}, + {"namespace-denied", otherNamespace, "allowed", "https", "403"}, + {"rbac-denied", namespace, "no-rbac", "https", "403"}, + {"cleartext-denied", namespace, "allowed", "http", "403"}, + } { + t.Run(tc.name, func(t *testing.T) { + actor := &ateapipb.ObjectRef{Atespace: namespace, Name: tc.name} + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actor}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actor}) + _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: namespace, Name: tc.name}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: namespace, Name: template.GetMetadata().GetName()}, + }}) + require.NoError(t, err) + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: actor}) + _, err := clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: actor}) + if err != nil { + t.Errorf("delete actor %s: %v", tc.name, err) + } + }) + rule := e2e.EgressAllowHostnames(host) + if tc.secret != "" { + rule.Hostnames.Effects = &ateapipb.EgressRuleEffects{InjectStaticHeaders: []*ateapipb.CredentialHeaderInjection{{ + Header: "authorization", Prefix: "Bearer ", + CredentialUri: fmt.Sprintf("ate-secret://kubernetes.io/%s/%s/token", tc.secretNamespace, tc.secret), + }}} + } + e2e.EnsureEgressPolicy(t, ctx, clients, actor, rule) + _, err = clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actor}) + require.NoError(t, err) + path := "/fetch?roots=bundle&url=" + url.QueryEscape(tc.scheme+"://"+host+"/credential") + // ConfigMap projections and route discovery are asynchronous. Each + // retry still requires the precise result; transport errors never pass. + deadline := time.Now().Add(90 * time.Second) + for { + resp, err := router.Get(ctx, resources.ActorRef{Atespace: namespace, Name: tc.name}, path) + var body []byte + if err == nil { + body, err = io.ReadAll(resp.Body) + resp.Body.Close() + var result struct{ Status, Error string } + if err == nil && resp.StatusCode == http.StatusOK && json.Unmarshal(body, &result) == nil && result.Error == "" && result.Status == tc.want { + break + } + } + if time.Now().After(deadline) { + t.Fatalf("want origin status %s; last response: %s; error: %v", tc.want, body, err) + } + time.Sleep(2 * time.Second) + } + }) + } +} + +// Only the local origin needs a private CA. Keep the production route, actor +// authentication, and credential-provider TLS settings intact, and restore the +// gateway configuration after the test. +func trustOriginCA(t *testing.T, clients *e2e.Clients) { + t.Helper() + configMaps := clients.K8s.CoreV1().ConfigMaps("ate-system") + config, err := configMaps.Get(t.Context(), "atenet-egress-agentgateway-config", metav1.GetOptions{}) + require.NoError(t, err) + original := config.Data["config.yaml"] + require.Equal(t, 1, strings.Count(original, "backendTLS: {}"), "expected one dynamic HTTPS upstream") + config.Data["config.yaml"] = strings.Replace(original, "backendTLS: {}", + "backendTLS: {root: /run/servicedns.podcert.ate.dev/trust-bundle.pem}", 1) + _, err = configMaps.Update(t.Context(), config, metav1.UpdateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + config, err := configMaps.Get(ctx, config.Name, metav1.GetOptions{}) + if err == nil { + config.Data["config.yaml"] = original + _, err = configMaps.Update(ctx, config, metav1.UpdateOptions{}) + } + if err != nil { + t.Errorf("restore gateway configuration: %v", err) + } + }) +} diff --git a/internal/e2e/suites/credentials/testmain_test.go b/internal/e2e/suites/credentials/testmain_test.go new file mode 100644 index 0000000000..6b8549a17a --- /dev/null +++ b/internal/e2e/suites/credentials/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credentials + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/credentials/values.yaml b/internal/e2e/suites/credentials/values.yaml new file mode 100644 index 0000000000..e5619662f3 --- /dev/null +++ b/internal/e2e/suites/credentials/values.yaml @@ -0,0 +1,20 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ateApi: + credentialProvider: + enabled: true + namespacePolicies: + - atespace: ate-e2e-probe-credentials + allowedNamespaces: [ate-e2e-probe-credentials] From fbf6b9f3a4b4e0cf7f20399f6676f823494c9ef3 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 19:23:03 +0000 Subject: [PATCH 5/7] Deploy the Kubernetes credential provider independently of ateapi Follow the upstream package and Deployment layout with a dedicated ServiceAccount and Service. Consolidate gateway integration coverage into the cluster E2E suite, including cache isolation across atespaces. Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 4 +- Makefile | 2 +- charts/substrate/README.md | 4 +- .../substrate/templates/ate-api-server.yaml | 52 ---- charts/substrate/templates/atenet-egress.yaml | 8 +- .../templates/credential-provider-policy.yaml | 26 -- .../templates/k8s-credential-provider.yaml | 145 ++++++++++ charts/substrate/values.yaml | 17 +- cmd/ate-setup/internal/images/images.go | 2 +- .../kubernetes-secrets/kubeprovider.go} | 30 +- .../kubernetes-secrets/kubeprovider_test.go} | 26 +- .../kubernetes-secrets}/main.go | 6 +- .../kubernetes-secrets}/main_test.go | 2 +- .../kubernetes-secrets}/manifests_test.go | 62 +++-- .../kubernetes-secrets/nsauthz.go} | 14 +- .../agentgateway_test.go | 263 ------------------ docs/kubernetes-credential-provider.md | 202 +++++--------- .../suites/credentials/credentials_test.go | 32 ++- internal/e2e/suites/credentials/values.yaml | 11 +- .../kustomization.yaml | 2 +- .../credential-provider/kustomization.yaml | 25 -- .../credential-provider/sidecar.yaml | 82 ------ .../k8s-credential-provider.yaml | 134 +++++++++ .../kustomization.yaml | 9 +- .../namespace-policy.yaml} | 2 +- 25 files changed, 470 insertions(+), 692 deletions(-) delete mode 100644 charts/substrate/templates/credential-provider-policy.yaml create mode 100644 charts/substrate/templates/k8s-credential-provider.yaml rename cmd/{k8s-credential-provider/provider.go => credential-provider/kubernetes-secrets/kubeprovider.go} (88%) rename cmd/{k8s-credential-provider/provider_test.go => credential-provider/kubernetes-secrets/kubeprovider_test.go} (94%) rename cmd/{k8s-credential-provider => credential-provider/kubernetes-secrets}/main.go (97%) rename cmd/{k8s-credential-provider => credential-provider/kubernetes-secrets}/main_test.go (99%) rename cmd/{k8s-credential-provider => credential-provider/kubernetes-secrets}/manifests_test.go (72%) rename cmd/{k8s-credential-provider/policy.go => credential-provider/kubernetes-secrets/nsauthz.go} (87%) delete mode 100644 cmd/k8s-credential-provider/agentgateway_test.go delete mode 100644 manifests/ate-install/components/credential-provider/kustomization.yaml delete mode 100644 manifests/ate-install/components/credential-provider/sidecar.yaml create mode 100644 manifests/egress-credential-injection/k8s-credential-provider.yaml rename manifests/{ate-install/kubernetes-credentials => egress-credential-injection}/kustomization.yaml (80%) rename manifests/{ate-install/components/credential-provider/policy.yaml => egress-credential-injection/namespace-policy.yaml} (87%) diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 0943951a47..2b2812eb58 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -57,8 +57,8 @@ jobs: kubectl apply -f manifests/ate-install/kind/prometheus.yaml - name: Build chart images run: | - for component in ateapi atecontroller atelet podcertcontroller atenet k8s-credential-provider; do - KO_DOCKER_REPO="localhost:5001/${component}" \ + for component in ateapi atecontroller atelet podcertcontroller atenet credential-provider/kubernetes-secrets; do + KO_DOCKER_REPO="localhost:5001/${component##*/}" \ ./hack/run-tool.sh ko build --bare --tags helm-e2e \ --platform linux/amd64 "./cmd/${component}" done diff --git a/Makefile b/Makefile index 8c24d567dc..b04750a7dc 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ CONTROL_PLANE_IMAGES := ./cmd/ateapi \ ./cmd/atecontroller \ ./cmd/atelet \ ./cmd/atenet \ - ./cmd/k8s-credential-provider \ + ./cmd/credential-provider/kubernetes-secrets \ ./cmd/podcertcontroller WORKER_IMAGES := ./cmd/ateom-gvisor \ ./cmd/ateom-microvm diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 106d174dd6..a54bac5abe 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,8 +42,8 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | -| `ateApi.credentialProvider.enabled` | `false` | Add the credential-provider sidecar and AGW HTTPS injection; requires a MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | -| `ateApi.credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; Kubernetes Secret RBAC is configured separately | +| `credentialProvider.enabled` | `false` | Deploy the Kubernetes credential provider and AGW HTTPS injection; requires a MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | +| `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; Kubernetes Secret RBAC is configured separately | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 23fc11b638..267232eb8d 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -74,9 +74,6 @@ spec: annotations: prometheus.io/scrape: "true" prometheus.io/port: "9090" -{{- if .Values.ateApi.credentialProvider.enabled }} - checksum/credential-provider-policy: {{ toJson .Values.ateApi.credentialProvider.namespacePolicies | sha256sum }} -{{- end }} spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} terminationGracePeriodSeconds: 40 @@ -158,50 +155,7 @@ spec: port: 9090 initialDelaySeconds: 10 periodSeconds: 10 -{{- if .Values.ateApi.credentialProvider.enabled }} - - name: credential-provider - image: {{ include "substrate.componentImage" (list "k8s-credential-provider" .) }} - args: - - --listen-address=:50051 - - --metrics-address=:9091 - - --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem - - --client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem - - --injector-spiffe-id=spiffe://cluster.local/ns/{{ .Release.Namespace }}/sa/{{ include "substrate.fullname" (list "atenet-egress" .) }} - - --namespace-policy-file=/etc/credential-provider/policy.yaml - ports: - - name: credentials - containerPort: 50051 - - name: cred-health - containerPort: 9091 - readinessProbe: - httpGet: - path: /readyz - port: cred-health - periodSeconds: 2 - livenessProbe: - httpGet: - path: /healthz - port: cred-health - initialDelaySeconds: 10 - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - volumeMounts: - - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev, readOnly: true } - - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } - - { name: credential-provider-policy, mountPath: /etc/credential-provider, readOnly: true } -{{- end }} volumes: -{{- if .Values.ateApi.credentialProvider.enabled }} - - name: credential-provider-policy - configMap: - name: {{ include "substrate.fullname" (list "credential-provider-policy" .) }} -{{- end }} - name: servicedns projected: sources: @@ -271,9 +225,3 @@ spec: protocol: TCP port: 443 targetPort: 443 -{{- if .Values.ateApi.credentialProvider.enabled }} - - name: credentials - protocol: TCP - port: 50051 - targetPort: credentials -{{- end }} diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 2aef864714..abcd65610b 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -56,7 +56,7 @@ data: - mode: internal protocol: AUTO listeners: -{{- if .Values.ateApi.credentialProvider.enabled }} +{{- if .Values.credentialProvider.enabled }} - protocol: HTTPS tls: mode: dynamicCa @@ -78,7 +78,7 @@ data: credentialProviders: - uriAuthority: kubernetes.io target: - host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:50051 + host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 policies: backendTLS: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem @@ -162,7 +162,7 @@ spec: port: readiness periodSeconds: 1 volumeMounts: -{{- if .Values.ateApi.credentialProvider.enabled }} +{{- if .Values.credentialProvider.enabled }} - name: egress-mitm mountPath: /run/egress-mitm readOnly: true @@ -228,7 +228,7 @@ spec: - name: drain-signal mountPath: /var/run/atenet volumes: -{{- if .Values.ateApi.credentialProvider.enabled }} +{{- if .Values.credentialProvider.enabled }} - name: egress-mitm secret: secretName: egress-mitm-ca-pool diff --git a/charts/substrate/templates/credential-provider-policy.yaml b/charts/substrate/templates/credential-provider-policy.yaml deleted file mode 100644 index b65aa297d7..0000000000 --- a/charts/substrate/templates/credential-provider-policy.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{/* -Copyright 2026 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.ateApi.credentialProvider.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "substrate.fullname" (list "credential-provider-policy" .) }} - namespace: {{ .Release.Namespace }} -data: - policy.yaml: | - policies: {{ toJson .Values.ateApi.credentialProvider.namespacePolicies }} -{{- end }} diff --git a/charts/substrate/templates/k8s-credential-provider.yaml b/charts/substrate/templates/k8s-credential-provider.yaml new file mode 100644 index 0000000000..87a59b8b6e --- /dev/null +++ b/charts/substrate/templates/k8s-credential-provider.yaml @@ -0,0 +1,145 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.credentialProvider.enabled }} +# Secret read RBAC must be granted separately in each allowed namespace. +# The credential provider: a gRPC service that resolves ate-secret:// URIs +# of the kubernetes.io class to Kubernetes Secret values. It is the ONLY +# component in the egress credential-injection path with Kubernetes access; the +# egress gateway and the injector never read Secrets. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + template: + metadata: + annotations: + checksum/namespace-policy: {{ toJson .Values.credentialProvider.namespacePolicies | sha256sum }} + labels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + spec: + serviceAccountName: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + containers: + - name: k8s-credential-provider + image: {{ include "substrate.componentImage" (list "kubernetes-secrets" .) }} + args: + - "--listen-address=:50051" + - "--metrics-address=:9090" + # Use this Service's DNS certificate; only the egress injector may call. + - "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + # Enforce the atespace→namespace authorization policy (default-deny). + - "--namespace-policy-file=/etc/k8s-credential-provider/namespace-policy.yaml" + - "--injector-spiffe-id=spiffe://cluster.local/ns/{{ .Release.Namespace }}/sa/{{ include "substrate.fullname" (list "atenet-egress" .) }}" + - "--log-level=info" + ports: + - name: grpc + containerPort: 50051 + - name: metrics + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: metrics + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: namespace-policy + mountPath: /etc/k8s-credential-provider + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + volumes: + - name: namespace-policy + configMap: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-namespace-policy" .) }} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + ports: + - name: grpc + port: 50051 + targetPort: grpc + protocol: TCP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-namespace-policy" .) }} + namespace: {{ .Release.Namespace }} +data: + namespace-policy.yaml: | + policies: {{ toJson .Values.credentialProvider.namespacePolicies }} +{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 37ed45af23..2e41ad1a9d 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -57,15 +57,14 @@ atelet: # Additional arguments appended to the ateapi defaults. ateApi: extraArgs: [] - # Optional Kubernetes Secret provider sharing ateapi's Pod and ServiceAccount. - # Enables AGW HTTPS interception and credential injection with the pinned image. - # Requires the egress-mitm-ca-pool Secret (see docs). - # Secret read access must be granted separately with namespace-scoped RBAC. - credentialProvider: - enabled: false - namespacePolicies: [] - # - atespace: team-a - # allowedNamespaces: [team-a-secrets] + +# Optional Kubernetes Secret provider and AGW HTTPS credential injection. +# Requires egress-mitm-ca-pool; grant Secret reads separately to the provider SA. +credentialProvider: + enabled: false + namespacePolicies: [] + # - atespace: team-a + # allowedNamespaces: [team-a-secrets] # Name of a ConfigMap in the release namespace that supplies per-environment # overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). diff --git a/cmd/ate-setup/internal/images/images.go b/cmd/ate-setup/internal/images/images.go index 93a6522a13..f462addf1f 100644 --- a/cmd/ate-setup/internal/images/images.go +++ b/cmd/ate-setup/internal/images/images.go @@ -44,7 +44,7 @@ var Components = []string{ "cmd/atecontroller", "cmd/atelet", "cmd/atenet", - "cmd/k8s-credential-provider", + "cmd/credential-provider/kubernetes-secrets", "cmd/ateom-gvisor", "cmd/ateom-microvm", "cmd/podcertcontroller", diff --git a/cmd/k8s-credential-provider/provider.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go similarity index 88% rename from cmd/k8s-credential-provider/provider.go rename to cmd/credential-provider/kubernetes-secrets/kubeprovider.go index ebe9f442f5..3d07ae2866 100644 --- a/cmd/k8s-credential-provider/provider.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go @@ -44,10 +44,10 @@ const ProviderName = "kubernetes.io" // uriScheme is the only scheme a credential URI may carry. const uriScheme = "ate-secret" -// secretRef is a parsed ate-secret:// URI for the kubernetes.io provider. +// SecretRef is a parsed ate-secret:// URI for the kubernetes.io provider. // // ate-secret://kubernetes.io//[/] -type secretRef struct { +type SecretRef struct { Namespace string Name string // Key is the data key within the Secret, or "" when the URI omits it (only @@ -55,37 +55,37 @@ type secretRef struct { Key string } -// parseURI parses a ate-secret:// URI of the kubernetes.io provider. It +// ParseURI parses a ate-secret:// URI of the kubernetes.io provider. It // rejects any other scheme or provider name. -func parseURI(raw string) (secretRef, error) { +func ParseURI(raw string) (SecretRef, error) { u, err := url.Parse(raw) if err != nil { - return secretRef{}, fmt.Errorf("parsing credential URI %q: %w", raw, err) + return SecretRef{}, fmt.Errorf("parsing credential URI %q: %w", raw, err) } if u.Scheme != uriScheme { - return secretRef{}, fmt.Errorf("malformed credential URI %q: scheme is %q, want %q", raw, u.Scheme, uriScheme) + return SecretRef{}, fmt.Errorf("malformed credential URI %q: scheme is %q, want %q", raw, u.Scheme, uriScheme) } if u.Host != ProviderName { - return secretRef{}, fmt.Errorf("credential URI %q: provider is %q, this provider serves %q", raw, u.Host, ProviderName) + return SecretRef{}, fmt.Errorf("credential URI %q: provider is %q, this provider serves %q", raw, u.Host, ProviderName) } if u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(raw, "#") { - return secretRef{}, fmt.Errorf("credential URI must not contain user info, a query, or a fragment") + return SecretRef{}, fmt.Errorf("credential URI must not contain user info, a query, or a fragment") } segments := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") // / is the minimum; an optional 3rd segment is the data // key. if len(segments) < 2 || len(segments) > 3 { - return secretRef{}, fmt.Errorf("credential URI %q: want /[/], got %d path segments", raw, len(segments)) + return SecretRef{}, fmt.Errorf("credential URI %q: want /[/], got %d path segments", raw, len(segments)) } for i, s := range segments { if s == "" { - return secretRef{}, fmt.Errorf("credential URI %q: empty path segment %d", raw, i) + return SecretRef{}, fmt.Errorf("credential URI %q: empty path segment %d", raw, i) } } - ref := secretRef{ + ref := SecretRef{ Namespace: segments[0], Name: segments[1], } @@ -93,7 +93,7 @@ func parseURI(raw string) (secretRef, error) { ref.Key = segments[2] } if len(validation.IsDNS1123Label(ref.Namespace)) != 0 || len(validation.IsDNS1123Subdomain(ref.Name)) != 0 || (ref.Key != "" && len(validation.IsConfigMapKey(ref.Key)) != 0) { - return secretRef{}, fmt.Errorf("credential URI contains an invalid namespace, secret name, or key") + return SecretRef{}, fmt.Errorf("credential URI contains an invalid namespace, secret name, or key") } return ref, nil } @@ -105,17 +105,17 @@ type Server struct { client kubernetes.Interface // nsAuth restricts which namespaces an atespace may resolve secrets from. - nsAuth *namespaceAuthorizer + nsAuth *NamespaceAuthorizer } // NewServer builds a Kubernetes credential provider with a default-deny policy. -func NewServer(client kubernetes.Interface, nsAuth *namespaceAuthorizer) *Server { +func NewServer(client kubernetes.Interface, nsAuth *NamespaceAuthorizer) *Server { return &Server{client: client, nsAuth: nsAuth} } // FetchSecret resolves one ate-secret:// URI to its Secret value. func (s *Server) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecretRequest) (*credproviderpb.FetchSecretResponse, error) { - ref, err := parseURI(req.GetUri()) + ref, err := ParseURI(req.GetUri()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } diff --git a/cmd/k8s-credential-provider/provider_test.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go similarity index 94% rename from cmd/k8s-credential-provider/provider_test.go rename to cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go index 3363438a12..f4832714dc 100644 --- a/cmd/k8s-credential-provider/provider_test.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go @@ -40,18 +40,18 @@ func TestParseURI(t *testing.T) { tests := []struct { name string uri string - want secretRef + want SecretRef wantErr bool }{ { name: "with key", uri: "ate-secret://kubernetes.io/ns1/example-api/token", - want: secretRef{Namespace: "ns1", Name: "example-api", Key: "token"}, + want: SecretRef{Namespace: "ns1", Name: "example-api", Key: "token"}, }, { name: "without key", uri: "ate-secret://kubernetes.io/ns1/example-api", - want: secretRef{Namespace: "ns1", Name: "example-api"}, + want: SecretRef{Namespace: "ns1", Name: "example-api"}, }, {name: "wrong scheme", uri: "https://kubernetes.io/ns1/example-api", wantErr: true}, {name: "wrong provider", uri: "ate-secret://vault.io/ns1/example-api", wantErr: true}, @@ -72,18 +72,18 @@ func TestParseURI(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, err := parseURI(tc.uri) + got, err := ParseURI(tc.uri) if tc.wantErr { if err == nil { - t.Fatalf("parseURI(%q) = %+v, want error", tc.uri, got) + t.Fatalf("ParseURI(%q) = %+v, want error", tc.uri, got) } return } if err != nil { - t.Fatalf("parseURI(%q) unexpected error: %v", tc.uri, err) + t.Fatalf("ParseURI(%q) unexpected error: %v", tc.uri, err) } if got != tc.want { - t.Errorf("parseURI(%q) = %+v, want %+v", tc.uri, got, tc.want) + t.Errorf("ParseURI(%q) = %+v, want %+v", tc.uri, got, tc.want) } }) } @@ -267,7 +267,7 @@ func TestFetchSecret(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { client := fake.NewSimpleClientset(secret, multiKey) - srv := NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/my-actor"}) if tc.wantCode != codes.OK { if status.Code(err) != tc.wantCode { @@ -303,16 +303,16 @@ func TestLoadNamespaceAuthorizer(t *testing.T) { if err := os.WriteFile(path, []byte(tc.policy), 0600); err != nil { t.Fatal(err) } - auth, err := loadNamespaceAuthorizer(path) + auth, err := LoadNamespaceAuthorizer(path) if (err != nil) != tc.wantErr { - t.Fatalf("loadNamespaceAuthorizer: %v", err) + t.Fatalf("LoadNamespaceAuthorizer: %v", err) } if err == nil && auth.Allowed("team-a", "ns1") != (tc.name == "valid") { t.Fatal("unexpected namespace grant") } }) } - if _, err := loadNamespaceAuthorizer(filepath.Join(t.TempDir(), "absent")); err == nil { + if _, err := LoadNamespaceAuthorizer(filepath.Join(t.TempDir(), "absent")); err == nil { t.Fatal("missing policy accepted") } } @@ -329,7 +329,7 @@ func TestFetchSecretKubernetesErrors(t *testing.T) { t.Run(tc.name, func(t *testing.T) { client := fake.NewSimpleClientset() client.PrependReactor("get", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, tc.err }) - srv := NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) _, err := srv.FetchSecret(t.Context(), &credproviderpb.FetchSecretRequest{ Uri: "ate-secret://kubernetes.io/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a", }) @@ -346,7 +346,7 @@ func TestFetchSecretKubernetesErrors(t *testing.T) { func TestFetchSecretObservesRotation(t *testing.T) { secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("first")}} client := fake.NewSimpleClientset(secret) - srv := NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) req := &credproviderpb.FetchSecretRequest{Uri: "ate-secret://kubernetes.io/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a"} first, err := srv.FetchSecret(t.Context(), req) if err != nil || string(first.GetOpaqueBytes()) != "first" { diff --git a/cmd/k8s-credential-provider/main.go b/cmd/credential-provider/kubernetes-secrets/main.go similarity index 97% rename from cmd/k8s-credential-provider/main.go rename to cmd/credential-provider/kubernetes-secrets/main.go index f82e622b0b..1d4d8dccd2 100644 --- a/cmd/k8s-credential-provider/main.go +++ b/cmd/credential-provider/kubernetes-secrets/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Command k8s-credential-provider is the Kubernetes Secrets credential-provider +// Command kubernetes-secrets is the Kubernetes-Secrets credential-provider // plugin: a gRPC service that resolves ate-secret:// URIs of the kubernetes.io // provider to Kubernetes Secret values. It is the only component in the egress // credential-injection path with Kubernetes access; the egress gateway and its @@ -47,7 +47,7 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" ) -const serviceName = "k8s-credential-provider" +const serviceName = "credprovider" var ( injectorSPIFFEID = pflag.String("injector-spiffe-id", "spiffe://cluster.local/ns/ate-system/sa/atenet-egress", "SPIFFE identity of the egress injector allowed to fetch credentials") @@ -99,7 +99,7 @@ func run(ctx context.Context) error { return fmt.Errorf("--namespace-policy-file is required") } - nsAuth, err := loadNamespaceAuthorizer(*nsPolicyFile) + nsAuth, err := LoadNamespaceAuthorizer(*nsPolicyFile) if err != nil { return fmt.Errorf("namespace policy: %w", err) } diff --git a/cmd/k8s-credential-provider/main_test.go b/cmd/credential-provider/kubernetes-secrets/main_test.go similarity index 99% rename from cmd/k8s-credential-provider/main_test.go rename to cmd/credential-provider/kubernetes-secrets/main_test.go index 03542b263b..483861f95d 100644 --- a/cmd/k8s-credential-provider/main_test.go +++ b/cmd/credential-provider/kubernetes-secrets/main_test.go @@ -132,7 +132,7 @@ func TestProviderMTLS(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("credential")}, }) srv := grpc.NewServer(grpc.Creds(creds)) - credproviderpb.RegisterCredentialProviderServer(srv, NewServer(client, &namespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}})) + credproviderpb.RegisterCredentialProviderServer(srv, NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}})) lis, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) diff --git a/cmd/k8s-credential-provider/manifests_test.go b/cmd/credential-provider/kubernetes-secrets/manifests_test.go similarity index 72% rename from cmd/k8s-credential-provider/manifests_test.go rename to cmd/credential-provider/kubernetes-secrets/manifests_test.go index 51fa0cfb9f..eb460de23d 100644 --- a/cmd/k8s-credential-provider/manifests_test.go +++ b/cmd/credential-provider/kubernetes-secrets/manifests_test.go @@ -28,17 +28,17 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" ) -func TestSidecarManifests(t *testing.T) { +func TestProviderManifests(t *testing.T) { for _, tc := range []struct { name, tool, namespace, prefix string args []string enabled bool }{ - {name: "disabled", tool: "helm", namespace: "ate-system", args: []string{"template", "substrate", "../../charts/substrate", "-n", "ate-system"}}, + {name: "disabled", tool: "helm", namespace: "ate-system", args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, {name: "custom release", tool: "helm", namespace: "custom", prefix: "test-", enabled: true, - args: []string{"template", "test", "../../charts/substrate", "-n", "custom", "--set", "ateApi.credentialProvider.enabled=true", "--set", "ateApi.credentialProvider.namespacePolicies[0].atespace=team-a", "--set", "ateApi.credentialProvider.namespacePolicies[0].allowedNamespaces[0]=ns1"}}, + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom", "--set", "credentialProvider.enabled=true", "--set", "credentialProvider.namespacePolicies[0].atespace=team-a", "--set", "credentialProvider.namespacePolicies[0].allowedNamespaces[0]=ns1"}}, {name: "kustomize", tool: "kubectl", namespace: "ate-system", enabled: true, - args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../manifests/ate-install/kubernetes-credentials"}}, + args: []string{"kustomize", "../../../manifests/egress-credential-injection"}}, } { t.Run(tc.name, func(t *testing.T) { if _, err := exec.LookPath(tc.tool); err != nil { @@ -49,7 +49,7 @@ func TestSidecarManifests(t *testing.T) { t.Fatalf("render: %v\n%s", err, data) } decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) - var sidecarFound, portFound, policyFound bool + var providerFound, portFound, policyFound, accountFound bool for { var doc struct { Kind string @@ -67,55 +67,57 @@ func TestSidecarManifests(t *testing.T) { t.Fatal(err) } switch doc.Kind { + case "ServiceAccount": + if doc.Metadata.Name == tc.prefix+"k8s-credential-provider" { + accountFound = true + } case "Deployment": - if doc.Metadata.Name != tc.prefix+"ate-api-server" { + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider" { continue } pod := doc.Spec.Template.Spec - if pod.ServiceAccountName != tc.prefix+"ate-api-server" { + if pod.ServiceAccountName != tc.prefix+"k8s-credential-provider" { t.Fatalf("unexpected ServiceAccount %q", pod.ServiceAccountName) } for _, container := range pod.Containers { - if container.Name != "credential-provider" { + if container.Name != "k8s-credential-provider" { continue } - sidecarFound = true + providerFound = true args := strings.Join(container.Args, " ") for _, required := range []string{ - "--listen-address=:50051", "--metrics-address=:9091", + "--listen-address=:50051", "--metrics-address=:9090", "--injector-spiffe-id=spiffe://cluster.local/ns/" + tc.namespace + "/sa/" + tc.prefix + "atenet-egress", "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem", "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem", } { + if tc.tool == "kubectl" && strings.HasPrefix(required, "--injector-spiffe-id=") { + continue + } if !strings.Contains(args, required) { - t.Errorf("sidecar missing %s", required) + t.Errorf("provider missing %s", required) } } - if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet.Port.StrVal != "cred-health" { + if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet.Port.StrVal != "metrics" { t.Fatal("missing dedicated readiness probe") } - for _, port := range container.Ports { - if port.ContainerPort == 443 || port.ContainerPort == 9090 { - t.Fatalf("sidecar conflicts with ateapi on %d", port.ContainerPort) - } - } } case "Service": - if doc.Metadata.Name != tc.prefix+"api" { + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider" { continue } for _, port := range doc.Spec.Ports { - if port.Port == 50051 && port.TargetPort.StrVal == "credentials" { + if port.Port == 50051 && port.TargetPort.StrVal == "grpc" { portFound = true } } case "ConfigMap": - if !strings.HasPrefix(doc.Metadata.Name, tc.prefix+"credential-provider-policy") { + if !strings.HasPrefix(doc.Metadata.Name, tc.prefix+"k8s-credential-provider-namespace-policy") { continue } policyFound = true var policy namespacePolicyFile - if err := yaml.UnmarshalStrict([]byte(doc.Data["policy.yaml"]), &policy); err != nil { + if err := yaml.UnmarshalStrict([]byte(doc.Data["namespace-policy.yaml"]), &policy); err != nil { t.Fatal(err) } auth, err := newNamespaceAuthorizer(policy) @@ -126,20 +128,20 @@ func TestSidecarManifests(t *testing.T) { t.Fatal("unexpected namespace policy") } case "ClusterRole": - if doc.Metadata.Name != tc.prefix+"ate-api-server-role" && doc.Metadata.Name != "ate-api-server" { + if !strings.Contains(doc.Metadata.Name, "k8s-credential-provider") && doc.Metadata.Name != tc.prefix+"ate-api-server-role" { continue } for _, rule := range doc.Rules { for _, resource := range rule.Resources { if resource == "secrets" || resource == "*" { - t.Fatal("sidecar grants cluster-wide Secret access") + t.Fatal("provider grants cluster-wide Secret access") } } } } } - if sidecarFound != tc.enabled || portFound != tc.enabled || policyFound != tc.enabled { - t.Fatalf("sidecar=%v port=%v policy=%v, enabled=%v", sidecarFound, portFound, policyFound, tc.enabled) + if providerFound != tc.enabled || portFound != tc.enabled || policyFound != tc.enabled || accountFound != tc.enabled { + t.Fatalf("provider=%v port=%v policy=%v account=%v, enabled=%v", providerFound, portFound, policyFound, accountFound, tc.enabled) } }) } @@ -151,11 +153,11 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { args []string enabled bool }{ - {name: "disabled", tool: "helm", args: []string{"template", "substrate", "../../charts/substrate", "-n", "ate-system"}}, - {name: "helm", tool: "helm", host: "test-api.custom.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", enabled: true, - args: []string{"template", "test", "../../charts/substrate", "-n", "custom", "--set", "ateApi.credentialProvider.enabled=true"}}, - {name: "kustomize", tool: "kubectl", host: "api.ate-system.svc:50051", roots: "/run/servicedns-ca/trust-bundle.pem", enabled: true, - args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../manifests/ate-install/agentgateway-egress-mitm"}}, + {name: "disabled", tool: "helm", args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, + {name: "helm", tool: "helm", host: "test-k8s-credential-provider.custom.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", enabled: true, + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom", "--set", "credentialProvider.enabled=true"}}, + {name: "kustomize", tool: "kubectl", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns-ca/trust-bundle.pem", enabled: true, + args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../../manifests/ate-install/agentgateway-egress-mitm"}}, } { t.Run(tc.name, func(t *testing.T) { if _, err := exec.LookPath(tc.tool); err != nil { diff --git a/cmd/k8s-credential-provider/policy.go b/cmd/credential-provider/kubernetes-secrets/nsauthz.go similarity index 87% rename from cmd/k8s-credential-provider/policy.go rename to cmd/credential-provider/kubernetes-secrets/nsauthz.go index 8ebcc5e4a6..c648220898 100644 --- a/cmd/k8s-credential-provider/policy.go +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz.go @@ -35,17 +35,17 @@ type atespaceNamespacePolicy struct { AllowedNamespaces []string `json:"allowedNamespaces"` } -// namespaceAuthorizer decides whether an atespace may resolve secrets in a given +// NamespaceAuthorizer decides whether an atespace may resolve secrets in a given // Kubernetes namespace. It is default-deny: an atespace absent from the mapping // can resolve nothing. -type namespaceAuthorizer struct { +type NamespaceAuthorizer struct { // allowed maps atespace -> set of permitted namespaces. allowed map[string]map[string]struct{} } -// loadNamespaceAuthorizer reads the YAML policy file at path and builds an +// LoadNamespaceAuthorizer reads the YAML policy file at path and builds an // authorizer, so a malformed file fails startup rather than the first request. -func loadNamespaceAuthorizer(path string) (*namespaceAuthorizer, error) { +func LoadNamespaceAuthorizer(path string) (*NamespaceAuthorizer, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("reading namespace policy file %q: %w", path, err) @@ -59,7 +59,7 @@ func loadNamespaceAuthorizer(path string) (*namespaceAuthorizer, error) { // newNamespaceAuthorizer builds an authorizer over a parsed policy file, // validating that each grant names an atespace. -func newNamespaceAuthorizer(file namespacePolicyFile) (*namespaceAuthorizer, error) { +func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, error) { allowed := make(map[string]map[string]struct{}) for i, p := range file.Policies { if !resources.IsValidResourceName(p.Atespace) { @@ -77,13 +77,13 @@ func newNamespaceAuthorizer(file namespacePolicyFile) (*namespaceAuthorizer, err set[ns] = struct{}{} } } - return &namespaceAuthorizer{allowed: allowed}, nil + return &NamespaceAuthorizer{allowed: allowed}, nil } // Allowed reports whether atespace may resolve secrets in namespace. Default // deny: an atespace absent from the mapping, or a namespace not in its list, is // refused. -func (a *namespaceAuthorizer) Allowed(atespace, namespace string) bool { +func (a *NamespaceAuthorizer) Allowed(atespace, namespace string) bool { if a == nil { return false } diff --git a/cmd/k8s-credential-provider/agentgateway_test.go b/cmd/k8s-credential-provider/agentgateway_test.go deleted file mode 100644 index 4ae6c8cffb..0000000000 --- a/cmd/k8s-credential-provider/agentgateway_test.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bufio" - "bytes" - "context" - "crypto" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" - "io" - "net" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/client-go/kubernetes/fake" - - "github.com/agent-substrate/substrate/internal/localca" - "github.com/agent-substrate/substrate/internal/substratex509" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" -) - -// TestAgentgatewayInjection exercises the rendered Helm route with a real AGW -// image and provider. Only Kubernetes storage and the ateapi actor lookup are -// faked. Docker must run locally on Linux for host networking. -func TestAgentgatewayInjection(t *testing.T) { - image := os.Getenv("AGENTGATEWAY_TEST_IMAGE") - if image == "" { - t.Skip("set AGENTGATEWAY_TEST_IMAGE to run the Docker integration test") - } - require.Equal(t, "linux", runtime.GOOS, "test requires Linux host networking") - dir := t.TempDir() - ca, err := localca.GenerateCA("integration", localca.KeyTypeECDSAP256, time.Hour) - require.NoError(t, err) - roots := x509.NewCertPool() - roots.AddCert(ca.RootCertificate) - write := func(name string, data []byte) string { - path := filepath.Join(dir, name) - require.NoError(t, os.WriteFile(path, data, 0600)) - return path - } - caPEM, err := ca.TLSCertificateChainPEM() - require.NoError(t, err) - caPath := write("ca.pem", caPEM) - caKey, err := ca.TLSPrivateKeyPEM() - require.NoError(t, err) - write("ca-key.pem", caKey) - servingCert := issueCertificate(t, ca, "") - writeBundle := func(name string, cert tls.Certificate) string { - key, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) - require.NoError(t, err) - bundle := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}) - bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]})...) - return write(name, bundle) - } - oldBundle, oldCAFile, oldInjector := *serverBundle, *clientCAFile, *injectorSPIFFEID - t.Cleanup(func() { *serverBundle, *clientCAFile, *injectorSPIFFEID = oldBundle, oldCAFile, oldInjector }) - *serverBundle, *clientCAFile = writeBundle("server.pem", servingCert), caPath - *injectorSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" - writeBundle("injector.pem", issueCertificate(t, ca, *injectorSPIFFEID)) - creds, err := buildServerCreds(t.Context()) - require.NoError(t, err) - kube := fake.NewSimpleClientset(&corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "team-a-secrets"}, - Data: map[string][]byte{"token": []byte("injected-token")}, - }) - rpc := grpc.NewServer(grpc.Creds(creds)) - credproviderpb.RegisterCredentialProviderServer(rpc, NewServer(kube, &namespaceAuthorizer{ - allowed: map[string]map[string]struct{}{"team-a": {"team-a-secrets": {}}}, - })) - ateapipb.RegisterControlServer(rpc, &credentialTestControl{}) - lis, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - go func() { _ = rpc.Serve(lis) }() - t.Cleanup(rpc.Stop) - - var upstreamCalls atomic.Int32 - origin := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamCalls.Add(1) - if r.Header.Get("Authorization") != "Bearer injected-token" { - w.WriteHeader(http.StatusUnauthorized) - return - } - w.WriteHeader(http.StatusNoContent) - })) - origin.TLS = &tls.Config{Certificates: []tls.Certificate{servingCert}, MinVersion: tls.VersionTLS13} - origin.StartTLS() - t.Cleanup(origin.Close) - _, originPort, err := net.SplitHostPort(origin.Listener.Addr().String()) - require.NoError(t, err) - target := "localhost:" + originPort - - // Keep the chart's route and TLS policies; substitute only local endpoints - // and test certificates, including the TLS origin's private CA. - rendered, err := exec.CommandContext(t.Context(), "helm", "template", "substrate", "../../charts/substrate", - "-n", "ate-system", "--set", "ateApi.credentialProvider.enabled=true").CombinedOutput() - require.NoError(t, err, "%s", rendered) - decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) - var config string - for { - var doc struct { - Kind string - Data map[string]string - } - err := decoder.Decode(&doc) - if err == io.EOF { - break - } - require.NoError(t, err) - if doc.Kind == "ConfigMap" && strings.Contains(doc.Data["config.yaml"], "credentialProviders:") { - config = doc.Data["config.yaml"] - } - } - require.NotEmpty(t, config) - portReservation, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - gatewayAddr := portReservation.Addr().String() - _, gatewayPort, err := net.SplitHostPort(gatewayAddr) - require.NoError(t, err) - require.NoError(t, portReservation.Close()) - config = strings.NewReplacer( - "api.ate-system.svc:443", lis.Addr().String(), - "api.ate-system.svc:50051", lis.Addr().String(), - "port: 8443", "port: "+gatewayPort, - "/run/servicedns.podcert.ate.dev/credential-bundle.pem", "/config/server.pem", - "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "/config/injector.pem", - "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "/config/ca.pem", - "/run/actor-id-ca-certs/ca.crt", "/config/ca.pem", - "/run/egress-mitm/tls.crt", "/config/ca.pem", - "/run/egress-mitm/tls.key", "/config/ca-key.pem", - "backendTLS: {}", "backendTLS: {root: /config/ca.pem}", - ).Replace(config) - write("config.yaml", []byte("config:\n adminAddr: 127.0.0.1:0\n statsAddr: 127.0.0.1:0\n readinessAddr: 127.0.0.1:0\n"+config)) - command := exec.CommandContext(t.Context(), "docker", "run", "--detach", "--rm", "--network=host", - "--user=0:0", "--volume", dir+":/config:ro", image, "-f", "/config/config.yaml") - var stderr bytes.Buffer - command.Stderr = &stderr - container, err := command.Output() - require.NoError(t, err, "%s", stderr.String()) - id := strings.TrimSpace(string(container)) - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - if t.Failed() { - logs, _ := exec.CommandContext(ctx, "docker", "logs", id).CombinedOutput() - t.Logf("AGW logs:\n%s", logs) - } - out, err := exec.CommandContext(ctx, "docker", "stop", "--timeout=1", id).CombinedOutput() - if err != nil { - t.Errorf("stop AGW: %v: %s", err, out) - } - }) - require.Eventually(t, func() bool { - conn, err := net.DialTimeout("tcp", gatewayAddr, 100*time.Millisecond) - if err != nil { - return false - } - _ = conn.Close() - return true - }, 20*time.Second, 100*time.Millisecond, "AGW did not start") - - for _, tc := range []struct { - name, atespace string - useTLS bool - wantStatus int - }{ - {"allowed", "team-a", true, http.StatusNoContent}, - {"different atespace cannot use cached secret", "team-b", true, http.StatusForbidden}, - {"cleartext cannot receive secrets", "team-a", false, http.StatusForbidden}, - } { - t.Run(tc.name, func(t *testing.T) { - beforeKube, beforeOrigin := len(kube.Actions()), upstreamCalls.Load() - actorCert := issueCertificate(t, ca, "") - template, err := x509.ParseCertificate(actorCert.Certificate[0]) - require.NoError(t, err) - require.NoError(t, substratex509.AddActorIdentityToCertificate(&substratex509.ActorIdentity{ - Atespace: tc.atespace, ActorName: "actor", ActorUid: "uid-1", Purpose: substratex509.ActorIdentityPurposeAtunnel, - }, template)) - der, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, - actorCert.PrivateKey.(crypto.Signer).Public(), ca.SigningKey) - require.NoError(t, err) - actorCert.Certificate = [][]byte{der} - outer, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", gatewayAddr, &tls.Config{ - RootCAs: roots, Certificates: []tls.Certificate{actorCert}, MinVersion: tls.VersionTLS13, - }) - require.NoError(t, err) - defer outer.Close() - require.NoError(t, outer.SetDeadline(time.Now().Add(10*time.Second))) - _, err = fmt.Fprintf(outer, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", target, target) - require.NoError(t, err) - response, err := http.ReadResponse(bufio.NewReader(outer), &http.Request{Method: http.MethodConnect}) - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode, "actor CONNECT authorization") - var tunnel net.Conn = outer - if tc.useTLS { - tunnel = tls.Client(outer, &tls.Config{RootCAs: roots, ServerName: "localhost", MinVersion: tls.VersionTLS13}) - } - _, err = fmt.Fprintf(tunnel, "GET / HTTP/1.1\r\nHost: %s\r\nAuthorization: actor-supplied\r\nConnection: close\r\n\r\n", target) - require.NoError(t, err) - response, err = http.ReadResponse(bufio.NewReader(tunnel), &http.Request{Method: http.MethodGet}) - require.NoError(t, err) - defer response.Body.Close() - require.Equal(t, tc.wantStatus, response.StatusCode) - if tc.wantStatus == http.StatusNoContent { - require.Len(t, kube.Actions(), beforeKube+1) - require.Equal(t, beforeOrigin+1, upstreamCalls.Load()) - } else { - require.Len(t, kube.Actions(), beforeKube, "denied request must not read Kubernetes") - require.Equal(t, beforeOrigin, upstreamCalls.Load(), "denied request must not reach upstream") - } - }) - } -} - -type credentialTestControl struct { - ateapipb.UnimplementedControlServer -} - -func (*credentialTestControl) GetActor(_ context.Context, _ *ateapipb.GetActorRequest) (*ateapipb.Actor, error) { - return &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Uid: "uid-1"}, - Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, - }, nil -} - -func (*credentialTestControl) GetActorEgressPolicy(_ context.Context, _ *ateapipb.GetActorEgressPolicyRequest) (*ateapipb.EgressPolicy, error) { - return &ateapipb.EgressPolicy{Rules: []*ateapipb.EgressRule{{Hostnames: &ateapipb.HostnameRule{ - Patterns: []string{"localhost"}, - Effects: &ateapipb.EgressRuleEffects{InjectStaticHeaders: []*ateapipb.CredentialHeaderInjection{{ - Header: "authorization", Prefix: "Bearer ", CredentialUri: "ate-secret://kubernetes.io/team-a-secrets/api/token", - }}}, - }}}}, nil -} diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md index 75c18a36e8..11e3b193fb 100644 --- a/docs/kubernetes-credential-provider.md +++ b/docs/kubernetes-credential-provider.md @@ -1,107 +1,74 @@ # Kubernetes credential provider -The optional `k8s-credential-provider` sidecar runs in ateapi's Pod. It serves -`CredentialProvider.FetchSecret` on `api.ate-system.svc:50051`, using ateapi's -serving certificate and ServiceAccount. ateapi's own gRPC service stays on 443. -No additional Deployment or ServiceAccount is needed. +The optional `k8s-credential-provider` Deployment follows the provider from +[upstream](https://github.com/agent-substrate/substrate/pull/1335). It serves +`CredentialProvider.FetchSecret` at `k8s-credential-provider.ate-system.svc:50051` +with its own ServiceAccount and projected serving certificate. AGW calls it +directly over mTLS on the HTTPS interception route. -A URI such as `ate-secret://kubernetes.io/team-a-secrets/example-api/token` -resolves the `token` entry in that Kubernetes Secret. Omitting the key is allowed -only for a Secret with exactly one data entry. Each request reads Kubernetes, -so Secret rotation is visible on the next provider fetch. AGW currently caches -successful credentials per actor and URI for five minutes, so injection may -continue using a cached value until it expires. -Secret values are neither persisted nor logged by the provider. +`ate-secret://kubernetes.io/team-a-secrets/example-api/token` resolves the `token` +entry in that Kubernetes Secret. Omitting the key requires exactly one data entry. +The provider reads Kubernetes on every fetch and never persists or logs values. +AGW caches successful credentials per actor and URI for five minutes, so rotation +can take that long to reach injected requests. -Access requires all three checks: +Each request requires a trusted injector certificate with the configured SPIFFE +identity, an explicit atespace-to-namespace grant for the attested actor, and +Kubernetes `get` permission for the provider's ServiceAccount. Empty policies +deny all requests. Secret permissions are granted separately from ateapi. -- The caller presents a trusted mTLS certificate with the configured egress - injector SPIFFE identity. A different trusted workload is still rejected. -- The actor SPIFFE identity attested by that injector belongs to an atespace - explicitly granted access to the Secret's namespace. Empty policies deny all. -- ateapi's ServiceAccount has Kubernetes `get` permission on that Secret. +## Enable the provider -The sidecar shares ateapi's Kubernetes identity and Pod failure domain. It is -process separation, not a separate Kubernetes authorization boundary. +Keep the pinned `images.agentgateway` image. It includes the +[protocol update](https://github.com/agentgateway/agentgateway/pull/3524) from +[this build](https://github.com/agentgateway/agentgateway/actions/runs/35238449333) +and implements the current [FetchSecret contract](../pkg/proto/credproviderpb/credprovider.proto). -## Agentgateway compatibility - -The pinned AGW nightly includes the -[credential protocol update](https://github.com/agentgateway/agentgateway/pull/3524) -from [this build](https://github.com/agentgateway/agentgateway/actions/runs/35238449333). -It implements the current Substrate credential protocol: - -- RPC: `/credprovider.CredentialProvider/FetchSecret`. -- Request: `uri` (field 1), `actor_spiffe_id` (field 2). -- Response: `opaque_bytes` (field 1). -- URI scheme: `ate-secret://`. - -The contract is [credprovider.proto](../pkg/proto/credproviderpb/credprovider.proto). -The provider does not implement the old `RequestSecret` RPC or -`substrate-secret://` scheme. - -## Enable the sidecar - -First create the MITM CA Secret using the existing installation tooling: +Create the MITM CA Secret using the existing installation tooling: ```sh hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret ``` -The Secret is named `egress-mitm-ca-pool` and must contain `tls.crt` and `tls.key` -in the gateway's namespace. For a non-default namespace, provision the same -Secret there. Actors must trust this CA; see the -[MITM trust bundle guide](egress-trust-bundle.md). +The gateway needs `egress-mitm-ca-pool` with `tls.crt` and `tls.key` in its namespace. +Actors must trust this CA; see the [MITM trust bundle guide](egress-trust-bundle.md). -For Helm, keep the pinned `images.agentgateway` image and add these values to -your existing release configuration: +For Helm, add these values to your release configuration: ```yaml -ateApi: - credentialProvider: - enabled: true - namespacePolicies: - - atespace: team-a - allowedNamespaces: [team-a-secrets] +credentialProvider: + enabled: true + namespacePolicies: + - atespace: team-a + allowedNamespaces: [team-a-secrets] ``` -The chart derives the injector identity and Service name from the release name -and namespace. For example, release `demo` in namespace `platform` serves at -`demo-api.platform.svc:50051` and accepts only +The feature is disabled by default. Enabling it deploys the provider and configures +AGW's HTTPS interception route. Cleartext egress cannot receive injected Secrets. +Policy changes roll the provider's Pods. Resource names and the injector identity +follow the release: release `demo` in namespace `platform` uses ServiceAccount +`demo-k8s-credential-provider`, endpoint +`demo-k8s-credential-provider.platform.svc:50051`, and injector identity `spiffe://cluster.local/ns/platform/sa/demo-atenet-egress`. -Enabling it also configures AGW's HTTPS interception route to fetch credentials -from the sidecar over mTLS. Credential providers are configured only on the HTTPS -route, so cleartext egress cannot receive injected Secrets. The feature is -disabled by default. Policy changes through Helm roll the ateapi -Pods so each sidecar loads the new policy. - -For the manifest installer, add -`manifests/ate-install/components/credential-provider` to your existing ateapi -Kustomization's `components`. Set the grants in its `policy.yaml`: -```yaml -policies: -- atespace: team-a - allowedNamespaces: [team-a-secrets] -``` - -A ready-made overlay of the repository's ateapi defaults is also available: +For the manifest installer, set your grants in +`manifests/egress-credential-injection/namespace-policy.yaml`, then deploy: ```sh -kubectl kustomize --load-restrictor=LoadRestrictionsNone \ - manifests/ate-install/kubernetes-credentials | ko apply -f - +kubectl kustomize manifests/egress-credential-injection | ko apply -f - +hack/install-ate.sh --deploy-atenet \ + --atenet-dataplane=agentgateway --experimental-use-sdsmint ``` -Preserve any existing installation-specific ateapi patches in your overlay. -The component's generated ConfigMap name changes with its policy, rolling the -Pods on reapplication. Direct edits to a mounted ConfigMap require a rollout -restart: the policy and client CA bundle are loaded at startup. The serving -credential bundle follows the existing certificate loader's rotation behavior. +The policy file uses `policies:` with the same list of grants as the Helm values. +Its generated ConfigMap name changes with the policy, rolling the provider on +reapplication. Direct ConfigMap edits require a rollout restart: policy and client +CA files are loaded at startup. Serving certificates rotate through the existing +certificate loader. ## Grant Secret access -Create the Secret in `team-a-secrets`, then grant only the required Secret reads. -For the default installation, this Role and RoleBinding allow the example above: +Create the Secret, then bind only the required reads to the provider: ```yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -126,76 +93,37 @@ roleRef: name: egress-credentials subjects: - kind: ServiceAccount - name: ate-api-server + name: k8s-credential-provider namespace: ate-system ``` -For a named Helm release, use its prefixed ateapi ServiceAccount and namespace. -Neither the chart nor the overlay grants cluster-wide Secret access. - -## Connect agentgateway - -The Helm configuration above wires `substrateEgress.credentialProviders` to the -sidecar on the HTTPS interception route. For the manifest installer, keep the -pinned AGW image, install the sidecar overlay, and deploy the AGW MITM -configuration: - -```sh -hack/install-ate.sh --deploy-atenet \ - --atenet-dataplane=agentgateway \ - --experimental-use-sdsmint -``` - -The AGW MITM component points `uriAuthority: kubernetes.io` at -`api.ate-system.svc:50051`, using its existing pod identity certificate and -service DNS CA bundle. No ext_proc injector is needed. - -Set an egress policy header injection's credential URI to -`ate-secret://kubernetes.io/team-a-secrets/example-api/token`, for example with -header `authorization` and prefix `Bearer `. Namespace grants alone do not -create an egress policy. +For a named Helm release, use its prefixed provider ServiceAccount and namespace. +Neither installer grants cluster-wide Secret access. -## Validate an AGW image locally +Set an actor's egress policy header injection to use credential URI +`ate-secret://kubernetes.io/team-a-secrets/example-api/token`, header +`authorization`, and prefix `Bearer `. Namespace grants alone do not create an +egress policy. No ext_proc injector is needed. -On Linux with a local Docker daemon and Helm installed, set -`AGENTGATEWAY_TEST_IMAGE` to the image reference from `images.agentgateway` in -`charts/substrate/values.yaml`, then run: +## Tests -```sh -go test -race ./cmd/k8s-credential-provider -run '^TestAgentgatewayInjection$' -count=1 -v -``` - -The test skips unless that environment variable is exported. It uses the -rendered Helm egress configuration, an authenticated actor CONNECT tunnel, TLS -interception, the real credential-provider server over mTLS, and a TLS upstream. -It verifies header injection, denial for an ungranted atespace after a successful -fetch, and denial of credential injection on cleartext egress. Kubernetes Secret -storage and ateapi's actor/policy RPCs are faked; it does not validate cluster -RBAC, certificate provisioning, or Pod deployment. +The Helm PR workflow runs `internal/e2e/suites/credentials` with real actors, +Secrets, RBAC, projected certificates, AGW, and the deployed provider. It checks +the exact injected token, an unauthenticated-origin control, namespace-policy +denial, cache isolation between atespaces, Kubernetes RBAC denial, and cleartext +denial. SubjectAccessReviews verify +the permission assumptions. -## Cluster E2E test - -The Helm CI workflow builds the sidecar image and runs -`internal/e2e/suites/credentials` after enabling the feature. The test deploys a -real actor and an in-cluster HTTPS origin, creates Secrets and namespace-scoped -RBAC, and exercises the actual ateapi, AGW, and provider. It checks the exact -injected token, an unauthenticated-origin control, namespace-policy denial, -Kubernetes RBAC denial, and cleartext denial. SubjectAccessReviews verify the -permission assumptions behind each denial. - -To run it on a dedicated Helm-installed Kind cluster with this branch's images -(including `k8s-credential-provider`) available: +On a dedicated Helm-installed Kind cluster with this branch's images, including +`kubernetes-secrets`, and the MITM CA Secret: ```sh -hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret helm upgrade substrate charts/substrate --namespace ate-system \ - --reuse-values -f internal/e2e/suites/credentials/values.yaml \ - --wait --timeout=5m + --reuse-values -f internal/e2e/suites/credentials/values.yaml --wait --timeout=5m E2E_ATENET_DATAPLANE=agentgateway E2E_CREDENTIAL_PROVIDER=1 \ hack/run-e2e-kind.sh ./internal/e2e/suites/credentials -v -args --no-color ``` -This configuration enables TLS interception cluster-wide, so run it after tests -that require passthrough egress. The test temporarily trusts the cluster's serving -CA for the local HTTPS origin, restoring that gateway configuration on cleanup. -It does not modify the actor or credential-provider authentication settings. +Enabling interception changes cluster egress TLS, so run this after tests that +require passthrough. The test temporarily trusts the cluster serving CA for its +local HTTPS origin and restores gateway configuration on cleanup. diff --git a/internal/e2e/suites/credentials/credentials_test.go b/internal/e2e/suites/credentials/credentials_test.go index 03fe439f73..67ac64a1fa 100644 --- a/internal/e2e/suites/credentials/credentials_test.go +++ b/internal/e2e/suites/credentials/credentials_test.go @@ -49,10 +49,11 @@ func TestKubernetesCredentialInjection(t *testing.T) { ctx := t.Context() clients := e2e.GetClients() namespace, template := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials", e2e.WithTrustBundle()) + deniedAtespace, deniedTemplate := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials-denied", e2e.WithTrustBundle()) otherNamespace := e2e.CreateNamespace(t).Name - api, err := clients.K8s.AppsV1().Deployments("ate-system").Get(ctx, "ate-api-server", metav1.GetOptions{}) + provider, err := clients.K8s.AppsV1().Deployments("ate-system").Get(ctx, "k8s-credential-provider", metav1.GetOptions{}) require.NoError(t, err) - serviceAccount := api.Spec.Template.Spec.ServiceAccountName + serviceAccount := provider.Spec.Template.Spec.ServiceAccountName require.NotEmpty(t, serviceAccount) for _, secret := range []struct{ namespace, name string }{ @@ -119,32 +120,47 @@ func TestKubernetesCredentialInjection(t *testing.T) { router, err := e2e.NewRouterClient(ctx) require.NoError(t, err) t.Cleanup(router.Close) + // Keep the successful actor alive through the cache-isolation check. + suite := t for _, tc := range []struct { name, secretNamespace, secret, scheme string want string }{ {"without-injection", "", "", "https", "401"}, {"allowed", namespace, "allowed", "https", "204"}, + {"atespace-denied", namespace, "allowed", "https", "403"}, {"namespace-denied", otherNamespace, "allowed", "https", "403"}, {"rbac-denied", namespace, "no-rbac", "https", "403"}, {"cleartext-denied", namespace, "allowed", "http", "403"}, } { t.Run(tc.name, func(t *testing.T) { - actor := &ateapipb.ObjectRef{Atespace: namespace, Name: tc.name} + atespace, actorTemplate := namespace, template + actorName := tc.name + if tc.name == "atespace-denied" { + // Use the already-fetched URI from an ungranted atespace so an + // incorrectly shared gateway cache cannot bypass authorization. + atespace, actorTemplate = deniedAtespace, deniedTemplate + actorName = "allowed" + } + actor := &ateapipb.ObjectRef{Atespace: atespace, Name: actorName} _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actor}) _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actor}) _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: namespace, Name: tc.name}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: namespace, Name: template.GetMetadata().GetName()}, + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: actorName}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: actorTemplate.GetMetadata().GetName()}, }}) require.NoError(t, err) - t.Cleanup(func() { + cleanupTest := t + if tc.name == "allowed" { + cleanupTest = suite + } + cleanupTest.Cleanup(func() { cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: actor}) _, err := clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: actor}) if err != nil { - t.Errorf("delete actor %s: %v", tc.name, err) + cleanupTest.Errorf("delete actor %s/%s: %v", atespace, actorName, err) } }) rule := e2e.EgressAllowHostnames(host) @@ -162,7 +178,7 @@ func TestKubernetesCredentialInjection(t *testing.T) { // retry still requires the precise result; transport errors never pass. deadline := time.Now().Add(90 * time.Second) for { - resp, err := router.Get(ctx, resources.ActorRef{Atespace: namespace, Name: tc.name}, path) + resp, err := router.Get(ctx, resources.ActorRef{Atespace: atespace, Name: actorName}, path) var body []byte if err == nil { body, err = io.ReadAll(resp.Body) diff --git a/internal/e2e/suites/credentials/values.yaml b/internal/e2e/suites/credentials/values.yaml index e5619662f3..22eb76aa53 100644 --- a/internal/e2e/suites/credentials/values.yaml +++ b/internal/e2e/suites/credentials/values.yaml @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -ateApi: - credentialProvider: - enabled: true - namespacePolicies: - - atespace: ate-e2e-probe-credentials - allowedNamespaces: [ate-e2e-probe-credentials] +credentialProvider: + enabled: true + namespacePolicies: + - atespace: ate-e2e-probe-credentials + allowedNamespaces: [ate-e2e-probe-credentials] diff --git a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml index 51b33fcef3..ce3a1dfa61 100644 --- a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml @@ -75,7 +75,7 @@ patches: credentialProviders: - uriAuthority: kubernetes.io target: - host: api.ate-system.svc:50051 + host: k8s-credential-provider.ate-system.svc:50051 policies: backendTLS: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem diff --git a/manifests/ate-install/components/credential-provider/kustomization.yaml b/manifests/ate-install/components/credential-provider/kustomization.yaml deleted file mode 100644 index cd18534422..0000000000 --- a/manifests/ate-install/components/credential-provider/kustomization.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kustomize.config.k8s.io/v1alpha1 -kind: Component - -configMapGenerator: -- name: credential-provider-policy - namespace: ate-system - files: - - policy.yaml - -patches: -- path: sidecar.yaml diff --git a/manifests/ate-install/components/credential-provider/sidecar.yaml b/manifests/ate-install/components/credential-provider/sidecar.yaml deleted file mode 100644 index 071cce3045..0000000000 --- a/manifests/ate-install/components/credential-provider/sidecar.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ate-api-server - namespace: ate-system -spec: - template: - spec: - containers: - - name: credential-provider - image: ko://github.com/agent-substrate/substrate/cmd/k8s-credential-provider - args: - - --listen-address=:50051 - - --metrics-address=:9091 - - --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem - - --client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem - - --injector-spiffe-id=spiffe://cluster.local/ns/ate-system/sa/atenet-egress - - --namespace-policy-file=/etc/credential-provider/policy.yaml - ports: - - name: credentials - containerPort: 50051 - - name: cred-health - containerPort: 9091 - readinessProbe: - httpGet: - path: /readyz - port: cred-health - periodSeconds: 2 - livenessProbe: - httpGet: - path: /healthz - port: cred-health - initialDelaySeconds: 10 - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - volumeMounts: - - name: servicedns - mountPath: /run/servicedns.podcert.ate.dev - readOnly: true - - name: podidentity - mountPath: /run/podidentity.podcert.ate.dev - readOnly: true - - name: credential-provider-policy - mountPath: /etc/credential-provider - readOnly: true - volumes: - - name: credential-provider-policy - configMap: - name: credential-provider-policy ---- -apiVersion: v1 -kind: Service -metadata: - name: api - namespace: ate-system -spec: - ports: - - name: credentials - protocol: TCP - port: 50051 - targetPort: credentials diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml new file mode 100644 index 0000000000..1c4c7410d7 --- /dev/null +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -0,0 +1,134 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Secret read RBAC must be granted separately in each allowed namespace. +# The credential provider: a gRPC service that resolves ate-secret:// URIs +# of the kubernetes.io class to Kubernetes Secret values. It is the ONLY +# component in the egress credential-injection path with Kubernetes access; the +# egress gateway and the injector never read Secrets. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8s-credential-provider + namespace: ate-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k8s-credential-provider + namespace: ate-system + labels: + app: k8s-credential-provider +spec: + replicas: 1 + selector: + matchLabels: + app: k8s-credential-provider + template: + metadata: + labels: + app: k8s-credential-provider + spec: + serviceAccountName: k8s-credential-provider + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + containers: + - name: k8s-credential-provider + image: ko://github.com/agent-substrate/substrate/cmd/credential-provider/kubernetes-secrets + args: + - "--listen-address=:50051" + - "--metrics-address=:9090" + # Serve with the pod's servicedns identity (SAN k8s-credential-provider.ate-system.svc) + # and require the injector to present a podidentity client cert whose chain + # verifies against the trust bundle. The provider additionally pins the + # caller's SAN to the egress gateway's identity + # (spiffe://cluster.local/ns/ate-system/sa/atenet-egress), so no other + # CA-trusted workload can fetch secrets. + - "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + # Enforce the atespace→namespace authorization policy (default-deny). + - "--namespace-policy-file=/etc/k8s-credential-provider/namespace-policy.yaml" + - "--log-level=info" + ports: + - name: grpc + containerPort: 50051 + - name: metrics + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: metrics + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: namespace-policy + mountPath: /etc/k8s-credential-provider + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + volumes: + - name: namespace-policy + configMap: + name: k8s-credential-provider-namespace-policy + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: k8s-credential-provider + namespace: ate-system +spec: + type: ClusterIP + selector: + app: k8s-credential-provider + ports: + - name: grpc + port: 50051 + targetPort: grpc + protocol: TCP diff --git a/manifests/ate-install/kubernetes-credentials/kustomization.yaml b/manifests/egress-credential-injection/kustomization.yaml similarity index 80% rename from manifests/ate-install/kubernetes-credentials/kustomization.yaml rename to manifests/egress-credential-injection/kustomization.yaml index 71da96655d..19bf054fef 100644 --- a/manifests/ate-install/kubernetes-credentials/kustomization.yaml +++ b/manifests/egress-credential-injection/kustomization.yaml @@ -16,7 +16,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../ate-api-server.yaml +- k8s-credential-provider.yaml -components: -- ../components/credential-provider +configMapGenerator: +- name: k8s-credential-provider-namespace-policy + namespace: ate-system + files: + - namespace-policy.yaml diff --git a/manifests/ate-install/components/credential-provider/policy.yaml b/manifests/egress-credential-injection/namespace-policy.yaml similarity index 87% rename from manifests/ate-install/components/credential-provider/policy.yaml rename to manifests/egress-credential-injection/namespace-policy.yaml index 603d3898c5..78747a0d26 100644 --- a/manifests/ate-install/components/credential-provider/policy.yaml +++ b/manifests/egress-credential-injection/namespace-policy.yaml @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Default deny. Add explicit atespace-to-namespace grants before enabling injection. +# Default-deny atespace-to-namespace grants. Secret RBAC is configured separately. policies: [] From 7d3392167676b659d9a2300228ecd387a4523ea2 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 19:53:49 +0000 Subject: [PATCH 6/7] Test credential injection through the installed HTTP route Install the provider's get-only Secret RBAC with the chart and standalone manifests. Enable injection on HTTP alongside HTTPS so the E2E can use the installed gateway configuration without modifying ConfigMaps or provisioning origin certificates. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 4 +- charts/substrate/templates/atenet-egress.yaml | 11 ++ .../templates/k8s-credential-provider.yaml | 24 +++- charts/substrate/values.yaml | 4 +- .../kubernetes-secrets/manifests_test.go | 60 +++++++--- docs/kubernetes-credential-provider.md | 66 ++++------- internal/e2e/fixtures/testserver/http.go | 12 +- internal/e2e/serverpod.go | 8 +- internal/e2e/serverpod_test.go | 14 --- .../suites/credentials/credentials_test.go | 111 +++--------------- .../kustomization.yaml | 9 ++ .../k8s-credential-provider.yaml | 24 +++- 12 files changed, 152 insertions(+), 195 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index a54bac5abe..6ece4426f3 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,8 +42,8 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | -| `credentialProvider.enabled` | `false` | Deploy the Kubernetes credential provider and AGW HTTPS injection; requires a MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | -| `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; Kubernetes Secret RBAC is configured separately | +| `credentialProvider.enabled` | `false` | Deploy the Kubernetes credential provider and AGW HTTP/HTTPS injection; requires a MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | +| `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index ae17e81f4e..019374f886 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -105,6 +105,17 @@ data: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- if .Values.credentialProvider.enabled }} + credentialProviders: + - uriAuthority: kubernetes.io + target: + host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- end }} - protocol: TCP tcpRoutes: - backends: diff --git a/charts/substrate/templates/k8s-credential-provider.yaml b/charts/substrate/templates/k8s-credential-provider.yaml index ae90099f6d..bdf8aa98b1 100644 --- a/charts/substrate/templates/k8s-credential-provider.yaml +++ b/charts/substrate/templates/k8s-credential-provider.yaml @@ -15,7 +15,6 @@ limitations under the License. */}} {{- if .Values.credentialProvider.enabled }} -# Secret read RBAC must be granted separately in each allowed namespace. # The credential provider: a gRPC service that resolves ate-secret:// URIs # of the kubernetes.io class to Kubernetes Secret values. It is the ONLY # component in the egress credential-injection path with Kubernetes access; the @@ -26,6 +25,29 @@ metadata: name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} namespace: {{ .Release.Namespace }} --- +# The provider checks the actor's atespace-to-namespace grant before reading. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index b1328a1f87..4789b65633 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -58,8 +58,8 @@ atelet: ateApi: extraArgs: [] -# Optional Kubernetes Secret provider and AGW HTTPS credential injection. -# Requires egress-mitm-ca-pool; grant Secret reads separately to the provider SA. +# Optional Kubernetes Secret provider and AGW HTTP/HTTPS credential injection. +# Requires egress-mitm-ca-pool; includes get-only Secret RBAC for the provider SA. credentialProvider: enabled: false namespacePolicies: [] diff --git a/cmd/credential-provider/kubernetes-secrets/manifests_test.go b/cmd/credential-provider/kubernetes-secrets/manifests_test.go index 60ba57c4c9..8200b5a90f 100644 --- a/cmd/credential-provider/kubernetes-secrets/manifests_test.go +++ b/cmd/credential-provider/kubernetes-secrets/manifests_test.go @@ -19,6 +19,7 @@ import ( "errors" "io" "os/exec" + "reflect" "slices" "strings" "testing" @@ -60,6 +61,7 @@ func TestProviderManifests(t *testing.T) { } decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) var providerFound, portFound, policyFound, accountFound bool + var roleFound, bindingFound bool for { var doc struct { Kind string @@ -68,8 +70,10 @@ func TestProviderManifests(t *testing.T) { Template corev1.PodTemplateSpec Ports []corev1.ServicePort } - Data map[string]string - Rules []rbacv1.PolicyRule + Data map[string]string + Rules []rbacv1.PolicyRule + RoleRef rbacv1.RoleRef + Subjects []rbacv1.Subject } if err := decoder.Decode(&doc); errors.Is(err, io.EOF) { break @@ -151,21 +155,32 @@ func TestProviderManifests(t *testing.T) { t.Fatal("unexpected namespace policy") } case "ClusterRole": - if !strings.Contains(doc.Metadata.Name, "k8s-credential-provider") && doc.Metadata.Name != tc.prefix+"ate-api-server-role" { + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider-secret-reader" { continue } - for _, rule := range doc.Rules { - for _, resource := range rule.Resources { - if resource == "secrets" || resource == "*" { - t.Fatal("provider grants cluster-wide Secret access") - } - } + roleFound = true + want := []rbacv1.PolicyRule{{APIGroups: []string{""}, Resources: []string{"secrets"}, Verbs: []string{"get"}}} + if !reflect.DeepEqual(doc.Rules, want) { + t.Fatalf("provider rules = %#v, want get-only Secret access", doc.Rules) + } + case "ClusterRoleBinding": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider-secret-reader" { + continue + } + bindingFound = true + wantRef := rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: tc.prefix + "k8s-credential-provider-secret-reader"} + wantSubjects := []rbacv1.Subject{{Kind: "ServiceAccount", Name: tc.prefix + "k8s-credential-provider", Namespace: tc.namespace}} + if doc.RoleRef != wantRef || !reflect.DeepEqual(doc.Subjects, wantSubjects) { + t.Fatalf("unexpected provider binding: roleRef=%+v subjects=%+v", doc.RoleRef, doc.Subjects) } } } if providerFound != tc.enabled || portFound != tc.enabled || policyFound != tc.enabled || accountFound != tc.enabled { t.Fatalf("provider=%v port=%v policy=%v account=%v, enabled=%v", providerFound, portFound, policyFound, accountFound, tc.enabled) } + if roleFound != tc.enabled || bindingFound != tc.enabled { + t.Fatalf("role=%v binding=%v, enabled=%v", roleFound, bindingFound, tc.enabled) + } }) } } @@ -191,7 +206,8 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { t.Fatalf("render: %v\n%s", err, data) } decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) - providers, mitmMounts := 0, 0 + providers := map[string]int{} + mitmMounts := 0 for { var doc struct { Kind string @@ -224,6 +240,10 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { Protocol string TLS struct{ Mode, Cert, Key string } Routes []struct { + Backends []struct { + Dynamic map[string]any + Policies struct{ BackendTLS map[string]any } + } Policies struct { SubstrateEgress struct { CredentialProviders []struct { @@ -248,12 +268,16 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { for _, listener := range bind.Listeners { for _, route := range listener.Routes { for _, provider := range route.Policies.SubstrateEgress.CredentialProviders { - providers++ - if listener.Protocol != "HTTPS" || listener.TLS.Mode != "dynamicCa" { - t.Fatal("credentials enabled outside TLS interception") - } - if listener.TLS.Cert != "/run/egress-mitm/tls.crt" || listener.TLS.Key != "/run/egress-mitm/tls.key" { - t.Fatal("incorrect MITM certificate paths") + providers[listener.Protocol]++ + if listener.Protocol == "HTTPS" { + if listener.TLS.Mode != "dynamicCa" || listener.TLS.Cert != "/run/egress-mitm/tls.crt" || listener.TLS.Key != "/run/egress-mitm/tls.key" { + t.Fatal("incorrect MITM configuration") + } + if len(route.Backends) != 1 || route.Backends[0].Dynamic == nil || len(route.Backends[0].Dynamic) != 0 || route.Backends[0].Policies.BackendTLS == nil || len(route.Backends[0].Policies.BackendTLS) != 0 { + t.Fatal("HTTPS must use a dynamic destination with default public TLS trust") + } + } else if listener.Protocol != "HTTP" { + t.Fatalf("credentials enabled on unexpected protocol %q", listener.Protocol) } if provider.URIAuthority != "kubernetes.io" || provider.Target.Host != tc.host { t.Fatalf("incorrect provider: %+v", provider) @@ -271,8 +295,8 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { if tc.enabled { want = 1 } - if providers != want || mitmMounts != want { - t.Fatalf("providers=%d MITM mounts=%d, want %d", providers, mitmMounts, want) + if providers["HTTP"] != want || providers["HTTPS"] != want || mitmMounts != want { + t.Fatalf("providers=%v MITM mounts=%d, want %d per protocol and %d mounts", providers, mitmMounts, want, want) } }) } diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md index 11e3b193fb..188acd451d 100644 --- a/docs/kubernetes-credential-provider.md +++ b/docs/kubernetes-credential-provider.md @@ -4,7 +4,7 @@ The optional `k8s-credential-provider` Deployment follows the provider from [upstream](https://github.com/agent-substrate/substrate/pull/1335). It serves `CredentialProvider.FetchSecret` at `k8s-credential-provider.ate-system.svc:50051` with its own ServiceAccount and projected serving certificate. AGW calls it -directly over mTLS on the HTTPS interception route. +directly over mTLS to inject credentials into HTTP and intercepted HTTPS requests. `ate-secret://kubernetes.io/team-a-secrets/example-api/token` resolves the `token` entry in that Kubernetes Secret. Omitting the key requires exactly one data entry. @@ -14,8 +14,10 @@ can take that long to reach injected requests. Each request requires a trusted injector certificate with the configured SPIFFE identity, an explicit atespace-to-namespace grant for the attested actor, and -Kubernetes `get` permission for the provider's ServiceAccount. Empty policies -deny all requests. Secret permissions are granted separately from ateapi. +Kubernetes `get` permission for the provider's ServiceAccount. Both installers +include the upstream get-only Secret ClusterRole and bind it to that ServiceAccount. +The provider can read Secrets across namespaces; its namespace policy controls +which namespaces each actor may use. Empty policies deny all requests. ## Enable the provider @@ -31,7 +33,8 @@ hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret ``` The gateway needs `egress-mitm-ca-pool` with `tls.crt` and `tls.key` in its namespace. -Actors must trust this CA; see the [MITM trust bundle guide](egress-trust-bundle.md). +Actors making HTTPS requests must trust this CA; see the +[MITM trust bundle guide](egress-trust-bundle.md). For Helm, add these values to your release configuration: @@ -44,13 +47,19 @@ credentialProvider: ``` The feature is disabled by default. Enabling it deploys the provider and configures -AGW's HTTPS interception route. Cleartext egress cannot receive injected Secrets. +AGW's HTTP route and HTTPS interception route. Policy changes roll the provider's Pods. Resource names and the injector identity follow the release: release `demo` in namespace `platform` uses ServiceAccount `demo-k8s-credential-provider`, endpoint `demo-k8s-credential-provider.platform.svc:50051`, and injector identity `spiffe://cluster.local/ns/platform/sa/demo-atenet-egress`. +HTTPS uses a dynamic backend: AGW selects the destination from the request and +validates its certificate using the system CA roots (`backendTLS: {}`). Public +APIs such as OpenAI and Anthropic need no per-backend certificates. The single +MITM CA lets AGW generate actor-facing certificates as needed. HTTP also travels +through the authenticated CONNECT tunnel, then leaves AGW over plaintext HTTP. + For the manifest installer, set your grants in `manifests/egress-credential-injection/namespace-policy.yaml`, then deploy: @@ -66,41 +75,9 @@ reapplication. Direct ConfigMap edits require a rollout restart: policy and clie CA files are loaded at startup. Serving certificates rotate through the existing certificate loader. -## Grant Secret access - -Create the Secret, then bind only the required reads to the provider: - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: egress-credentials - namespace: team-a-secrets -rules: -- apiGroups: [""] - resources: ["secrets"] - resourceNames: [example-api] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: egress-credentials - namespace: team-a-secrets -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: egress-credentials -subjects: -- kind: ServiceAccount - name: k8s-credential-provider - namespace: ate-system -``` - -For a named Helm release, use its prefixed provider ServiceAccount and namespace. -Neither installer grants cluster-wide Secret access. +## Configure injection -Set an actor's egress policy header injection to use credential URI +Create the Secret and set an actor's egress policy header injection to use credential URI `ate-secret://kubernetes.io/team-a-secrets/example-api/token`, header `authorization`, and prefix `Bearer `. Namespace grants alone do not create an egress policy. No ext_proc injector is needed. @@ -108,11 +85,10 @@ egress policy. No ext_proc injector is needed. ## Tests The Helm PR workflow runs `internal/e2e/suites/credentials` with real actors, -Secrets, RBAC, projected certificates, AGW, and the deployed provider. It checks +Secrets, chart-managed RBAC, AGW, and the deployed provider. It checks the exact injected token, an unauthenticated-origin control, namespace-policy -denial, cache isolation between atespaces, Kubernetes RBAC denial, and cleartext -denial. SubjectAccessReviews verify -the permission assumptions. +denial and cache isolation between atespaces. The local origin serves HTTP; +the suite uses the installed gateway configuration without modifying ConfigMaps. On a dedicated Helm-installed Kind cluster with this branch's images, including `kubernetes-secrets`, and the MITM CA Secret: @@ -125,5 +101,5 @@ E2E_ATENET_DATAPLANE=agentgateway E2E_CREDENTIAL_PROVIDER=1 \ ``` Enabling interception changes cluster egress TLS, so run this after tests that -require passthrough. The test temporarily trusts the cluster serving CA for its -local HTTPS origin and restores gateway configuration on cleanup. +require passthrough. This suite tests HTTP credential injection; the manifest +tests also check HTTPS interception and default public CA trust. diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index a4a3d06436..5eaf72806d 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -15,22 +15,19 @@ package main import ( - "crypto/tls" "log" "net/http" "os" "time" - "github.com/agent-substrate/substrate/internal/credbundle" "github.com/spf13/cobra" ) // newHTTPCmd is a plain HTTP/1.1 origin an Actor's egress lands on. It exists so // a test can assert the destination port is recovered from SO_ORIGINAL_DST. -// It can also serve TLS and verify an injected Authorization header against a -// mounted token for the credential-provider E2E test. +// It can also verify an injected Authorization header against a mounted token. func newHTTPCmd() *cobra.Command { - var listenAddress, tlsBundle, authorizationFile string + var listenAddress, authorizationFile string cmd := &cobra.Command{ Use: "http", Short: "Serve a plain HTTP/1.1 origin answering /healthz.", @@ -51,15 +48,10 @@ func newHTTPCmd() *cobra.Command { WriteTimeout: 2 * time.Minute, } log.Printf("testserver http: listening on %s", listenAddress) - if tlsBundle != "" { - server.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS13, GetCertificate: credbundle.Loader(tlsBundle)} - return server.ListenAndServeTLS("", "") - } return server.ListenAndServe() }, } cmd.Flags().StringVar(&listenAddress, "listen", ":8080", "Address the HTTP origin listens on.") - cmd.Flags().StringVar(&tlsBundle, "tls-bundle", "", "Serve HTTPS using this certificate and key bundle.") cmd.Flags().StringVar(&authorizationFile, "authorization-file", "", "Enable /credential, requiring a Bearer token matching this file.") return cmd } diff --git a/internal/e2e/serverpod.go b/internal/e2e/serverpod.go index a9727de00d..ea4d186337 100644 --- a/internal/e2e/serverpod.go +++ b/internal/e2e/serverpod.go @@ -68,8 +68,6 @@ type ServerPod struct { // an HTTP GET. A gRPC server answers an HTTP request with a protocol error, // so a server speaking grpc must set this and register the health service. GRPCProbe bool - // HTTPSProbe uses HTTPS for readiness. Ignored when GRPCProbe is set. - HTTPSProbe bool // HealthPath is the HTTP readiness path, defaulting to /healthz. Ignored // when GRPCProbe is set. HealthPath string @@ -181,9 +179,5 @@ func serverReadinessProbe(spec ServerPod, targetPort string) string { if path == "" { path = "/healthz" } - probe := fmt.Sprintf(" httpGet:\n path: %s\n port: %s", path, targetPort) - if spec.HTTPSProbe { - probe += "\n scheme: HTTPS" - } - return probe + return fmt.Sprintf(" httpGet:\n path: %s\n port: %s", path, targetPort) } diff --git a/internal/e2e/serverpod_test.go b/internal/e2e/serverpod_test.go index 429ef95678..53848d371e 100644 --- a/internal/e2e/serverpod_test.go +++ b/internal/e2e/serverpod_test.go @@ -156,20 +156,6 @@ func TestRenderServerPod_HTTPProbe(t *testing.T) { } } -func TestRenderServerPod_HTTPSProbe(t *testing.T) { - pod, service := renderServerPodDocs(t, ServerPod{ - Name: "tlsorigin", ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", - Args: []string{"http"}, Port: 443, TargetPort: 8443, HTTPSProbe: true, - }) - probe := pod.Spec.Containers[0].ReadinessProbe.HTTPGet - if probe == nil || probe.Scheme != corev1.URISchemeHTTPS || probe.Port.IntValue() != 8443 { - t.Fatalf("HTTPS readiness probe = %+v, want HTTPS on container port 8443", probe) - } - if service.Spec.Ports[0].Port != 443 { - t.Fatal("origin Service must expose port 443") - } -} - // TestRenderServerPod covers where each port lands: every field kubelet or // the binary reaches follows the listener, while the Service alone keeps the // published port. diff --git a/internal/e2e/suites/credentials/credentials_test.go b/internal/e2e/suites/credentials/credentials_test.go index 67ac64a1fa..6a1d16f42d 100644 --- a/internal/e2e/suites/credentials/credentials_test.go +++ b/internal/e2e/suites/credentials/credentials_test.go @@ -22,14 +22,11 @@ import ( "net/http" "net/url" "os" - "strings" "testing" "time" "github.com/stretchr/testify/require" - authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/agent-substrate/substrate/internal/e2e" @@ -37,9 +34,8 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) -// TestKubernetesCredentialInjection requires the Helm values in values.yaml and -// an egress MITM CA. It uses real actors, Kubernetes RBAC, and the deployed -// provider. The origin uses the cluster's serving CA to keep all traffic local. +// TestKubernetesCredentialInjection uses the Helm values in values.yaml, real +// actors, and a local HTTP origin with the installed gateway configuration and RBAC. func TestKubernetesCredentialInjection(t *testing.T) { if os.Getenv("E2E_CREDENTIAL_PROVIDER") == "" { t.Skip("enable the credential-provider Helm E2E values and set E2E_CREDENTIAL_PROVIDER=1") @@ -48,16 +44,12 @@ func TestKubernetesCredentialInjection(t *testing.T) { require.NoError(t, err) ctx := t.Context() clients := e2e.GetClients() - namespace, template := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials", e2e.WithTrustBundle()) - deniedAtespace, deniedTemplate := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials-denied", e2e.WithTrustBundle()) + namespace, template := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials") + deniedAtespace, deniedTemplate := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials-denied") otherNamespace := e2e.CreateNamespace(t).Name - provider, err := clients.K8s.AppsV1().Deployments("ate-system").Get(ctx, "k8s-credential-provider", metav1.GetOptions{}) - require.NoError(t, err) - serviceAccount := provider.Spec.Template.Spec.ServiceAccountName - require.NotEmpty(t, serviceAccount) for _, secret := range []struct{ namespace, name string }{ - {namespace, "allowed"}, {namespace, "no-rbac"}, {otherNamespace, "allowed"}, + {namespace, "allowed"}, {otherNamespace, "allowed"}, } { _, err := clients.K8s.CoreV1().Secrets(secret.namespace).Create(ctx, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: secret.name}, @@ -65,56 +57,16 @@ func TestKubernetesCredentialInjection(t *testing.T) { }, metav1.CreateOptions{}) require.NoError(t, err) } - for _, ns := range []string{namespace, otherNamespace} { - _, err := clients.K8s.RbacV1().Roles(ns).Create(ctx, &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{Name: "credential-reader"}, - Rules: []rbacv1.PolicyRule{{APIGroups: []string{""}, Resources: []string{"secrets"}, - ResourceNames: []string{"allowed"}, Verbs: []string{"get"}}}, - }, metav1.CreateOptions{}) - require.NoError(t, err) - _, err = clients.K8s.RbacV1().RoleBindings(ns).Create(ctx, &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "credential-reader"}, - RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "credential-reader"}, - Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Namespace: "ate-system", Name: serviceAccount}}, - }, metav1.CreateOptions{}) - require.NoError(t, err) - } - // Prove the negative controls isolate different boundaries: Kubernetes - // permits the other namespace, but the provider policy does not; within - // the granted namespace, Kubernetes itself refuses the second Secret. - for _, tc := range []struct { - namespace, secret string - allowed bool - }{{namespace, "allowed", true}, {otherNamespace, "allowed", true}, {namespace, "no-rbac", false}} { - require.Eventually(t, func() bool { - review, err := clients.K8s.AuthorizationV1().SubjectAccessReviews().Create(ctx, &authorizationv1.SubjectAccessReview{ - Spec: authorizationv1.SubjectAccessReviewSpec{ - User: "system:serviceaccount:ate-system:" + serviceAccount, - Groups: []string{"system:serviceaccounts", "system:serviceaccounts:ate-system", "system:authenticated"}, - ResourceAttributes: &authorizationv1.ResourceAttributes{ - Namespace: tc.namespace, Verb: "get", Resource: "secrets", Name: tc.secret, - }, - }, - }, metav1.CreateOptions{}) - return err == nil && review.Status.Allowed == tc.allowed - }, 30*time.Second, time.Second, "unexpected Secret RBAC for %s/%s", tc.namespace, tc.secret) - } - trustOriginCA(t, clients) e2e.DeployServerPod(t, ctx, e2e.ServerPod{ Name: "credential-origin", Namespace: namespace, ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", - Args: []string{"http", "--tls-bundle=/run/tls/bundle.pem", "--authorization-file=/run/token/token"}, - Port: 443, TargetPort: 8443, HTTPSProbe: true, + Args: []string{"http", "--authorization-file=/run/token/token"}, + Port: 80, TargetPort: 8080, Volumes: []corev1.Volume{ {Name: "token", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "allowed"}}}, - {Name: "tls", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{{PodCertificate: &corev1.PodCertificateProjection{ - SignerName: "servicedns.podcert.ate.dev/identity", KeyType: "ECDSAP256", CredentialBundlePath: "bundle.pem", - }}}, - }}}, }, - VolumeMounts: []corev1.VolumeMount{{Name: "tls", MountPath: "/run/tls", ReadOnly: true}, {Name: "token", MountPath: "/run/token", ReadOnly: true}}, + VolumeMounts: []corev1.VolumeMount{{Name: "token", MountPath: "/run/token", ReadOnly: true}}, }) host := "credential-origin." + namespace + ".svc" router, err := e2e.NewRouterClient(ctx) @@ -123,15 +75,12 @@ func TestKubernetesCredentialInjection(t *testing.T) { // Keep the successful actor alive through the cache-isolation check. suite := t for _, tc := range []struct { - name, secretNamespace, secret, scheme string - want string + name, secretNamespace, secret, want string }{ - {"without-injection", "", "", "https", "401"}, - {"allowed", namespace, "allowed", "https", "204"}, - {"atespace-denied", namespace, "allowed", "https", "403"}, - {"namespace-denied", otherNamespace, "allowed", "https", "403"}, - {"rbac-denied", namespace, "no-rbac", "https", "403"}, - {"cleartext-denied", namespace, "allowed", "http", "403"}, + {"without-injection", "", "", "401"}, + {"allowed", namespace, "allowed", "204"}, + {"atespace-denied", namespace, "allowed", "403"}, + {"namespace-denied", otherNamespace, "allowed", "403"}, } { t.Run(tc.name, func(t *testing.T) { atespace, actorTemplate := namespace, template @@ -173,9 +122,9 @@ func TestKubernetesCredentialInjection(t *testing.T) { e2e.EnsureEgressPolicy(t, ctx, clients, actor, rule) _, err = clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actor}) require.NoError(t, err) - path := "/fetch?roots=bundle&url=" + url.QueryEscape(tc.scheme+"://"+host+"/credential") - // ConfigMap projections and route discovery are asynchronous. Each - // retry still requires the precise result; transport errors never pass. + path := "/fetch?roots=system&url=" + url.QueryEscape("http://"+host+"/credential") + // Route discovery is asynchronous. Retries require the precise status; + // transport errors never pass. deadline := time.Now().Add(90 * time.Second) for { resp, err := router.Get(ctx, resources.ActorRef{Atespace: atespace, Name: actorName}, path) @@ -196,31 +145,3 @@ func TestKubernetesCredentialInjection(t *testing.T) { }) } } - -// Only the local origin needs a private CA. Keep the production route, actor -// authentication, and credential-provider TLS settings intact, and restore the -// gateway configuration after the test. -func trustOriginCA(t *testing.T, clients *e2e.Clients) { - t.Helper() - configMaps := clients.K8s.CoreV1().ConfigMaps("ate-system") - config, err := configMaps.Get(t.Context(), "atenet-egress-agentgateway-config", metav1.GetOptions{}) - require.NoError(t, err) - original := config.Data["config.yaml"] - require.Equal(t, 1, strings.Count(original, "backendTLS: {}"), "expected one dynamic HTTPS upstream") - config.Data["config.yaml"] = strings.Replace(original, "backendTLS: {}", - "backendTLS: {root: /run/servicedns.podcert.ate.dev/trust-bundle.pem}", 1) - _, err = configMaps.Update(t.Context(), config, metav1.UpdateOptions{}) - require.NoError(t, err) - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - config, err := configMaps.Get(ctx, config.Name, metav1.GetOptions{}) - if err == nil { - config.Data["config.yaml"] = original - _, err = configMaps.Update(ctx, config, metav1.UpdateOptions{}) - } - if err != nil { - t.Errorf("restore gateway configuration: %v", err) - } - }) -} diff --git a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml index ce3a1dfa61..c40e6de9da 100644 --- a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml @@ -94,6 +94,15 @@ patches: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem + credentialProviders: + - uriAuthority: kubernetes.io + target: + host: k8s-credential-provider.ate-system.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem - protocol: TCP tcpRoutes: - backends: diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml index 1c4c7410d7..50fa38eaba 100644 --- a/manifests/egress-credential-injection/k8s-credential-provider.yaml +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Secret read RBAC must be granted separately in each allowed namespace. # The credential provider: a gRPC service that resolves ate-secret:// URIs # of the kubernetes.io class to Kubernetes Secret values. It is the ONLY # component in the egress credential-injection path with Kubernetes access; the @@ -23,6 +22,29 @@ metadata: name: k8s-credential-provider namespace: ate-system --- +# The provider checks the actor's atespace-to-namespace grant before reading. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8s-credential-provider-secret-reader +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8s-credential-provider-secret-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8s-credential-provider-secret-reader +subjects: +- kind: ServiceAccount + name: k8s-credential-provider + namespace: ate-system +--- apiVersion: apps/v1 kind: Deployment metadata: From 3da1c21b01879f9bc5b1048842a7ed31404cfa2f Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 20:13:33 +0000 Subject: [PATCH 7/7] Always install credential injection and HTTPS interception Configure the provider and MITM gateway on every Helm install, and run credential and MITM E2E alongside the standard suites using the initial configuration. Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 17 ++---- charts/substrate/README.md | 6 ++- charts/substrate/templates/atenet-egress.yaml | 15 ------ .../templates/k8s-credential-provider.yaml | 2 - charts/substrate/values.yaml | 5 +- .../kubernetes-secrets/manifests_test.go | 54 ++++++++++--------- docs/kubernetes-credential-provider.md | 28 +++++----- hack/render-manifests.sh | 2 + .../suites/credentials/credentials_test.go | 2 +- internal/e2e/suites/credentials/values.yaml | 1 - 10 files changed, 58 insertions(+), 74 deletions(-) diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 8ec69ff877..6a7438a89c 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -25,6 +25,8 @@ jobs: env: VERSION: helm-e2e E2E_ATENET_DATAPLANE: agentgateway + E2E_CREDENTIAL_PROVIDER: "1" + E2E_EGRESS_MITM: "1" steps: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 @@ -72,6 +74,7 @@ jobs: helm upgrade --install substrate charts/substrate \ --namespace ate-system \ --create-namespace \ + -f internal/e2e/suites/credentials/values.yaml \ --set image.registry=localhost:5001 \ --set image.tag=helm-e2e \ --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ @@ -84,6 +87,7 @@ jobs: hack/install-ate-kind.sh --create-actor-id-ca-pool-secret hack/install-ate-kind.sh --create-actor-id-ca-certs-secret hack/install-ate-kind.sh --create-api-authentication-config + hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret - name: Wait for Helm install run: | helm upgrade substrate charts/substrate \ @@ -104,24 +108,13 @@ jobs: - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demo - run: hack/install-ate-kind.sh --deploy-demo-egress + run: hack/install-ate-kind.sh --deploy-demo-egress-mitm - name: Run E2E tests (gVisor) run: hack/run-e2e-kind.sh -v -args --no-color - name: Run E2E tests (micro-VM) env: E2E_SANDBOX_CLASS: microvm run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color - - name: Enable Kubernetes credential injection - # Interception changes egress TLS, so enable it after the standard lanes. - run: | - hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret - helm upgrade substrate charts/substrate --namespace ate-system \ - --reuse-values -f internal/e2e/suites/credentials/values.yaml \ - --wait --timeout=5m - - name: Run E2E tests (Kubernetes credentials) - env: - E2E_CREDENTIAL_PROVIDER: "1" - run: hack/run-e2e-kind.sh ./internal/e2e/suites/credentials -v -args --no-color - name: Dump diagnostics on failure if: failure() run: | diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 6ece4426f3..2803a5aa63 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -19,6 +19,11 @@ By default, component images are pulled from `ghcr.io/kagent-dev/substrate` using the chart `appVersion` as the tag. Override `image.registry` and `image.tag` to install from a different image repository or tag. +The chart installs the Kubernetes credential provider and enables HTTPS egress +interception. Create the `egress-mitm-ca-pool` Secret and configure actor trust +as described in the [credential provider setup](../../docs/kubernetes-credential-provider.md). +Namespace grants default to an empty list, denying credential access. + ## Render manifests without applying ```bash @@ -42,7 +47,6 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | -| `credentialProvider.enabled` | `false` | Deploy the Kubernetes credential provider and AGW HTTP/HTTPS injection; requires a MITM CA Secret; see [setup](../../docs/kubernetes-credential-provider.md) | | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 019374f886..3ae521e96d 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -56,7 +56,6 @@ data: - mode: internal protocol: AUTO listeners: -{{- if .Values.credentialProvider.enabled }} - protocol: HTTPS tls: mode: dynamicCa @@ -84,14 +83,6 @@ data: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem -{{- else }} - - protocol: TLS - hostname: "*" - tcpRoutes: - - backends: - - dynamic: - target: source.connectHeaders["host"] -{{- end }} - protocol: HTTP routes: - backends: @@ -105,7 +96,6 @@ data: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem -{{- if .Values.credentialProvider.enabled }} credentialProviders: - uriAuthority: kubernetes.io target: @@ -115,7 +105,6 @@ data: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem -{{- end }} - protocol: TCP tcpRoutes: - backends: @@ -174,11 +163,9 @@ spec: port: readiness periodSeconds: 1 volumeMounts: -{{- if .Values.credentialProvider.enabled }} - name: egress-mitm mountPath: /run/egress-mitm readOnly: true -{{- end }} - name: config mountPath: /etc/agentgateway readOnly: true @@ -240,7 +227,6 @@ spec: - name: drain-signal mountPath: /var/run/atenet volumes: -{{- if .Values.credentialProvider.enabled }} - name: egress-mitm secret: secretName: egress-mitm-ca-pool @@ -249,7 +235,6 @@ spec: path: tls.crt - key: tls.key path: tls.key -{{- end }} - name: config configMap: name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} diff --git a/charts/substrate/templates/k8s-credential-provider.yaml b/charts/substrate/templates/k8s-credential-provider.yaml index bdf8aa98b1..753cbc9511 100644 --- a/charts/substrate/templates/k8s-credential-provider.yaml +++ b/charts/substrate/templates/k8s-credential-provider.yaml @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.credentialProvider.enabled }} # The credential provider: a gRPC service that resolves ate-secret:// URIs # of the kubernetes.io class to Kubernetes Secret values. It is the ONLY # component in the egress credential-injection path with Kubernetes access; the @@ -166,4 +165,3 @@ metadata: data: namespace-policy.yaml: | policies: {{ toJson .Values.credentialProvider.namespacePolicies }} -{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 4789b65633..fef293a355 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -58,10 +58,9 @@ atelet: ateApi: extraArgs: [] -# Optional Kubernetes Secret provider and AGW HTTP/HTTPS credential injection. -# Requires egress-mitm-ca-pool; includes get-only Secret RBAC for the provider SA. +# Kubernetes Secret provider and AGW HTTP/HTTPS credential injection. +# Includes get-only Secret RBAC. HTTPS requires egress-mitm-ca-pool and actor trust. credentialProvider: - enabled: false namespacePolicies: [] # - atespace: team-a # allowedNamespaces: [team-a-secrets] diff --git a/cmd/credential-provider/kubernetes-secrets/manifests_test.go b/cmd/credential-provider/kubernetes-secrets/manifests_test.go index 8200b5a90f..75a99bb6c0 100644 --- a/cmd/credential-provider/kubernetes-secrets/manifests_test.go +++ b/cmd/credential-provider/kubernetes-secrets/manifests_test.go @@ -35,19 +35,18 @@ func TestProviderManifests(t *testing.T) { name, tool, namespace, prefix string image string args []string - enabled bool }{ - {name: "disabled", tool: "helm", namespace: "ate-system", args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, - {name: "custom release", tool: "helm", namespace: "custom", prefix: "test-", enabled: true, - args: []string{"template", "test", "../../../charts/substrate", "-n", "custom", "--set", "credentialProvider.enabled=true", "--set", "credentialProvider.namespacePolicies[0].atespace=team-a", "--set", "credentialProvider.namespacePolicies[0].allowedNamespaces[0]=ns1"}}, - {name: "kustomize", tool: "kubectl", namespace: "ate-system", enabled: true, + {name: "default", tool: "helm", namespace: "ate-system", args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, + {name: "custom release", tool: "helm", namespace: "custom", prefix: "test-", + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom", "--set", "credentialProvider.namespacePolicies[0].atespace=team-a", "--set", "credentialProvider.namespacePolicies[0].allowedNamespaces[0]=ns1"}}, + {name: "kustomize", tool: "kubectl", namespace: "ate-system", args: []string{"kustomize", "../../../manifests/egress-credential-injection"}}, - {name: "CI images", tool: "helm", namespace: "ate-system", enabled: true, + {name: "CI images", tool: "helm", namespace: "ate-system", image: "localhost:5001/kagent-dev/substrate/kubernetes-secrets:helm-e2e", - args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", "--set", "credentialProvider.enabled=true", "--set", "image.registry=localhost:5001", "--set", "image.tag=helm-e2e"}}, - {name: "global images", tool: "helm", namespace: "ate-system", enabled: true, + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", "--set", "image.registry=localhost:5001", "--set", "image.tag=helm-e2e"}}, + {name: "global images", tool: "helm", namespace: "ate-system", image: "mirror.example/custom/substrate/kubernetes-secrets:test", - args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", "--set", "credentialProvider.enabled=true", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", "--set", "image.repository=custom/substrate", "--set", "image.tag=test", "--set", "global.imageRegistry=mirror.example", "--set", "imagePullSecrets[0].name=local", "--set", "global.imagePullSecrets[0].name=global", "--set", "global.imagePullPolicy=Always"}}, } { @@ -175,11 +174,11 @@ func TestProviderManifests(t *testing.T) { } } } - if providerFound != tc.enabled || portFound != tc.enabled || policyFound != tc.enabled || accountFound != tc.enabled { - t.Fatalf("provider=%v port=%v policy=%v account=%v, enabled=%v", providerFound, portFound, policyFound, accountFound, tc.enabled) + if !providerFound || !portFound || !policyFound || !accountFound { + t.Fatalf("provider=%v port=%v policy=%v account=%v", providerFound, portFound, policyFound, accountFound) } - if roleFound != tc.enabled || bindingFound != tc.enabled { - t.Fatalf("role=%v binding=%v, enabled=%v", roleFound, bindingFound, tc.enabled) + if !roleFound || !bindingFound { + t.Fatalf("role=%v binding=%v", roleFound, bindingFound) } }) } @@ -189,12 +188,12 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { for _, tc := range []struct { name, tool, host, roots string args []string - enabled bool }{ - {name: "disabled", tool: "helm", args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, - {name: "helm", tool: "helm", host: "test-k8s-credential-provider.custom.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", enabled: true, - args: []string{"template", "test", "../../../charts/substrate", "-n", "custom", "--set", "credentialProvider.enabled=true"}}, - {name: "kustomize", tool: "kubectl", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns-ca/trust-bundle.pem", enabled: true, + {name: "default", tool: "helm", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, + {name: "custom release", tool: "helm", host: "test-k8s-credential-provider.custom.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom"}}, + {name: "kustomize", tool: "kubectl", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns-ca/trust-bundle.pem", args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../../manifests/ate-install/agentgateway-egress-mitm"}}, } { t.Run(tc.name, func(t *testing.T) { @@ -207,7 +206,7 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { } decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) providers := map[string]int{} - mitmMounts := 0 + mitmMounts, mitmVolumes, passthroughListeners := 0, 0, 0 for { var doc struct { Kind string @@ -220,6 +219,11 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { t.Fatal(err) } if doc.Kind == "Deployment" { + for _, volume := range doc.Spec.Template.Spec.Volumes { + if volume.Secret != nil && volume.Secret.SecretName == "egress-mitm-ca-pool" { + mitmVolumes++ + } + } for _, container := range doc.Spec.Template.Spec.Containers { if container.Name != "agentgateway" { continue @@ -266,6 +270,9 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { } for _, bind := range config.Binds { for _, listener := range bind.Listeners { + if listener.Protocol == "TLS" { + passthroughListeners++ + } for _, route := range listener.Routes { for _, provider := range route.Policies.SubstrateEgress.CredentialProviders { providers[listener.Protocol]++ @@ -291,12 +298,11 @@ func TestAgentgatewayCredentialConfiguration(t *testing.T) { } } } - want := 0 - if tc.enabled { - want = 1 + if providers["HTTP"] != 1 || providers["HTTPS"] != 1 { + t.Fatalf("providers=%v, want one per HTTP/HTTPS route", providers) } - if providers["HTTP"] != want || providers["HTTPS"] != want || mitmMounts != want { - t.Fatalf("providers=%v MITM mounts=%d, want %d per protocol and %d mounts", providers, mitmMounts, want, want) + if mitmMounts != 1 || mitmVolumes != 1 || passthroughListeners != 0 { + t.Fatalf("MITM mounts=%d volumes=%d passthrough listeners=%d", mitmMounts, mitmVolumes, passthroughListeners) } }) } diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md index 188acd451d..cfc1879480 100644 --- a/docs/kubernetes-credential-provider.md +++ b/docs/kubernetes-credential-provider.md @@ -1,6 +1,6 @@ # Kubernetes credential provider -The optional `k8s-credential-provider` Deployment follows the provider from +The `k8s-credential-provider` Deployment follows the provider from [upstream](https://github.com/agent-substrate/substrate/pull/1335). It serves `CredentialProvider.FetchSecret` at `k8s-credential-provider.ate-system.svc:50051` with its own ServiceAccount and projected serving certificate. AGW calls it @@ -19,7 +19,7 @@ include the upstream get-only Secret ClusterRole and bind it to that ServiceAcco The provider can read Secrets across namespaces; its namespace policy controls which namespaces each actor may use. Empty policies deny all requests. -## Enable the provider +## Configure the provider Keep the pinned `images.agentgateway` image. It includes the [protocol update](https://github.com/agentgateway/agentgateway/pull/3524) from @@ -40,14 +40,13 @@ For Helm, add these values to your release configuration: ```yaml credentialProvider: - enabled: true namespacePolicies: - atespace: team-a allowedNamespaces: [team-a-secrets] ``` -The feature is disabled by default. Enabling it deploys the provider and configures -AGW's HTTP route and HTTPS interception route. +The Helm chart always deploys the provider and configures AGW's HTTP route and +HTTPS interception route. Namespace grants default to an empty list. Policy changes roll the provider's Pods. Resource names and the injector identity follow the release: release `demo` in namespace `platform` uses ServiceAccount `demo-k8s-credential-provider`, endpoint @@ -84,22 +83,21 @@ egress policy. No ext_proc injector is needed. ## Tests -The Helm PR workflow runs `internal/e2e/suites/credentials` with real actors, +The Helm PR workflow installs the provider and MITM gateway from the start and +runs `internal/e2e/suites/credentials` alongside the standard suites with real actors, Secrets, chart-managed RBAC, AGW, and the deployed provider. It checks the exact injected token, an unauthenticated-origin control, namespace-policy denial and cache isolation between atespaces. The local origin serves HTTP; the suite uses the installed gateway configuration without modifying ConfigMaps. -On a dedicated Helm-installed Kind cluster with this branch's images, including -`kubernetes-secrets`, and the MITM CA Secret: +Include `-f internal/e2e/suites/credentials/values.yaml` in the initial Helm +installation to grant the test atespace access. After deploying the standard +MITM egress fixtures, run the suites together: ```sh -helm upgrade substrate charts/substrate --namespace ate-system \ - --reuse-values -f internal/e2e/suites/credentials/values.yaml --wait --timeout=5m -E2E_ATENET_DATAPLANE=agentgateway E2E_CREDENTIAL_PROVIDER=1 \ - hack/run-e2e-kind.sh ./internal/e2e/suites/credentials -v -args --no-color +E2E_ATENET_DATAPLANE=agentgateway E2E_CREDENTIAL_PROVIDER=1 E2E_EGRESS_MITM=1 \ + hack/run-e2e-kind.sh -v -args --no-color ``` -Enabling interception changes cluster egress TLS, so run this after tests that -require passthrough. This suite tests HTTP credential injection; the manifest -tests also check HTTPS interception and default public CA trust. +The credential suite tests HTTP injection. The existing MITM suite checks HTTPS +interception and actor trust against a public HTTPS origin using the same install. diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index 2187044970..e75108a1cb 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -38,6 +38,8 @@ PRESERVED_FILES=( atenet-egress-with-sdsmint.yaml atenet-router.yaml atenet-router-monitoring.yaml + # The provider's upstream manifest lives in manifests/egress-credential-injection. + k8s-credential-provider.yaml pod-certificate-controller.yaml postgres.yaml sandboxconfig-gvisor.yaml diff --git a/internal/e2e/suites/credentials/credentials_test.go b/internal/e2e/suites/credentials/credentials_test.go index 6a1d16f42d..5f5d1d2bb7 100644 --- a/internal/e2e/suites/credentials/credentials_test.go +++ b/internal/e2e/suites/credentials/credentials_test.go @@ -38,7 +38,7 @@ import ( // actors, and a local HTTP origin with the installed gateway configuration and RBAC. func TestKubernetesCredentialInjection(t *testing.T) { if os.Getenv("E2E_CREDENTIAL_PROVIDER") == "" { - t.Skip("enable the credential-provider Helm E2E values and set E2E_CREDENTIAL_PROVIDER=1") + t.Skip("requires credential E2E namespace grants and E2E_CREDENTIAL_PROVIDER=1") } env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") require.NoError(t, err) diff --git a/internal/e2e/suites/credentials/values.yaml b/internal/e2e/suites/credentials/values.yaml index 22eb76aa53..27ff95125a 100644 --- a/internal/e2e/suites/credentials/values.yaml +++ b/internal/e2e/suites/credentials/values.yaml @@ -13,7 +13,6 @@ # limitations under the License. credentialProvider: - enabled: true namespacePolicies: - atespace: ate-e2e-probe-credentials allowedNamespaces: [ate-e2e-probe-credentials]