Separate PostgresQL runtime and schema ownership

cmd/ate-setup/differences.md

@@ -277,5 +277,5 @@ to the in-cluster database, leaving behind an orphaned proxy. Use
 
 The shell installer had no tests. `cmd/ate-setup` has unit tests for template
 rendering, overlay selection, config resolution, the authentication config, the
-apiserver environment ConfigMap, delegated script arguments, manifest deletion,
+apiserver environment ConfigMap and Secret, delegated script arguments, manifest deletion,
 per-demo rendering, and image reference rewriting.

cmd/ate-setup/internal/config/config.go

@@ -43,12 +43,6 @@ const (
 // DefaultRolloutTimeout is the default wait timeout for workload rollouts.
 const DefaultRolloutTimeout = 60 * time.Second
 
-// DefaultPostgresConnectionString mirrors default_postgres_connection_string in
-// the shell installer: the apiserver reaches PostgreSQL over mTLS using the
-// podcertificate controller's projected servicedns trust bundle and its own
-// podidentity credential bundle.
-const DefaultPostgresConnectionString = "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem"
-
 // DefaultPostgresSchema mirrors the shell installer's default for
 // ATE_API_POSTGRES_SCHEMA, the PostgreSQL schema holding the Substrate tables.
 const DefaultPostgresSchema = "public"
@@ -98,8 +92,12 @@ type Config struct {
 	// Router selects the atenet router dataplane.
 	Router string
 	// PostgresConnectionString is the apiserver's store connection string.
-	// Empty means use DefaultPostgresConnectionString.
+	// Empty means use the bundled PostgreSQL runtime role.
 	PostgresConnectionString string
+	// PostgresDDLConnectionString is the optional schema-owner connection
+	// string. Empty means use the runtime string for an external database; a
+	// non-empty value requires PostgresConnectionString.
+	PostgresDDLConnectionString string
 	// PostgresSchema is the PostgreSQL schema for the Substrate tables
 	// (ATE_API_POSTGRES_SCHEMA). Empty means DefaultPostgresSchema.
 	PostgresSchema string
@@ -232,6 +230,7 @@ func Load(opts Options) (*Config, error) {
 		KODefaultPlatforms:                    env["KO_DEFAULTPLATFORMS"],
 		Images:                                loadImageSource(opts, env),
 		PostgresConnectionString:              env["ATE_API_POSTGRES_CONNECTION_STRING"],
+		PostgresDDLConnectionString:           env["ATE_API_POSTGRES_DDL_CONNECTION_STRING"],
 		PostgresSchema:                        env["ATE_API_POSTGRES_SCHEMA"],
 		RolloutTimeout:                        rolloutTimeout,
 		rolloutTimeoutSet:                     timeoutStr != "",
@@ -281,6 +280,9 @@ func validate(cfg *Config) error {
 	if err := cfg.Images.Validate(); err != nil {
 		return err
 	}
+	if cfg.PostgresDDLConnectionString != "" && cfg.PostgresConnectionString == "" {
+		return fmt.Errorf("ATE_API_POSTGRES_DDL_CONNECTION_STRING requires ATE_API_POSTGRES_CONNECTION_STRING")
+	}
 	switch cfg.Router {
 	case RouterEnvoy, RouterAgentgateway:
 	default:
@@ -333,15 +335,6 @@ func validateExtprocService(spec string) error {
 	return nil
 }
 
-// PostgresConnString returns the configured connection string, falling back to
-// the in-cluster default.
-func (c *Config) PostgresConnString() string {
-	if c.PostgresConnectionString != "" {
-		return c.PostgresConnectionString
-	}
-	return DefaultPostgresConnectionString
-}
-
 // PostgresSchemaName returns the configured schema, falling back to the
 // shell installer's default. ate-api-server rejects an empty value.
 func (c *Config) PostgresSchemaName() string {

cmd/ate-setup/internal/config/config_test.go

@@ -44,6 +44,7 @@ func loadEnv(t *testing.T) {
 		"ATE_CREDENTIAL_PROVIDER_ADDRESS",
 		"ATE_CREDENTIAL_PROVIDER_NAME",
 		"ATE_API_POSTGRES_CONNECTION_STRING",
+		"ATE_API_POSTGRES_DDL_CONNECTION_STRING",
 		"ATE_API_POSTGRES_SCHEMA",
 		"ATE_ATENET_DATAPLANE",
 		"ATE_EXPERIMENTAL_USE_SDSMINT",
@@ -78,8 +79,8 @@ func TestLoadDefaults(t *testing.T) {
 	if cfg.Router != RouterEnvoy {
 		t.Errorf("Router = %q, want %q", cfg.Router, RouterEnvoy)
 	}
-	if cfg.PostgresConnString() != DefaultPostgresConnectionString {
-		t.Errorf("PostgresConnString() = %q, want %q", cfg.PostgresConnString(), DefaultPostgresConnectionString)
+	if cfg.PostgresConnectionString != "" {
+		t.Errorf("PostgresConnectionString = %q, want bundled PostgreSQL", cfg.PostgresConnectionString)
 	}
 	if cfg.RolloutTimeout != DefaultRolloutTimeout {
 		t.Errorf("RolloutTimeout = %v, want %v", cfg.RolloutTimeout, DefaultRolloutTimeout)
@@ -114,8 +115,34 @@ func TestLoadPostgresConnectionStringOverride(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Load() error = %v", err)
 	}
-	if cfg.PostgresConnString() != dsn {
-		t.Errorf("PostgresConnString() = %q, want %q", cfg.PostgresConnString(), dsn)
+	if cfg.PostgresConnectionString != dsn {
+		t.Errorf("PostgresConnectionString = %q, want %q", cfg.PostgresConnectionString, dsn)
+	}
+}
+
+func TestLoadPostgresDDLConnectionString(t *testing.T) {
+	loadEnv(t)
+	const runtimeDSN = "postgresql://runtime@db.example:5432/atepg?sslmode=disable"
+	const dsn = "postgresql://owner@db.example:5432/atepg?sslmode=disable"
+	t.Setenv("ATE_API_POSTGRES_CONNECTION_STRING", runtimeDSN)
+	t.Setenv("ATE_API_POSTGRES_DDL_CONNECTION_STRING", dsn)
+
+	cfg, err := Load(Options{})
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+	if cfg.PostgresDDLConnectionString != dsn {
+		t.Errorf("PostgresDDLConnectionString = %q, want %q", cfg.PostgresDDLConnectionString, dsn)
+	}
+}
+
+func TestLoadRejectsPostgresDDLConnectionStringWithoutRuntime(t *testing.T) {
+	loadEnv(t)
+	t.Setenv("ATE_API_POSTGRES_DDL_CONNECTION_STRING", "postgresql://owner@db.example:5432/atepg")
+
+	_, err := Load(Options{})
+	if err == nil || !strings.Contains(err.Error(), "requires ATE_API_POSTGRES_CONNECTION_STRING") {
+		t.Fatalf("Load() error = %v, want missing runtime DSN error", err)
 	}
 }
 

cmd/ate-setup/internal/steps/create.go

@@ -16,12 +16,14 @@ package steps
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/agent-substrate/substrate/cmd/ate-setup/internal/config"
 	"github.com/agent-substrate/substrate/cmd/ate-setup/internal/log"
@@ -37,8 +39,11 @@ const (
 	SecretServiceDNSCA     = "service-dns-ca-pool"
 	SecretPodIdentityCA    = "pod-identity-ca-pool"
 	SecretEgressMITMCAPool = "egress-mitm-ca-pool"
+	SecretAPIServerEnvVars = "ate-api-server-secret-envvars"
+	SecretPostgresRoles    = "postgres-role-passwords"
 	ConfigMapAPIEnvVars    = "ate-api-server-envvars"
 	ConfigMapAPIAuthn      = "ate-api-authentication"
+	apiServerEnvHashKey    = "ate.dev/env-hash"
 	// poolKeyID is the identifier given to the first CA and JWT key in a new
 	// pool, matching the --ca-id/--key-id the shell scripts passed.
 	poolKeyID = "1"
@@ -119,35 +124,61 @@ func (e *Env) CreateActorIDCACertsSecret(ctx context.Context) error {
 	})
 }
 
-// CreateAPIServerEnvVars writes the ConfigMap that tells ate-api-server how to
-// reach its PostgreSQL store. ate-api-server.yaml pulls it in via an optional
-// envFrom and resolves --postgres-connection-string=@env and
-// --postgres-schema=@env from it.
+// CreateAPIServerEnvVars writes the PostgreSQL schema to a ConfigMap and the
+// credential-bearing connection strings to a Secret.
 func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error {
 	log.Step("create_api_server_env_vars")
 	if err := e.Kube.EnsureNamespace(ctx, NamespaceAteSystem); err != nil {
 		return err
 	}
 
-	connString := e.Cfg.PostgresConnString()
-	log.Infof("POSTGRES_CONNECTION_STRING: %s", connString)
+	runtimeDSN, ddlDSN, err := e.postgresConnectionStrings(ctx)
+	if err != nil {
+		return err
+	}
+	log.Infof("POSTGRES_CONNECTION_STRING: configured")
+	log.Infof("POSTGRES_DDL_CONNECTION_STRING: configured")
 
-	return e.Kube.ApplyConfigMap(ctx, NamespaceAteSystem, ConfigMapAPIEnvVars,
-		buildAPIServerEnvVars(connString, e.Cfg.PostgresSchemaName()))
+	configVars := map[string]string{
+		"ATE_API_POSTGRES_SCHEMA": e.Cfg.PostgresSchemaName(),
+	}
+	secretVars := buildAPIServerSecretEnvVars(runtimeDSN, ddlDSN)
+	if err := e.Kube.ApplyConfigMap(ctx, NamespaceAteSystem, ConfigMapAPIEnvVars, configVars); err != nil {
+		return err
+	}
+	if err := e.Kube.ApplySecret(ctx, NamespaceAteSystem, SecretAPIServerEnvVars, secretVars); err != nil {
+		return err
+	}
+	return e.annotateAPIServerEnvHash(ctx, apiServerEnvHash(configVars, secretVars))
 }
 
-// buildAPIServerEnvVars is the ConfigMap payload. ate-api-server takes the
-// connection string and the schema from it, and exits on an empty schema; an
-// unrecognized key here reaches the container as a stray environment variable,
-// so the set stays exactly what the shell installer's
-// create_api_server_env_vars writes.
-func buildAPIServerEnvVars(connString, schema string) map[string]string {
+func buildAPIServerSecretEnvVars(runtimeDSN, ddlDSN string) map[string]string {
 	return map[string]string{
-		"ATE_API_POSTGRES_CONNECTION_STRING": connString,
-		"ATE_API_POSTGRES_SCHEMA":            schema,
+		"ATE_API_POSTGRES_CONNECTION_STRING":     runtimeDSN,
+		"ATE_API_POSTGRES_DDL_CONNECTION_STRING": ddlDSN,
 	}
 }
 
+func apiServerEnvHash(configVars, secretVars map[string]string) string {
+	payload := configVars["ATE_API_POSTGRES_SCHEMA"] + "\x00" +
+		secretVars["ATE_API_POSTGRES_CONNECTION_STRING"] + "\x00" +
+		secretVars["ATE_API_POSTGRES_DDL_CONNECTION_STRING"]
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(payload)))
+}
+
+func (e *Env) annotateAPIServerEnvHash(ctx context.Context, hash string) error {
+	exists, err := e.Kube.DeploymentExists(ctx, NamespaceAteSystem, "ate-api-server")
+	if err != nil || !exists {
+		return err
+	}
+	patch := fmt.Sprintf(`{"spec":{"template":{"metadata":{"annotations":{"%s":%q}}}}}`, apiServerEnvHashKey, hash)
+	if _, err := e.Kube.Typed.AppsV1().Deployments(NamespaceAteSystem).Patch(
+		ctx, "ate-api-server", types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
+		return fmt.Errorf("while annotating ate-api-server environment hash: %w", err)
+	}
+	return nil
+}
+
 // CreateAPIAuthenticationConfig writes the default ate-api-server
 // authentication config, pointing it at the cluster's service account issuer.
 func (e *Env) CreateAPIAuthenticationConfig(ctx context.Context) error {

cmd/ate-setup/internal/steps/create_test.go

@@ -15,33 +15,83 @@
 package steps
 
 import (
+	"context"
 	"crypto/x509"
 	"maps"
 	"slices"
 	"testing"
 
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+
+	"github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube"
 	"github.com/agent-substrate/substrate/internal/localca"
 )
 
-// ate-api-server resolves --postgres-connection-string=@env and
-// --postgres-schema=@env from this ConfigMap. These are the keys the shell
-// installer writes, and an empty value for either makes the apiserver exit
-// ("--postgres-connection-string is required", "PostgreSQL schema must not be
-// empty"), so both the key set and the values are pinned here.
-func TestBuildAPIServerEnvVars(t *testing.T) {
-	const dsn = "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full"
+func TestBuildAPIServerSecretEnvVars(t *testing.T) {
+	const runtimeDSN = "postgresql://runtime@postgres:5432/atepg"
+	const ddlDSN = "postgresql://ddl@postgres:5432/atepg"
 
-	got := buildAPIServerEnvVars(dsn, "public")
+	got := buildAPIServerSecretEnvVars(runtimeDSN, ddlDSN)
 
-	want := []string{"ATE_API_POSTGRES_CONNECTION_STRING", "ATE_API_POSTGRES_SCHEMA"}
+	want := []string{"ATE_API_POSTGRES_CONNECTION_STRING", "ATE_API_POSTGRES_DDL_CONNECTION_STRING"}
 	if keys := slices.Sorted(maps.Keys(got)); !slices.Equal(keys, want) {
 		t.Errorf("keys = %v, want %v", keys, want)
 	}
-	if got["ATE_API_POSTGRES_CONNECTION_STRING"] != dsn {
-		t.Errorf("ATE_API_POSTGRES_CONNECTION_STRING = %q, want %q", got["ATE_API_POSTGRES_CONNECTION_STRING"], dsn)
+	if got["ATE_API_POSTGRES_CONNECTION_STRING"] != runtimeDSN {
+		t.Errorf("ATE_API_POSTGRES_CONNECTION_STRING = %q, want %q", got["ATE_API_POSTGRES_CONNECTION_STRING"], runtimeDSN)
+	}
+	if got["ATE_API_POSTGRES_DDL_CONNECTION_STRING"] != ddlDSN {
+		t.Errorf("ATE_API_POSTGRES_DDL_CONNECTION_STRING = %q, want %q", got["ATE_API_POSTGRES_DDL_CONNECTION_STRING"], ddlDSN)
+	}
+}
+
+func TestAPIServerEnvHash(t *testing.T) {
+	configVars := map[string]string{"ATE_API_POSTGRES_SCHEMA": "substrate"}
+	secretVars := buildAPIServerSecretEnvVars("runtime", "ddl")
+
+	want := apiServerEnvHash(configVars, secretVars)
+	if got := apiServerEnvHash(configVars, secretVars); got != want {
+		t.Fatalf("stable inputs produced hashes %q and %q", want, got)
+	}
+	for name, values := range map[string][2]map[string]string{
+		"schema": {
+			{"ATE_API_POSTGRES_SCHEMA": "other"},
+			secretVars,
+		},
+		"connection string": {
+			configVars,
+			buildAPIServerSecretEnvVars("other", "ddl"),
+		},
+		"DDL connection string": {
+			configVars,
+			buildAPIServerSecretEnvVars("runtime", "other"),
+		},
+	} {
+		if got := apiServerEnvHash(values[0], values[1]); got == want {
+			t.Errorf("changing %s did not change the hash", name)
+		}
+	}
+}
+
+func TestAnnotateAPIServerEnvHash(t *testing.T) {
+	deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{
+		Name:      "ate-api-server",
+		Namespace: NamespaceAteSystem,
+	}}
+	e := &Env{Kube: &kube.Client{Typed: fake.NewSimpleClientset(deployment)}}
+
+	if err := e.annotateAPIServerEnvHash(context.Background(), "new-hash"); err != nil {
+		t.Fatal(err)
+	}
+	got, err := e.Kube.Typed.AppsV1().Deployments(NamespaceAteSystem).Get(
+		context.Background(), "ate-api-server", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
 	}
-	if got["ATE_API_POSTGRES_SCHEMA"] != "public" {
-		t.Errorf("ATE_API_POSTGRES_SCHEMA = %q, want %q", got["ATE_API_POSTGRES_SCHEMA"], "public")
+	if got.Spec.Template.Annotations[apiServerEnvHashKey] != "new-hash" {
+		t.Errorf("environment hash annotation = %q, want new-hash", got.Spec.Template.Annotations[apiServerEnvHashKey])
 	}
 }
 

cmd/ate-setup/internal/steps/deploy.go

@@ -234,6 +234,14 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error {
 	if err := e.applyOtelConfig(ctx); err != nil {
 		return err
 	}
+	if e.useBundledPostgres() {
+		if err := e.applyBundledPostgres(ctx); err != nil {
+			return err
+		}
+		if err := e.Kube.RolloutStatus(ctx, kube.KindStatefulSet, NamespaceAteSystem, "postgres", e.Cfg.RolloutTimeout); err != nil {
+			return err
+		}
+	}
 	if err := e.ResolveAndApply(ctx, e.Cfg.Manifest("ate-api-server.yaml")); err != nil {
 		return err
 	}