From 50d3283a5863b5c264ea1f5f63b311947f289cb1 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Thu, 17 Sep 2026 14:02:56 -0700 Subject: [PATCH] ate-setup: keep an existing SandboxConfig on deploy Deploy ate-system re-applied sandboxconfig-gvisor.yaml every run, so an upgrade could overwrite the operator's SandboxConfig and change the sandbox runtime with it. For GA a system upgrade does not touch the runtime, so the default is now created only when missing. --- cmd/ate-setup/internal/kube/apply.go | 21 ++++++ cmd/ate-setup/internal/kube/apply_test.go | 79 +++++++++++++++++++++++ cmd/ate-setup/internal/steps/deploy.go | 24 +++++-- 3 files changed, 119 insertions(+), 5 deletions(-) diff --git a/cmd/ate-setup/internal/kube/apply.go b/cmd/ate-setup/internal/kube/apply.go index 1947c81107..f685201139 100644 --- a/cmd/ate-setup/internal/kube/apply.go +++ b/cmd/ate-setup/internal/kube/apply.go @@ -118,6 +118,27 @@ func (c *Client) ApplyTolerant(ctx context.Context, objs []*unstructured.Unstruc return nil } +// ApplyMissing applies only the objects that are not present yet, reporting +// each object it leaves alone through onKeep. Unlike Apply it never touches an +// existing object, so an operator's edits to it survive a redeploy. +func (c *Client) ApplyMissing(ctx context.Context, objs []*unstructured.Unstructured, onKeep func(obj *unstructured.Unstructured)) error { + var missing []*unstructured.Unstructured + for _, obj := range objs { + present, err := c.Exists(ctx, obj.GroupVersionKind(), obj.GetNamespace(), obj.GetName()) + if err != nil { + return err + } + if present { + if onKeep != nil { + onKeep(obj) + } + continue + } + missing = append(missing, obj) + } + return c.Apply(ctx, missing) +} + // Delete removes every object, ignoring those that are already gone. This is // the `kubectl delete --ignore-not-found -f` equivalent. func (c *Client) Delete(ctx context.Context, objs []*unstructured.Unstructured) error { diff --git a/cmd/ate-setup/internal/kube/apply_test.go b/cmd/ate-setup/internal/kube/apply_test.go index da08f9f86b..4ae0d043c5 100644 --- a/cmd/ate-setup/internal/kube/apply_test.go +++ b/cmd/ate-setup/internal/kube/apply_test.go @@ -18,7 +18,15 @@ import ( "context" "os" "path/filepath" + "slices" "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" ) // Teardown walks a fixed list of manifests covering every install shape, so a @@ -53,3 +61,74 @@ func TestDeletePathReportsUnparseableManifest(t *testing.T) { t.Error("DeletePath() = nil, want an error for an unparseable manifest") } } + +// fakeResource is the slice of the dynamic client ApplyMissing reaches: Get to +// decide, Apply to create. It records applies so the test can tell a kept +// object from a rewritten one. +type fakeResource struct { + dynamic.NamespaceableResourceInterface + objs map[string]*unstructured.Unstructured + applied []string +} + +func (f *fakeResource) Namespace(string) dynamic.ResourceInterface { return f } + +func (f *fakeResource) Get(_ context.Context, name string, _ metav1.GetOptions, _ ...string) (*unstructured.Unstructured, error) { + obj, ok := f.objs[name] + if !ok { + return nil, apierrors.NewNotFound(schema.GroupResource{Group: "ate.dev", Resource: "sandboxconfigs"}, name) + } + return obj, nil +} + +func (f *fakeResource) Apply(_ context.Context, name string, obj *unstructured.Unstructured, _ metav1.ApplyOptions, _ ...string) (*unstructured.Unstructured, error) { + f.applied = append(f.applied, name) + f.objs[name] = obj + return obj, nil +} + +type fakeDynamic struct{ res *fakeResource } + +func (f fakeDynamic) Resource(schema.GroupVersionResource) dynamic.NamespaceableResourceInterface { + return f.res +} + +func sandboxConfig(name, marker string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "ate.dev", Version: "v1alpha1", Kind: "SandboxConfig"}) + obj.SetName(name) + _ = unstructured.SetNestedField(obj.Object, marker, "spec", "marker") + return obj +} + +// The default SandboxConfig is installed once and then belongs to the +// operator, so a redeploy must create what is missing and leave the rest as it +// found it, edits included. +func TestApplyMissingKeepsExistingObjects(t *testing.T) { + gvk := schema.GroupVersionKind{Group: "ate.dev", Version: "v1alpha1", Kind: "SandboxConfig"} + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{gvk.GroupVersion()}) + mapper.Add(gvk, meta.RESTScopeRoot) + res := &fakeResource{objs: map[string]*unstructured.Unstructured{ + "gvisor-default": sandboxConfig("gvisor-default", "operator-edited"), + }} + c := &Client{Dynamic: fakeDynamic{res}, mapper: mapper} + + var kept []string + err := c.ApplyMissing(context.Background(), + []*unstructured.Unstructured{sandboxConfig("gvisor-default", "release"), sandboxConfig("extra", "release")}, + func(obj *unstructured.Unstructured) { kept = append(kept, obj.GetName()) }) + if err != nil { + t.Fatalf("ApplyMissing() = %v", err) + } + + if want := []string{"gvisor-default"}; !slices.Equal(kept, want) { + t.Errorf("kept = %v, want %v", kept, want) + } + if want := []string{"extra"}; !slices.Equal(res.applied, want) { + t.Errorf("applied = %v, want %v", res.applied, want) + } + marker, _, _ := unstructured.NestedString(res.objs["gvisor-default"].Object, "spec", "marker") + if marker != "operator-edited" { + t.Errorf("gvisor-default spec.marker = %q, want the operator's value kept", marker) + } +} diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e7ea78b212..bdd287ec47 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -23,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/agent-substrate/substrate/cmd/ate-setup/internal/kube" @@ -106,11 +107,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - // Install the cluster-wide sandbox config. Sandbox binaries live on - // cluster-scoped SandboxConfigs each ActorTemplate names via - // sandboxConfig.configName; gVisor templates name this one unless they - // create their own SandboxConfig. - if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")); err != nil { + if err := e.ensureDefaultSandboxConfig(ctx); err != nil { return err } @@ -175,6 +172,23 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return nil } +// ensureDefaultSandboxConfig installs the cluster-wide sandbox config where it +// is missing. Sandbox binaries live on cluster-scoped SandboxConfigs each +// ActorTemplate names via sandboxConfig.configName; gVisor templates name this +// one unless they create their own. +// +// An existing config is left alone: for GA a system upgrade never changes the +// sandbox runtime, that stays an explicit SandboxConfig edit by the user. +func (e *Env) ensureDefaultSandboxConfig(ctx context.Context) error { + objs, err := kube.LoadPath(e.Cfg.Manifest("sandboxconfig-gvisor.yaml")) + if err != nil { + return err + } + return e.Kube.ApplyMissing(ctx, objs, func(obj *unstructured.Unstructured) { + log.Infof("Keeping existing %s; edit it to change the sandbox runtime", kube.Describe(obj)) + }) +} + // applyPodcertWorkersOverride sets WORKERS_PER_SIGNER on podcertificate-controller if configured. func (e *Env) applyPodcertWorkersOverride(ctx context.Context) error { if e.Cfg.PodcertWorkersPerSigner <= 0 {