🌱 Add uninstall feature test

test/e2e/features/uninstall.feature

@@ -0,0 +1,40 @@
+Feature: Uninstall ClusterExtension
+
+  As an OLM user I would like to uninstall a cluster extension.
+
+  Background:
+    Given OLM is available
+    And ClusterCatalog "test" serves bundles
+    And ServiceAccount "olm-sa" with needed permissions is available in ${TEST_NAMESPACE}
+    And ClusterExtension is applied
+      """
+      apiVersion: olm.operatorframework.io/v1
+      kind: ClusterExtension
+      metadata:
+        name: ${NAME}
+      spec:
+        namespace: ${TEST_NAMESPACE}
+        serviceAccount:
+          name: olm-sa
+        source:
+          sourceType: Catalog
+          catalog:
+            packageName: test
+            selector:
+              matchLabels:
+                "olm.operatorframework.io/metadata.name": test-catalog
+      """
+    And bundle "test-operator.1.2.0" is installed in version "1.2.0"
+    And ClusterExtension is rolled out
+
+  Scenario: Uninstall ClusterExtension
+    When resource "clusterextension/${NAME}" is removed
+    Then ClusterExtension is uninstalled
+
+  Scenario: ClusterExtension resources are cleaned up even if the ServiceAccount is no longer present
+    When resource "serviceaccount/olm-sa" is removed
+    # Ensure service account is gone before checking to ensure resources are cleaned up whether the service account
+    # and its permissions are present on the cluster or not
+    And resource "serviceaccount/olm-sa" is eventually not found
+    And resource "clusterextension/${NAME}" is removed
+    Then ClusterExtension is uninstalled

test/e2e/steps/hooks.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/component-base/featuregate"
 	"k8s.io/klog/v2/textlogger"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/operator-framework/operator-controller/internal/operator-controller/features"
 )
@@ -32,6 +33,9 @@ type scenarioContext struct {
 	removedResources     []unstructured.Unstructured
 	backGroundCmds       []*exec.Cmd
 	metricsResponse      map[string]string
+
+	// set by ClusterExtensionIsRolledOut
+	extensionObjects []client.Object
 }
 
 type contextKey string

test/e2e/steps/steps.go

@@ -2,6 +2,7 @@ package steps
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"encoding/json"
@@ -18,21 +19,24 @@ import (
 
 	"github.com/cucumber/godog"
 	jsonpatch "github.com/evanphx/json-patch"
-	"github.com/google/go-cmp/cmp"
+	diff "github.com/google/go-cmp/cmp"
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/common/model"
 	"github.com/spf13/pflag"
 	"github.com/stretchr/testify/require"
+	"helm.sh/helm/v3/pkg/release"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/yaml"
 
 	ocv1 "github.com/operator-framework/operator-controller/api/v1"
+	"github.com/operator-framework/operator-controller/internal/operator-controller/features"
 )
 
 const (
@@ -56,6 +60,7 @@ func RegisterSteps(sc *godog.ScenarioContext) {
 	sc.Step(`^(?i)ClusterExtension is updated(?:\s+.*)?$`, ResourceIsApplied)
 	sc.Step(`^(?i)ClusterExtension is available$`, ClusterExtensionIsAvailable)
 	sc.Step(`^(?i)ClusterExtension is rolled out$`, ClusterExtensionIsRolledOut)
+	sc.Step(`^(?i)ClusterExtension is uninstalled$`, ClusterExtensionIsUninstalled)
 	sc.Step(`^(?i)ClusterExtension reports "([^"]+)" as active revision(s?)$`, ClusterExtensionReportsActiveRevisions)
 	sc.Step(`^(?i)ClusterExtension reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+) and Message:$`, ClusterExtensionReportsCondition)
 	sc.Step(`^(?i)ClusterExtension reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+) and Message includes:$`, ClusterExtensionReportsConditionWithMessageFragment)
@@ -68,6 +73,7 @@ func RegisterSteps(sc *godog.ScenarioContext) {
 	sc.Step(`^(?i)resource "([^"]+)" is installed$`, ResourceAvailable)
 	sc.Step(`^(?i)resource "([^"]+)" is available$`, ResourceAvailable)
 	sc.Step(`^(?i)resource "([^"]+)" is removed$`, ResourceRemoved)
+	sc.Step(`^(?i)resource "([^"]+)" is eventually not found$`, ResourceEventuallyNotFound)
 	sc.Step(`^(?i)resource "([^"]+)" exists$`, ResourceAvailable)
 	sc.Step(`^(?i)resource is applied$`, ResourceIsApplied)
 	sc.Step(`^(?i)resource "deployment/test-operator" reports as (not ready|ready)$`, MarkTestOperatorNotReady)
@@ -260,6 +266,35 @@ func ClusterExtensionIsRolledOut(ctx context.Context) error {
 		}
 		return condition["status"] == "True" && condition["reason"] == "Succeeded" && condition["type"] == "Progressing"
 	}, timeout, tick)
+
+	// Get ClusterExtension resources
+	objs, err := listExtensionResources(sc.clusterExtensionName)
+	require.NoError(godog.T(ctx), err, "failed to list extension resources")
+
+	// Ensure they are all labeled and available on the cluster
+	for _, obj := range objs {
+		labels := obj.GetLabels()
+		require.NotNil(godog.T(ctx), labels)
+		require.Equal(godog.T(ctx), "ClusterExtension", labels["olm.operatorframework.io/owner-kind"])
+		require.Equal(godog.T(ctx), sc.clusterExtensionName, labels["olm.operatorframework.io/owner-name"])
+		if err := ResourceAvailable(ctx, fmt.Sprintf("%s/%s", obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName())); err != nil {
+			return err
+		}
+	}
+
+	// Store extension resources in test context
+	sc.extensionObjects = objs
+	return nil
+}
+
+func ClusterExtensionIsUninstalled(ctx context.Context) error {
+	sc := scenarioCtx(ctx)
+	require.NotNil(godog.T(ctx), sc.extensionObjects)
+	for _, obj := range sc.extensionObjects {
+		if err := ResourceEventuallyNotFound(ctx, fmt.Sprintf("%s/%s", obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName())); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -398,24 +433,25 @@ func ClusterExtensionRevisionIsArchived(ctx context.Context, revisionName string
 func ResourceAvailable(ctx context.Context, resource string) error {
 	sc := scenarioCtx(ctx)
 	resource = substituteScenarioVars(resource, sc)
-	rtype, name, found := strings.Cut(resource, "/")
+	kind, name, found := strings.Cut(resource, "/")
 	if !found {
-		return fmt.Errorf("resource %s is not in the format <type>/<name>", resource)
+		return fmt.Errorf("resource %s is not in the format <kind>/<name>", resource)
 	}
 	waitFor(ctx, func() bool {
-		_, err := k8sClient("get", rtype, name, "-n", sc.namespace)
+		_, err := k8sClient("get", kind, name, "-n", sc.namespace)
 		return err == nil
 	})
 	return nil
 }
 
 func ResourceRemoved(ctx context.Context, resource string) error {
 	sc := scenarioCtx(ctx)
-	rtype, name, found := strings.Cut(resource, "/")
+	resource = substituteScenarioVars(resource, sc)
+	kind, name, found := strings.Cut(resource, "/")
 	if !found {
-		return fmt.Errorf("resource %s is not in the format <type>/<name>", resource)
+		return fmt.Errorf("resource %s is not in the format <kind>/<name>", resource)
 	}
-	yaml, err := k8sClient("get", rtype, name, "-n", sc.namespace, "-o", "yaml")
+	yaml, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "yaml")
 	if err != nil {
 		return err
 	}
@@ -424,23 +460,38 @@ func ResourceRemoved(ctx context.Context, resource string) error {
 		return err
 	}
 	sc.removedResources = append(sc.removedResources, *obj)
-	_, err = k8sClient("delete", rtype, name, "-n", sc.namespace)
+	_, err = k8sClient("delete", kind, name, "-n", sc.namespace)
 	return err
 }
 
+func ResourceEventuallyNotFound(ctx context.Context, resource string) error {
+	sc := scenarioCtx(ctx)
+	resource = substituteScenarioVars(resource, sc)
+	kind, name, found := strings.Cut(resource, "/")
+	if !found {
+		return fmt.Errorf("resource %s is not in the format <kind>/<name>", resource)
+	}
+
+	require.Eventually(godog.T(ctx), func() bool {
+		obj, err := k8sClient("get", kind, name, "-n", sc.namespace, "--ignore-not-found", "-o", "yaml")
+		return err == nil && obj == ""
+	}, timeout, tick)
+	return nil
+}
+
 func ResourceMatches(ctx context.Context, resource string, requiredContentTemplate *godog.DocString) error {
 	sc := scenarioCtx(ctx)
 	resource = substituteScenarioVars(resource, sc)
-	rtype, name, found := strings.Cut(resource, "/")
+	kind, name, found := strings.Cut(resource, "/")
 	if !found {
-		return fmt.Errorf("resource %s is not in the format <type>/<name>", resource)
+		return fmt.Errorf("resource %s is not in the format <kind>/<name>", resource)
 	}
 	requiredContent, err := toUnstructured(substituteScenarioVars(requiredContentTemplate.Content, sc))
 	if err != nil {
 		return fmt.Errorf("failed to parse required resource yaml: %v", err)
 	}
 	waitFor(ctx, func() bool {
-		objJson, err := k8sClient("get", rtype, name, "-n", sc.namespace, "-o", "json")
+		objJson, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "json")
 		if err != nil {
 			return false
 		}
@@ -461,19 +512,20 @@ func ResourceMatches(ctx context.Context, resource string, requiredContentTempla
 			return false
 		}
 
-		return len(cmp.Diff(upd.Object, obj.Object)) == 0
+		return len(diff.Diff(upd.Object, obj.Object)) == 0
 	})
 	return nil
 }
 
 func ResourceRestored(ctx context.Context, resource string) error {
 	sc := scenarioCtx(ctx)
-	rtype, name, found := strings.Cut(resource, "/")
+	resource = substituteScenarioVars(resource, sc)
+	kind, name, found := strings.Cut(resource, "/")
 	if !found {
-		return fmt.Errorf("resource %s is not in the format <type>/<name>", resource)
+		return fmt.Errorf("resource %s is not in the format <kind>/<name>", resource)
 	}
 	waitFor(ctx, func() bool {
-		yaml, err := k8sClient("get", rtype, name, "-n", sc.namespace, "-o", "yaml")
+		yaml, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "yaml")
 		if err != nil {
 			return false
 		}
@@ -486,7 +538,7 @@ func ResourceRestored(ctx context.Context, resource string) error {
 		for i, removed := range sc.removedResources {
 			rct := removed.GetCreationTimestamp()
 			if removed.GetName() == obj.GetName() && removed.GetKind() == obj.GetKind() && rct.Before(&ct) {
-				switch rtype {
+				switch kind {
 				case "configmap":
 					if !reflect.DeepEqual(removed.Object["data"], obj.Object["data"]) {
 						return false
@@ -853,3 +905,138 @@ func extendMap(m map[string]string, keyValue ...string) map[string]string {
 	}
 	return m
 }
+
+// listExtensionResources returns a slice of client.Object containing all resources for a ClusterExtension
+// this method is best called when the extension has been installed successfully. An error is returned if there was
+// any issue in determining the extension's resources.
+func listExtensionResources(extName string) ([]client.Object, error) {
+	if enabled, found := featureGates[features.BoxcutterRuntime]; found && enabled {
+		return listExtensionRevisionResources(extName)
+	}
+	return listHelmReleaseResources(extName)
+}
+
+// listHelmReleaseResources returns a slice of client.Object containing all resources for a ClusterExtension's
+// Helm release. Note: The current implementation does not support chunked release secrets.
+func listHelmReleaseResources(extName string) ([]client.Object, error) {
+	secret, err := helmReleaseSecretForExtension(extName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get helm release secret for extension %s: %v", extName, err)
+	}
+
+	rel, err := helmReleaseFromSecret(secret)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get helm release from secret for cluster extension '%s': %w", extName, err)
+	}
+
+	objs, err := collectHelmReleaseObjects(rel)
+	if err != nil {
+		return nil, fmt.Errorf("failed to collect helm release objects for cluster extension '%s': %w", extName, err)
+	}
+	return objs, nil
+}
+
+// helmReleaseSecretForExtension returns the Helm release secret for the extension with name extName
+func helmReleaseSecretForExtension(extName string) (*corev1.Secret, error) {
+	out, err := k8sClient("get", "secrets", "-n", "olmv1-system", "-l", fmt.Sprintf("name=%s", extName), "--field-selector", "type=operatorframework.io/index.v1", "-o", "json")
+	if err != nil {
+		return nil, err
+	}
+	if strings.TrimSpace(out) == "" {
+		return nil, err
+	}
+
+	var secretList corev1.SecretList
+	if err = json.Unmarshal([]byte(out), &secretList); err != nil {
+		return nil, err
+	}
+	if len(secretList.Items) != 1 {
+		return nil, err
+	}
+	return &secretList.Items[0], nil
+}
+
+// helmReleaseFromSecret returns the Helm Release object encoded in the secret. Note: this function does not yet support
+// releases chunked over multiple Secrets
+func helmReleaseFromSecret(secret *corev1.Secret) (*release.Release, error) {
+	gzReader, err := gzip.NewReader(strings.NewReader(string(secret.Data["chunk"])))
+	if err != nil {
+		return nil, err
+	}
+	defer gzReader.Close()
+
+	releaseJsonBytes, err := io.ReadAll(gzReader)
+	if err != nil {
+		return nil, err
+	}
+
+	var rel release.Release
+	if err = json.Unmarshal(releaseJsonBytes, &rel); err != nil {
+		return nil, err
+	}
+	return &rel, nil
+}
+
+// collectHelmReleaseObjects returns a slice of client.Object containing the manifests in rel
+func collectHelmReleaseObjects(rel *release.Release) ([]client.Object, error) {
+	objs := []client.Object{}
+	menifests := strings.Split(rel.Manifest, "\n")
+	for _, manifest := range menifests {
+		if manifest == "" {
+			continue
+		}
+		u := &unstructured.Unstructured{}
+		if err := yaml.Unmarshal([]byte(manifest), u); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal manifest: %w", err)
+		}
+		objs = append(objs, u)
+	}
+	return objs, nil
+}
+
+func listExtensionRevisionResources(extName string) ([]client.Object, error) {
+	rev, err := latestActiveRevisionForExtension(extName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get latest active revision for extension %s: %w", extName, err)
+	}
+
+	var objs []client.Object
+	for _, phase := range rev.Spec.Phases {
+		for _, obj := range phase.Objects {
+			objs = append(objs, &obj.Object)
+		}
+	}
+
+	return objs, nil
+}
+
+// latestActiveRevisionForExtension returns the latest active revision for the extension called extName
+func latestActiveRevisionForExtension(extName string) (*ocv1.ClusterExtensionRevision, error) {
+	out, err := k8sClient("get", "clusterextensionrevisions", "-l", fmt.Sprintf("olm.operatorframework.io/owner-name=%s", extName), "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("error listing revisions for extension '%s': %w", extName, err)
+	}
+	if strings.TrimSpace(out) == "" {
+		return nil, fmt.Errorf("no revisions found for extension '%s'", extName)
+	}
+	var revisionList ocv1.ClusterExtensionRevisionList
+	if err := json.Unmarshal([]byte(out), &revisionList); err != nil {
+		return nil, fmt.Errorf("error unmarshalling revisions for extension '%s': %w", extName, err)
+	}
+
+	var latest *ocv1.ClusterExtensionRevision
+	for _, rev := range revisionList.Items {
+		if rev.Spec.LifecycleState != ocv1.ClusterExtensionRevisionLifecycleStateActive {
+			continue
+		}
+		if latest == nil || rev.Spec.Revision > latest.Spec.Revision {
+			latest = &rev
+		}
+	}
+
+	if latest == nil {
+		return nil, fmt.Errorf("no active revisions found for extension '%s'", extName)
+	}
+
+	return latest, nil
+}