diff --git a/test/e2e/features/uninstall.feature b/test/e2e/features/uninstall.feature new file mode 100644 index 000000000..08c25e71f --- /dev/null +++ b/test/e2e/features/uninstall.feature @@ -0,0 +1,40 @@ +Feature: Uninstall ClusterExtension + + As an OLM user I would like to uninstall a cluster extension. + + Background: + Given OLM is available + And ClusterCatalog "test" serves bundles + And ServiceAccount "olm-sa" with needed permissions is available in ${TEST_NAMESPACE} + And ClusterExtension is applied + """ + apiVersion: olm.operatorframework.io/v1 + kind: ClusterExtension + metadata: + name: ${NAME} + spec: + namespace: ${TEST_NAMESPACE} + serviceAccount: + name: olm-sa + source: + sourceType: Catalog + catalog: + packageName: test + selector: + matchLabels: + "olm.operatorframework.io/metadata.name": test-catalog + """ + And bundle "test-operator.1.2.0" is installed in version "1.2.0" + And ClusterExtension is rolled out + + Scenario: Uninstall ClusterExtension + When resource "clusterextension/${NAME}" is removed + Then ClusterExtension is uninstalled + + Scenario: ClusterExtension resources are cleaned up even if the ServiceAccount is no longer present + When resource "serviceaccount/olm-sa" is removed + # Ensure service account is gone before checking to ensure resources are cleaned up whether the service account + # and its permissions are present on the cluster or not + And resource "serviceaccount/olm-sa" is eventually not found + And resource "clusterextension/${NAME}" is removed + Then ClusterExtension is uninstalled diff --git a/test/e2e/steps/hooks.go b/test/e2e/steps/hooks.go index c91072947..6d7a06c66 100644 --- a/test/e2e/steps/hooks.go +++ b/test/e2e/steps/hooks.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/component-base/featuregate" "k8s.io/klog/v2/textlogger" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/operator-framework/operator-controller/internal/operator-controller/features" ) @@ -32,6 +33,9 @@ type scenarioContext struct { removedResources []unstructured.Unstructured backGroundCmds []*exec.Cmd metricsResponse map[string]string + + // set by ClusterExtensionIsRolledOut + extensionObjects []client.Object } type contextKey string diff --git a/test/e2e/steps/steps.go b/test/e2e/steps/steps.go index 9ae772ce8..e540f370b 100644 --- a/test/e2e/steps/steps.go +++ b/test/e2e/steps/steps.go @@ -2,6 +2,7 @@ package steps import ( "bytes" + "compress/gzip" "context" "crypto/tls" "encoding/json" @@ -18,21 +19,24 @@ import ( "github.com/cucumber/godog" jsonpatch "github.com/evanphx/json-patch" - "github.com/google/go-cmp/cmp" + diff "github.com/google/go-cmp/cmp" "github.com/google/go-containerregistry/pkg/crane" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" "github.com/spf13/pflag" "github.com/stretchr/testify/require" + "helm.sh/helm/v3/pkg/release" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" ocv1 "github.com/operator-framework/operator-controller/api/v1" + "github.com/operator-framework/operator-controller/internal/operator-controller/features" ) const ( @@ -56,6 +60,7 @@ func RegisterSteps(sc *godog.ScenarioContext) { sc.Step(`^(?i)ClusterExtension is updated(?:\s+.*)?$`, ResourceIsApplied) sc.Step(`^(?i)ClusterExtension is available$`, ClusterExtensionIsAvailable) sc.Step(`^(?i)ClusterExtension is rolled out$`, ClusterExtensionIsRolledOut) + sc.Step(`^(?i)ClusterExtension is uninstalled$`, ClusterExtensionIsUninstalled) sc.Step(`^(?i)ClusterExtension reports "([^"]+)" as active revision(s?)$`, ClusterExtensionReportsActiveRevisions) sc.Step(`^(?i)ClusterExtension reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+) and Message:$`, ClusterExtensionReportsCondition) sc.Step(`^(?i)ClusterExtension reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+) and Message includes:$`, ClusterExtensionReportsConditionWithMessageFragment) @@ -68,6 +73,7 @@ func RegisterSteps(sc *godog.ScenarioContext) { sc.Step(`^(?i)resource "([^"]+)" is installed$`, ResourceAvailable) sc.Step(`^(?i)resource "([^"]+)" is available$`, ResourceAvailable) sc.Step(`^(?i)resource "([^"]+)" is removed$`, ResourceRemoved) + sc.Step(`^(?i)resource "([^"]+)" is eventually not found$`, ResourceEventuallyNotFound) sc.Step(`^(?i)resource "([^"]+)" exists$`, ResourceAvailable) sc.Step(`^(?i)resource is applied$`, ResourceIsApplied) sc.Step(`^(?i)resource "deployment/test-operator" reports as (not ready|ready)$`, MarkTestOperatorNotReady) @@ -260,6 +266,35 @@ func ClusterExtensionIsRolledOut(ctx context.Context) error { } return condition["status"] == "True" && condition["reason"] == "Succeeded" && condition["type"] == "Progressing" }, timeout, tick) + + // Get ClusterExtension resources + objs, err := listExtensionResources(sc.clusterExtensionName) + require.NoError(godog.T(ctx), err, "failed to list extension resources") + + // Ensure they are all labeled and available on the cluster + for _, obj := range objs { + labels := obj.GetLabels() + require.NotNil(godog.T(ctx), labels) + require.Equal(godog.T(ctx), "ClusterExtension", labels["olm.operatorframework.io/owner-kind"]) + require.Equal(godog.T(ctx), sc.clusterExtensionName, labels["olm.operatorframework.io/owner-name"]) + if err := ResourceAvailable(ctx, fmt.Sprintf("%s/%s", obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName())); err != nil { + return err + } + } + + // Store extension resources in test context + sc.extensionObjects = objs + return nil +} + +func ClusterExtensionIsUninstalled(ctx context.Context) error { + sc := scenarioCtx(ctx) + require.NotNil(godog.T(ctx), sc.extensionObjects) + for _, obj := range sc.extensionObjects { + if err := ResourceEventuallyNotFound(ctx, fmt.Sprintf("%s/%s", obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName())); err != nil { + return err + } + } return nil } @@ -398,12 +433,12 @@ func ClusterExtensionRevisionIsArchived(ctx context.Context, revisionName string func ResourceAvailable(ctx context.Context, resource string) error { sc := scenarioCtx(ctx) resource = substituteScenarioVars(resource, sc) - rtype, name, found := strings.Cut(resource, "/") + kind, name, found := strings.Cut(resource, "/") if !found { - return fmt.Errorf("resource %s is not in the format /", resource) + return fmt.Errorf("resource %s is not in the format /", resource) } waitFor(ctx, func() bool { - _, err := k8sClient("get", rtype, name, "-n", sc.namespace) + _, err := k8sClient("get", kind, name, "-n", sc.namespace) return err == nil }) return nil @@ -411,11 +446,12 @@ func ResourceAvailable(ctx context.Context, resource string) error { func ResourceRemoved(ctx context.Context, resource string) error { sc := scenarioCtx(ctx) - rtype, name, found := strings.Cut(resource, "/") + resource = substituteScenarioVars(resource, sc) + kind, name, found := strings.Cut(resource, "/") if !found { - return fmt.Errorf("resource %s is not in the format /", resource) + return fmt.Errorf("resource %s is not in the format /", resource) } - yaml, err := k8sClient("get", rtype, name, "-n", sc.namespace, "-o", "yaml") + yaml, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "yaml") if err != nil { return err } @@ -424,23 +460,38 @@ func ResourceRemoved(ctx context.Context, resource string) error { return err } sc.removedResources = append(sc.removedResources, *obj) - _, err = k8sClient("delete", rtype, name, "-n", sc.namespace) + _, err = k8sClient("delete", kind, name, "-n", sc.namespace) return err } +func ResourceEventuallyNotFound(ctx context.Context, resource string) error { + sc := scenarioCtx(ctx) + resource = substituteScenarioVars(resource, sc) + kind, name, found := strings.Cut(resource, "/") + if !found { + return fmt.Errorf("resource %s is not in the format /", resource) + } + + require.Eventually(godog.T(ctx), func() bool { + obj, err := k8sClient("get", kind, name, "-n", sc.namespace, "--ignore-not-found", "-o", "yaml") + return err == nil && obj == "" + }, timeout, tick) + return nil +} + func ResourceMatches(ctx context.Context, resource string, requiredContentTemplate *godog.DocString) error { sc := scenarioCtx(ctx) resource = substituteScenarioVars(resource, sc) - rtype, name, found := strings.Cut(resource, "/") + kind, name, found := strings.Cut(resource, "/") if !found { - return fmt.Errorf("resource %s is not in the format /", resource) + return fmt.Errorf("resource %s is not in the format /", resource) } requiredContent, err := toUnstructured(substituteScenarioVars(requiredContentTemplate.Content, sc)) if err != nil { return fmt.Errorf("failed to parse required resource yaml: %v", err) } waitFor(ctx, func() bool { - objJson, err := k8sClient("get", rtype, name, "-n", sc.namespace, "-o", "json") + objJson, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "json") if err != nil { return false } @@ -461,19 +512,20 @@ func ResourceMatches(ctx context.Context, resource string, requiredContentTempla return false } - return len(cmp.Diff(upd.Object, obj.Object)) == 0 + return len(diff.Diff(upd.Object, obj.Object)) == 0 }) return nil } func ResourceRestored(ctx context.Context, resource string) error { sc := scenarioCtx(ctx) - rtype, name, found := strings.Cut(resource, "/") + resource = substituteScenarioVars(resource, sc) + kind, name, found := strings.Cut(resource, "/") if !found { - return fmt.Errorf("resource %s is not in the format /", resource) + return fmt.Errorf("resource %s is not in the format /", resource) } waitFor(ctx, func() bool { - yaml, err := k8sClient("get", rtype, name, "-n", sc.namespace, "-o", "yaml") + yaml, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "yaml") if err != nil { return false } @@ -486,7 +538,7 @@ func ResourceRestored(ctx context.Context, resource string) error { for i, removed := range sc.removedResources { rct := removed.GetCreationTimestamp() if removed.GetName() == obj.GetName() && removed.GetKind() == obj.GetKind() && rct.Before(&ct) { - switch rtype { + switch kind { case "configmap": if !reflect.DeepEqual(removed.Object["data"], obj.Object["data"]) { return false @@ -853,3 +905,138 @@ func extendMap(m map[string]string, keyValue ...string) map[string]string { } return m } + +// listExtensionResources returns a slice of client.Object containing all resources for a ClusterExtension +// this method is best called when the extension has been installed successfully. An error is returned if there was +// any issue in determining the extension's resources. +func listExtensionResources(extName string) ([]client.Object, error) { + if enabled, found := featureGates[features.BoxcutterRuntime]; found && enabled { + return listExtensionRevisionResources(extName) + } + return listHelmReleaseResources(extName) +} + +// listHelmReleaseResources returns a slice of client.Object containing all resources for a ClusterExtension's +// Helm release. Note: The current implementation does not support chunked release secrets. +func listHelmReleaseResources(extName string) ([]client.Object, error) { + secret, err := helmReleaseSecretForExtension(extName) + if err != nil { + return nil, fmt.Errorf("failed to get helm release secret for extension %s: %v", extName, err) + } + + rel, err := helmReleaseFromSecret(secret) + if err != nil { + return nil, fmt.Errorf("failed to get helm release from secret for cluster extension '%s': %w", extName, err) + } + + objs, err := collectHelmReleaseObjects(rel) + if err != nil { + return nil, fmt.Errorf("failed to collect helm release objects for cluster extension '%s': %w", extName, err) + } + return objs, nil +} + +// helmReleaseSecretForExtension returns the Helm release secret for the extension with name extName +func helmReleaseSecretForExtension(extName string) (*corev1.Secret, error) { + out, err := k8sClient("get", "secrets", "-n", "olmv1-system", "-l", fmt.Sprintf("name=%s", extName), "--field-selector", "type=operatorframework.io/index.v1", "-o", "json") + if err != nil { + return nil, err + } + if strings.TrimSpace(out) == "" { + return nil, err + } + + var secretList corev1.SecretList + if err = json.Unmarshal([]byte(out), &secretList); err != nil { + return nil, err + } + if len(secretList.Items) != 1 { + return nil, err + } + return &secretList.Items[0], nil +} + +// helmReleaseFromSecret returns the Helm Release object encoded in the secret. Note: this function does not yet support +// releases chunked over multiple Secrets +func helmReleaseFromSecret(secret *corev1.Secret) (*release.Release, error) { + gzReader, err := gzip.NewReader(strings.NewReader(string(secret.Data["chunk"]))) + if err != nil { + return nil, err + } + defer gzReader.Close() + + releaseJsonBytes, err := io.ReadAll(gzReader) + if err != nil { + return nil, err + } + + var rel release.Release + if err = json.Unmarshal(releaseJsonBytes, &rel); err != nil { + return nil, err + } + return &rel, nil +} + +// collectHelmReleaseObjects returns a slice of client.Object containing the manifests in rel +func collectHelmReleaseObjects(rel *release.Release) ([]client.Object, error) { + objs := []client.Object{} + menifests := strings.Split(rel.Manifest, "\n") + for _, manifest := range menifests { + if manifest == "" { + continue + } + u := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(manifest), u); err != nil { + return nil, fmt.Errorf("failed to unmarshal manifest: %w", err) + } + objs = append(objs, u) + } + return objs, nil +} + +func listExtensionRevisionResources(extName string) ([]client.Object, error) { + rev, err := latestActiveRevisionForExtension(extName) + if err != nil { + return nil, fmt.Errorf("failed to get latest active revision for extension %s: %w", extName, err) + } + + var objs []client.Object + for _, phase := range rev.Spec.Phases { + for _, obj := range phase.Objects { + objs = append(objs, &obj.Object) + } + } + + return objs, nil +} + +// latestActiveRevisionForExtension returns the latest active revision for the extension called extName +func latestActiveRevisionForExtension(extName string) (*ocv1.ClusterExtensionRevision, error) { + out, err := k8sClient("get", "clusterextensionrevisions", "-l", fmt.Sprintf("olm.operatorframework.io/owner-name=%s", extName), "-o", "json") + if err != nil { + return nil, fmt.Errorf("error listing revisions for extension '%s': %w", extName, err) + } + if strings.TrimSpace(out) == "" { + return nil, fmt.Errorf("no revisions found for extension '%s'", extName) + } + var revisionList ocv1.ClusterExtensionRevisionList + if err := json.Unmarshal([]byte(out), &revisionList); err != nil { + return nil, fmt.Errorf("error unmarshalling revisions for extension '%s': %w", extName, err) + } + + var latest *ocv1.ClusterExtensionRevision + for _, rev := range revisionList.Items { + if rev.Spec.LifecycleState != ocv1.ClusterExtensionRevisionLifecycleStateActive { + continue + } + if latest == nil || rev.Spec.Revision > latest.Spec.Revision { + latest = &rev + } + } + + if latest == nil { + return nil, fmt.Errorf("no active revisions found for extension '%s'", extName) + } + + return latest, nil +}