diff --git a/pipelines/test/disconnected/kustomization.yaml b/pipelines/test/disconnected/kustomization.yaml new file mode 100644 index 0000000..541627a --- /dev/null +++ b/pipelines/test/disconnected/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +labels: + - pairs: + deployment: kuadrant-qe-pipeline + app: test-disconnected + +components: + - ../../../tasks/login/ + - ../../../tasks/test/ + - ../../../tasks/disconnected/ + +resources: + - pipeline.yaml diff --git a/pipelines/test/disconnected/pipeline.yaml b/pipelines/test/disconnected/pipeline.yaml new file mode 100644 index 0000000..417fe06 --- /dev/null +++ b/pipelines/test/disconnected/pipeline.yaml @@ -0,0 +1,203 @@ +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-disconnected +spec: + params: + - default: 'quay.io/kuadrant/testsuite:unstable' + description: Testsuite image to run tests on + name: testsuite-image + type: string + - description: API URL of the Openshift cluster + name: kube-api + type: string + - description: Secret name with cluster credentials + name: cluster-credentials + type: string + default: openshift-pipelines-credentials + - description: Kubernetes secret name (dockerconfigjson) with source registry credentials for image mirroring + name: registry-credentials + type: string + default: registry-redhat-io-pull-secret + - default: kuadrant-system + description: Name of the Openshift project + name: project + type: string + - default: disconnected + description: Makefile target for tests + name: make-target + type: string + - default: "" + description: Pytest flags to use with Make (flags="${pytest-flags}" make disconnected) + name: pytest-flags + type: string + - default: pipeline-settings + description: Config Map with settings for the testsuite + name: settings-cm + type: string + - default: "" + description: Additional env for testsuite container separated with spaces (e.g. KUADRANT_CONTROL_PLANE__provider_secret=gcp-credentials KUADRANT_KEYCLOAK__url="https://my-sso.net") + name: additional-env + type: string + - default: "true" + description: Set to "true" for downstream (RHCL) operator, any other value for upstream (Kuadrant) + name: downstream + type: string + - default: quay.io/kuadrant/kuadrant-operator-catalog:latest + description: Upstream Kuadrant operator catalog image (only used when downstream != "true") + name: index-image + type: string + - default: "true" + description: Run cleanup after tests (reconnect cluster and remove all resources) + name: cleanup + type: string + - default: rhcl + description: Prefix of the launch name saved in report portal (nightly, username, manual, etc.). In case of release candidate testing use kuadrant-v, rhcl-v, or authorino-v + name: launch-name + type: string + - default: "" + description: Optional launch description for Report Portal + name: launch-description + type: string + - default: testsuite + description: Report Portal Project to store test results (e.g. testsuite, nightly-testsuite, releases) + name: rp-project + type: string + - default: "true" + description: Upload test results to Report Portal + name: upload-results + type: string + - default: 'quay.io/kuadrant/rptool:unstable' + description: rptool image for uploading results to Report Portal + name: rptool-image + type: string + tasks: + - name: kubectl-login + params: + - name: testsuite-image + value: $(params.testsuite-image) + - name: kube-api + value: $(params.kube-api) + - name: cluster-credentials + value: $(params.cluster-credentials) + taskRef: + kind: Task + name: kubectl-login + workspaces: + - name: shared-workspace + - name: uninstall-operators + params: + - name: kubeconfig-path + value: $(tasks.kubectl-login.results.kubeconfig-path) + runAfter: + - kubectl-login + taskRef: + kind: Task + name: disconnected-uninstall-operators + workspaces: + - name: shared-workspace + - name: setup-disconnected + params: + - name: kubeconfig-path + value: $(tasks.kubectl-login.results.kubeconfig-path) + - name: registry-credentials + value: $(params.registry-credentials) + - name: downstream + value: $(params.downstream) + - name: index-image + value: $(params.index-image) + runAfter: + - uninstall-operators + taskRef: + kind: Task + name: disconnected-setup + workspaces: + - name: shared-workspace + - name: cluster-disconnect + params: + - name: kubeconfig-path + value: $(tasks.kubectl-login.results.kubeconfig-path) + - name: action + value: disconnect + - name: downstream + value: $(params.downstream) + runAfter: + - setup-disconnected + taskRef: + kind: Task + name: disconnected-cluster-network + workspaces: + - name: shared-workspace + - name: install-operator + params: + - name: kubeconfig-path + value: $(tasks.kubectl-login.results.kubeconfig-path) + runAfter: + - cluster-disconnect + taskRef: + kind: Task + name: disconnected-install-operator + workspaces: + - name: shared-workspace + - name: run-tests + params: + - name: testsuite-image + value: $(params.testsuite-image) + - name: project + value: $(params.project) + - name: make-target + value: $(params.make-target) + - name: pytest-flags + value: $(params.pytest-flags) + - name: settings-cm + value: $(params.settings-cm) + - name: additional-env + value: $(params.additional-env) + - name: kubeconfig-path + value: $(tasks.kubectl-login.results.kubeconfig-path) + - name: cluster-credentials + value: $(params.cluster-credentials) + runAfter: + - install-operator + taskRef: + kind: Task + name: run-tests + workspaces: + - name: shared-workspace + finally: + - name: disconnected-cleanup + when: + - input: $(params.cleanup) + operator: in + values: ["true"] + params: + - name: kubeconfig-path + value: $(tasks.kubectl-login.results.kubeconfig-path) + taskRef: + kind: Task + name: disconnected-cleanup + workspaces: + - name: shared-workspace + - name: rptool-upload + when: + - input: $(params.upload-results) + operator: in + values: ["true"] + params: + - name: launch-name + value: $(params.launch-name) + - name: launch-description + value: $(params.launch-description) + - name: rptool-image + value: $(params.rptool-image) + - name: make-target + value: $(params.make-target) + - name: rp-project + value: $(params.rp-project) + taskRef: + kind: Task + name: rptool-upload + workspaces: + - name: shared-workspace + workspaces: + - name: shared-workspace diff --git a/tasks/disconnected/cluster-disconnect.yaml b/tasks/disconnected/cluster-disconnect.yaml new file mode 100644 index 0000000..327fe42 --- /dev/null +++ b/tasks/disconnected/cluster-disconnect.yaml @@ -0,0 +1,229 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: disconnected-cluster-network +spec: + description: 'Disconnect or reconnect cluster nodes from external network via iptables' + params: + - name: kubeconfig-path + type: string + - name: action + type: string + description: 'Action to perform: disconnect, reconnect, or status' + default: disconnect + - name: downstream + type: string + description: 'Set to "true" for downstream (RHCL) operator, any other value for upstream (Kuadrant)' + default: "true" + steps: + - name: cluster-network + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + imagePullPolicy: Always + timeout: 15m + env: + - name: KUBECONFIG + value: $(params.kubeconfig-path) + - name: ACTION + value: $(params.action) + - name: DOWNSTREAM + value: $(params.downstream) + script: | + #!/usr/bin/env bash + set -euo pipefail + + NODE_NAMES=$(oc get nodes -o name | cut -d'/' -f2) + + if [ -z "$NODE_NAMES" ]; then + echo "ERROR: Could not find any nodes in the cluster" + exit 1 + fi + + NODE_COUNT=$(echo "$NODE_NAMES" | wc -l) + + if [ "$DOWNSTREAM" = "true" ]; then + EXTERNAL_REGISTRIES="registry\.redhat\.io/rhcl-1/|registry\.access\.redhat\.com/rhcl-1/" + else + EXTERNAL_REGISTRIES="quay\.io/kuadrant/" + fi + + case "$ACTION" in + disconnect) + echo "==> Disconnecting cluster from external network" + echo " Nodes (${NODE_COUNT}):" + for NODE_NAME in $NODE_NAMES; do + echo " - ${NODE_NAME}" + done + echo "" + + echo "Applying iptables rules to block external access..." + echo "" + + for NODE_NAME in $NODE_NAMES; do + echo " Node: ${NODE_NAME}" + + oc debug -n default node/${NODE_NAME} -- chroot /host /bin/bash -c " + # Save current rules + iptables-save > /tmp/iptables-before-disconnect.rules + + # Clear OUTPUT chain + iptables -F OUTPUT + + # Allow established connections and loopback + iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + iptables -A OUTPUT -o lo -j ACCEPT + + # Allow DNS to cluster DNS (needed for internal resolution) + iptables -A OUTPUT -p udp --dport 53 -d 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 53 -d 10.0.0.0/8 -j ACCEPT + + # Allow access to local networks (RFC 1918 private networks) + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 172.16.0.0/12 -j ACCEPT + iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT + + # Allow access to link-local (needed for some internal services) + iptables -A OUTPUT -d 169.254.0.0/16 -j ACCEPT + + # Log blocked external attempts (optional, for debugging) + iptables -A OUTPUT -m limit --limit 5/min -j LOG --log-prefix 'BLOCKED-EXT: ' --log-level 4 + + # Block everything else (external internet) + iptables -A OUTPUT -j REJECT --reject-with icmp-host-unreachable + + echo ' iptables rules applied' + " + done + + echo "" + echo "==> Cleaning cached operator images (preserving platform images)" + echo " This ensures operator pods must pull from the mirror registry" + echo "" + + for NODE_NAME in $NODE_NAMES; do + echo " Node: ${NODE_NAME}" + oc debug -n default node/${NODE_NAME} -- chroot /host /bin/bash -c " + REMOVED=0 + # Remove operator images by repo tag + for img in \$(crictl images -o json | jq -r '.images[].repoTags[]? // empty' | grep -E '^(${EXTERNAL_REGISTRIES})' || true); do + crictl rmi \"\$img\" 2>/dev/null && REMOVED=\$((REMOVED + 1)) || true + done + # Remove operator images by repo digest (tagless images) + for img in \$(crictl images -o json | jq -r '.images[] | select((.repoTags == null) or (.repoTags | length == 0)) | .repoDigests[]? // empty' | grep -E '^(${EXTERNAL_REGISTRIES})' || true); do + crictl rmi \"\$img\" 2>/dev/null && REMOVED=\$((REMOVED + 1)) || true + done + echo \" Removed \${REMOVED} cached operator image(s)\" + " + done + + echo "" + echo "==> Testing disconnected state" + + DISCONNECT_OK=true + for NODE_NAME in $NODE_NAMES; do + echo -n " ${NODE_NAME} - external access: " + if oc debug -n default node/${NODE_NAME} -- chroot /host timeout 5 curl -I https://quay.io 2>/dev/null > /dev/null 2>&1; then + echo "NOT BLOCKED" + DISCONNECT_OK=false + else + echo "BLOCKED (as expected)" + fi + done + + echo -n " Internal (Kubernetes API): " + INTERNAL_TEST=$(oc get --raw /healthz 2>/dev/null && echo "ok" || echo "failed") + if [ "$INTERNAL_TEST" = "ok" ]; then + echo "WORKS (as expected)" + else + echo "FAILED - Internal access blocked!" + fi + + echo "" + if [ "$DISCONNECT_OK" = true ]; then + echo "Cluster Disconnected (all ${NODE_COUNT} nodes)" + else + echo "WARNING: Cluster Partially Disconnected" + fi + echo "" + ;; + + reconnect) + echo "==> Reconnecting cluster to external network" + echo " Nodes (${NODE_COUNT}):" + for NODE_NAME in $NODE_NAMES; do + echo " - ${NODE_NAME}" + done + echo "" + + echo "Restoring original iptables rules..." + + for NODE_NAME in $NODE_NAMES; do + echo " Node: ${NODE_NAME}" + + oc debug -n default node/${NODE_NAME} -- chroot /host /bin/bash -c " + if [ -f /tmp/iptables-before-disconnect.rules ]; then + iptables-restore < /tmp/iptables-before-disconnect.rules + rm -f /tmp/iptables-before-disconnect.rules + echo ' Original iptables rules restored' + else + # If backup doesn't exist, just flush OUTPUT chain (restore default ACCEPT) + iptables -F OUTPUT + iptables -P OUTPUT ACCEPT + echo ' iptables OUTPUT chain flushed (default ACCEPT policy)' + fi + " + done + + echo "" + echo "==> Testing reconnected state" + + RECONNECT_OK=true + for NODE_NAME in $NODE_NAMES; do + echo -n " ${NODE_NAME} - external access: " + if oc debug -n default node/${NODE_NAME} -- chroot /host timeout 5 curl -I https://quay.io 2>/dev/null > /dev/null 2>&1; then + echo "WORKS (reconnected)" + else + echo "STILL BLOCKED" + RECONNECT_OK=false + fi + done + + echo "" + if [ "$RECONNECT_OK" = true ]; then + echo "Cluster Reconnected (all ${NODE_COUNT} nodes)" + else + echo "WARNING: Cluster Partially Reconnected" + fi + echo "" + ;; + + status) + echo "==> Checking network connectivity" + echo " Nodes (${NODE_COUNT}):" + echo "" + + for NODE_NAME in $NODE_NAMES; do + echo -n " ${NODE_NAME} - external (quay.io): " + if oc debug -n default node/${NODE_NAME} -- chroot /host timeout 5 curl -I https://quay.io 2>/dev/null > /dev/null 2>&1; then + echo "CONNECTED" + else + echo "DISCONNECTED" + fi + done + + echo "" + echo -n " Internal (Kubernetes API): " + if oc get --raw /healthz &>/dev/null; then + echo "CONNECTED" + else + echo "DISCONNECTED" + fi + echo "" + ;; + + *) + echo "ERROR: Unknown action '${ACTION}'. Use: disconnect, reconnect, or status" + exit 1 + ;; + esac + workspaces: + - name: shared-workspace diff --git a/tasks/disconnected/disconnected-cleanup.yaml b/tasks/disconnected/disconnected-cleanup.yaml new file mode 100644 index 0000000..241ba72 --- /dev/null +++ b/tasks/disconnected/disconnected-cleanup.yaml @@ -0,0 +1,254 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: disconnected-cleanup +spec: + description: 'Reconnect cluster and clean up all disconnected install resources' + params: + - name: kubeconfig-path + type: string + steps: + - name: reconnect + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + imagePullPolicy: Always + timeout: 15m + env: + - name: KUBECONFIG + value: $(params.kubeconfig-path) + script: | + #!/usr/bin/env bash + set -euo pipefail + + echo "==> Reconnecting cluster to external network" + + NODE_NAMES=$(oc get nodes -o name | cut -d'/' -f2) + + if [ -z "$NODE_NAMES" ]; then + echo "WARNING: Could not find any nodes (skipping reconnect)" + exit 0 + fi + + NODE_COUNT=$(echo "$NODE_NAMES" | wc -l) + echo " Nodes (${NODE_COUNT}):" + for NODE_NAME in $NODE_NAMES; do + echo " - ${NODE_NAME}" + done + echo "" + + NEEDS_RESTORE=false + for NODE_NAME in $NODE_NAMES; do + if ! oc debug -n default node/${NODE_NAME} -- chroot /host timeout 5 curl -I https://quay.io 2>/dev/null > /dev/null 2>&1; then + NEEDS_RESTORE=true + break + fi + done + + if [ "$NEEDS_RESTORE" = false ]; then + echo " All nodes already connected to external network" + exit 0 + fi + + echo " Disconnected nodes detected - restoring connectivity..." + + for NODE_NAME in $NODE_NAMES; do + echo " Restoring node: ${NODE_NAME}..." + oc debug -n default node/${NODE_NAME} -- chroot /host /bin/bash -c " + if [ -f /tmp/iptables-before-disconnect.rules ]; then + iptables-restore < /tmp/iptables-before-disconnect.rules + rm -f /tmp/iptables-before-disconnect.rules + echo ' Original iptables rules restored' + else + iptables -F OUTPUT + iptables -P OUTPUT ACCEPT + echo ' iptables OUTPUT chain flushed (default ACCEPT policy)' + fi + " 2>/dev/null || echo " WARNING: Failed to restore iptables on ${NODE_NAME}" + done + + sleep 2 + RECONNECT_OK=true + for NODE_NAME in $NODE_NAMES; do + if ! oc debug -n default node/${NODE_NAME} -- chroot /host timeout 5 curl -I https://quay.io 2>/dev/null > /dev/null 2>&1; then + echo " WARNING: ${NODE_NAME} may still be disconnected" + RECONNECT_OK=false + fi + done + + if [ "$RECONNECT_OK" = true ]; then + echo " All nodes reconnected to external network" + else + echo " WARNING: Some nodes may still be disconnected" + fi + echo "" + - name: cleanup + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + imagePullPolicy: Always + timeout: 45m + env: + - name: KUBECONFIG + value: $(params.kubeconfig-path) + - name: WORK_DIR + value: $(workspaces.shared-workspace.path)/disconnected-install + script: | + #!/usr/bin/env bash + set -euo pipefail + + MIRROR_NAMESPACE="mirror-registry" + + echo "==> Cleanup Disconnected Installation Test Resources" + echo "" + + if ! oc whoami &>/dev/null; then + echo "OpenShift cluster not accessible (skipping cluster cleanup)" + # Still clean local files + if [ -d "$WORK_DIR" ]; then + rm -rf "$WORK_DIR" + echo "Removed ${WORK_DIR}" + fi + exit 0 + fi + + echo "OpenShift cluster accessible" + echo "" + + # Find and remove Kuadrant disconnected test CatalogSources + echo "==> Removing CatalogSources" + CATALOG_NAMES=$(oc get catalogsource -n openshift-marketplace -l kuadrant.io/disconnected-test=true -o name 2>/dev/null | \ + cut -d'/' -f2 || echo "") + + if [ -n "$CATALOG_NAMES" ]; then + for catalog_name in ${CATALOG_NAMES}; do + echo " Removing: ${catalog_name}" + oc delete catalogsource ${catalog_name} -n openshift-marketplace --ignore-not-found=true + done + else + echo " No Kuadrant CatalogSources found" + fi + + # Remove kuadrant-system namespace + echo "" + echo "==> Removing kuadrant-system namespace" + if oc get namespace kuadrant-system &>/dev/null; then + if [ -f "${WORK_DIR}/install/uninstall.sh" ]; then + echo " Running automated uninstall script..." + chmod +x "${WORK_DIR}/install/uninstall.sh" + bash "${WORK_DIR}/install/uninstall.sh" || true + fi + # Ensure namespace resources are cleaned up even if uninstall script partially failed + oc delete subscription --all -n kuadrant-system --ignore-not-found=true 2>/dev/null || true + oc delete csv --all -n kuadrant-system --ignore-not-found=true 2>/dev/null || true + oc delete operatorgroup --all -n kuadrant-system --ignore-not-found=true 2>/dev/null || true + oc wait --for=delete pod --all -n kuadrant-system --timeout=60s 2>/dev/null || true + oc delete namespace kuadrant-system --ignore-not-found=true --timeout=120s 2>/dev/null || true + if oc get namespace kuadrant-system &>/dev/null 2>&1; then + echo " WARNING: kuadrant-system namespace still exists (may have stuck finalizers)" + else + echo " Removed kuadrant-system namespace" + fi + else + echo " Namespace kuadrant-system not found" + fi + + # Remove cluster-scoped mirror resources (IDMS/ITMS/ICSP) created by this test. + echo "" + echo "==> Removing image mirror resources created by this test" + CLEANUP_MANIFEST="${WORK_DIR}/applied-cluster-resources.txt" + MIRROR_REMOVED=false + if [ -f "$CLEANUP_MANIFEST" ]; then + while IFS= read -r resource; do + [ -z "$resource" ] && continue + echo " Removing: ${resource}" + oc delete "${resource}" --ignore-not-found=true 2>/dev/null || true + MIRROR_REMOVED=true + done < "$CLEANUP_MANIFEST" + if [ "$MIRROR_REMOVED" = false ]; then + echo " Manifest is empty - no mirror resources to remove" + fi + else + echo " No cleanup manifest found at ${CLEANUP_MANIFEST}" + echo " Skipping (setup recorded no mirror resources; not using --all on a shared cluster)" + fi + + # Wait for cluster to stabilize after removing mirror configuration + if [ "$MIRROR_REMOVED" = true ]; then + echo "" + echo "==> Waiting for cluster to stabilize after removing image mirror configuration" + + NODE_COUNT=$(oc get nodes --no-headers 2>/dev/null | wc -l || echo "0") + if [ "$NODE_COUNT" -eq 1 ]; then + echo " (Single-node cluster detected)" + fi + + sleep 5 + + WAIT_TIMEOUT=180 + APISERVER_STABLE=false + START_TIME=$(date +%s) + + while [ $(($(date +%s) - START_TIME)) -lt $WAIT_TIMEOUT ]; do + APISERVER_AVAILABLE=$(oc get co openshift-apiserver -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null) + APISERVER_PROGRESSING=$(oc get co openshift-apiserver -o jsonpath='{.status.conditions[?(@.type=="Progressing")].status}' 2>/dev/null) + + if [ "$APISERVER_AVAILABLE" = "True" ] && [ "$APISERVER_PROGRESSING" = "False" ]; then + APISERVER_STABLE=true + echo " Cluster stabilized" + break + fi + + # Auto-fix stuck pods on single-node clusters + if [ "$NODE_COUNT" -eq 1 ]; then + PENDING_PODS=$(oc get pods -n openshift-apiserver --no-headers 2>/dev/null | grep -c "Pending" || echo "0") + TERMINATING_PODS=$(oc get pods -n openshift-apiserver --no-headers 2>/dev/null | grep -c "Terminating" || echo "0") + + if [ "$PENDING_PODS" -gt 0 ] && [ "$TERMINATING_PODS" -gt 0 ]; then + echo " Auto-fixing stuck apiserver pods..." + oc get pods -n openshift-apiserver --no-headers 2>/dev/null | grep "Terminating" | awk '{print $1}' | while read pod; do + oc delete pod -n openshift-apiserver "$pod" --force --grace-period=0 2>/dev/null || true + done + sleep 10 + fi + fi + + sleep 5 + done + + if [ "$APISERVER_STABLE" = false ]; then + echo " WARNING: Cluster did not stabilize within ${WAIT_TIMEOUT}s (MachineConfig may still be updating)" + fi + fi + + # Remove mirror-registry namespace + echo "" + echo "==> Removing mirror-registry namespace" + if oc get namespace ${MIRROR_NAMESPACE} &>/dev/null; then + oc delete namespace ${MIRROR_NAMESPACE} + echo " Removed ${MIRROR_NAMESPACE} namespace" + else + echo " Namespace ${MIRROR_NAMESPACE} not found" + fi + + # Re-enable default OperatorHub sources + echo "" + echo "==> Checking OperatorHub default sources" + SOURCES_DISABLED=$(oc get operatorhub cluster -o jsonpath='{.spec.disableAllDefaultSources}' 2>/dev/null || echo "false") + + if [ "$SOURCES_DISABLED" = "true" ]; then + oc patch operatorhub cluster --type json \ + -p '[{"op": "replace", "path": "/spec/disableAllDefaultSources", "value": false}]' + echo " Default OperatorHub sources re-enabled" + else + echo " Default OperatorHub sources already enabled" + fi + + # Clean local files + echo "" + echo "==> Cleaning up workspace files" + if [ -d "$WORK_DIR" ]; then + rm -rf "$WORK_DIR" + echo " Removed ${WORK_DIR}" + fi + + echo "" + echo "Cleanup Complete" + workspaces: + - name: shared-workspace diff --git a/tasks/disconnected/install-operator.yaml b/tasks/disconnected/install-operator.yaml new file mode 100644 index 0000000..f720bda --- /dev/null +++ b/tasks/disconnected/install-operator.yaml @@ -0,0 +1,37 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: disconnected-install-operator +spec: + description: 'Install RHCL operator from generated disconnected install manifests' + params: + - name: kubeconfig-path + type: string + steps: + - name: install-operator + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + imagePullPolicy: Always + timeout: 30m + env: + - name: KUBECONFIG + value: $(params.kubeconfig-path) + - name: WORK_DIR + value: $(workspaces.shared-workspace.path)/disconnected-install + script: | + #!/usr/bin/env bash + set -euo pipefail + + INSTALL_SCRIPT="${WORK_DIR}/install/install.sh" + + if [ ! -f "$INSTALL_SCRIPT" ]; then + echo "ERROR: Install script not found at ${INSTALL_SCRIPT}" + echo " The setup-disconnected task must run first to generate install manifests." + ls -la "${WORK_DIR}/" 2>/dev/null || echo " Work directory does not exist." + exit 1 + fi + + echo "==> Running generated install script" + chmod +x "$INSTALL_SCRIPT" + bash "$INSTALL_SCRIPT" + workspaces: + - name: shared-workspace diff --git a/tasks/disconnected/kustomization.yaml b/tasks/disconnected/kustomization.yaml new file mode 100644 index 0000000..0cca818 --- /dev/null +++ b/tasks/disconnected/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - uninstall-operators.yaml + - setup-disconnected.yaml + - cluster-disconnect.yaml + - install-operator.yaml + - disconnected-cleanup.yaml diff --git a/tasks/disconnected/setup-disconnected.yaml b/tasks/disconnected/setup-disconnected.yaml new file mode 100644 index 0000000..0b7821e --- /dev/null +++ b/tasks/disconnected/setup-disconnected.yaml @@ -0,0 +1,891 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: disconnected-setup +spec: + description: 'Set up in-cluster mirror registry, mirror operator images, configure cluster for disconnected install' + params: + - name: kubeconfig-path + type: string + - name: registry-credentials + type: string + description: 'Kubernetes secret name (dockerconfigjson) with source registry credentials for image mirroring' + default: registry-redhat-io-pull-secret + - name: downstream + type: string + description: 'Set to "true" for downstream (RHCL) operator, any other value for upstream (Kuadrant)' + default: "true" + - name: index-image + type: string + description: 'Upstream Kuadrant operator catalog image (only used when downstream != "true")' + default: quay.io/kuadrant/kuadrant-operator-catalog:latest + steps: + - name: setup-disconnected + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + imagePullPolicy: Always + timeout: 2h + env: + - name: KUBECONFIG + value: $(params.kubeconfig-path) + - name: WORK_DIR + value: $(workspaces.shared-workspace.path)/disconnected-install + - name: DOWNSTREAM + value: $(params.downstream) + - name: INDEX_IMAGE + value: $(params.index-image) + - name: DISABLE_DEFAULT_SOURCES + value: "true" + volumeMounts: + - name: registry-creds + mountPath: /var/registry-auth + readOnly: true + script: | + #!/usr/bin/env bash + set -euo pipefail + + echo "==> Testing Disconnected Installation with oc-mirror" + echo "" + + # Verify cluster access + if ! oc whoami &>/dev/null; then + echo "ERROR: Not logged into OpenShift cluster" + exit 1 + fi + + # Install oc-mirror if not present + if ! command -v oc-mirror &>/dev/null; then + echo "==> Installing oc-mirror..." + OCP_VERSION=$(oc get clusterversion version -o jsonpath='{.status.desired.version}' 2>/dev/null || echo "") + if [ -z "$OCP_VERSION" ]; then + echo "ERROR: Could not detect OCP version for oc-mirror download" + exit 1 + fi + curl -sL "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${OCP_VERSION}/oc-mirror.rhel9.tar.gz" \ + -o /tmp/oc-mirror.tar.gz + tar xzf /tmp/oc-mirror.tar.gz -C /usr/local/bin/ + chmod +x /usr/local/bin/oc-mirror + rm -f /tmp/oc-mirror.tar.gz + echo " oc-mirror installed" + echo "" + fi + + # Set up registry auth from mounted secret + if [ -f /var/registry-auth/.dockerconfigjson ]; then + mkdir -p "$HOME/.docker" + cp /var/registry-auth/.dockerconfigjson "$HOME/.docker/config.json" + echo "Registry auth configured from secret" + else + echo "WARNING: No registry credentials found at /var/registry-auth/.dockerconfigjson" + echo " Downstream mirroring from registry.redhat.io may fail" + fi + echo "" + + # Configuration + mkdir -p "${WORK_DIR}" + + # Catalog image and operator package name depend on mode + if [ "$DOWNSTREAM" = "true" ]; then + # Downstream RHCL: use Red Hat operator index, auto-detect OCP version + OCP_VERSION=$(oc get clusterversion version -o jsonpath='{.status.desired.version}' 2>/dev/null || echo "") + if [ -z "$OCP_VERSION" ]; then + echo "ERROR: Could not detect OCP version from cluster" + exit 1 + fi + OCP_MINOR=$(echo "$OCP_VERSION" | cut -d. -f1,2) + CATALOG_IMG="registry.redhat.io/redhat/redhat-operator-index:v${OCP_MINOR}" + OPERATOR_PACKAGE="rhcl-operator" + else + # Upstream Kuadrant: use catalog image from parameter + CATALOG_IMG="${INDEX_IMAGE}" + OPERATOR_PACKAGE="kuadrant-operator" + fi + + MIRROR_NAMESPACE="mirror-registry" + + echo "==> Configuration:" + echo " Mode: $([ "$DOWNSTREAM" = "true" ] && echo "downstream (RHCL)" || echo "upstream (Kuadrant)")" + echo " Working Directory: ${WORK_DIR}" + echo " Catalog Image: ${CATALOG_IMG}" + echo " Operator Package: ${OPERATOR_PACKAGE}" + echo " Mirror Namespace: ${MIRROR_NAMESPACE}" + echo "" + + # Step 1: Deploy simple Docker registry + echo "==> Setting up simple Docker registry for mirroring" + + echo " Ensuring namespace exists..." + oc new-project ${MIRROR_NAMESPACE} 2>/dev/null || echo " Namespace ${MIRROR_NAMESPACE} already exists" + + # Check if Docker registry already deployed + if oc get deployment docker-registry -n ${MIRROR_NAMESPACE} &>/dev/null; then + echo " Docker registry already deployed" + else + echo " Deploying simple Docker registry..." + cat </dev/null; then + oc create route edge docker-registry \ + --service=docker-registry \ + --port=5000 \ + -n ${MIRROR_NAMESPACE} + fi + + # Wait for registry to be ready + echo " Waiting for registry to be ready..." + oc wait --for=condition=available deployment/docker-registry \ + -n ${MIRROR_NAMESPACE} \ + --timeout=5m + fi + + # Get registry hostname + REGISTRY_HOSTNAME=$(oc get route docker-registry -n ${MIRROR_NAMESPACE} -o jsonpath='{.spec.host}') + + if [ -z "$REGISTRY_HOSTNAME" ]; then + echo " ERROR: Could not determine registry hostname" + oc get routes -n ${MIRROR_NAMESPACE} + exit 1 + fi + + echo " Docker registry route: ${REGISTRY_HOSTNAME}" + echo "" + + # Wait for registry HTTP endpoint + echo " Waiting for registry HTTP endpoint to respond (90s timeout)..." + TIMEOUT=90 + ELAPSED=0 + REGISTRY_READY=false + while [ $ELAPSED -lt $TIMEOUT ]; do + HTTP_STATUS=$(curl -k -s -o /dev/null -w "%{http_code}" https://${REGISTRY_HOSTNAME}/v2/ 2>/dev/null || echo "000") + if [ "$HTTP_STATUS" = "200" ] || [ "$HTTP_STATUS" = "401" ]; then + echo " Registry HTTP endpoint ready (HTTP ${HTTP_STATUS})" + REGISTRY_READY=true + break + fi + sleep 3 + ELAPSED=$((ELAPSED + 3)) + if [ $((ELAPSED % 15)) -eq 0 ]; then + echo " Still waiting... (${ELAPSED}s) [HTTP ${HTTP_STATUS}]" + fi + done + + if [ "$REGISTRY_READY" = false ]; then + echo " Registry HTTP endpoint not ready after ${TIMEOUT}s" + oc get deployment docker-registry -n ${MIRROR_NAMESPACE} + oc get pods -n ${MIRROR_NAMESPACE} -l app=docker-registry + oc logs -n ${MIRROR_NAMESPACE} -l app=docker-registry --tail=20 + exit 1 + fi + echo "" + + # Step 2: Configure CA trust for mirror registry + echo "==> Configuring CA trust for mirror registry" + + # Get the ingress CA certificate + INGRESS_CA=$(oc get configmap -n openshift-config-managed default-ingress-cert -o jsonpath='{.data.ca-bundle\.crt}') + + if [ -n "$INGRESS_CA" ]; then + # Check if registry-certs configmap exists + if oc get configmap registry-certs -n openshift-config &>/dev/null; then + echo " Updating existing registry-certs configmap..." + TEMP_CA_FILE="${WORK_DIR}/ca-bundle.crt" + echo "${INGRESS_CA}" > "${TEMP_CA_FILE}" + oc patch configmap registry-certs -n openshift-config --type merge \ + --patch "{\"data\":{\"${REGISTRY_HOSTNAME}\":$(cat ${TEMP_CA_FILE} | jq -Rs .)}}" + rm -f "${TEMP_CA_FILE}" + else + echo " Creating registry-certs configmap..." + TEMP_CA_FILE="${WORK_DIR}/ca-bundle.crt" + echo "${INGRESS_CA}" > "${TEMP_CA_FILE}" + oc create configmap registry-certs -n openshift-config \ + --from-file="${REGISTRY_HOSTNAME}=${TEMP_CA_FILE}" + rm -f "${TEMP_CA_FILE}" + fi + + # Update image.config.openshift.io/cluster to reference the configmap + if ! oc get image.config.openshift.io/cluster -o yaml | grep -q "registry-certs"; then + echo " Configuring cluster to use registry-certs..." + oc patch image.config.openshift.io/cluster --type merge \ + -p '{"spec":{"additionalTrustedCA":{"name":"registry-certs"}}}' + fi + + echo " CA trust configured for ${REGISTRY_HOSTNAME}" + else + echo " WARNING: Could not get ingress CA certificate" + fi + echo "" + + # Step 3: Detect gateway (Istio/OSSM) images needed on the disconnected cluster + echo "==> Detecting gateway (Istio/OSSM) images for mirroring" + + # The OCP-managed Gateway API (OCP 4.19+ built-in gateway) runs istiod/proxy images that + # need to be extracted and mirrored separately + ISTIO_IMAGES="" + + if oc get deployment istiod-openshift-gateway -n openshift-ingress &>/dev/null 2>&1; then + GW_PILOT=$(oc get deployment istiod-openshift-gateway -n openshift-ingress \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + if [ -n "$GW_PILOT" ]; then + ISTIO_IMAGES="${ISTIO_IMAGES}${GW_PILOT}"$'\n' + fi + GW_PROXY=$(oc get cm istio-sidecar-injector-openshift-gateway -n openshift-ingress -o json 2>/dev/null | \ + jq -r '.data.values' 2>/dev/null | jq -r '.global.proxy.image // empty' 2>/dev/null || true) + if [ -n "$GW_PROXY" ]; then + ISTIO_IMAGES="${ISTIO_IMAGES}${GW_PROXY}"$'\n' + fi + fi + + # The OSSM3 / Sail-operator gateway (istio-system) uses istiod/proxy images that + # differ from the OCP built-in gateway; the disconnected tests use this provider + if oc get deployment istiod -n istio-system &>/dev/null 2>&1; then + SAIL_PILOT=$(oc get deployment istiod -n istio-system \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + if [ -n "$SAIL_PILOT" ]; then + ISTIO_IMAGES="${ISTIO_IMAGES}${SAIL_PILOT}"$'\n' + fi + SAIL_PROXY=$(oc get cm istio-sidecar-injector -n istio-system -o json 2>/dev/null | \ + jq -r '.data.values' 2>/dev/null | jq -r '.global.proxy.image // empty' 2>/dev/null || true) + if [ -n "$SAIL_PROXY" ]; then + ISTIO_IMAGES="${ISTIO_IMAGES}${SAIL_PROXY}"$'\n' + fi + fi + + # Deduplicate + ISTIO_IMAGES=$(echo "$ISTIO_IMAGES" | sort -u | grep -v '^$' || true) + + if [ -n "$ISTIO_IMAGES" ]; then + echo " Detected Istio images to mirror:" + while IFS= read -r img; do + echo " - ${img}" + done <<< "$ISTIO_IMAGES" + else + echo " No Istio images detected" + fi + echo "" + + # Step 4: Create ImageSetConfiguration + echo "==> Creating ImageSetConfiguration" + IMAGESET_CONFIG="${WORK_DIR}/imageset-config.yaml" + + # Header + operators list + cat < ${IMAGESET_CONFIG} + kind: ImageSetConfiguration + apiVersion: mirror.openshift.io/v2alpha1 + mirror: + operators: + EOF + + if [ "$DOWNSTREAM" = "true" ]; then + cat <> ${IMAGESET_CONFIG} + - catalog: ${CATALOG_IMG} + packages: + - name: rhcl-operator + - name: authorino-operator + - name: limitador-operator + - name: dns-operator + EOF + else + cat <> ${IMAGESET_CONFIG} + - catalog: ${CATALOG_IMG} + full: true + EOF + fi + + # additionalImages: mockserver + any OCP-managed gateway images (Source 2) + cat <> ${IMAGESET_CONFIG} + additionalImages: + - name: quay.io/rhn_support_azgabur/mockserver:latest + EOF + + if [ -n "$ISTIO_IMAGES" ]; then + while IFS= read -r img; do + echo " - name: ${img}" >> ${IMAGESET_CONFIG} + done <<< "$ISTIO_IMAGES" + fi + + cat ${IMAGESET_CONFIG} + echo "" + + # Step 5: Run oc-mirror + echo "==> Running oc-mirror v2 (this may take several minutes)..." + echo " Source Catalog: ${CATALOG_IMG}" + echo " Destination: ${REGISTRY_HOSTNAME}" + echo "" + + oc-mirror --v2 --config ${IMAGESET_CONFIG} \ + --workspace file://${WORK_DIR}/oc-mirror-workspace \ + --dest-tls-verify=false \ + --image-timeout 30m \ + docker://${REGISTRY_HOSTNAME} + + echo "" + echo "==> oc-mirror completed" + echo "" + + # Step 6: Apply image mirror configuration + echo "==> Applying image mirror configuration" + + # oc-mirror v2 puts files in working-dir/cluster-resources + RESULTS_DIR="${WORK_DIR}/oc-mirror-workspace/working-dir/cluster-resources" + + if [ ! -d "$RESULTS_DIR" ]; then + echo "ERROR: Results directory not found at ${RESULTS_DIR}" + exit 1 + fi + + # oc-mirror v2 can generate IDMS, ITMS, or ICSP depending on how images were mirrored + IDMS_FILE="${RESULTS_DIR}/idms-oc-mirror.yaml" + ITMS_FILE="${RESULTS_DIR}/itms-oc-mirror.yaml" + ICSP_FILE="${RESULTS_DIR}/icsp-oc-mirror.yaml" + + APPLIED_ANY=false + + # Record cluster-scoped resources we create so cleanup can remove exactly these by name. + CLEANUP_MANIFEST="${WORK_DIR}/applied-cluster-resources.txt" + : > "${CLEANUP_MANIFEST}" + + if [ -f "$IDMS_FILE" ]; then + echo " Applying ImageDigestMirrorSet..." + oc apply -f ${IDMS_FILE} -o name | tee -a "${CLEANUP_MANIFEST}" + APPLIED_ANY=true + fi + + if [ -f "$ITMS_FILE" ]; then + echo " Applying ImageTagMirrorSet..." + oc apply -f ${ITMS_FILE} -o name | tee -a "${CLEANUP_MANIFEST}" + APPLIED_ANY=true + fi + + if [ -f "$ICSP_FILE" ]; then + echo " Applying ImageContentSourcePolicy..." + oc apply -f ${ICSP_FILE} -o name | tee -a "${CLEANUP_MANIFEST}" + APPLIED_ANY=true + fi + + if [ "$APPLIED_ANY" = false ]; then + echo "ERROR: No image mirror configuration found" + ls -la ${RESULTS_DIR}/ + exit 1 + fi + echo "" + + # Step 7: Wait for cluster to stabilize after IDMS/ITMS changes + echo "==> Waiting for cluster to stabilize after image mirror configuration" + echo "" + + # Give the cluster a moment to start processing the changes + sleep 5 + + # Check if this is a single-node cluster + NODE_COUNT=$(oc get nodes --no-headers 2>/dev/null | wc -l) + IS_SINGLE_NODE=false + if [ "$NODE_COUNT" -eq 1 ]; then + IS_SINGLE_NODE=true + echo " Detected single-node cluster" + fi + echo "" + + # Wait for openshift-apiserver to stabilize (critical for API access) + echo " Waiting for openshift-apiserver to stabilize..." + APISERVER_TIMEOUT=300 + APISERVER_STABLE=false + START_TIME=$(date +%s) + + while [ $(($(date +%s) - START_TIME)) -lt $APISERVER_TIMEOUT ]; do + APISERVER_AVAILABLE=$(oc get co openshift-apiserver -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null) + APISERVER_PROGRESSING=$(oc get co openshift-apiserver -o jsonpath='{.status.conditions[?(@.type=="Progressing")].status}' 2>/dev/null) + APISERVER_DEGRADED=$(oc get co openshift-apiserver -o jsonpath='{.status.conditions[?(@.type=="Degraded")].status}' 2>/dev/null) + + if [ "$APISERVER_AVAILABLE" = "True" ] && [ "$APISERVER_PROGRESSING" = "False" ] && [ "$APISERVER_DEGRADED" = "False" ]; then + APISERVER_STABLE=true + echo " openshift-apiserver is stable" + break + fi + + # On single-node clusters, check for stuck apiserver pods due to anti-affinity + if [ "$IS_SINGLE_NODE" = true ]; then + PENDING_PODS=$(oc get pods -n openshift-apiserver --no-headers 2>/dev/null | grep -c "Pending" || echo "0") + TERMINATING_PODS=$(oc get pods -n openshift-apiserver --no-headers 2>/dev/null | grep -c "Terminating" || echo "0") + + if [ "$PENDING_PODS" -gt 0 ] && [ "$TERMINATING_PODS" -gt 0 ]; then + echo " Detected stuck pod transition (anti-affinity on single-node)" + echo " Force-deleting terminating pods..." + oc get pods -n openshift-apiserver --no-headers 2>/dev/null | grep "Terminating" | awk '{print $1}' | while read pod; do + oc delete pod -n openshift-apiserver "$pod" --force --grace-period=0 2>/dev/null || true + done + sleep 10 + fi + fi + + echo " Waiting... (Available: $APISERVER_AVAILABLE, Progressing: $APISERVER_PROGRESSING, Degraded: $APISERVER_DEGRADED)" + sleep 10 + done + + if [ "$APISERVER_STABLE" = false ]; then + echo " WARNING: openshift-apiserver did not stabilize within ${APISERVER_TIMEOUT}s" + echo " Continuing anyway (pipeline context)..." + fi + echo "" + + # Step 8: Wait for MachineConfig rollout + echo "==> Checking MachineConfig status" + + # The IDMS/ITMS applied above make the machine-config-operator render a new + # registries.conf MachineConfig, but rendering and rollout start lag the apply + # by up to a couple of minutes. Poll for the rollout to begin before concluding + # none is needed, so the node reboots happen here (while still connected) + MCP_START_TIMEOUT=120 + echo " Waiting up to ${MCP_START_TIMEOUT}s for a MachineConfig rollout to start..." + ROLLOUT_STARTED=false + START_TIME=$(date +%s) + while [ $(($(date +%s) - START_TIME)) -lt $MCP_START_TIMEOUT ]; do + if oc get mcp -o json | jq -e '.items[] | select(.status.conditions[] | select(.type=="Updating" and .status=="True"))' &>/dev/null; then + ROLLOUT_STARTED=true + break + fi + sleep 10 + done + + if [ "$ROLLOUT_STARTED" = true ]; then + echo " MachineConfigPool is updating..." + echo " Waiting for rollout (timeout: 30m)..." + oc wait mcp --all --for=condition=Updated --timeout=30m || { + echo " WARNING: MachineConfig rollout timed out" + } + else + echo " No MachineConfigPool updates needed (none started within ${MCP_START_TIMEOUT}s)" + fi + echo "" + + # Step 9: Apply CatalogSource + echo "==> Creating CatalogSource" + + # oc-mirror v2 generates CatalogSource file for the catalog + CATALOG_FILE=$(find ${RESULTS_DIR} -name "cs-*.yaml" -o -name "catalogSource-*.yaml" 2>/dev/null | head -1) + + if [ -z "$CATALOG_FILE" ]; then + echo " ERROR: No CatalogSource file found in ${RESULTS_DIR}" + ls -la ${RESULTS_DIR}/ + exit 1 + fi + + # Rename CatalogSource for clarity + ORIGINAL_NAME=$(grep '^ name:' ${CATALOG_FILE} | head -1 | awk '{print $2}') + NEW_NAME="kuadrant-disconnected-operator-catalog" + + echo " Renaming CatalogSource: ${ORIGINAL_NAME} -> ${NEW_NAME}" + + # Update the name and add a label for easy identification + sed -i "s/name: ${ORIGINAL_NAME}/name: ${NEW_NAME}/" ${CATALOG_FILE} + # Add disconnected-test label for easy cleanup + sed -i '/^ name: '"${NEW_NAME}"'/a\ labels:\n kuadrant.io/disconnected-test: "true"' ${CATALOG_FILE} + + oc apply -f ${CATALOG_FILE} + echo "" + + # Step 10: Wait for CatalogSource to be ready + echo "==> Waiting for CatalogSource to be ready" + + # Give OLM a moment to process the CatalogSource + sleep 5 + + # Find the catalog name (using the label we added) + CATALOG_NAME=$(oc get catalogsource -n openshift-marketplace -l kuadrant.io/disconnected-test=true -o name | cut -d'/' -f2) + + if [ -z "$CATALOG_NAME" ]; then + echo " ERROR: Kuadrant CatalogSource not found" + oc get catalogsource -n openshift-marketplace + exit 1 + fi + + echo " Waiting for catalog: ${CATALOG_NAME}" + + # CatalogSource doesn't use standard conditions - check connectionState instead + CATALOG_TIMEOUT=300 + CATALOG_READY=false + START_TIME=$(date +%s) + + while [ $(($(date +%s) - START_TIME)) -lt $CATALOG_TIMEOUT ]; do + LAST_STATE=$(oc get catalogsource ${CATALOG_NAME} -n openshift-marketplace \ + -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null || echo "") + + if [ "$LAST_STATE" = "READY" ]; then + CATALOG_READY=true + echo " READY" + break + fi + + sleep 5 + done + + if [ "$CATALOG_READY" = false ]; then + echo " ERROR: Timed out waiting for catalog to be READY" + echo " Current state: ${LAST_STATE}" + exit 1 + fi + echo "" + + # Step 11: Disable default OperatorHub sources + if [ "$DISABLE_DEFAULT_SOURCES" = "true" ]; then + echo "==> Disabling default OperatorHub sources" + + # Check current state + CURRENT_STATE=$(oc get operatorhub cluster -o jsonpath='{.spec.disableAllDefaultSources}' 2>/dev/null || echo "false") + + if [ "$CURRENT_STATE" = "true" ]; then + echo " Default sources already disabled" + else + oc patch operatorhub cluster --type json \ + -p '[{"op": "add", "path": "/spec/disableAllDefaultSources", "value": true}]' + echo " Default OperatorHub sources disabled" + fi + echo "" + fi + + # Step 12: Verify packages are available + echo "==> Verifying operator packages are available" + + if [ "$DOWNSTREAM" = "true" ]; then + EXPECTED_PACKAGES="${OPERATOR_PACKAGE}" + else + EXPECTED_PACKAGES="${OPERATOR_PACKAGE} authorino-operator limitador-operator dns-operator" + fi + PACKAGE_TIMEOUT=120 + ALL_FOUND=false + START_TIME=$(date +%s) + + echo " Waiting for PackageManifests to appear from disconnected catalogs..." + + while [ $(($(date +%s) - START_TIME)) -lt $PACKAGE_TIMEOUT ]; do + ALL_FOUND=true + + for package in ${EXPECTED_PACKAGES}; do + FOUND_COUNT=$(oc get packagemanifest -n openshift-marketplace -o json 2>/dev/null | \ + jq -r ".items[] | select(.metadata.name==\"${package}\") | select(.status.catalogSource | startswith(\"kuadrant-disconnected-\")) | .metadata.name" 2>/dev/null | wc -l || echo "0") + + if [ "$FOUND_COUNT" -eq 0 ]; then + ALL_FOUND=false + break + fi + done + + if [ "$ALL_FOUND" = true ]; then + break + fi + + sleep 5 + done + + echo "" + for package in ${EXPECTED_PACKAGES}; do + CATALOG_SOURCE=$(oc get packagemanifest -n openshift-marketplace -o json 2>/dev/null | \ + jq -r ".items[] | select(.metadata.name==\"${package}\") | select(.status.catalogSource | startswith(\"kuadrant-disconnected-\")) | .status.catalogSource" 2>/dev/null | head -1) + + if [ -n "$CATALOG_SOURCE" ]; then + echo " ${package} (from ${CATALOG_SOURCE})" + else + echo " ${package} (not found in disconnected catalogs)" + ALL_FOUND=false + fi + done + + if [ "$ALL_FOUND" = false ]; then + echo "" + echo " ERROR: Not all packages available from disconnected catalogs" + exit 1 + fi + echo "" + + # Step 13: Generate installation manifests + INSTALL_DIR="${WORK_DIR}/install" + mkdir -p "${INSTALL_DIR}" + + # Use the catalog name (should be kuadrant-disconnected-operator-catalog) + KUADRANT_CATALOG="${CATALOG_NAME}" + + # Generate Namespace manifest + cat > "${INSTALL_DIR}/01-namespace.yaml" < "${INSTALL_DIR}/02-operatorgroup.yaml" </dev/null || echo "") + + if [ -z "$DEFAULT_CHANNEL" ]; then + # Fallback: try to get channels from packagemanifest + AVAILABLE_CHANNELS=$(oc get packagemanifest ${OPERATOR_PACKAGE} -n openshift-marketplace -o jsonpath='{.status.channels[*].name}' 2>/dev/null || echo "") + if [ -n "$AVAILABLE_CHANNELS" ]; then + DEFAULT_CHANNEL=$(echo "$AVAILABLE_CHANNELS" | awk '{print $1}') + echo " No default channel set, using first available: ${DEFAULT_CHANNEL}" + else + DEFAULT_CHANNEL="alpha" + echo " WARNING: Could not detect channel, defaulting to: ${DEFAULT_CHANNEL}" + fi + else + echo " Using default channel: ${DEFAULT_CHANNEL}" + fi + echo "" + + # Generate Subscription manifest + cat > "${INSTALL_DIR}/03-subscription.yaml" < "${INSTALL_DIR}/04-kuadrant.yaml" < "${INSTALL_DIR}/install.sh" <<'SCRIPT' + #!/bin/bash + set -euo pipefail + + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + + echo "==> Installing Kuadrant from disconnected catalogs" + echo "" + + echo "Step 1: Creating namespace..." + oc apply -f "${SCRIPT_DIR}/01-namespace.yaml" + echo "" + + echo "Step 2: Creating OperatorGroup..." + oc apply -f "${SCRIPT_DIR}/02-operatorgroup.yaml" + echo "" + + echo "Step 3: Creating Subscription..." + oc apply -f "${SCRIPT_DIR}/03-subscription.yaml" + echo "" + + echo "Step 4: Waiting for operator installation..." + echo " (This may take 2-3 minutes)" + echo "" + + OPERATOR_PACKAGE_NAME=$(grep '^\s*name:' "${SCRIPT_DIR}/03-subscription.yaml" | tail -1 | awk '{print $2}') + + echo " Waiting for ${OPERATOR_PACKAGE_NAME} CSV..." + CSV_NAME="" + for i in {1..60}; do + CSV_NAME=$(oc get csv -n kuadrant-system -o name 2>/dev/null | grep "${OPERATOR_PACKAGE_NAME}" | head -1 || true) + if [ -n "$CSV_NAME" ]; then + break + fi + sleep 5 + done + + if [ -z "$CSV_NAME" ]; then + echo " ERROR: ${OPERATOR_PACKAGE_NAME} CSV did not appear within timeout" + oc get csv -n kuadrant-system 2>/dev/null || true + exit 1 + fi + + echo " Found: ${CSV_NAME}" + echo " Waiting for ${CSV_NAME} to succeed..." + oc wait --for=jsonpath='{.status.phase}'=Succeeded ${CSV_NAME} -n kuadrant-system --timeout=600s + + echo " Waiting for all dependency operators to be ready..." + oc wait --for=jsonpath='{.status.phase}'=Succeeded csv -l operators.coreos.com/${OPERATOR_PACKAGE_NAME}.kuadrant-system="" -n kuadrant-system --timeout=300s 2>/dev/null || \ + oc wait --for=jsonpath='{.status.phase}'=Succeeded csv --all -n kuadrant-system --timeout=300s + echo "" + + # OLM installs the dependency operators (Limitador, Authorino, DNS) in no + # guaranteed order, and the main operator detects these optional + # dependencies only once, at startup. It can therefore boot before their + # CRDs are registered and permanently skip them -- RateLimitPolicy then + # fails with MissingDependency and no Limitador instance is ever created. + # All dependency CSVs have Succeeded by now, so restart the operator so it + # re-detects them before we create the Kuadrant CR. (OLM reverts + # 'oc rollout restart', so delete the pods to force the restart.) + echo "Step 5: Restarting operator to re-detect dependencies..." + MAIN_DEPLOY=$(oc get csv "${CSV_NAME##*/}" -n kuadrant-system -o jsonpath='{.spec.install.spec.deployments[*].name}' 2>/dev/null | tr ' ' '\n' | grep -i 'controller-manager' | head -1 || true) + [ -z "$MAIN_DEPLOY" ] && MAIN_DEPLOY="kuadrant-operator-controller-manager" + echo " Restarting ${MAIN_DEPLOY}..." + oc get pods -n kuadrant-system -o name 2>/dev/null | grep "/${MAIN_DEPLOY}-" | xargs -r oc delete -n kuadrant-system + oc wait --for=jsonpath='{.status.readyReplicas}'=1 deployment/${MAIN_DEPLOY} -n kuadrant-system --timeout=180s || true + echo "" + + echo "Step 6: Creating Kuadrant instance..." + + echo " Waiting for Kuadrant CRD to be registered..." + for i in {1..60}; do + if oc get crd kuadrants.kuadrant.io &>/dev/null; then + break + fi + sleep 5 + done + + if ! oc get crd kuadrants.kuadrant.io &>/dev/null; then + echo " ERROR: Kuadrant CRD not registered within timeout" + oc get crd | grep kuadrant || true + exit 1 + fi + + oc apply -f "${SCRIPT_DIR}/04-kuadrant.yaml" + echo "" + + echo " Waiting for Kuadrant CR to be ready..." + oc wait --for=jsonpath='{.status.state}'=Ready kuadrant/kuadrant -n kuadrant-system --timeout=300s 2>/dev/null || sleep 10 + + echo "" + echo "Installation Complete" + echo "" + + echo "Installed operators:" + oc get csv -n kuadrant-system + echo "" + + echo "Running pods:" + oc get pods -n kuadrant-system + echo "" + + echo "Verify images are from mirror registry:" + oc get pods -n kuadrant-system -o jsonpath='{range .items[*]}{"\n"}{.metadata.name}{"\n"}{range .spec.containers[*]}{" "}{.image}{"\n"}{end}{end}' + echo "" + SCRIPT + + chmod +x "${INSTALL_DIR}/install.sh" + + # Generate uninstall script + cat > "${INSTALL_DIR}/uninstall.sh" <<'SCRIPT' + #!/bin/bash + set -euo pipefail + + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + + echo "==> Uninstalling Kuadrant" + echo "" + + echo "Deleting Kuadrant CR..." + oc delete -f "${SCRIPT_DIR}/04-kuadrant.yaml" --ignore-not-found + + echo "Deleting Subscription..." + oc delete -f "${SCRIPT_DIR}/03-subscription.yaml" --ignore-not-found + + echo "Deleting CSVs..." + oc delete csv --all -n kuadrant-system --ignore-not-found + + echo "Deleting OperatorGroup..." + oc delete -f "${SCRIPT_DIR}/02-operatorgroup.yaml" --ignore-not-found + + echo "Waiting for pods to terminate..." + oc wait --for=delete pod --all -n kuadrant-system --timeout=60s 2>/dev/null || true + + echo "Deleting namespace..." + oc delete -f "${SCRIPT_DIR}/01-namespace.yaml" --ignore-not-found + + echo "" + echo "Uninstall complete" + SCRIPT + + chmod +x "${INSTALL_DIR}/uninstall.sh" + + echo "" + echo "==========================================" + echo "Setup Complete" + echo "==========================================" + echo "" + echo " ImageDigestMirrorSet created" + echo " ImageTagMirrorSet created" + echo " CatalogSource ready (${CATALOG_NAME})" + echo " Installation manifests: ${INSTALL_DIR}/" + if [ "$DISABLE_DEFAULT_SOURCES" = "true" ]; then + echo " Default OperatorHub sources disabled" + fi + echo "" + volumes: + - name: registry-creds + secret: + secretName: $(params.registry-credentials) + workspaces: + - name: shared-workspace diff --git a/tasks/disconnected/uninstall-operators.yaml b/tasks/disconnected/uninstall-operators.yaml new file mode 100644 index 0000000..d9ab52d --- /dev/null +++ b/tasks/disconnected/uninstall-operators.yaml @@ -0,0 +1,65 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: disconnected-uninstall-operators +spec: + description: 'Fully uninstall Kuadrant/RHCL operators and all related cluster resources' + params: + - name: kubeconfig-path + type: string + steps: + - name: uninstall-operators + image: quay.io/kuadrant/testsuite-pipelines-tools:latest + imagePullPolicy: Always + timeout: 10m + env: + - name: KUBECONFIG + value: $(params.kubeconfig-path) + script: | + #!/usr/bin/env bash + set -euo pipefail + + NAMESPACE="kuadrant-system" + + echo "==> Uninstalling Kuadrant/RHCL operators" + echo "" + + # Step 1: Delete Kuadrant CR (let operator process finalizers) + if oc get kuadrant -n ${NAMESPACE} &>/dev/null 2>&1; then + echo "Deleting Kuadrant CR..." + oc delete kuadrant --all -n ${NAMESPACE} --timeout=60s 2>/dev/null || true + fi + + # Step 2: Delete subscriptions and CSVs in kuadrant-system + echo "Deleting subscriptions and CSVs in ${NAMESPACE}..." + oc delete subscription --all -n ${NAMESPACE} --ignore-not-found 2>/dev/null || true + oc delete csv --all -n ${NAMESPACE} --ignore-not-found 2>/dev/null || true + + # Step 3: Clean up leftover operator subscriptions and CSVs in openshift-operators + echo "Cleaning up leftover operator resources in openshift-operators..." + oc get subscription -n openshift-operators -o name 2>/dev/null | grep -iE 'authorino|limitador|dns-operator|rhcl-operator|kuadrant' | xargs -r oc delete -n openshift-operators --ignore-not-found 2>/dev/null || true + oc get csv -n openshift-operators -o name 2>/dev/null | grep -iE 'authorino|limitador|dns-operator|rhcl-operator|kuadrant' | xargs -r oc delete -n openshift-operators --ignore-not-found 2>/dev/null || true + + # Step 4: Wait for pods to terminate, then delete namespace + if oc get namespace ${NAMESPACE} &>/dev/null 2>&1; then + echo "Deleting namespace ${NAMESPACE}..." + oc delete namespace ${NAMESPACE} --timeout=120s 2>/dev/null || true + fi + + # Step 5: Delete CRDs (everything under *.kuadrant.io) + echo "Deleting CRDs..." + oc get crd -o name | grep '\.kuadrant\.io$' | xargs -r oc delete --ignore-not-found 2>/dev/null || true + + # Step 6: Delete console plugin + echo "Deleting console plugin..." + oc get consoleplugin -o name 2>/dev/null | grep -i kuadrant | xargs -r oc delete --ignore-not-found 2>/dev/null || true + + # Step 7: Delete cluster-scoped RBAC (OLM-generated, match by kuadrant-related names) + echo "Deleting cluster roles and bindings..." + oc get clusterrolebinding -o name 2>/dev/null | grep -E 'kuadrant|authorino|limitador|rhcl-operator|dns-operator\.' | grep -v openshift-dns-operator | xargs -r oc delete --ignore-not-found 2>/dev/null || true + oc get clusterrole -o name 2>/dev/null | grep -E 'kuadrant|authorino|limitador|rhcl-operator|dns-operator\.' | grep -v openshift-dns-operator | xargs -r oc delete --ignore-not-found 2>/dev/null || true + + echo "" + echo "Uninstall complete" + workspaces: + - name: shared-workspace