From 39ebaee9dc67a529e8dbc5eb71b47f961c12562f Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Thu, 7 May 2026 23:16:31 +0200 Subject: [PATCH] WIP --- .tekton/images-mirror-set.yaml | 145 ++++ .../pipelines/argocd-image-e2e.yaml | 407 ++++++++++ ...tops-bundle-integration-test-pipeline.yaml | 669 +++++++++++++++++ .tekton/tasks/build-ginkgo-test-image.yaml | 242 ++++++ .tekton/test-image/Dockerfile | 51 ++ .tekton/test-image/Dockerfile.base | 38 + .tekton/test-image/Dockerfile.testsuites | 30 + .../scripts/collect-and-upload-logs.sh | 303 ++++++++ .../scripts/collect-logs-sidecar.sh | 169 +++++ .tekton/test-image/scripts/go-cache.sh | 87 +++ .tekton/test-image/scripts/install-bundle.sh | 171 +++++ .../test-image/scripts/install-operator.sh | 351 +++++++++ .../test-image/scripts/run-and-save-logs.sh | 62 ++ .../scripts/run-argocd-e2e-tests.sh | 700 ++++++++++++++++++ .tekton/test-image/scripts/run-e2e-tests.sh | 100 +++ .../test-image/scripts/run-parallel-tests.sh | 24 + .../test-image/scripts/run-rollouts-tests.sh | 220 ++++++ .../scripts/run-sequential-tests.sh | 24 + .../test-image/scripts/run-ui-e2e-tests.sh | 164 ++++ .../test-image/scripts/send-slack-message.py | 335 +++++++++ .tekton/test-image/scripts/verify-images.sh | 162 ++++ .tekton/test-image/skip-argocd.txt | 183 +++++ .tekton/test-image/skip-parallel.txt | 12 + .tekton/test-image/skip-sequential.txt | 37 + .tekton/test-image/skip-ui-e2e.txt | 3 + containers/argocd/Dockerfile | 2 +- containers/gitops-operator-bundle/Dockerfile | 3 +- 27 files changed, 4692 insertions(+), 2 deletions(-) create mode 100644 .tekton/images-mirror-set.yaml create mode 100644 .tekton/integration-tests/pipelines/argocd-image-e2e.yaml create mode 100644 .tekton/integration-tests/pipelines/gitops-bundle-integration-test-pipeline.yaml create mode 100644 .tekton/tasks/build-ginkgo-test-image.yaml create mode 100644 .tekton/test-image/Dockerfile create mode 100644 .tekton/test-image/Dockerfile.base create mode 100644 .tekton/test-image/Dockerfile.testsuites create mode 100644 .tekton/test-image/scripts/collect-and-upload-logs.sh create mode 100644 .tekton/test-image/scripts/collect-logs-sidecar.sh create mode 100644 .tekton/test-image/scripts/go-cache.sh create mode 100755 .tekton/test-image/scripts/install-bundle.sh create mode 100644 .tekton/test-image/scripts/install-operator.sh create mode 100644 .tekton/test-image/scripts/run-and-save-logs.sh create mode 100644 .tekton/test-image/scripts/run-argocd-e2e-tests.sh create mode 100644 .tekton/test-image/scripts/run-e2e-tests.sh create mode 100644 .tekton/test-image/scripts/run-parallel-tests.sh create mode 100644 .tekton/test-image/scripts/run-rollouts-tests.sh create mode 100644 .tekton/test-image/scripts/run-sequential-tests.sh create mode 100755 .tekton/test-image/scripts/run-ui-e2e-tests.sh create mode 100644 .tekton/test-image/scripts/send-slack-message.py create mode 100644 .tekton/test-image/scripts/verify-images.sh create mode 100644 .tekton/test-image/skip-argocd.txt create mode 100644 .tekton/test-image/skip-parallel.txt create mode 100644 .tekton/test-image/skip-sequential.txt create mode 100644 .tekton/test-image/skip-ui-e2e.txt diff --git a/.tekton/images-mirror-set.yaml b/.tekton/images-mirror-set.yaml new file mode 100644 index 000000000..7884684e2 --- /dev/null +++ b/.tekton/images-mirror-set.yaml @@ -0,0 +1,145 @@ +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: gitops-catalog-test-idmr +spec: + imageDigestMirrors: + # ------------------------- + # GitOps Backend + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/gitops-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/gitops-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/gitops-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/gitops-rhel9 + + # ------------------------- + # Console Plugin + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/console-plugin-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/console-plugin-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/console-plugin-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/console-plugin-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/console-plugin-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/console-plugin-rhel9 + + # ------------------------- + # Dex + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/dex-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/dex-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/dex-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/dex-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/dex-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/dex-rhel9 + + # ------------------------- + # Must Gather + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/must-gather-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/must-gather-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/must-gather-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/must-gather-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/must-gather-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/must-gather-rhel9 + + # ------------------------- + # Argo CD + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/argocd-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-rhel8 + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-rhel8-tmp + - registry.stage.redhat.io/openshift-gitops-1/argocd-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/argocd-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-rhel9 + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-rhel9-tmp + - registry.stage.redhat.io/openshift-gitops-1/argocd-rhel9 + + # ------------------------- + # Argo CD Agent + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/argocd-agent-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-agent-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/argocd-agent-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/argocd-agent-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-agent-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/argocd-agent-rhel9 + + # ------------------------- + # Argo Rollouts + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/argo-rollouts-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argo-rollouts-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/argo-rollouts-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/argo-rollouts-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argo-rollouts-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/argo-rollouts-rhel9 + + # ------------------------- + # Operator Bundle + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/gitops-operator-bundle + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-operator-bundle + - registry.stage.redhat.io/openshift-gitops-1/gitops-operator-bundle + + # ------------------------- + # GitOps Operator Controller + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/gitops-rhel8-operator + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-rhel8-operator + - registry.stage.redhat.io/openshift-gitops-1/gitops-rhel8-operator + + - source: registry.redhat.io/openshift-gitops-1/gitops-rhel9-operator + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-rhel9-operator + - registry.stage.redhat.io/openshift-gitops-1/gitops-rhel9-operator + + # ------------------------- + # Argo CD Extensions + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/argocd-extensions-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-extensions-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/argocd-extensions-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/argocd-extensions-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-extensions-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/argocd-extensions-rhel9 + + # ------------------------- + # Image Updater + # ------------------------- + - source: registry.redhat.io/openshift-gitops-1/argocd-image-updater-rhel8 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-image-updater-rhel8 + - registry.stage.redhat.io/openshift-gitops-1/argocd-image-updater-rhel8 + + - source: registry.redhat.io/openshift-gitops-1/argocd-image-updater-rhel9 + mirrors: + - quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-image-updater-rhel9 + - registry.stage.redhat.io/openshift-gitops-1/argocd-image-updater-rhel9 diff --git a/.tekton/integration-tests/pipelines/argocd-image-e2e.yaml b/.tekton/integration-tests/pipelines/argocd-image-e2e.yaml new file mode 100644 index 000000000..248bc1317 --- /dev/null +++ b/.tekton/integration-tests/pipelines/argocd-image-e2e.yaml @@ -0,0 +1,407 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: argocd-image-e2e +spec: + description: | + Integration test for the argocd-rhel9 image. Provisions an ephemeral + HyperShift cluster, deploys ArgoCD directly from upstream manifests + (no operator), and runs the upstream ArgoCD E2E test suite against the + freshly built image. + + Intended to be attached as an IntegrationTestScenario to the argocd-rhel9 + component in the release repository, catching image-level regressions + (missing binaries, wrong paths, arch issues) at image build time. + params: + - description: Snapshot of the application + name: SNAPSHOT + default: '{"components": [{"name":"argocd-rhel9", "containerImage": "quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-rhel9:latest"}]}' + type: string + - description: OpenShift version to provision + name: OPENSHIFT_VERSION + default: "4.14" + type: string + - description: Enable FIPS mode for the ephemeral cluster + name: FIPS_ENABLED + default: "false" + type: string + - description: EaaS cluster instance type + name: CLUSTER_INSTANCE_TYPE + default: "m6g.2xlarge" + type: string + - description: Git URL of the release repository (for task definitions) + name: RELEASE_REPO_URL + default: "https://github.com/rh-gitops-midstream/release.git" + type: string + - description: Git revision of the release repository + name: RELEASE_REPO_REVISION + default: "main" + type: string + - description: Upstream argo-cd repo URL (for test source and install manifests) + name: ARGOCD_REPO_URL + default: "https://github.com/argoproj/argo-cd.git" + type: string + - description: Upstream argo-cd branch/tag to test against + name: ARGOCD_REPO_BRANCH + default: "master" + type: string + + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: build-test-image + runAfter: + - parse-metadata + taskRef: + resolver: git + params: + - name: url + value: $(params.RELEASE_REPO_URL) + - name: revision + value: $(params.RELEASE_REPO_REVISION) + - name: pathInRepo + value: .tekton/tasks/build-ginkgo-test-image.yaml + params: + - name: SOURCE_URL + value: $(tasks.parse-metadata.results.source-git-url) + - name: SOURCE_REVISION + value: $(tasks.parse-metadata.results.source-git-revision) + - name: IMAGE_EXPIRES_AFTER + value: "7d" + + - name: provision-eaas-space + runAfter: + - parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + params: + - name: ownerName + value: $(context.pipelineRun.name) + - name: ownerUid + value: $(context.pipelineRun.uid) + + - name: provision-cluster + runAfter: + - provision-eaas-space + params: + - name: openshiftVersion + value: $(params.OPENSHIFT_VERSION) + - name: fipsEnabled + value: $(params.FIPS_ENABLED) + - name: clusterInstanceType + value: $(params.CLUSTER_INSTANCE_TYPE) + taskSpec: + params: + - name: openshiftVersion + type: string + - name: fipsEnabled + type: string + - name: clusterInstanceType + type: string + results: + - name: clusterName + value: "$(steps.create-cluster.results.clusterName)" + - name: resolvedVersion + value: "$(steps.resolve-openshift-version.results.resolvedVersion)" + steps: + - name: get-supported-versions + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-supported-ephemeral-cluster-versions/0.1/eaas-get-supported-ephemeral-cluster-versions.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: resolve-openshift-version + image: docker.io/python:3-alpine + results: + - name: resolvedVersion + script: | + #!/usr/bin/env python3 + import urllib.request, json, re + + version = "$(params.openshiftVersion)" + + if re.match(r'^\d+\.\d+\.\d+$', version): + with open("$(step.results.resolvedVersion.path)", "w") as f: + f.write(version) + print(f"Using exact version: {version}") + raise SystemExit(0) + + url = f"https://quay.io/api/v1/repository/openshift-release-dev/ocp-release/tag/?filter_tag_name=like:{version}.&limit=100&onlyActiveTags=true" + data = json.loads(urllib.request.urlopen(url).read()) + + candidates = [] + for tag in data.get("tags", []): + name = tag["name"] + if name.endswith("-multi"): + bare = name.removesuffix("-multi") + if re.match(rf'^{re.escape(version)}\.\d+$', bare): + candidates.append(bare) + + if not candidates: + print(f"ERROR: Could not resolve minor version {version}") + raise SystemExit(1) + + candidates.sort(key=lambda v: list(map(int, v.split(".")))) + latest = candidates[-1] + + with open("$(step.results.resolvedVersion.path)", "w") as f: + f.write(latest) + print(f"Resolved {version} -> {latest}") + - name: create-cluster + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: timeout + value: "60m" + - name: version + value: "$(steps.resolve-openshift-version.results.resolvedVersion)" + - name: fips + value: "$(params.fipsEnabled)" + - name: instanceType + value: "$(params.clusterInstanceType)" + + - name: run-argocd-e2e + runAfter: + - provision-cluster + - build-test-image + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: argocdImage + value: "$(tasks.parse-metadata.results.component-container-image)" + - name: argocdRepoUrl + value: "$(params.ARGOCD_REPO_URL)" + - name: argocdRepoBranch + value: "$(params.ARGOCD_REPO_BRANCH)" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: testImageUrl + value: "$(tasks.build-test-image.results.IMAGE_URL)" + taskSpec: + params: + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: argocdImage + type: string + - name: argocdRepoUrl + type: string + - name: argocdRepoBranch + type: string + - name: openshiftVersion + type: string + - name: testImageUrl + type: string + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: get-kubeconfig + onError: continue + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: run-tests + image: $(params.testImageUrl) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: CI + value: "konflux" + - name: ARGOCD_IMAGE + value: "$(params.argocdImage)" + - name: TEST_REPO_URL + value: "$(params.argocdRepoUrl)" + - name: BRANCH + value: "$(params.argocdRepoBranch)" + - name: TASK_LOG_NAME + value: "argocd-e2e" + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + - name: OPENSHIFT_VERSION + value: "$(params.openshiftVersion)" + - name: DEPLOY_MODE + value: "standalone" + computeResources: + requests: + memory: "4Gi" + cpu: "1000m" + limits: + memory: "8Gi" + cpu: "2000m" + script: | + #!/bin/bash + KUBECONFIG=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + if [[ -z "$KUBECONFIG" ]]; then + echo "ERROR: No kubeconfig found" + exit 1 + fi + export KUBECONFIG + + API_SERVER=$(oc whoami --show-server 2>/dev/null || true) + PASS_FILE=$(find /credentials -name "*password" -type f 2>/dev/null | head -1) + if [[ -n "$API_SERVER" && -n "$PASS_FILE" ]]; then + echo "========================================" + echo "DEBUG: oc login $API_SERVER -u kubeadmin -p $(cat "$PASS_FILE") --insecure-skip-tls-verify" + echo "========================================" + fi + + /usr/local/bin/run-and-save-logs.sh /usr/local/bin/run-argocd-e2e-tests.sh + sidecars: + - name: log-collector + image: $(params.testImageUrl) + workingDir: /var/log-workspace + env: + - name: KUBECONFIG_NAME + value: "auto" + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: BRANCH_NAME + value: "logs" + - name: NAMESPACE + value: "argocd-e2e" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + command: ["/bin/bash", "-c", "/usr/local/bin/collect-logs-sidecar.sh"] + + finally: + - name: upload-logs + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testImageUrl + value: "$(tasks.build-test-image.results.IMAGE_URL)" + taskSpec: + params: + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: testImageUrl + type: string + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: get-kubeconfig + onError: continue + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: collect-and-upload-logs + image: $(params.testImageUrl) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: NAMESPACE + value: "argocd-e2e" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + - name: QUAY_CREDENTIALS_PATH + value: "/quay-credentials/.dockerconfigjson" + - name: TASK_NAMES + value: "argocd-e2e logs" + script: | + #!/bin/bash + KUBECONFIG=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + export KUBECONFIG="${KUBECONFIG:-}" + /usr/local/bin/collect-and-upload-logs.sh diff --git a/.tekton/integration-tests/pipelines/gitops-bundle-integration-test-pipeline.yaml b/.tekton/integration-tests/pipelines/gitops-bundle-integration-test-pipeline.yaml new file mode 100644 index 000000000..998ec6c7d --- /dev/null +++ b/.tekton/integration-tests/pipelines/gitops-bundle-integration-test-pipeline.yaml @@ -0,0 +1,669 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: gitops-bundle-integration-test-pipeline +spec: + description: | + An integration test which provisions an ephemeral Hypershift cluster, deploys GitOps Operator from a bundle, and runs parallel e2e tests. + params: + - description: Snapshot of the application + name: SNAPSHOT + default: '{"components": [{"name":"gitops-operator-bundle-main", "containerImage": "quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-operator-bundle:latest"}]}' + type: string + - description: Duration to wait for bundle installation to complete before failing. + name: INSTALL_TIMEOUT + default: 25m + type: string + - description: Git URL of release repository (used to resolve task definitions only; source URL for builds is auto-derived from SNAPSHOT) + name: BUNDLE_TASK_URL + default: "https://github.com/rh-gitops-midstream/release.git" + type: string + - description: Git revision for task definitions (used to resolve task definitions only; source revision for builds is auto-derived from SNAPSHOT) + name: BUNDLE_TASK_REVISION + default: "main" + type: string + - description: OpenShift version to provision (minor version e.g. "4.14", or full patch version e.g. "4.14.57") + name: OPENSHIFT_VERSION + default: "4.14" + type: string + - description: Enable FIPS mode for the ephemeral cluster + name: FIPS_ENABLED + default: "false" + type: string + - description: Git URL of the test repository + name: TEST_REPO_URL + default: "https://github.com/rh-gitops-release-qa/gitops-operator.git" + type: string + - description: Git branch or revision of the test repository + name: TEST_REPO_BRANCH + default: "master" + type: string + - description: Test runner script name (e.g., run-parallel-tests.sh, run-sequential-tests.sh, run-rollouts-tests.sh) + name: TEST_SCRIPT + default: "run-parallel-tests.sh" + type: string + - description: AWS instance type for the ephemeral cluster (e.g., m6g.large, m6g.xlarge, m6g.2xlarge) + name: CLUSTER_INSTANCE_TYPE + default: "m6g.large" + type: string + finally: + - name: upload-logs + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testImageUrl + value: "$(tasks.build-ginkgo-test-image.results.IMAGE_URL)" + taskSpec: + params: + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: testImageUrl + type: string + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: get-kubeconfig + onError: continue + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: collect-and-upload-logs + image: $(params.testImageUrl) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: NAMESPACE + value: "openshift-gitops-operator" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + - name: QUAY_CREDENTIALS_PATH + value: "/quay-credentials/.dockerconfigjson" + - name: TASK_NAMES + value: "install-operator test-operator logs" + script: | + #!/bin/bash + # Find kubeconfig — get-kubeconfig step may have failed (cluster deprovisioned) + KUBECONFIG=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + export KUBECONFIG="${KUBECONFIG:-}" + /usr/local/bin/collect-and-upload-logs.sh + + - name: send-slack-notification + params: + - name: aggregateTasksStatus + value: "$(tasks.status)" + - name: testImageUrl + value: "$(tasks.build-ginkgo-test-image.results.IMAGE_URL)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testScript + value: "$(params.TEST_SCRIPT)" + - name: testRepoUrl + value: "$(params.TEST_REPO_URL)" + - name: testRepoBranch + value: "$(params.TEST_REPO_BRANCH)" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: fipsEnabled + value: "$(params.FIPS_ENABLED)" + taskSpec: + params: + - name: aggregateTasksStatus + type: string + - name: testImageUrl + type: string + - name: pipelineRunName + type: string + - name: testScript + type: string + - name: testRepoUrl + type: string + - name: testRepoBranch + type: string + - name: openshiftVersion + type: string + - name: fipsEnabled + type: string + volumes: + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: send-slack-message + image: $(params.testImageUrl) + volumeMounts: + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: + name: slack-gitops-test-notification-url + key: hook-url + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: AGGREGATE_STATUS + value: "$(params.aggregateTasksStatus)" + - name: LOG_URL + value: "https://konflux-ui.apps.stone-prd-rh01.pg1f.p1.openshiftapps.com/ns/rh-openshift-gitops-tenant/applications/gitops-main/pipelineruns/$(params.pipelineRunName)" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + - name: QUAY_CREDENTIALS_PATH + value: "/quay-credentials/.dockerconfigjson" + - name: TASK_NAMES + value: "install-operator test-operator" + - name: TEST_SCRIPT + value: "$(params.testScript)" + - name: TEST_REPO_URL + value: "$(params.testRepoUrl)" + - name: TEST_REPO_BRANCH + value: "$(params.testRepoBranch)" + - name: OPENSHIFT_VERSION + value: "$(params.openshiftVersion)" + - name: FIPS_ENABLED + value: "$(params.fipsEnabled)" + script: | + #!/bin/bash + /usr/local/bin/send-slack-message.py + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: extract-bundle-image + runAfter: + - parse-metadata + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + taskSpec: + params: + - name: SNAPSHOT + type: string + results: + - name: bundleImage + description: Bundle image reference from SNAPSHOT + steps: + - name: parse + image: quay.io/devtools_gitops/test_image:integrated_ginkgo + env: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + script: | + #!/usr/bin/env python3 + import json, os, sys + + snapshot = json.loads(os.environ["SNAPSHOT"]) + bundle = next( + (c for c in snapshot.get("components", []) if "bundle" in c.get("name", "")), + None, + ) + if not bundle: + print("ERROR: No bundle component found in SNAPSHOT") + sys.exit(1) + + image = bundle.get("containerImage") + if not image: + print("ERROR: Bundle component has no containerImage") + sys.exit(1) + + with open("$(results.bundleImage.path)", "w") as f: + f.write(image) + print(f"Bundle image: {image}") + + - name: build-ginkgo-test-image + runAfter: + - parse-metadata + taskRef: + resolver: git + params: + - name: url + value: $(params.BUNDLE_TASK_URL) + - name: revision + value: $(params.BUNDLE_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/build-ginkgo-test-image.yaml + params: + - name: SOURCE_URL + value: $(tasks.parse-metadata.results.source-git-url) + - name: SOURCE_REVISION + value: $(tasks.parse-metadata.results.source-git-revision) + - name: IMAGE_EXPIRES_AFTER + value: "7d" + + - name: provision-eaas-space + runAfter: + - parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + params: + - name: ownerName + value: $(context.pipelineRun.name) + - name: ownerUid + value: $(context.pipelineRun.uid) + + - name: provision-cluster + runAfter: + - provision-eaas-space + params: + - name: bundleSourceUrl + value: $(tasks.parse-metadata.results.source-git-url) + - name: bundleSourceRevision + value: $(tasks.parse-metadata.results.source-git-revision) + - name: openshiftVersion + value: $(params.OPENSHIFT_VERSION) + - name: fipsEnabled + value: $(params.FIPS_ENABLED) + taskSpec: + params: + - name: bundleSourceUrl + type: string + - name: bundleSourceRevision + type: string + - name: openshiftVersion + type: string + - name: fipsEnabled + type: string + results: + - name: clusterName + value: "$(steps.create-cluster.results.clusterName)" + - name: resolvedVersion + value: "$(steps.resolve-openshift-version.results.resolvedVersion)" + steps: + - name: get-supported-versions + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-supported-ephemeral-cluster-versions/0.1/eaas-get-supported-ephemeral-cluster-versions.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: extract-image-content-sources + image: docker.io/python:3-alpine + script: | + #!/bin/sh + set -e + apk add --no-cache git >/dev/null 2>&1 + pip install pyyaml --quiet >/dev/null 2>&1 + + WORK=$(mktemp -d) + git clone $(params.bundleSourceUrl) "$WORK/release" + cd "$WORK/release" + git checkout $(params.bundleSourceRevision) + + python3 -c " + import yaml, json, sys + with open('.tekton/images-mirror-set.yaml') as f: + data = yaml.safe_load(f) + json.dump(data['spec']['imageDigestMirrors'], sys.stdout) + " > /workspace/imageContentSources.json + + echo "Successfully extracted imageContentSources from release repository" + - name: resolve-openshift-version + image: docker.io/python:3-alpine + results: + - name: resolvedVersion + script: | + #!/usr/bin/env python3 + import urllib.request, json, re, os + + version = "$(params.openshiftVersion)" + + if re.match(r'^\d+\.\d+\.\d+$', version): + with open("$(step.results.resolvedVersion.path)", "w") as f: + f.write(version) + print(f"Using exact version: {version}") + raise SystemExit(0) + + url = f"https://quay.io/api/v1/repository/openshift-release-dev/ocp-release/tag/?filter_tag_name=like:{version}.&limit=100&onlyActiveTags=true" + data = json.loads(urllib.request.urlopen(url).read()) + + candidates = [] + for tag in data.get("tags", []): + name = tag["name"] + if name.endswith("-multi"): + bare = name.removesuffix("-multi") + if re.match(rf'^{re.escape(version)}\.\d+$', bare): + candidates.append(bare) + + if not candidates: + print(f"ERROR: Could not resolve minor version {version} to a patch release") + raise SystemExit(1) + + candidates.sort(key=lambda v: list(map(int, v.split(".")))) + latest = candidates[-1] + + with open("$(step.results.resolvedVersion.path)", "w") as f: + f.write(latest) + print(f"Resolved {version} -> {latest}") + - name: create-cluster + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: imageContentSourcesFile + value: "/workspace/imageContentSources.json" + + - name: timeout + value: "60m" + - name: version + value: "$(steps.resolve-openshift-version.results.resolvedVersion)" + - name: fips + value: "$(params.fipsEnabled)" + - name: instanceType + value: "$(params.CLUSTER_INSTANCE_TYPE)" + + - name: install-operator + runAfter: + - provision-cluster + - extract-bundle-image + params: + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: namespace + value: "openshift-gitops-operator" + - name: installTimeout + value: "$(params.INSTALL_TIMEOUT)" + - name: bundleImage + value: "$(tasks.extract-bundle-image.results.bundleImage)" + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + taskSpec: + params: + - name: openshiftVersion + type: string + - name: namespace + type: string + - name: installTimeout + type: string + - name: bundleImage + type: string + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + results: + - name: installedCSV + value: "$(steps.get-installed-version.results.installedCSV)" + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: get-kubeconfig + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: install-bundle + image: $(tasks.build-ginkgo-test-image.results.IMAGE_URL) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: BUNDLE_IMAGE + value: "$(params.bundleImage)" + - name: NAMESPACE + value: "$(params.namespace)" + - name: INSTALL_TIMEOUT + value: "$(params.installTimeout)" + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: TASK_LOG_NAME + value: "install-operator" + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + script: | + #!/bin/bash + # Print login command for debugging the EaaS test cluster + API_SERVER=$(oc whoami --show-server 2>/dev/null || true) + PASS_FILE=$(find /credentials -name "*password" -type f 2>/dev/null | head -1) + if [[ -n "$API_SERVER" && -n "$PASS_FILE" ]]; then + echo "========================================" + echo "DEBUG: To log in to the test cluster:" + echo " oc login $API_SERVER -u kubeadmin -p $(cat "$PASS_FILE") --insecure-skip-tls-verify" + echo "========================================" + fi + /usr/local/bin/run-and-save-logs.sh /usr/local/bin/install-bundle.sh + - name: get-installed-version + image: $(tasks.build-ginkgo-test-image.results.IMAGE_URL) + results: + - name: installedCSV + volumeMounts: + - name: credentials + mountPath: /credentials + env: + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: NAMESPACE + value: "$(params.namespace)" + script: | + #!/bin/bash + CSV=$(oc get csv -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "unknown") + printf "%s" "$CSV" > "$(step.results.installedCSV.path)" + echo "Installed CSV: $CSV" + + - name: test-operator + runAfter: + - install-operator + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testRepoUrl + value: "$(params.TEST_REPO_URL)" + - name: testRepoBranch + value: "$(params.TEST_REPO_BRANCH)" + - name: testScript + value: "$(params.TEST_SCRIPT)" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + taskSpec: + params: + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: testRepoUrl + type: string + - name: testRepoBranch + type: string + - name: testScript + type: string + - name: openshiftVersion + type: string + results: + - name: LOG_ARTIFACT_TAG + description: Tag of uploaded log artifact from sidecar + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: get-kubeconfig + onError: continue + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: run-tests + image: $(tasks.build-ginkgo-test-image.results.IMAGE_URL) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: CI + value: "konflux" + - name: TEST_REPO_URL + value: "$(params.testRepoUrl)" + - name: BRANCH + value: "$(params.testRepoBranch)" + - name: TASK_LOG_NAME + value: "test-operator" + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + - name: OPENSHIFT_VERSION + value: "$(params.openshiftVersion)" + computeResources: + requests: + memory: "4Gi" + cpu: "1000m" + limits: + memory: "8Gi" + cpu: "2000m" + script: | + #!/bin/bash + # Find kubeconfig — get-kubeconfig step may have failed + KUBECONFIG=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + if [[ -z "$KUBECONFIG" ]]; then + echo "ERROR: No kubeconfig found in /credentials — get-kubeconfig step may have failed" + ls -la /credentials/ 2>/dev/null || true + echo "Tests cannot run without cluster access." + exit 1 + fi + export KUBECONFIG + + # Print login command for debugging the EaaS test cluster + API_SERVER=$(oc whoami --show-server 2>/dev/null || true) + PASS_FILE=$(find /credentials -name "*password" -type f 2>/dev/null | head -1) + if [[ -n "$API_SERVER" && -n "$PASS_FILE" ]]; then + echo "========================================" + echo "DEBUG: To log in to the test cluster:" + echo " oc login $API_SERVER -u kubeadmin -p $(cat "$PASS_FILE") --insecure-skip-tls-verify" + echo "========================================" + fi + + /usr/local/bin/run-and-save-logs.sh /usr/local/bin/$(params.testScript) + sidecars: + - name: log-collector + image: $(tasks.build-ginkgo-test-image.results.IMAGE_URL) + workingDir: /var/log-workspace + env: + - name: KUBECONFIG_NAME + value: "auto" + - name: PIPELINE_RUN_NAME + value: "$(params.pipelineRunName)" + - name: BRANCH_NAME + value: "logs" + - name: NAMESPACE + value: "openshift-gitops-operator" + - name: QUAY_REPO + value: "quay.io/devtools_gitops/test_image" + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + command: ["/bin/bash", "-c", "/usr/local/bin/collect-logs-sidecar.sh"] diff --git a/.tekton/tasks/build-ginkgo-test-image.yaml b/.tekton/tasks/build-ginkgo-test-image.yaml new file mode 100644 index 000000000..e4db989e1 --- /dev/null +++ b/.tekton/tasks/build-ginkgo-test-image.yaml @@ -0,0 +1,242 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: build-ginkgo-test-image +spec: + description: | + Builds the gitops-ginkgo-test-runner image in three layers: + 1. Base image (Dockerfile.base) - CLI tools and Go toolchain + 2. Testsuites image (Dockerfile.testsuites) - pre-compiled Ginkgo tests + 3. Final image (Dockerfile) - helper scripts, rebuilt per commit + + Each layer is tagged by a content hash and skipped if already present. + params: + - name: SOURCE_URL + type: string + description: Git URL of the catalog repo (from pipeline context) + - name: SOURCE_REVISION + type: string + description: Git commit SHA - used for tagging + - name: IMAGE_DESTINATION + type: string + default: "quay.io/devtools_gitops/test_image" + description: Base image repository (without tag) + - name: DOCKERFILE_PATH + type: string + default: ".tekton/test-image/Dockerfile" + - name: CONTEXT_DIR + type: string + default: ".tekton/test-image" + - name: IMAGE_EXPIRES_AFTER + type: string + default: "7d" + description: "Quay expiration label (e.g., 1h, 7d, never)" + results: + - name: IMAGE_DIGEST + description: Digest of the built image + - name: IMAGE_URL + description: Full image reference with tag (repo:tag) + - name: IMAGE_TAG + description: Just the tag portion (short SHA) + volumes: + - name: source + emptyDir: {} + - name: docker-credentials + secret: + secretName: gitops-test-runner-image-push + - name: cache-state + emptyDir: {} + steps: + - name: clone-source + image: docker.io/alpine/git:latest + volumeMounts: + - name: source + mountPath: /workspace/source + script: | + #!/bin/sh + set -e + echo "Cloning $(params.SOURCE_URL) @ $(params.SOURCE_REVISION)" + git clone "$(params.SOURCE_URL)" /workspace/source + cd /workspace/source + git checkout "$(params.SOURCE_REVISION)" + + COMMIT_SHA=$(git rev-parse HEAD) + echo "Commit SHA: ${COMMIT_SHA}" + git log -1 --oneline + echo -n "${COMMIT_SHA}" > /workspace/source/COMMIT_SHA + + - name: check-cache + image: quay.io/skopeo/stable:latest + volumeMounts: + - name: source + mountPath: /workspace/source + - name: docker-credentials + mountPath: /credentials + - name: cache-state + mountPath: /cache-state + script: | + #!/bin/bash + set -euo pipefail + + COMMIT_SHA=$(cat /workspace/source/COMMIT_SHA) + SHORT_SHA=$(echo "${COMMIT_SHA}" | cut -c1-8) + CONTEXT="/workspace/source/$(params.CONTEXT_DIR)" + AUTH="--authfile=/credentials/.dockerconfigjson" + BUILD_ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + + DESTINATION="$(params.IMAGE_DESTINATION):${SHORT_SHA}" + BASE_HASH=$(sha256sum "${CONTEXT}/Dockerfile.base" | cut -c1-12) + BASE_IMAGE="$(params.IMAGE_DESTINATION):base-${BUILD_ARCH}-${BASE_HASH}" + TESTSUITES_HASH=$(cat "${CONTEXT}/Dockerfile.base" "${CONTEXT}/Dockerfile.testsuites" | sha256sum | cut -c1-12) + TESTSUITES_IMAGE="$(params.IMAGE_DESTINATION):testsuites-${BUILD_ARCH}-${TESTSUITES_HASH}" + + echo "Cache check — final: ${DESTINATION}" + echo "Cache check — base: ${BASE_IMAGE}" + echo "Cache check — testsuites: ${TESTSUITES_IMAGE}" + + # Check final image + if skopeo inspect ${AUTH} "docker://${DESTINATION}" > /cache-state/final-inspect.json 2>&1; then + echo "FINAL IMAGE EXISTS — skipping entire build" + DIGEST=$(sed -n 's/.*"Digest": "\([^"]*\)".*/\1/p' /cache-state/final-inspect.json | head -1) + echo -n "${DIGEST}" > /cache-state/final-digest + echo "hit" > /cache-state/final + else + echo "Final image not found, will build" + cat /cache-state/final-inspect.json 2>/dev/null || true + echo "miss" > /cache-state/final + fi + + # Check base image + if skopeo inspect ${AUTH} "docker://${BASE_IMAGE}" >/dev/null 2>&1; then + echo "Base image exists — will skip base build" + echo "hit" > /cache-state/base + else + echo "Base image not found — will rebuild" + echo "miss" > /cache-state/base + fi + + # Check testsuites image + if skopeo inspect ${AUTH} "docker://${TESTSUITES_IMAGE}" >/dev/null 2>&1; then + echo "Testsuites image exists — will skip testsuites build" + echo "hit" > /cache-state/testsuites + else + echo "Testsuites image not found — will rebuild" + echo "miss" > /cache-state/testsuites + fi + + - name: build-and-push + image: quay.io/buildah/stable:latest + workingDir: /workspace/source + volumeMounts: + - name: source + mountPath: /workspace/source + - name: docker-credentials + mountPath: /credentials + - name: cache-state + mountPath: /cache-state + securityContext: + capabilities: + add: + - SETFCAP + computeResources: + requests: + memory: "4Gi" + cpu: "1000m" + limits: + memory: "8Gi" + cpu: "2000m" + script: | + #!/bin/bash + set -euo pipefail + + COMMIT_SHA=$(cat /workspace/source/COMMIT_SHA) + SHORT_SHA=$(echo "${COMMIT_SHA}" | cut -c1-8) + DESTINATION="$(params.IMAGE_DESTINATION):${SHORT_SHA}" + CONTEXT="/workspace/source/$(params.CONTEXT_DIR)" + + export REGISTRY_AUTH_FILE=/credentials/.dockerconfigjson + BUILD_ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + + # Check if final image already exists (from check-cache step) + if [ "$(cat /cache-state/final)" = "hit" ]; then + echo "Image ${DESTINATION} already exists, skipping build." + cp /cache-state/final-digest /tekton/results/IMAGE_DIGEST + echo -n "${DESTINATION}" > /tekton/results/IMAGE_URL + echo -n "${SHORT_SHA}" > /tekton/results/IMAGE_TAG + exit 0 + fi + + # --- Layer 1: Base image (tools + Go) --- + BASE_DOCKERFILE="${CONTEXT}/Dockerfile.base" + BASE_HASH=$(sha256sum "${BASE_DOCKERFILE}" | cut -c1-12) + BASE_IMAGE="$(params.IMAGE_DESTINATION):base-${BUILD_ARCH}-${BASE_HASH}" + + if [ "$(cat /cache-state/base)" = "hit" ]; then + echo "Base image up to date, skipping base build." + else + echo "Building base image (tag: base-${BUILD_ARCH}-${BASE_HASH})" + buildah bud \ + --storage-driver=vfs \ + --format=oci \ + --file="${BASE_DOCKERFILE}" \ + --tag="${BASE_IMAGE}" \ + "${CONTEXT}" + + buildah push \ + --storage-driver=vfs \ + --authfile="${REGISTRY_AUTH_FILE}" \ + "${BASE_IMAGE}" + + echo "Base image pushed: ${BASE_IMAGE}" + fi + + # --- Layer 2: Testsuites image (pre-compiled tests) --- + TESTSUITES_DOCKERFILE="${CONTEXT}/Dockerfile.testsuites" + TESTSUITES_HASH=$(cat "${BASE_DOCKERFILE}" "${TESTSUITES_DOCKERFILE}" | sha256sum | cut -c1-12) + TESTSUITES_IMAGE="$(params.IMAGE_DESTINATION):testsuites-${BUILD_ARCH}-${TESTSUITES_HASH}" + + if [ "$(cat /cache-state/testsuites)" = "hit" ]; then + echo "Testsuites image up to date, skipping testsuites build." + else + echo "Building testsuites image (tag: testsuites-${BUILD_ARCH}-${TESTSUITES_HASH})" + buildah bud \ + --storage-driver=vfs \ + --format=oci \ + --build-arg="BASE_IMAGE=${BASE_IMAGE}" \ + --file="${TESTSUITES_DOCKERFILE}" \ + --tag="${TESTSUITES_IMAGE}" \ + "${CONTEXT}" + + buildah push \ + --storage-driver=vfs \ + --authfile="${REGISTRY_AUTH_FILE}" \ + "${TESTSUITES_IMAGE}" + + echo "Testsuites image pushed: ${TESTSUITES_IMAGE}" + fi + + # --- Layer 3: Final image (scripts overlay) --- + echo "Building final image: ${DESTINATION}" + buildah bud \ + --storage-driver=vfs \ + --format=oci \ + --build-arg="BASE_IMAGE=${TESTSUITES_IMAGE}" \ + --file="/workspace/source/$(params.DOCKERFILE_PATH)" \ + --tag="${DESTINATION}" \ + --label="quay.expires-after=$(params.IMAGE_EXPIRES_AFTER)" \ + --label="git.commit=${COMMIT_SHA}" \ + --label="base.image=${BASE_IMAGE}" \ + --label="testsuites.image=${TESTSUITES_IMAGE}" \ + "${CONTEXT}" + + buildah push \ + --storage-driver=vfs \ + --authfile="${REGISTRY_AUTH_FILE}" \ + --digestfile=/tekton/results/IMAGE_DIGEST \ + "${DESTINATION}" + + echo -n "${DESTINATION}" > /tekton/results/IMAGE_URL + echo -n "${SHORT_SHA}" > /tekton/results/IMAGE_TAG + + echo "Done: ${DESTINATION}" + echo "Digest: $(cat /tekton/results/IMAGE_DIGEST)" diff --git a/.tekton/test-image/Dockerfile b/.tekton/test-image/Dockerfile new file mode 100644 index 000000000..2f5b37740 --- /dev/null +++ b/.tekton/test-image/Dockerfile @@ -0,0 +1,51 @@ +ARG BASE_IMAGE=quay.io/devtools_gitops/test_image:testsuites +FROM ${BASE_IMAGE} + +LABEL name="gitops-ginkgo-test-runner" \ + description="GitOps integration test runner with helper scripts" \ + maintainer="GitOps Team" \ + io.openshift.tags="gitops,testing,ginkgo,integration" + +# Node.js + Playwright for UI E2E tests +ARG NODE_VERSION=v20.18.0 +RUN ARCH=$(uname -m) && \ + case "$ARCH" in x86_64) NODE_ARCH="x64";; aarch64) NODE_ARCH="arm64";; esac && \ + curl -fsSL "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz" \ + | tar -xz -C /usr/local --strip-components=1 && \ + node --version && npm --version + +# Pre-compile argo-cd E2E test binary + CLI for arm64 (EaaS HyperShift clusters). +# Compilation requires ~6Gi RAM which exceeds the test step's limits, so we do it +# here where the build system has more resources. The test step uses the pre-built +# artifacts when branch and target arch match, otherwise recompiles using the warm +# Go module and build caches baked into the image. +ARG ARGOCD_E2E_REPO=https://github.com/argoproj/argo-cd.git +ARG ARGOCD_E2E_BRANCH=master +RUN mkdir -p /prebuilt/argocd-e2e /usr/local/go-cache/build /usr/local/go-cache/mod && \ + git clone --depth 1 -b "${ARGOCD_E2E_BRANCH}" "${ARGOCD_E2E_REPO}" /tmp/argo-cd && \ + cd /tmp/argo-cd && \ + export GOCACHE=/usr/local/go-cache/build GOMODCACHE=/usr/local/go-cache/mod && \ + export GOARCH=arm64 GOOS=linux CGO_ENABLED=0 GOTOOLCHAIN=auto && \ + go mod download && \ + CLIENT_VERSION=$(cat VERSION 2>/dev/null || echo "${ARGOCD_E2E_BRANCH}") && \ + CLIENT_VERSION="${CLIENT_VERSION#v}" && \ + echo "Compiling e2e.test for arm64..." && \ + go test -c \ + -ldflags "-X github.com/argoproj/argo-cd/v3/common.version=${CLIENT_VERSION}" \ + -o /prebuilt/argocd-e2e/e2e.test ./test/e2e && \ + echo "Compiling argocd CLI for arm64..." && \ + go build \ + -ldflags "-X github.com/argoproj/argo-cd/v3/common.version=${CLIENT_VERSION}" \ + -o /prebuilt/argocd-e2e/argocd ./cmd && \ + tar czf /prebuilt/argocd-e2e/test-fixtures.tar.gz test/ && \ + echo "${ARGOCD_E2E_BRANCH}" > /prebuilt/argocd-e2e/BRANCH && \ + echo "arm64" > /prebuilt/argocd-e2e/GOARCH && \ + rm -rf /tmp/argo-cd + +# Copy helper scripts and config files +COPY scripts/ /usr/local/bin/ +COPY skip-*.txt /usr/local/bin/ +RUN chmod +x /usr/local/bin/*.sh /usr/local/bin/*.py + +WORKDIR /workspace +ENTRYPOINT ["/bin/bash"] diff --git a/.tekton/test-image/Dockerfile.base b/.tekton/test-image/Dockerfile.base new file mode 100644 index 000000000..609f77a17 --- /dev/null +++ b/.tekton/test-image/Dockerfile.base @@ -0,0 +1,38 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +LABEL name="gitops-ginkgo-test-runner-base" \ + description="Base image with CLI tools and Go toolchain" \ + maintainer="GitOps Team" + +ARG OC_VERSION=4.14 +ARG ORAS_VERSION=1.2.0 + +RUN dnf -y install \ + jq \ + git \ + golang \ + python3-pyyaml \ + skopeo \ + make \ + tar \ + gzip \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +# Install OpenShift CLI (oc) +RUN curl -Lo /tmp/openshift-client.tar.gz \ + https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable-${OC_VERSION}/openshift-client-linux.tar.gz \ + && tar -xzf /tmp/openshift-client.tar.gz -C /usr/local/bin/ oc kubectl \ + && rm /tmp/openshift-client.tar.gz \ + && oc version --client + +# Install ORAS for log artifact upload +RUN ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) \ + && curl -Lo /tmp/oras.tar.gz \ + https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_${ARCH}.tar.gz \ + && tar -xzf /tmp/oras.tar.gz -C /usr/local/bin/ oras \ + && rm /tmp/oras.tar.gz \ + && oras version + +WORKDIR /workspace +ENTRYPOINT ["/bin/bash"] diff --git a/.tekton/test-image/Dockerfile.testsuites b/.tekton/test-image/Dockerfile.testsuites new file mode 100644 index 000000000..80e113610 --- /dev/null +++ b/.tekton/test-image/Dockerfile.testsuites @@ -0,0 +1,30 @@ +ARG BASE_IMAGE=quay.io/devtools_gitops/test_image:base +FROM ${BASE_IMAGE} + +LABEL name="gitops-ginkgo-test-runner-testsuites" \ + description="Base image with pre-compiled Ginkgo test suites" \ + maintainer="GitOps Team" + +ARG TEST_REPO_URL=https://github.com/rh-gitops-release-qa/gitops-operator.git +ARG TEST_REPO_BRANCH=master + +WORKDIR /testsuites + +RUN git clone --depth 1 --branch ${TEST_REPO_BRANCH} ${TEST_REPO_URL} gitops-operator + +WORKDIR /testsuites/gitops-operator + +RUN make ginkgo && go mod download + +RUN ./bin/ginkgo build -r ./test/openshift/e2e/ginkgo/parallel/ + +RUN ./bin/ginkgo build -r ./test/openshift/e2e/ginkgo/sequential/ + +RUN go clean -cache \ + && rm -rf /root/go/pkg/mod/cache/download \ + && rm -rf /tmp/* \ + && rm -rf .git \ + && git init && git remote add origin ${TEST_REPO_URL} + +WORKDIR /workspace +ENTRYPOINT ["/bin/bash"] diff --git a/.tekton/test-image/scripts/collect-and-upload-logs.sh b/.tekton/test-image/scripts/collect-and-upload-logs.sh new file mode 100644 index 000000000..5de7f188a --- /dev/null +++ b/.tekton/test-image/scripts/collect-and-upload-logs.sh @@ -0,0 +1,303 @@ +#!/bin/bash +set -u + +# Environment variables expected: +# - KUBECONFIG +# - PIPELINE_RUN_NAME +# - NAMESPACE +# - QUAY_REPO +# - QUAY_CREDENTIALS_PATH (path to .dockerconfigjson) +# - TASK_NAMES (space-separated list of task names whose logs to pull, e.g. "install-operator test-operator") + +LOGS_DIR="logs" +IMAGE_TAG="${PIPELINE_RUN_NAME}-logs" +TASK_NAMES="${TASK_NAMES:-}" +ERRORS=() + +collect_error() { + ERRORS+=("$1") + echo "WARNING: $1" +} + +echo "==========================================" +echo "Collecting logs and debug info" +echo "Pipeline: ${PIPELINE_RUN_NAME}" +echo "Namespace: ${NAMESPACE}" +echo "==========================================" + +# Configure Docker credentials for oras (needed for both pulling task logs and final push) +if [ -f "${QUAY_CREDENTIALS_PATH}" ]; then + TEMP_DOCKER_CONFIG="$(mktemp -d)" + cp "${QUAY_CREDENTIALS_PATH}" "$TEMP_DOCKER_CONFIG/config.json" + export DOCKER_CONFIG="$TEMP_DOCKER_CONFIG" +else + collect_error "Quay credentials not found at ${QUAY_CREDENTIALS_PATH}" +fi + +# Create logs directory structure +mkdir -p "${LOGS_DIR}/tasks" +mkdir -p "${LOGS_DIR}/cluster-pods" +mkdir -p "${LOGS_DIR}/debug" +mkdir -p "${LOGS_DIR}/results" + +# ----------------------------------------------------------- +# 1. Pull per-task log artifacts uploaded by earlier tasks +# ----------------------------------------------------------- +if [ -n "${TASK_NAMES}" ]; then + echo "" + echo "==========================================" + echo "Pulling per-task log artifacts" + echo "==========================================" + for TASK_NAME in ${TASK_NAMES}; do + TASK_TAG="${PIPELINE_RUN_NAME}-task-${TASK_NAME}" + TASK_REF="${QUAY_REPO}:${TASK_TAG}" + TASK_DIR="${LOGS_DIR}/tasks/${TASK_NAME}" + mkdir -p "${TASK_DIR}" + + echo "Pulling task logs: ${TASK_REF}" + PULL_DIR="/tmp/pull-${TASK_NAME}" + mkdir -p "${PULL_DIR}" + if oras pull --no-tty -o "${PULL_DIR}" "${TASK_REF}" 2>/dev/null; then + # Extract any tarballs pulled from oras + for tarball in "${PULL_DIR}"/*.tar.gz; do + [ -f "$tarball" ] && tar xzf "$tarball" -C "${TASK_DIR}" && rm -f "$tarball" + done + # Copy any remaining files (non-tarball artifacts) + find "${PULL_DIR}" -type f -exec cp {} "${TASK_DIR}/" \; 2>/dev/null || true + # Move test result files to the results directory + find "${TASK_DIR}" -name "*.xml" -exec cp {} "${LOGS_DIR}/results/" \; 2>/dev/null || true + find "${TASK_DIR}" -name "*.json" -exec cp {} "${LOGS_DIR}/results/" \; 2>/dev/null || true + else + collect_error "Could not pull logs for task ${TASK_NAME} (may not have uploaded)" + fi + rm -rf "${PULL_DIR}" + done +fi + +# ----------------------------------------------------------- +# 2-4. Collect cluster debug info (only if cluster is reachable) +# ----------------------------------------------------------- +CLUSTER_REACHABLE=false +if [ -n "${KUBECONFIG:-}" ] && [ -f "${KUBECONFIG:-}" ]; then + export KUBECONFIG + if oc whoami --request-timeout=10s &>/dev/null; then + CLUSTER_REACHABLE=true + else + collect_error "Cluster unreachable (kubeconfig exists but oc commands fail)" + fi +else + collect_error "Kubeconfig not available (file: ${KUBECONFIG:-unset})" +fi + +if [ "$CLUSTER_REACHABLE" = true ]; then + echo "" + echo "Collecting cluster debug information..." + + { + echo "--- Cluster Version ---" + oc version 2>&1 || true + echo "" + echo "--- Cluster Operators ---" + oc get co 2>&1 || true + echo "" + echo "--- Nodes ---" + oc get nodes -o wide 2>&1 || true + } > "${LOGS_DIR}/debug/cluster-info.txt" 2>&1 || collect_error "Failed to collect cluster info" + + echo "Collecting namespace debug information..." + + { + echo "--- All Resources in ${NAMESPACE} ---" + oc get all -n "${NAMESPACE}" -o wide 2>&1 || true + echo "" + echo "--- Subscriptions ---" + oc get subscriptions -n "${NAMESPACE}" -o yaml 2>&1 || true + echo "" + echo "--- ClusterServiceVersions ---" + oc get csv -n "${NAMESPACE}" -o yaml 2>&1 || true + echo "" + echo "--- InstallPlans ---" + oc get installplans -n "${NAMESPACE}" -o yaml 2>&1 || true + } > "${LOGS_DIR}/debug/namespace-resources.txt" 2>&1 || collect_error "Failed to collect namespace resources" + + { + echo "--- Events ---" + oc get events -n "${NAMESPACE}" --sort-by='.lastTimestamp' 2>&1 || true + } > "${LOGS_DIR}/debug/events.txt" 2>&1 || collect_error "Failed to collect events" + + { + echo "--- CatalogSource ---" + oc get catalogsource -n openshift-marketplace -o yaml 2>&1 || true + } > "${LOGS_DIR}/debug/catalogsource.txt" 2>&1 || collect_error "Failed to collect catalogsource" + + echo "Collecting pod logs..." + TASK_COUNT=0 + oc get pods -n "${NAMESPACE}" -o json 2>/dev/null | jq -r '.items[].metadata.name' 2>/dev/null | while read -r pod; do + TASK_COUNT=$((TASK_COUNT + 1)) + TASK_LOG_FILE="${LOGS_DIR}/cluster-pods/$(printf '%02d' ${TASK_COUNT})-${pod}.log" + + { + echo "=== Pod: ${pod} ===" + echo "=== Namespace: ${NAMESPACE} ===" + echo "=== Collection Time: $(date -u +"%Y-%m-%d %H:%M:%S UTC") ===" + echo "" + + echo "--- Pod Description ---" + oc describe pod "${pod}" -n "${NAMESPACE}" 2>&1 || true + echo "" + + oc get pod "${pod}" -n "${NAMESPACE}" -o json 2>/dev/null | jq -r '.spec.containers[]?.name' 2>/dev/null | while read -r container; do + echo "--- Container: ${container} ---" + oc logs "${pod}" -c "${container}" -n "${NAMESPACE}" 2>&1 || echo "(no logs available)" + echo "" + + echo "--- Container: ${container} (previous) ---" + oc logs "${pod}" -c "${container}" -n "${NAMESPACE}" --previous 2>&1 || echo "(no previous logs)" + echo "" + done + } > "${TASK_LOG_FILE}" + done || collect_error "Failed to collect some pod logs" +else + echo "Skipping cluster debug collection (cluster unreachable)" +fi + +# ----------------------------------------------------------- +# 5. Parse JUnit test summary (if available) +# ----------------------------------------------------------- +TEST_SUMMARY="" +JUNIT_FILE="${LOGS_DIR}/results/junit-results.xml" +if [ -f "$JUNIT_FILE" ]; then + TESTS=$(grep -oP 'tests="\K[0-9]+' "$JUNIT_FILE" | head -1 || echo "?") + FAILURES=$(grep -oP 'failures="\K[0-9]+' "$JUNIT_FILE" | head -1 || echo "?") + SKIPPED=$(grep -oP 'skipped="\K[0-9]+' "$JUNIT_FILE" | head -1 || echo "0") + ERRORS_COUNT=$(grep -oP 'errors="\K[0-9]+' "$JUNIT_FILE" | head -1 || echo "0") + TEST_SUMMARY="Tests: ${TESTS} total, $((TESTS - FAILURES - SKIPPED - ERRORS_COUNT)) passed, ${FAILURES} failed, ${SKIPPED} skipped, ${ERRORS_COUNT} errors" +fi + +# ----------------------------------------------------------- +# 6. Create README +# ----------------------------------------------------------- +{ + echo "Pipeline Run Logs - ${PIPELINE_RUN_NAME}" + echo "Namespace - ${NAMESPACE}" + echo "Collected - $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + echo "" + if [ -n "$TEST_SUMMARY" ]; then + echo "Test Summary: ${TEST_SUMMARY}" + echo "" + fi + echo "Structure:" + echo " - tasks/ : stdout/stderr from each pipeline task step" + echo " - results/ : test result files (JUnit XML, JSON reports)" + echo " - cluster-pods/ : pod logs from the ephemeral cluster" + echo " - debug/ : cluster and namespace debug information" + echo "" + echo "Files:" + find "${LOGS_DIR}/" -type f | sort | sed 's/^/ - /' + echo "" + if [ ${#ERRORS[@]} -gt 0 ]; then + echo "Collection warnings:" + for err in "${ERRORS[@]}"; do + echo " - ${err}" + done + echo "" + fi + echo "To extract these logs:" + echo " oras pull ${QUAY_REPO}:${IMAGE_TAG}" + echo " tar xzf ${PIPELINE_RUN_NAME}-logs.tar.gz" +} > "${LOGS_DIR}/README.txt" + +# ----------------------------------------------------------- +# 7. Create CLAUDE.md for self-documenting analysis +# ----------------------------------------------------------- +cat > "${LOGS_DIR}/CLAUDE.md" << 'CLAUDE_EOF' +# GitOps Operator Integration Test Logs + +These logs are from a Konflux pipeline that installs the GitOps operator +on an ephemeral EaaS HyperShift cluster and runs e2e tests. + +## Quick diagnosis + +1. **Test results**: Check `results/junit-results.xml` for pass/fail summary. + Count failures: `grep -c 'failure message' results/junit-results.xml` + +2. **Test output**: Check `tasks/test-operator/test-operator.log` for test + stdout/stderr. Search for `FAIL` or `--- FAIL` to find failing tests. + +3. **Operator install**: Check `tasks/install-operator/install-operator.log` + for operator deployment issues. + +4. **Pod health**: Check `cluster-pods/` for ArgoCD component logs. + +5. **Cluster events**: Check `debug/events.txt` for scheduling, image pull, + or crash loop issues. + +## Common failure patterns + +| Symptom | Where to look | Likely cause | +|---------|--------------|--------------| +| `ImagePullBackOff` in events | debug/events.txt, install-operator.log | Pull secret not propagated to HyperShift nodes | +| `exec format error` | test-operator.log | Architecture mismatch (ARM image on x86 or vice versa) | +| Test timeout (no results) | test-operator.log (last test name) | A test hung — check which test was running last | +| `FailedScheduling` | debug/events.txt | Node selector mismatch or insufficient resources | +| `MachineConfig` failures | test-operator.log | MCO not available on HyperShift — should be in skip list | +| 464/470 argo tests fail | tasks/test-operator/argocd-e2e.log | `argocd-delete` plugin missing — kubectl is wrong binary | +| `connection refused` | test-operator.log | ArgoCD server not ready or port-forward failed | + +## File structure + +``` +logs/ +├── CLAUDE.md ← you are here +├── README.txt ← pipeline run metadata and test summary +├── tasks/ ← per-task stdout/stderr from pipeline steps +│ ├── install-operator/ ← operator installation output +│ └── test-operator/ ← test execution output + JUnit/JSON results +├── results/ ← copies of JUnit XML and JSON reports +├── cluster-pods/ ← pod logs from the ephemeral test cluster +└── debug/ ← cluster state: events, resources, catalog +``` + +## Analysis workflow + +1. Read `README.txt` for the test summary line +2. If tests failed, read the test log to identify which tests failed and why +3. If operator install failed, check install log for image pull or timeout issues +4. Cross-reference with cluster events and pod logs for infrastructure problems +5. Check if failures match known HyperShift limitations (skip list candidates) +CLAUDE_EOF + +# ----------------------------------------------------------- +# 8. Upload combined logs to Quay +# ----------------------------------------------------------- +echo "" +echo "==========================================" +echo "Uploading combined logs to Quay" +echo "==========================================" + +FULL_OCI_REF="${QUAY_REPO}:${IMAGE_TAG}" + +echo "Uploading logs to ${FULL_OCI_REF}..." +TARBALL="${PIPELINE_RUN_NAME}-logs.tar.gz" +tar czf "/tmp/${TARBALL}" --transform "s,^,${PIPELINE_RUN_NAME}/," -C "${LOGS_DIR}" . + +if ( cd /tmp && oras push --no-tty \ + --artifact-type "application/vnd.konflux.logs.v1+tar" \ + "${FULL_OCI_REF}" \ + "${TARBALL}" ); then + echo "Successfully pushed combined log artifact to ${FULL_OCI_REF}" +else + echo "ERROR: Failed to push log artifact to ${FULL_OCI_REF}" + echo "Logs were collected locally but could not be uploaded." +fi + +echo "" +echo "Contents:" +find "${LOGS_DIR}/" -type f | sort | head -50 +echo "" +echo "Total size:" +du -sh "${LOGS_DIR}/" +if [ ${#ERRORS[@]} -gt 0 ]; then + echo "" + echo "Collection completed with ${#ERRORS[@]} warning(s)." +fi diff --git a/.tekton/test-image/scripts/collect-logs-sidecar.sh b/.tekton/test-image/scripts/collect-logs-sidecar.sh new file mode 100644 index 000000000..e186e69df --- /dev/null +++ b/.tekton/test-image/scripts/collect-logs-sidecar.sh @@ -0,0 +1,169 @@ +#!/bin/bash +set -u + +# Environment variables expected: +# - KUBECONFIG_NAME +# - PIPELINE_RUN_NAME +# - BRANCH_NAME (used as the task artifact name, e.g. "logs" → tag: ${PIPELINE_RUN_NAME}-task-logs) +# - NAMESPACE +# - QUAY_REPO + +LOGS_DIR="logs" +IMAGE_TAG="${PIPELINE_RUN_NAME}-task-${BRANCH_NAME}" +FULL_OCI_REF="${QUAY_REPO}:${IMAGE_TAG}" +COLLECT_INTERVAL=30 +UPLOAD_EVERY=10 # upload every 10 snapshots (10 * 30s = 5 min) + +TIMEOUT=300 +ELAPSED=0 + +if [ "${KUBECONFIG_NAME}" = "auto" ]; then + while [ $ELAPSED -lt $TIMEOUT ]; do + KUBECONFIG_PATH=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + if [ -n "${KUBECONFIG_PATH:-}" ]; then break; fi + sleep 5 + ELAPSED=$((ELAPSED + 5)) + done +else + KUBECONFIG_PATH="/credentials/${KUBECONFIG_NAME}" + while [ ! -f "${KUBECONFIG_PATH}" ] && [ $ELAPSED -lt $TIMEOUT ]; do + sleep 5 + ELAPSED=$((ELAPSED + 5)) + done +fi + +CLUSTER_AVAILABLE=false +if [ -n "${KUBECONFIG_PATH:-}" ] && [ -f "${KUBECONFIG_PATH:-}" ]; then + export KUBECONFIG="${KUBECONFIG_PATH}" + if oc whoami --request-timeout=10s &>/dev/null; then + CLUSTER_AVAILABLE=true + else + echo "WARNING: Kubeconfig found but cluster is unreachable" + fi +else + echo "WARNING: Kubeconfig not found after ${TIMEOUT}s — will skip cluster log collection" + echo "Contents of /credentials:" + ls -la /credentials/ 2>/dev/null || true +fi + +# Configure Docker credentials for oras (needed for periodic uploads) +AUTH_PATH="/quay-credentials/.dockerconfigjson" +if [ -f "$AUTH_PATH" ]; then + TEMP_DOCKER_CONFIG="$(mktemp -d)" + cp "$AUTH_PATH" "$TEMP_DOCKER_CONFIG/config.json" + export DOCKER_CONFIG="$TEMP_DOCKER_CONFIG" +fi + +mkdir -p "${LOGS_DIR}" + +# --- Helper functions --- + +collect_snapshot() { + if [ "$CLUSTER_AVAILABLE" != true ]; then return 0; fi + if ! oc get pods -n "${NAMESPACE}" &>/dev/null; then + return 0 + fi + + local timestamp + timestamp=$(date +%s) + local snapshot_dir="${LOGS_DIR}/snapshot-${timestamp}" + mkdir -p "${snapshot_dir}" + + oc get pods -n "${NAMESPACE}" -o json 2>/dev/null | jq -r '.items[].metadata.name' 2>/dev/null | while read -r pod; do + local pod_log_file="${snapshot_dir}/${pod}.log" + echo "=== Pod: ${pod} ===" > "${pod_log_file}" + echo "=== Timestamp: $(date -u +"%Y-%m-%d %H:%M:%S UTC") ===" >> "${pod_log_file}" + echo "" >> "${pod_log_file}" + + oc get pod "${pod}" -n "${NAMESPACE}" -o json 2>/dev/null | jq -r '.spec.containers[]?.name' 2>/dev/null | while read -r container; do + echo "--- Container: ${container} ---" >> "${pod_log_file}" + oc logs "${pod}" -c "${container}" -n "${NAMESPACE}" --tail=100 >> "${pod_log_file}" 2>&1 || echo "(no logs available)" >> "${pod_log_file}" + echo "" >> "${pod_log_file}" + done + done || true +} + +collect_final() { + if [ "$CLUSTER_AVAILABLE" != true ]; then return 0; fi + + FINAL_DIR="${LOGS_DIR}/final" + mkdir -p "${FINAL_DIR}" + + local count=0 + oc get pods -n "${NAMESPACE}" -o json 2>/dev/null | jq -r '.items[].metadata.name' 2>/dev/null | while read -r pod; do + count=$((count + 1)) + local log_file="${FINAL_DIR}/$(printf '%02d' ${count})-${pod}.log" + { + echo "=== Pod: ${pod} ===" + echo "=== Namespace: ${NAMESPACE} ===" + echo "=== Collection Time: $(date -u +"%Y-%m-%d %H:%M:%S UTC") ===" + echo "" + oc get pod "${pod}" -n "${NAMESPACE}" -o json 2>/dev/null | jq -r '.spec.containers[]?.name' 2>/dev/null | while read -r container; do + echo "--- Container: ${container} ---" + oc logs "${pod}" -c "${container}" -n "${NAMESPACE}" 2>&1 || echo "(no logs available)" + echo "" + done + } > "${log_file}" + done || true +} + +generate_readme() { + { + echo "Sidecar Logs - ${PIPELINE_RUN_NAME}" + echo "Namespace - ${NAMESPACE}" + echo "Collected - $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + echo "Cluster available: ${CLUSTER_AVAILABLE}" + echo "Upload #${UPLOAD_COUNT}" + echo "" + echo "Structure:" + echo " - snapshot-* directories: periodic snapshots during test execution" + echo " - final/ directory: complete logs at test completion (if present)" + echo "" + echo "Files:" + find "${LOGS_DIR}/" -type f -name "*.log" 2>/dev/null | sort | sed 's/^/ - /' + echo "" + echo "To extract these logs:" + echo " oras pull ${QUAY_REPO}:${IMAGE_TAG}" + echo " tar xzf ${PIPELINE_RUN_NAME}-task-${BRANCH_NAME}-logs.tar.gz" + } > "${LOGS_DIR}/README.txt" +} + +upload_logs() { + UPLOAD_COUNT=$((UPLOAD_COUNT + 1)) + generate_readme + + local tarball="${PIPELINE_RUN_NAME}-task-${BRANCH_NAME}-logs.tar.gz" + tar czf "/tmp/${tarball}" --transform "s,^,${PIPELINE_RUN_NAME}-task-${BRANCH_NAME}/," -C "${LOGS_DIR}" . + + if ( cd /tmp && oras push --no-tty \ + --artifact-type "application/vnd.konflux.logs.v1+tar" \ + "${FULL_OCI_REF}" \ + "${tarball}" ); then + echo "Sidecar upload #${UPLOAD_COUNT} pushed to ${FULL_OCI_REF} ($(du -h "/tmp/${tarball}" | cut -f1))" + else + echo "WARNING: Sidecar upload #${UPLOAD_COUNT} failed" + fi + rm -f "/tmp/${tarball}" +} + +# --- Main loop --- + +SNAPSHOT_COUNT=0 +UPLOAD_COUNT=0 + +while true; do + collect_snapshot + SNAPSHOT_COUNT=$((SNAPSHOT_COUNT + 1)) + + if (( SNAPSHOT_COUNT % UPLOAD_EVERY == 0 )); then + upload_logs + fi + + sleep ${COLLECT_INTERVAL} & + wait $! || break +done + +# Final comprehensive collection + upload (best effort) +echo "Sidecar exiting — collecting final logs and uploading" +collect_final +upload_logs diff --git a/.tekton/test-image/scripts/go-cache.sh b/.tekton/test-image/scripts/go-cache.sh new file mode 100644 index 000000000..bf0777f73 --- /dev/null +++ b/.tekton/test-image/scripts/go-cache.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Go build cache backed by OCI registry via oras. +# Source this file, then call go_cache_pull / go_cache_push. +# +# Usage: +# source /usr/local/bin/go-cache.sh +# go_cache_pull "argocd-v2.14" +# # ... compile ... +# go_cache_push "argocd-v2.14" +# +# Expects: QUAY_REPO env var, quay credentials at /quay-credentials/.dockerconfigjson + +QUAY_REPO="${QUAY_REPO:-quay.io/devtools_gitops/test_image}" + +_go_cache_setup_auth() { + local auth_path="/quay-credentials/.dockerconfigjson" + if [ -f "$auth_path" ] && [ -z "${DOCKER_CONFIG:-}" ]; then + local temp_config + temp_config=$(mktemp -d) + cp "$auth_path" "$temp_config/config.json" + export DOCKER_CONFIG="$temp_config" + fi +} + +_go_cache_tag() { + local suffix="${1:-default}" + local arch + arch=$(go env GOARCH 2>/dev/null || echo "amd64") + + local sum_hash="unknown" + if [ -f "go.sum" ]; then + sum_hash=$(sha256sum go.sum | cut -c1-12) + fi + + echo "go-cache-${arch}-${suffix}-${sum_hash}" +} + +go_cache_pull() { + local tag + tag="${QUAY_REPO}:$(_go_cache_tag "${1:-default}")" + + _go_cache_setup_auth + + local tmpdir + tmpdir=$(mktemp -d) + if (cd "$tmpdir" && oras pull --no-tty "$tag" 2>/dev/null); then + if [ -f "$tmpdir/go-cache.tar.gz" ]; then + tar xzf "$tmpdir/go-cache.tar.gz" -C / 2>/dev/null || true + echo "Go cache restored from ${tag}" + fi + else + echo "No Go cache found at ${tag}, building from scratch" + fi + rm -rf "$tmpdir" +} + +go_cache_push() { + local tag + tag="${QUAY_REPO}:$(_go_cache_tag "${1:-default}")" + + _go_cache_setup_auth + + local gocache gomodcache + gocache=$(go env GOCACHE 2>/dev/null) + gomodcache=$(go env GOMODCACHE 2>/dev/null) + + local paths=() + [ -d "$gocache" ] && paths+=("${gocache#/}") + [ -d "$gomodcache" ] && paths+=("${gomodcache#/}") + + if [ ${#paths[@]} -eq 0 ]; then + return 0 + fi + + local tmpdir + tmpdir=$(mktemp -d) + tar czf "$tmpdir/go-cache.tar.gz" -C / "${paths[@]}" 2>/dev/null || true + + (cd "$tmpdir" && oras push --no-tty \ + --artifact-type "application/vnd.go-cache.v1+tar" \ + "$tag" \ + "go-cache.tar.gz" 2>/dev/null) && \ + echo "Go cache pushed to ${tag}" || \ + echo "WARNING: Failed to push Go cache" + + rm -rf "$tmpdir" +} diff --git a/.tekton/test-image/scripts/install-bundle.sh b/.tekton/test-image/scripts/install-bundle.sh new file mode 100755 index 000000000..bc216d86c --- /dev/null +++ b/.tekton/test-image/scripts/install-bundle.sh @@ -0,0 +1,171 @@ +#!/bin/bash +set -ex + +# Environment variables expected: +# - BUNDLE_IMAGE (from SNAPSHOT, the bundle to install) +# - NAMESPACE (default: openshift-gitops-operator) +# - INSTALL_TIMEOUT (default: 25m) +# - KUBECONFIG + +NAMESPACE="${NAMESPACE:-openshift-gitops-operator}" +INSTALL_TIMEOUT="${INSTALL_TIMEOUT:-25m}" + +if [[ -z "$BUNDLE_IMAGE" ]]; then + echo "ERROR: BUNDLE_IMAGE environment variable is required" + exit 1 +fi + +echo "==========================================" +echo "Installing GitOps Operator Bundle" +echo "==========================================" +echo "Bundle image: ${BUNDLE_IMAGE}" +echo "Namespace: ${NAMESPACE}" +echo "Timeout: ${INSTALL_TIMEOUT}" +echo "" + +# 1. Install operator-sdk +echo "Installing operator-sdk..." +OPERATOR_SDK_VERSION=1.36.1 +ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n "$(uname -m)" ;; esac) +OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v${OPERATOR_SDK_VERSION} +curl -Lo /tmp/operator-sdk "${OPERATOR_SDK_DL_URL}/operator-sdk_linux_${ARCH}" +chmod +x /tmp/operator-sdk +/tmp/operator-sdk version +echo "" + +# 2. Verify cluster connectivity +echo "Verifying cluster connectivity..." +oc status +oc whoami +echo "" + +# 3. Create namespace and label it for cluster monitoring +echo "Creating namespace ${NAMESPACE}..." +oc create namespace "${NAMESPACE}" || true +oc label namespaces "${NAMESPACE}" openshift.io/cluster-monitoring=true --overwrite=true +echo "" + +# 4. Run operator-sdk run bundle +echo "Running operator-sdk run bundle..." +if ! /tmp/operator-sdk run bundle --timeout="$INSTALL_TIMEOUT" \ + --namespace "${NAMESPACE}" \ + "$BUNDLE_IMAGE" \ + --verbose; then + echo "ERROR: operator-sdk run bundle failed" + exit 1 +fi +echo "" + +# 5. Wait for the controller pod to appear +echo "Waiting for GitOps controller pod to appear..." +for i in {0..30}; do + sleep 3 + if oc get pod -n "${NAMESPACE}" | grep gitops > /dev/null 2>&1; then + break + fi + if [[ $i -eq 30 ]]; then + echo "ERROR: Controller pod did not appear after 90 seconds" + oc get pods -n "${NAMESPACE}" -o wide || true + exit 1 + fi +done + +controller_pod=$(oc get pod -n "${NAMESPACE}" -l control-plane=gitops-operator -o 'jsonpath={.items[0].metadata.name}' 2>/dev/null) +if [[ -z "$controller_pod" ]]; then + echo "ERROR: Could not find controller pod with label control-plane=gitops-operator" + oc get pods -n "${NAMESPACE}" -o wide || true + exit 1 +fi +echo "Controller pod: $controller_pod" +echo "" + +# 6. Wait for openshift-gitops namespace to become Active +echo "Waiting for openshift-gitops namespace to become Active..." +if ! oc wait --for=jsonpath='{.status.phase}'=Active ns/openshift-gitops --timeout=120s; then + echo "ERROR: openshift-gitops namespace did not become Active" + oc get ns openshift-gitops -o yaml 2>/dev/null || echo "Namespace does not exist" + exit 1 +fi +echo "" + +# 7. Wait for ArgoCD deployments to roll out +ARGOCD_NAMESPACE="openshift-gitops" +echo "Waiting for ArgoCD deployments in ${ARGOCD_NAMESPACE}..." + +mapfile -t deployments < <(oc get deployments -n "${ARGOCD_NAMESPACE}" --no-headers -o custom-columns=':metadata.name' 2>/dev/null) +if [[ ${#deployments[@]} -eq 0 ]]; then + echo "ERROR: No deployments found in ${ARGOCD_NAMESPACE}" + oc get all -n "${ARGOCD_NAMESPACE}" || true + exit 1 +fi + +for deployment in "${deployments[@]}"; do + echo " Waiting for deployment/${deployment}..." + if ! oc rollout status "deployment/${deployment}" -n "${ARGOCD_NAMESPACE}" --timeout=120s; then + echo "ERROR: deployment/${deployment} did not roll out successfully" + oc get deployment "${deployment}" -n "${ARGOCD_NAMESPACE}" -o wide || true + oc get pods -n "${ARGOCD_NAMESPACE}" -o wide || true + exit 1 + fi +done +echo "" + +# 8. Wait for the application-controller StatefulSet +echo "Waiting for openshift-gitops-application-controller StatefulSet to be created..." +for i in {0..30}; do + sleep 3 + if oc get statefulset -n "${ARGOCD_NAMESPACE}" | grep gitops > /dev/null 2>&1; then + break + fi + if [[ $i -eq 30 ]]; then + echo "ERROR: StatefulSet did not appear after 90 seconds" + oc get statefulsets -n "${ARGOCD_NAMESPACE}" -o wide || true + exit 1 + fi +done + +echo "Waiting for statefulset rollout..." +if ! oc rollout status statefulset/openshift-gitops-application-controller -n "${ARGOCD_NAMESPACE}" --timeout=120s; then + echo "ERROR: StatefulSet rollout failed" + oc get statefulset openshift-gitops-application-controller -n "${ARGOCD_NAMESPACE}" -o wide || true + oc get pods -n "${ARGOCD_NAMESPACE}" -o wide || true + exit 1 +fi +echo "" + +# 9. Wait for ArgoCD CR to reach Available phase +echo "Waiting for ArgoCD CR to reach Available phase..." +if ! oc wait argocd openshift-gitops -n "${ARGOCD_NAMESPACE}" --for=jsonpath='{.status.phase}'="Available" --timeout=600s; then + echo "ERROR: ArgoCD CR did not reach Available phase" + oc get argocd openshift-gitops -n "${ARGOCD_NAMESPACE}" -o yaml || true + exit 1 +fi +echo "" + +# 10. Output success and diagnostic information +echo "==========================================" +echo "✅ Bundle installation completed successfully!" +echo "==========================================" +echo "" + +echo "--- Installed CSV ---" +oc get csv -n "${NAMESPACE}" -o wide || true +echo "" + +echo "--- Operator Pods ---" +oc get pods -n "${NAMESPACE}" -o wide || true +echo "" + +echo "--- ArgoCD Pods ---" +oc get pods -n "${ARGOCD_NAMESPACE}" -o wide || true +echo "" + +echo "--- ArgoCD CR Status ---" +oc get argocd openshift-gitops -n "${ARGOCD_NAMESPACE}" -o jsonpath='{.status}' | python3 -m json.tool || true +echo "" + +echo "--- ImageDigestMirrorSet ---" +oc get imagedigestmirrorset -o yaml 2>/dev/null || echo "No IDMS found" +echo "" + +echo "==========================================" diff --git a/.tekton/test-image/scripts/install-operator.sh b/.tekton/test-image/scripts/install-operator.sh new file mode 100644 index 000000000..984a53b7e --- /dev/null +++ b/.tekton/test-image/scripts/install-operator.sh @@ -0,0 +1,351 @@ +#!/bin/bash +set -ex + +# Environment variables expected: +# - OPENSHIFT_VERSION (e.g. "4.20" or "4.20.19") +# - NAMESPACE +# - INSTALL_TIMEOUT +# - KUBECONFIG + +MINOR_VERSION=$(echo "${OPENSHIFT_VERSION}" | grep -oP '^\d+\.\d+') +CATALOG_IMAGE="quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/catalog:v${MINOR_VERSION}" + +echo "Installing GitOps Operator from catalog: ${CATALOG_IMAGE}" +echo "OpenShift version: ${OPENSHIFT_VERSION} (minor: ${MINOR_VERSION})" +echo "Target namespace: ${NAMESPACE}" + +# 1. Inject quay pull credentials into cluster +if [[ -f "/quay-pull-credentials/.dockerconfigjson" ]]; then + # 1a. Patch global pull-secret (may take time to propagate on HyperShift) + echo "Injecting quay pull credentials into cluster global pull-secret..." + EXISTING=$(oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d) + MERGED=$(echo "$EXISTING" | python3 -c " +import json, sys +existing = json.load(sys.stdin) +with open('/quay-pull-credentials/.dockerconfigjson') as f: + extra = json.load(f) +existing.setdefault('auths', {}).update(extra.get('auths', {})) +print(json.dumps(existing)) +") + oc set data secret/pull-secret -n openshift-config --from-literal=.dockerconfigjson="$MERGED" + echo "Injected $(echo "$MERGED" | python3 -c "import json,sys; print(len(json.load(sys.stdin)['auths']))" 2>/dev/null) registry credentials into cluster pull-secret" + + # 1b. Create additional-pull-secret in kube-system (HyperShift-native mechanism). + # The Hosted Cluster Config Operator detects this secret and deploys a DaemonSet + # that writes credentials to /var/lib/kubelet/config.json on each node. + echo "Creating additional-pull-secret in kube-system for HyperShift node credential injection..." + oc create secret generic additional-pull-secret \ + -n kube-system \ + --from-file=.dockerconfigjson=/quay-pull-credentials/.dockerconfigjson \ + --type=kubernetes.io/dockerconfigjson \ + --dry-run=client -o yaml | oc apply -f - + + # Wait for the syncer DaemonSet to appear and propagate to all nodes + echo "Waiting for pull-secret syncer DaemonSet..." + SYNC_TIMEOUT=300 + SYNC_START=$(date +%s) + while true; do + DS_NAME=$(oc get daemonset -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | grep -i 'pull-secret' || true) + + if [[ -n "$DS_NAME" ]]; then + DESIRED=$(oc get ds "$DS_NAME" -n kube-system -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0") + READY=$(oc get ds "$DS_NAME" -n kube-system -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") + if [[ "$DESIRED" -gt 0 && "$DESIRED" == "$READY" ]]; then + echo "Pull-secret syncer DaemonSet $DS_NAME is ready ($READY/$DESIRED nodes)" + break + fi + echo " Syncer DaemonSet $DS_NAME: $READY/$DESIRED nodes ready..." + fi + + ELAPSED=$(( $(date +%s) - SYNC_START )) + if [[ $ELAPSED -ge $SYNC_TIMEOUT ]]; then + echo "WARNING: Pull-secret syncer not fully ready within ${SYNC_TIMEOUT}s, continuing anyway" + oc get daemonset -n kube-system 2>/dev/null || true + break + fi + sleep 15 + done +else + echo "WARNING: No quay pull credentials found at /quay-pull-credentials/.dockerconfigjson" +fi + +# 2. Ensure the operator namespace exists +oc create namespace "${NAMESPACE}" --dry-run=client -o yaml | oc apply -f - + +# 3. Create CatalogSource +cat </dev/null; then + # Ensure pull secret exists in the namespace + oc create secret generic quay-mirror-pull \ + --from-file=.dockerconfigjson=/quay-pull-credentials/.dockerconfigjson \ + --type=kubernetes.io/dockerconfigjson \ + -n openshift-gitops \ + --dry-run=client -o yaml | oc apply -f - &>/dev/null + + # Link to all SAs that don't already have it + for sa in $(oc get sa -n openshift-gitops -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + if ! oc get sa "$sa" -n openshift-gitops -o jsonpath='{.imagePullSecrets[*].name}' 2>/dev/null | grep -q quay-mirror-pull; then + oc patch sa "$sa" -n openshift-gitops --type=json \ + -p '[{"op":"add","path":"/imagePullSecrets/-","value":{"name":"quay-mirror-pull"}}]' &>/dev/null || true + fi + done + + # Restart pods stuck on image pull errors so they pick up the new SA credentials + STUCK=$(oc get pods -n openshift-gitops 2>/dev/null | grep -E 'ImagePullBackOff|ErrImagePull' | awk '{print $1}') + for pod in $STUCK; do + echo " [pull-secret-injector] Restarting stuck pod: $pod" + oc delete pod "$pod" -n openshift-gitops --grace-period=0 &>/dev/null || true + done + fi + sleep 10 + done + ) & + SA_PATCH_PID=$! + echo "Background pull-secret injection started (PID: $SA_PATCH_PID)" +fi + +# 8. Verify all related images are available at mirrors +echo "" +echo "==========================================" +echo "Verifying related images are available" +echo "==========================================" +/usr/local/bin/verify-images.sh || { + echo "WARNING: Some images are not available at their mirror locations." + echo "ArgoCD pods may fail with ImagePullBackOff." +} + +echo "" +echo "==========================================" +echo "DEBUG INFO: Post-Installation State" +echo "==========================================" +echo "" + +echo "--- CatalogSource Status ---" +oc get catalogsource gitops-stage -n openshift-marketplace -o yaml || true +echo "" + +echo "--- Subscription Status ---" +oc get subscription gitops-operator-konflux -n "${NAMESPACE}" -o yaml || true +echo "" + +echo "--- ClusterServiceVersion Status ---" +oc get csv "${CSV_NAME}" -n "${NAMESPACE}" -o yaml || true +echo "" + +echo "--- All Pods in ${NAMESPACE} ---" +oc get pods -n "${NAMESPACE}" -o wide || true +echo "" + +echo "--- Events in ${NAMESPACE} ---" +oc get events -n "${NAMESPACE}" --sort-by='.lastTimestamp' || true +echo "" + +echo "--- Operator Deployment ---" +oc get deployments -n "${NAMESPACE}" -o wide || true +echo "" + +echo "==========================================" + +# 9. Verify default ArgoCD instance is ready +echo "" +echo "==========================================" +echo "Verifying default ArgoCD instance" +echo "==========================================" + +echo "Waiting for openshift-gitops namespace and ArgoCD deployments to appear..." +for _ in {1..60}; do + if oc get deployment openshift-gitops-server -n openshift-gitops &>/dev/null; then + break + fi + sleep 10 +done + +if ! oc get deployment openshift-gitops-server -n openshift-gitops &>/dev/null; then + echo "ERROR: ArgoCD deployments not created after 10 minutes" + oc get ns openshift-gitops 2>/dev/null || echo "Namespace openshift-gitops does not exist" + oc get argocd -n openshift-gitops 2>/dev/null || true + oc get pods -n openshift-gitops -o wide 2>/dev/null || true + oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + echo "--- IDMS on cluster ---" + oc get imagedigestmirrorset 2>/dev/null || echo "No IDMS" + echo "--- Operator logs ---" + oc logs deployment/openshift-gitops-operator-controller-manager -n openshift-gitops-operator -c manager --tail=50 2>/dev/null || true + exit 1 +fi + +echo "ArgoCD deployments found, waiting for them to become available..." +for deploy in openshift-gitops-server openshift-gitops-repo-server; do + if ! oc wait --for=condition=Available "deployment/$deploy" -n openshift-gitops --timeout=600s; then + echo "ERROR: deployment/$deploy did not become Available" + oc get deployment "$deploy" -n openshift-gitops -o wide 2>/dev/null || true + oc get pods -n openshift-gitops -o wide 2>/dev/null || true + echo "--- Pod details ---" + for pod in $(oc get pods -n openshift-gitops -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do + echo "=== $pod ===" + oc get pod "$pod" -n openshift-gitops -o jsonpath='{range .status.containerStatuses[*]}container={.name} ready={.ready} state={.state}{"\n"}{end}' 2>/dev/null || true + done + echo "--- Events ---" + oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + echo "--- IDMS ---" + oc get imagedigestmirrorset 2>/dev/null || echo "No IDMS" + exit 1 + fi + echo "$deploy is ready" +done + +# application-controller is a StatefulSet, not a Deployment +if oc get statefulset openshift-gitops-application-controller -n openshift-gitops &>/dev/null; then + if ! oc rollout status statefulset/openshift-gitops-application-controller -n openshift-gitops --timeout=600s; then + echo "ERROR: statefulset/openshift-gitops-application-controller did not become ready" + oc get pods -n openshift-gitops -o wide 2>/dev/null || true + oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + exit 1 + fi + echo "openshift-gitops-application-controller is ready" +else + echo "WARNING: openshift-gitops-application-controller statefulset not found, skipping" +fi + +echo "ArgoCD instance is ready" + +# Stop background pull-secret injection +if [[ -n "${SA_PATCH_PID}" ]]; then + kill $SA_PATCH_PID 2>/dev/null || true + wait $SA_PATCH_PID 2>/dev/null || true + echo "Stopped background pull-secret injection" +fi + +# 10. Collect cluster-wide debug info (on success) +echo "" +echo "==========================================" +echo "DEBUG INFO: Cluster Image Configuration" +echo "==========================================" + +echo "--- ImageDigestMirrorSet ---" +oc get imagedigestmirrorset -o yaml 2>/dev/null || echo "No IDMS found" +echo "" + +echo "--- ImageContentSourcePolicy ---" +oc get imagecontentsourcepolicy -o yaml 2>/dev/null || echo "No ICSP found" +echo "" + +echo "--- openshift-gitops namespace pods ---" +oc get pods -n openshift-gitops -o wide 2>/dev/null || true +echo "" + +echo "--- openshift-gitops namespace events (last 5 min) ---" +oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -40 || true +echo "" + +echo "--- openshift-gitops pod descriptions (non-Running) ---" +for pod in $(oc get pods -n openshift-gitops -o jsonpath='{range .items[?(@.status.phase!="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do + echo "=== Pod: $pod ===" + oc describe pod "$pod" -n openshift-gitops 2>/dev/null | grep -A5 -E 'State:|Image:|Warning|Error|Back-off|ImagePull' || true + echo "" +done + +echo "==========================================" + +# 11. Verify pull-secret propagated to nodes +if [[ -f "/quay-pull-credentials/.dockerconfigjson" ]]; then + echo "" + echo "==========================================" + echo "Verifying pull-secret propagation to nodes" + echo "==========================================" + EXPECTED_REPOS=$(python3 -c " +import json +with open('/quay-pull-credentials/.dockerconfigjson') as f: + d = json.load(f) +for k in sorted(d.get('auths', {})): + print(k) +" 2>/dev/null | head -3) + CLUSTER_SECRET=$(oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | base64 -d) + MISSING=0 + while IFS= read -r repo; do + if echo "$CLUSTER_SECRET" | python3 -c "import json,sys; d=json.load(sys.stdin); sys.exit(0 if '$repo' in d.get('auths',{}) else 1)" 2>/dev/null; then + echo " OK $repo" + else + echo " MISS $repo" + MISSING=$((MISSING + 1)) + fi + done <<< "$EXPECTED_REPOS" + if [[ $MISSING -gt 0 ]]; then + echo "WARNING: $MISSING repo(s) missing from cluster pull-secret" + else + echo "Pull-secret contains injected credentials" + fi +fi diff --git a/.tekton/test-image/scripts/run-and-save-logs.sh b/.tekton/test-image/scripts/run-and-save-logs.sh new file mode 100644 index 000000000..ac84ce121 --- /dev/null +++ b/.tekton/test-image/scripts/run-and-save-logs.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -o pipefail + +# Wrapper that runs a command, tees output to a log file, and uploads to Quay. +# The finally step later pulls these per-task artifacts and combines them. +# +# Environment variables expected: +# - TASK_LOG_NAME - name for the log artifact (e.g. "install-operator") +# - PIPELINE_RUN_NAME - pipeline run name, used in the artifact tag +# - QUAY_REPO - quay repository for log uploads +# - QUAY_CREDENTIALS_PATH - path to .dockerconfigjson (default: /quay-credentials/.dockerconfigjson) +# +# Usage: run-and-save-logs.sh [args...] + +QUAY_CREDENTIALS_PATH="${QUAY_CREDENTIALS_PATH:-/quay-credentials/.dockerconfigjson}" +LOG_DIR="/tmp/task-logs" +LOG_FILE="${LOG_DIR}/${TASK_LOG_NAME}.log" + +mkdir -p "${LOG_DIR}" + +# Run the command, tee to log file, preserve exit code +"$@" 2>&1 | tee "${LOG_FILE}" +EXIT_CODE=${PIPESTATUS[0]} + +# Collect any test result files (JUnit XML, JSON reports) produced by the command +for pattern in /tmp/task-logs/*.xml /tmp/task-logs/*.json; do + if [ -f "$pattern" ]; then + echo "Found test result file: $pattern" + fi +done + +# Upload logs to Quay if credentials are available +if [ -n "${QUAY_REPO}" ] && [ -n "${PIPELINE_RUN_NAME}" ] && [ -n "${TASK_LOG_NAME}" ]; then + echo "" + echo "==========================================" + echo "Uploading task logs: ${TASK_LOG_NAME}" + echo "==========================================" + + if [ -f "${QUAY_CREDENTIALS_PATH}" ]; then + TEMP_DOCKER_CONFIG="$(mktemp -d)" + cp "${QUAY_CREDENTIALS_PATH}" "${TEMP_DOCKER_CONFIG}/config.json" + export DOCKER_CONFIG="${TEMP_DOCKER_CONFIG}" + else + echo "Warning: Quay credentials not found at ${QUAY_CREDENTIALS_PATH}, skipping upload" + exit "${EXIT_CODE}" + fi + + IMAGE_TAG="${PIPELINE_RUN_NAME}-task-${TASK_LOG_NAME}" + FULL_OCI_REF="${QUAY_REPO}:${IMAGE_TAG}" + + # Tar and push log directory (oras expects files, not directories) + TARBALL="${TASK_LOG_NAME}-logs.tar.gz" + tar czf "/tmp/${TARBALL}" -C "${LOG_DIR}" . + ( cd /tmp && oras push --no-tty \ + --artifact-type "application/vnd.konflux.logs.v1+tar" \ + "${FULL_OCI_REF}" \ + "${TARBALL}" ) || echo "Warning: failed to upload task logs" + + echo "Task logs uploaded to ${FULL_OCI_REF}" +fi + +exit "${EXIT_CODE}" diff --git a/.tekton/test-image/scripts/run-argocd-e2e-tests.sh b/.tekton/test-image/scripts/run-argocd-e2e-tests.sh new file mode 100644 index 000000000..3dbfba4d4 --- /dev/null +++ b/.tekton/test-image/scripts/run-argocd-e2e-tests.sh @@ -0,0 +1,700 @@ +#!/usr/bin/env bash +set -u -o pipefail + +# Upstream ArgoCD E2E tests adapted from downstream-CI single-argocd-e2e-test. +# Deploys a test ArgoCD instance in a dedicated namespace, compiles and runs +# the upstream E2E test suite inside the cluster. +# +# Env vars expected: KUBECONFIG, TEST_REPO_URL, BRANCH +# TEST_REPO_URL should point to the argo-cd repo (default: https://github.com/argoproj/argo-cd.git) +# BRANCH should be a version tag (e.g. v2.14.0) or "master" + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +ROOT_DIR=$(mktemp -d) +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/argoproj/argo-cd.git}" +BRANCH="${BRANCH:-master}" + +SKIP_FILE=/usr/local/bin/skip-argocd.txt +if [[ -f "$SKIP_FILE" ]]; then + ARGOCD_E2E_SKIP=$(grep -v '^\s*#' "$SKIP_FILE" | grep -v '^\s*$' | paste -sd '|') +fi +ARGOCD_E2E_SKIP="${ARGOCD_E2E_SKIP:-TestCreateAndUseAccount|TestCanIGetLogs|TestAccountSessionToken}" + +ARGO_CD_DIR="${ROOT_DIR}/argo-cd" +export HOME="$ROOT_DIR" +export GIT_HTTP_LOW_SPEED_LIMIT=1000 +export GIT_TERMINAL_PROMPT=0 +export GODEBUG="tarinsecurepath=0,zipinsecurepath=0" +export GOTOOLCHAIN=auto + +# shellcheck disable=SC2034 # these are used indirectly via ${!pid_var} in cleanup +LOGGER_PID="" +GIT_FWD_PID="" +TEMP_PF_PID="" +PERMISSION_WATCHER_PID="" +COMPILE_HEARTBEAT_PID="" + +cleanup_resources() { + local exit_code=$? + echo "Cleaning up..." + for pid_var in LOGGER_PID GIT_FWD_PID TEMP_PF_PID PERMISSION_WATCHER_PID COMPILE_HEARTBEAT_PID; do + local pid=${!pid_var} + if [[ -n "$pid" ]]; then kill "$pid" 2>/dev/null || true; fi + done + + # Only restore operator in operator mode + if [[ "${DEPLOY_MODE:-operator}" != "standalone" ]]; then + OP_NS=$(oc get deployment -A -l app.kubernetes.io/name=openshift-gitops-operator \ + -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null || true) + OP_DEPLOY=$(oc get deployment -A -l app.kubernetes.io/name=openshift-gitops-operator \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + + if [[ -n "$OP_NS" && -n "$OP_DEPLOY" ]]; then + oc set env "deployment/$OP_DEPLOY" -n "$OP_NS" \ + DISABLE_DEFAULT_ARGOCD_INSTANCE- ARGOCD_CLUSTER_CONFIG_NAMESPACES- 2>/dev/null || true + oc scale deployment "$OP_DEPLOY" -n "$OP_NS" --replicas=1 2>/dev/null || true + oc rollout status "deployment/$OP_DEPLOY" -n "$OP_NS" --timeout=120s 2>/dev/null || true + fi + fi + + for ns in argocd-e2e argocd-e2e-external argocd-e2e-external-2; do + if oc get ns "$ns" >/dev/null 2>&1; then + oc get applications -n "$ns" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | \ + xargs -n1 -I{} oc patch application {} -n "$ns" --type merge \ + -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true + fi + done + oc delete argocd argocd-test -n argocd-e2e --timeout=30s --ignore-not-found 2>/dev/null || \ + oc patch argocd argocd-test -n argocd-e2e --type json \ + -p='[{"op":"remove","path":"/metadata/finalizers"}]' --ignore-not-found 2>/dev/null || true + oc delete ns -l e2e.argoproj.io=true --ignore-not-found --wait=false 2>/dev/null || true + oc delete project argocd-e2e argocd-e2e-external argocd-e2e-external-2 \ + --ignore-not-found --wait=false 2>/dev/null || true + for sa in argocd-test-applicationset-controller argocd-test-argocd-application-controller argocd-test-argocd-server; do + oc delete clusterrolebinding "full-admin-${sa}" --ignore-not-found 2>/dev/null || true + done + + exit $exit_code +} +trap cleanup_resources EXIT INT TERM + +wait_for_port_forward() { + local ip=$1 port=$2 + for _ in $(seq 1 30); do + if bash -c "/dev/null; then return 0; fi + sleep 1 + done + return 1 +} + +# --- Clone and compile --- + +git config --global user.name "Tekton Pipeline" +git config --global user.email "tekton@example.com" +git config --global --add safe.directory "*" + +# Detect target cluster architecture for cross-compilation +TARGET_ARCH=$(oc get nodes -o jsonpath='{.items[0].status.nodeInfo.architecture}' 2>/dev/null || echo "amd64") +echo "Target cluster architecture: ${TARGET_ARCH}" + +TAG="${BRANCH}" +if [[ "${BRANCH}" =~ ^v ]]; then + TAG="${BRANCH%%+*}" +fi + +IMAGE_TAG="${TAG}" +if [[ "${IMAGE_TAG}" == "master" || "${IMAGE_TAG}" == "main" ]]; then + IMAGE_TAG="latest" +fi + +PREBUILT_DIR="/prebuilt/argocd-e2e" +PREBUILT_BRANCH=$(cat "${PREBUILT_DIR}/BRANCH" 2>/dev/null || true) +PREBUILT_ARCH=$(cat "${PREBUILT_DIR}/GOARCH" 2>/dev/null || true) + +if [[ -f "${PREBUILT_DIR}/e2e.test" && -f "${PREBUILT_DIR}/argocd" \ + && "${PREBUILT_BRANCH}" == "${TAG}" && "${PREBUILT_ARCH}" == "${TARGET_ARCH}" ]]; then + echo "Using pre-built artifacts for ${TAG}/${TARGET_ARCH}" + mkdir -p "${ARGO_CD_DIR}/dist" + cp "${PREBUILT_DIR}/e2e.test" "${ARGO_CD_DIR}/e2e.test" + cp "${PREBUILT_DIR}/argocd" "${ARGO_CD_DIR}/dist/argocd" + cp "${PREBUILT_DIR}/test-fixtures.tar.gz" "${ARGO_CD_DIR}/test-fixtures.tar.gz" +else + echo "Pre-built artifacts not available (want ${TAG}/${TARGET_ARCH}, have ${PREBUILT_BRANCH:-none}/${PREBUILT_ARCH:-none})" + echo "Cloning argo-cd from ${TEST_REPO_URL} @ ${BRANCH}" + + git clone --depth 1 "${TEST_REPO_URL}" "${ARGO_CD_DIR}" 2>&1 + cd "${ARGO_CD_DIR}" || exit 1 + + if [[ "${BRANCH}" =~ ^v ]]; then + git fetch --depth 1 origin "tags/$TAG" 2>&1 + git checkout FETCH_HEAD 2>&1 + fi + + mkdir -p "${ROOT_DIR}/go-cache" "${ROOT_DIR}/go-mod" + export GOCACHE="${ROOT_DIR}/go-cache" + export GOMODCACHE="${ROOT_DIR}/go-mod" + export GOARCH="${TARGET_ARCH}" + export GOOS="linux" + + # Seed from image-baked caches if available + if [[ -d /usr/local/go-cache/build ]]; then + cp -a /usr/local/go-cache/build/* "${GOCACHE}/" 2>/dev/null || true + fi + if [[ -d /usr/local/go-cache/mod ]]; then + cp -a /usr/local/go-cache/mod/* "${GOMODCACHE}/" 2>/dev/null || true + fi + + # shellcheck source=/dev/null + source /usr/local/bin/go-cache.sh + go_cache_pull "argocd-${TAG}" + + go mod download + + CLIENT_VERSION=$(cat VERSION 2>/dev/null || echo "${TAG}") + CLIENT_VERSION="${CLIENT_VERSION#v}" + + echo "Compiling E2E test binary..." + ( while true; do echo "still compiling..."; sleep 60; done ) & + COMPILE_HEARTBEAT_PID=$! + + if ! go test -c -ldflags "-X github.com/argoproj/argo-cd/v3/common.version=${CLIENT_VERSION}" \ + -o e2e.test ./test/e2e 2>&1 | tee "${RESULTS_DIR}/compile.log"; then + kill "$COMPILE_HEARTBEAT_PID" 2>/dev/null || true + echo "ERROR: test compilation failed" + exit 1 + fi + kill "$COMPILE_HEARTBEAT_PID" 2>/dev/null || true + + go build -ldflags "-X github.com/argoproj/argo-cd/v3/common.version=${CLIENT_VERSION}" \ + -o "${ARGO_CD_DIR}/dist/argocd" ./cmd 2>&1 + + go_cache_push "argocd-${TAG}" +fi + +# --- Configure operator for test namespace (or standalone mode) --- + +DEPLOY_MODE="${DEPLOY_MODE:-operator}" + +OP_NS="" +OP_DEPLOY="" + +# Determine ArgoCD image to use +if [[ -n "${ARGOCD_IMAGE:-}" ]]; then + # ARGOCD_IMAGE explicitly provided (from SNAPSHOT in standalone mode) + echo "Using explicitly provided ArgoCD image: ${ARGOCD_IMAGE}" +elif [[ -n "${OPENSHIFT_VERSION:-}" ]]; then + # Build the Konflux argocd image reference from OPENSHIFT_VERSION + # Using a tag (not a digest) lets the container runtime pick the correct arch + MINOR_VERSION=$(echo "${OPENSHIFT_VERSION}" | grep -oP '^\d+\.\d+') + ARGOCD_IMAGE="quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-rhel9:v${MINOR_VERSION}" + echo "Built ArgoCD image from OPENSHIFT_VERSION: ${ARGOCD_IMAGE}" +else + ARGOCD_IMAGE="quay.io/argoproj/argocd:${IMAGE_TAG}" + echo "Using upstream ArgoCD image: ${ARGOCD_IMAGE}" +fi + +if [[ "${DEPLOY_MODE}" != "standalone" ]]; then + # Operator mode: discover and configure the operator + OP_NS=$(oc get deployment -A -l app.kubernetes.io/name=openshift-gitops-operator \ + -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null || true) + OP_DEPLOY=$(oc get deployment -A -l app.kubernetes.io/name=openshift-gitops-operator \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + + if [[ -n "$OP_NS" && -n "$OP_DEPLOY" ]]; then + oc set env "deployment/$OP_DEPLOY" -n "$OP_NS" \ + DISABLE_DEFAULT_ARGOCD_INSTANCE=true \ + ARGOCD_CLUSTER_CONFIG_NAMESPACES=openshift-gitops,argocd-e2e -c manager || true + oc rollout status "deployment/$OP_DEPLOY" -n "$OP_NS" --timeout=60s || true + fi +fi + +if oc get project argocd-e2e >/dev/null 2>&1; then + oc patch argocd argocd-test -n argocd-e2e -p '{"metadata":{"finalizers":[]}}' \ + --type=merge --ignore-not-found 2>/dev/null || true + oc delete project argocd-e2e --ignore-not-found --wait=true 2>/dev/null +fi +oc new-project argocd-e2e + +oc -n argocd-e2e adm policy add-scc-to-user privileged -z default 2>/dev/null || true +oc adm policy add-cluster-role-to-user cluster-admin -z default -n argocd-e2e 2>/dev/null || true + +# --- Deploy test ArgoCD instance --- + +echo "Deploying test ArgoCD instance (mode: ${DEPLOY_MODE})..." +oc create namespace argocd-e2e-external --dry-run=client -o yaml | oc apply -f - +oc label namespace argocd-e2e-external e2e.argoproj.io=true --overwrite 2>/dev/null || true + +if [[ "${DEPLOY_MODE}" == "standalone" ]]; then + # Standalone mode: deploy ArgoCD from upstream manifests without operator + echo "Standalone mode: deploying ArgoCD from upstream manifests" + + # Apply CRDs first + oc apply -f "${ARGO_CD_DIR}/manifests/crds/" -n argocd-e2e 2>/dev/null || true + + # Prepare install.yaml with our image and namespace + cp "${ARGO_CD_DIR}/manifests/install.yaml" "${ROOT_DIR}/install-patched.yaml" + + # Replace upstream image with SNAPSHOT image + sed -i "s|quay.io/argoproj/argocd:.*|${ARGOCD_IMAGE}|g" "${ROOT_DIR}/install-patched.yaml" + + # Replace namespace references + sed -i '/^ namespace: argocd$/s/argocd/argocd-e2e/' "${ROOT_DIR}/install-patched.yaml" + + # Apply install.yaml + oc apply -f "${ROOT_DIR}/install-patched.yaml" -n argocd-e2e + + # Wait for deployments + echo "Waiting for ArgoCD deployments..." + for _ in {1..30}; do + if oc get deployment argocd-server -n argocd-e2e >/dev/null 2>&1; then break; fi + sleep 2 + done + oc wait --for=condition=Available deployment/argocd-server -n argocd-e2e --timeout=300s || true + oc wait --for=condition=Available deployment/argocd-repo-server -n argocd-e2e --timeout=300s || true + + # Wait for application controller (can be deployment or statefulset) + oc rollout status deployment/argocd-application-controller -n argocd-e2e --timeout=300s 2>/dev/null || \ + oc rollout status statefulset/argocd-application-controller -n argocd-e2e --timeout=300s 2>/dev/null || true + + echo "Standalone ArgoCD deployed successfully" + +else + # Operator mode: deploy via ArgoCD CR + cat </dev/null 2>&1; then break; fi + sleep 2 + done + oc wait --for=condition=Available deployment/argocd-test-server -n argocd-e2e --timeout=180s || true + oc wait --for=condition=Available deployment/argocd-test-repo-server -n argocd-e2e --timeout=120s || true + + # Extract the actual image the operator used + OPERATOR_IMAGE=$(oc get deployment argocd-test-repo-server -n argocd-e2e \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + if [[ -n "$OPERATOR_IMAGE" ]]; then + echo "Overriding ARGOCD_IMAGE with operator-deployed image: ${OPERATOR_IMAGE}" + ARGOCD_IMAGE="${OPERATOR_IMAGE}" + else + echo "WARNING: Could not extract image from argocd-test-repo-server, using: ${ARGOCD_IMAGE}" + fi + + echo "Pausing operator reconciliation..." + oc annotate argocd argocd-test -n argocd-e2e \ + argocd.argoproj.io/operator-pause-reconciliation="true" --overwrite + + echo "Scaling down operators to lock config state..." + oc scale deploy -l app.kubernetes.io/name=openshift-gitops-operator -A --replicas=0 2>/dev/null || true + oc scale deploy -l app.kubernetes.io/part-of=argocd-operator -A --replicas=0 2>/dev/null || true + if [[ -n "$OP_NS" ]]; then + oc scale deploy -n "$OP_NS" --all --replicas=0 2>/dev/null || true + fi + sleep 5 +fi + +oc delete secret -n argocd-e2e -l argocd.argoproj.io/secret-type=cluster --ignore-not-found 2>/dev/null || true + +# Cluster secret name varies by deployment mode +CLUSTER_SECRET_NAME="argocd-test-default-cluster-config" +if [[ "${DEPLOY_MODE}" == "standalone" ]]; then + CLUSTER_SECRET_NAME="argocd-default-cluster-config" +fi + +cat </dev/null || true + oc patch configmap argocd-cmd-params-cm -n argocd-e2e --type merge \ + -p '{"data":{"server.insecure":"true"}}' 2>/dev/null || true +else + # Operator mode: create configmaps + cat </dev/null || true + +# ApplicationSet controller configuration +APPSET_DEPLOY="argocd-test-applicationset-controller" +if [[ "${DEPLOY_MODE}" == "standalone" ]]; then + APPSET_DEPLOY="argocd-applicationset-controller" +fi + +oc scale deployments "${APPSET_DEPLOY}" -n argocd-e2e --replicas=0 2>/dev/null || true +oc set env "deploy/${APPSET_DEPLOY}" -n argocd-e2e \ + -c argocd-applicationset-controller \ + ARGOCD_APPLICATIONSET_CONTROLLER_ALLOWED_SCM_PROVIDERS=https://github.com/ \ + ARGOCD_APPLICATIONSET_CONTROLLER_NAMESPACES=argocd-e2e-external,argocd-e2e 2>/dev/null || true +oc scale deployments "${APPSET_DEPLOY}" -n argocd-e2e --replicas=1 2>/dev/null || true + +cat </dev/null 2>&1; then + oc rollout status "statefulset/${APP_CONTROLLER}" -n argocd-e2e --timeout=300s +else + oc rollout status "deployment/${APP_CONTROLLER}" -n argocd-e2e --timeout=300s +fi +oc rollout status "deployment/${SERVER}" -n argocd-e2e --timeout=300s +oc rollout status "deployment/${APPSET_CONTROLLER}" -n argocd-e2e --timeout=300s + +# --- Set up git server --- + +cat </dev/null) +if [[ -n "$ARGOCD_BIN" ]]; then + echo "Copying argocd CLI from image (${ARGOCD_BIN}) to /tmp/argo-cd/dist/" + oc exec -n argocd-e2e e2e-test-runner -- cp "$ARGOCD_BIN" /tmp/argo-cd/dist/argocd + if ! oc exec -n argocd-e2e e2e-test-runner -- /tmp/argo-cd/dist/argocd version --client --short 2>/dev/null; then + echo "WARNING: argocd from image failed to execute (arch mismatch?), using compiled binary" + oc cp "${ARGO_CD_DIR}/dist/argocd" "argocd-e2e/e2e-test-runner:/tmp/argo-cd/dist/" + fi +else + echo "WARNING: argocd not found in image, falling back to compiled binary" + oc cp "${ARGO_CD_DIR}/dist/argocd" "argocd-e2e/e2e-test-runner:/tmp/argo-cd/dist/" +fi + +# Get a real kubectl for the test runner pod (must match cluster node arch, not pipeline pod arch). +# Pipeline pod is x86_64, cluster nodes are typically arm64 — never copy the pipeline pod's own binaries. +# Try well-known binary paths directly — `command -v` may not work in minimal containers. +KUBECTL_FOUND=false +for bin_candidate in /usr/local/bin/kubectl /usr/bin/kubectl /usr/local/bin/oc /usr/bin/oc; do + if oc exec -n argocd-e2e e2e-test-runner -- test -x "$bin_candidate" 2>/dev/null; then + echo "Found ${bin_candidate} in test runner image, copying to /tmp/bin/kubectl" + oc exec -n argocd-e2e e2e-test-runner -- cp "$bin_candidate" /tmp/bin/kubectl + KUBECTL_FOUND=true + break + fi +done + +if [[ "$KUBECTL_FOUND" != "true" ]]; then + echo "kubectl/oc not found in image, downloading kubectl v1.30.0 for ${TARGET_ARCH}" + curl -fsSL "https://dl.k8s.io/release/v1.30.0/bin/linux/${TARGET_ARCH}/kubectl" \ + -o /tmp/kubectl-download + oc cp /tmp/kubectl-download "argocd-e2e/e2e-test-runner:/tmp/bin/kubectl" + rm -f /tmp/kubectl-download +fi + +oc exec -n argocd-e2e e2e-test-runner -- chmod +x /tmp/bin/kubectl +if ! oc exec -n argocd-e2e e2e-test-runner -- /tmp/bin/kubectl version --client 2>&1; then + echo "ERROR: kubectl at /tmp/bin/kubectl failed to execute" + oc exec -n argocd-e2e e2e-test-runner -- file /tmp/bin/kubectl 2>/dev/null || true + exit 1 +fi + +# --- Verify git push works --- + +( while true; do oc -n argocd-e2e port-forward service/git-server 9418:9418 >/dev/null 2>&1; sleep 1; done ) & +# shellcheck disable=SC2034 # used via ${!pid_var} in cleanup +GIT_FWD_PID=$! +wait_for_port_forward "127.0.0.1" 9418 || exit 1 + +VERIFY_DIR=$(mktemp -d) +cd "$VERIFY_DIR" || exit 1 +git init && touch test-file && git add test-file && git commit -m "verify push" >/dev/null +git push "git://127.0.0.1:9418/testdata.git" master:verify-branch 2>&1 || true + +# --- Get API token --- + +# Admin password secret name and server service vary by deployment mode +if [[ "${DEPLOY_MODE}" == "standalone" ]]; then + ADMIN_SECRET="argocd-initial-admin-secret" + ADMIN_SECRET_KEY="password" + SERVER_SERVICE="argocd-server" + SERVER_PORT="8080" +else + ADMIN_SECRET="argocd-test-cluster" + ADMIN_SECRET_KEY="admin.password" + SERVER_SERVICE="argocd-test-server" + SERVER_PORT="80" +fi + +ADMIN_PASS=$(oc -n argocd-e2e get secrets "${ADMIN_SECRET}" -o jsonpath="{.data.${ADMIN_SECRET_KEY}}" | base64 -d) +( while true; do oc -n argocd-e2e port-forward "service/${SERVER_SERVICE}" "8080:${SERVER_PORT}" >/dev/null 2>&1; sleep 1; done ) & +TEMP_PF_PID=$! +wait_for_port_forward "127.0.0.1" 8080 || exit 1 + +TOKEN_JSON=$(curl -s -X POST -H "Content-Type: application/json" \ + -d "{\"username\":\"admin\",\"password\":\"$ADMIN_PASS\"}" \ + "http://127.0.0.1:8080/api/v1/session") +ARGOCD_AUTH_TOKEN=$(echo "$TOKEN_JSON" | grep -o '"token":"[^"]*"' | sed 's/"token"://g' | tr -d '"') +kill "$TEMP_PF_PID" 2>/dev/null || true +TEMP_PF_PID="" + +# --- Background namespace watcher --- + +( + while true; do + CURRENT_NS=$(oc get secret "${CLUSTER_SECRET_NAME}" -n argocd-e2e \ + -o jsonpath='{.data.namespaces}' 2>/dev/null | base64 -d 2>/dev/null || echo "argocd-e2e") + + for ns in $(oc get ns -l e2e.argoproj.io=true -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + if ! oc get ns "$ns" -o jsonpath='{.metadata.labels}' | grep -q 'argocd.argoproj.io/managed-by'; then + oc label ns "$ns" argocd.argoproj.io/managed-by=argocd-e2e --overwrite >/dev/null 2>&1 || true + fi + if [[ "$CURRENT_NS" != *"$ns"* ]]; then + echo "Whitelisting new test namespace: $ns" + NEW_NS="${CURRENT_NS},${ns}" + oc patch secret "${CLUSTER_SECRET_NAME}" -n argocd-e2e \ + --type='merge' -p="{\"stringData\":{\"namespaces\":\"$NEW_NS\"}}" >/dev/null 2>&1 || true + ( + sleep 2 + for app in $(oc get application -n argocd-e2e -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + oc annotate application "$app" -n argocd-e2e \ + argocd.argoproj.io/refresh="hard" --overwrite >/dev/null 2>&1 || true + done + ) & + fi + done + sleep 1 + done +) & +# shellcheck disable=SC2034 # used via ${!pid_var} in cleanup +PERMISSION_WATCHER_PID=$! + +# --- Run tests --- + +echo "Running all ArgoCD E2E tests (mode: ${DEPLOY_MODE})..." + +# Set env vars based on deployment mode +if [[ "${DEPLOY_MODE}" == "standalone" ]]; then + NAME_PREFIX="" + API_SERVER_URL="https://argocd-server" + ARGOCD_SERVER_ADDR="argocd-server.argocd-e2e.svc.cluster.local:8080" +else + NAME_PREFIX="argocd-test" + API_SERVER_URL="https://argocd-test-server" + ARGOCD_SERVER_ADDR="argocd-test-server.argocd-e2e.svc.cluster.local:80" +fi + +cat < "${ROOT_DIR}/run_test_remote.sh" +#!/usr/bin/env sh + +export PATH=/tmp/argo-cd/dist:/tmp/bin:\$PATH +export NAMESPACE=argocd-e2e +export ARGOCD_E2E_NAMESPACE=\$NAMESPACE +export ARGOCD_E2E_NAME_PREFIX=${NAME_PREFIX} +export ARGOCD_E2E_REMOTE=true +export ARGOCD_E2E_WAIT_TIMEOUT=120 + +export ARGOCD_E2E_SKIP_SETUP=true +export ARGOCD_E2E_REUSE_SERVER=true +export ARGOCD_E2E_APISERVER_URL="${API_SERVER_URL}" + +export ARGOCD_SERVER="${ARGOCD_SERVER_ADDR}" +export ARGOCD_SERVER_INSECURE=true +export ARGOCD_E2E_INSECURE=true + +export ARGOCD_AUTH_TOKEN="$ARGOCD_AUTH_TOKEN" +export ARGOCD_E2E_ADMIN_PASSWORD="$ADMIN_PASS" +export DIST_DIR="/tmp/argo-cd/dist" + +export ARGOCD_E2E_GIT_SERVICE="git://git-server.argocd-e2e.svc.cluster.local:9418/testdata.git" +export ARGOCD_E2E_REPO_DEFAULT="git://git-server.argocd-e2e.svc.cluster.local:9418/testdata.git" + +export ARGOCD_E2E_DIR="/tmp/argo-e2e" + +export ARGOCD_GPG_ENABLED=true +export ARGOCD_E2E_SKIP_GPG=true +export ARGOCD_E2E_SKIP_HELM2=true +export ARGOCD_E2E_SKIP_OPENSHIFT=true +export ARGOCD_E2E_SKIP_KSONNET=true +export GRPC_ENFORCE_ALPN_ENABLED=false +export NO_PROXY="*" + +git config --global user.email "test@example.com" +git config --global user.name "Test Runner" +git config --global --add safe.directory "*" + +cd /tmp/argo-cd/test/e2e + +SKIP_PATTERN="${ARGOCD_E2E_SKIP}" +if [ -n "\$SKIP_PATTERN" ]; then + /tmp/argo-cd/e2e.test -test.v -test.timeout 120m -test.skip "\$SKIP_PATTERN" +else + /tmp/argo-cd/e2e.test -test.v -test.timeout 120m +fi +REMOTE_SCRIPT + +oc cp "${ROOT_DIR}/run_test_remote.sh" "argocd-e2e/e2e-test-runner:/tmp/run_test_remote.sh" +TEST_EXIT_CODE=0 +oc exec -n argocd-e2e e2e-test-runner -- sh /tmp/run_test_remote.sh 2>&1 | tee "${RESULTS_DIR}/argocd-e2e.log" || TEST_EXIT_CODE=$? + +FAIL_COUNT=$(grep -c "^--- FAIL:" "${RESULTS_DIR}/argocd-e2e.log" 2>/dev/null || echo "0") +PASS_COUNT=$(grep -c "^--- PASS:" "${RESULTS_DIR}/argocd-e2e.log" 2>/dev/null || echo "0") + +for component in argocd-test-server application-controller applicationset-controller; do + POD=$(oc get pods -n argocd-e2e 2>/dev/null | grep "$component" | awk '{print $1}' | head -1) + if [[ -n "$POD" ]]; then + echo "--- $component logs (tail) ---" + oc logs -n argocd-e2e "$POD" --tail=50 2>/dev/null | tee "${RESULTS_DIR}/${component}.log" || true + fi +done + +echo "========================================" +echo "ArgoCD E2E results: ${PASS_COUNT} passed, ${FAIL_COUNT} failed (exit code ${TEST_EXIT_CODE})" +echo "========================================" + +if [[ "$TEST_EXIT_CODE" -ne 0 || "$FAIL_COUNT" -gt 0 ]]; then + exit 1 +fi diff --git a/.tekton/test-image/scripts/run-e2e-tests.sh b/.tekton/test-image/scripts/run-e2e-tests.sh new file mode 100644 index 000000000..7d5b81729 --- /dev/null +++ b/.tekton/test-image/scripts/run-e2e-tests.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -x + +# Environment variables expected: +# - TEST_REPO_URL (optional, defaults to the pre-baked repo remote) +# - BRANCH +# - TEST_DIR +# - TIMEOUT +# - PROCS +# - KUBECONFIG + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +CACHE_DIR=$(mktemp -d) +export GOCACHE="${CACHE_DIR}/go-cache" +export GOMODCACHE="${CACHE_DIR}/go-mod" +mkdir -p "$GOCACHE" "$GOMODCACHE" + +oc status + +# --- Ensure argocd CLI is available (some tests call `argocd login` etc.) --- +# Extract the Konflux-built argocd binary from the same image the operator deployed. +# The pipeline pod is x86_64 while cluster nodes are arm64, so we can't copy from +# a running pod — we pull the image for the local arch via oc image extract. +if ! command -v argocd &>/dev/null; then + ARGOCD_IMAGE=$(oc get deployment openshift-gitops-repo-server -n openshift-gitops \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + if [[ -n "$ARGOCD_IMAGE" ]]; then + echo "Extracting argocd CLI from ${ARGOCD_IMAGE}..." + EXTRACT_AUTH_DIR=$(mktemp -d) + oc get secret pull-secret -n openshift-config \ + -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | \ + base64 -d > "${EXTRACT_AUTH_DIR}/config.json" 2>/dev/null || true + + ARGOCD_BIN_DIR=$(mktemp -d) + EXTRACTED=false + for bin_path in /usr/local/bin/argocd /usr/bin/argocd; do + if DOCKER_CONFIG="${EXTRACT_AUTH_DIR}" oc image extract "${ARGOCD_IMAGE}" \ + --path "${bin_path}:${ARGOCD_BIN_DIR}/" --confirm 2>/dev/null; then + if [[ -f "${ARGOCD_BIN_DIR}/argocd" ]]; then + EXTRACTED=true + break + fi + fi + done + rm -rf "${EXTRACT_AUTH_DIR}" + + if [[ "$EXTRACTED" == "true" ]]; then + chmod +x "${ARGOCD_BIN_DIR}/argocd" + if "${ARGOCD_BIN_DIR}/argocd" version --client --short 2>/dev/null; then + export PATH="${ARGOCD_BIN_DIR}:${PATH}" + echo "argocd CLI installed: $(argocd version --client --short)" + else + echo "WARNING: Extracted argocd binary is not executable on this arch" + file "${ARGOCD_BIN_DIR}/argocd" 2>/dev/null || true + rm -rf "${ARGOCD_BIN_DIR}" + fi + else + echo "WARNING: Could not extract argocd binary from ${ARGOCD_IMAGE}" + rm -rf "${ARGOCD_BIN_DIR}" + fi + else + echo "WARNING: openshift-gitops-repo-server not found, argocd CLI unavailable" + fi +fi + +cd /testsuites/gitops-operator/ || exit 1 +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/rh-gitops-release-qa/gitops-operator.git}" +git remote set-url origin "${TEST_REPO_URL}" 2>/dev/null || git remote add origin "${TEST_REPO_URL}" +git fetch origin +git clean -fd +git checkout -B "${BRANCH}" "origin/${BRANCH}" + +# shellcheck source=/dev/null +source /usr/local/bin/go-cache.sh +go_cache_pull "operator-${BRANCH}" + +GINKGO_ARGS=() +if [[ -n "${GINKGO_SKIP:-}" ]]; then + GINKGO_ARGS+=("--skip=${GINKGO_SKIP}") + echo "Skipping tests matching: ${GINKGO_SKIP}" +fi + +# Enable parallel mode only when PROCS > 1 +PARALLEL_FLAG="" +if [[ "${PROCS:-1}" -gt 1 ]]; then + PARALLEL_FLAG="-p" +fi + +TEST_EXIT=0 +/testsuites/gitops-operator/bin/ginkgo -timeout "${TIMEOUT}" ${PARALLEL_FLAG} -procs="${PROCS}" --no-color -v --trace -r \ + "${GINKGO_ARGS[@]}" \ + --junit-report="${RESULTS_DIR}/junit-results.xml" \ + --json-report="${RESULTS_DIR}/test-results.json" \ + "${TEST_DIR}/." || TEST_EXIT=$? + +go_cache_push "operator-${BRANCH}" + +exit $TEST_EXIT diff --git a/.tekton/test-image/scripts/run-parallel-tests.sh b/.tekton/test-image/scripts/run-parallel-tests.sh new file mode 100644 index 000000000..e5a234ade --- /dev/null +++ b/.tekton/test-image/scripts/run-parallel-tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -x + +# Parallel ginkgo tests for the gitops-operator. +# Env vars expected: TEST_REPO_URL, BRANCH, KUBECONFIG + +export TEST_DIR="${TEST_DIR:-./test/openshift/e2e/ginkgo/parallel}" +export PROCS="${PROCS:-4}" +export TIMEOUT="${TIMEOUT:-90m}" + +SKIP_FILE="/usr/local/bin/skip-parallel.txt" +if [[ -f "$SKIP_FILE" ]]; then + SKIP_PATTERN=$(grep -v '^\s*#' "$SKIP_FILE" | grep -v '^\s*$' | paste -sd '|') + if [[ -n "$SKIP_PATTERN" ]]; then + if [[ -n "${GINKGO_SKIP:-}" ]]; then + export GINKGO_SKIP="${GINKGO_SKIP}|${SKIP_PATTERN}" + else + export GINKGO_SKIP="$SKIP_PATTERN" + fi + fi +fi + +/usr/local/bin/run-e2e-tests.sh +exit $? diff --git a/.tekton/test-image/scripts/run-rollouts-tests.sh b/.tekton/test-image/scripts/run-rollouts-tests.sh new file mode 100644 index 000000000..8e2a82471 --- /dev/null +++ b/.tekton/test-image/scripts/run-rollouts-tests.sh @@ -0,0 +1,220 @@ +#!/bin/bash +set -ex + +# Argo Rollouts E2E tests adapted from downstream-CI z-stream pipeline. +# Runs three test suites: +# 1. argo-rollouts-manager E2E (cluster-scoped + namespace-scoped) +# 2. upstream argoproj/argo-rollouts E2E +# 3. rollouts-plugin-trafficrouter-openshift E2E +# +# Env vars expected: KUBECONFIG +# Env vars optional: TEST_REPO_URL, BRANCH (used to resolve commit pins from gitops-operator go.mod) + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +exit_code=0 +failed=0 + +OPERATOR_NAMESPACE=$(oc get deployment openshift-gitops-operator-controller-manager \ + -n openshift-gitops-operator -o jsonpath='{.metadata.namespace}' --ignore-not-found) +OPERATOR_NAMESPACE=${OPERATOR_NAMESPACE:-"openshift-operators"} + +SUBSCRIPTION_NAME=$(oc get subscription -n "$OPERATOR_NAMESPACE" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "openshift-gitops-operator") +echo "Using subscription: ${SUBSCRIPTION_NAME} in ${OPERATOR_NAMESPACE}" + +# --- Helper functions --- + +wait_until_pods_running() { + local ns=$1 + echo "Waiting until operator pods in namespace $ns are ready" + + for _ in {1..150}; do + local pods + pods=$(oc get pods --no-headers -n "$ns" | grep openshift-gitops-operator-controller-manager 2>/dev/null) + local not_running + not_running=$(echo "${pods}" | grep -v Running | grep -vc Completed || true) + if [[ -n "${pods}" && ${not_running} -eq 0 ]]; then + local ready=true + while IFS= read -r pod; do + local current total + current=$(echo "$pod" | awk '{split($2,a,"/"); print a[1]}') + total=$(echo "$pod" | awk '{split($2,a,"/"); print a[2]}') + if [[ "$current" != "$total" || "$current" -lt 1 ]]; then + ready=false + break + fi + done <<< "$pods" + if $ready; then + echo "All pods are up" + return 0 + fi + fi + sleep 2 + done + echo "ERROR: timeout waiting for pods to come up" + return 1 +} + +enable_rollouts_cluster_scoped() { + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type merge --patch '{"spec": {"config": {"env": [{"name": "CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES", "value": "argo-rollouts,test-rom-ns-1,rom-ns-1"}]}}}' + + for _ in {1..30}; do + if oc get deployment openshift-gitops-operator-controller-manager -n "$OPERATOR_NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[0].env}' | grep -q 'CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES'; then + break + fi + echo "Waiting for CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES to be set" + sleep 5 + done + wait_until_pods_running "$OPERATOR_NAMESPACE" +} + +enable_rollouts_namespace_scoped() { + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type merge --patch '{"spec": {"config": {"env": [{"name": "CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES", "value": ""}]}}}' + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type merge --patch '{"spec": {"config": {"env": [{"name": "NAMESPACE_SCOPED_ARGO_ROLLOUTS", "value": "true"}]}}}' + + for _ in {1..30}; do + if oc get deployment openshift-gitops-operator-controller-manager -n "$OPERATOR_NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[0].env}' | grep -q 'NAMESPACE_SCOPED_ARGO_ROLLOUTS'; then + break + fi + echo "Waiting for NAMESPACE_SCOPED_ARGO_ROLLOUTS to be set" + sleep 5 + done + wait_until_pods_running "$OPERATOR_NAMESPACE" +} + +disable_rollouts_config() { + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type json --patch '[{"op": "remove", "path": "/spec/config"}]' || true + wait_until_pods_running "$OPERATOR_NAMESPACE" +} + +cleanup() { + disable_rollouts_config + oc delete rollouts -A --all 2>/dev/null || true + oc delete rolloutmanager -A --all 2>/dev/null || true +} +trap cleanup EXIT + +# --- Resolve commit pins from gitops-operator go.mod --- + +ROLLOUTS_TMP_DIR=$(mktemp -d) +cd "$ROLLOUTS_TMP_DIR" + +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/rh-gitops-release-qa/gitops-operator.git}" +BRANCH="${BRANCH:-master}" + +echo "Resolving rollouts commit pins from ${TEST_REPO_URL} @ ${BRANCH}" +git clone --depth 1 --branch "${BRANCH}" "${TEST_REPO_URL}" gitops-operator-src + +TARGET_ROLLOUT_MANAGER_COMMIT=$(grep 'argoproj-labs/argo-rollouts-manager' \ + gitops-operator-src/go.mod | awk '{print $2}' | sed 's/.*-//' | head -1) + +if [[ -z "$TARGET_ROLLOUT_MANAGER_COMMIT" ]]; then + echo "ERROR: Could not resolve argo-rollouts-manager commit from go.mod" + exit 1 +fi + +echo "argo-rollouts-manager commit: ${TARGET_ROLLOUT_MANAGER_COMMIT}" + +# --- 1. argo-rollouts-manager E2E tests --- + +git clone https://github.com/argoproj-labs/argo-rollouts-manager +cd "$ROLLOUTS_TMP_DIR/argo-rollouts-manager" +git checkout "$TARGET_ROLLOUT_MANAGER_COMMIT" + +TARGET_PLUGIN_COMMIT=$(grep 'rollouts-plugin-trafficrouter-openshift' \ + go.mod | awk '{print $2}' | sed 's/.*-//' | head -1 || true) +if [[ -z "$TARGET_PLUGIN_COMMIT" ]]; then + TARGET_PLUGIN_COMMIT="main" + echo "rollouts-plugin commit: not pinned in go.mod, using main" +else + echo "rollouts-plugin commit (from rollouts-manager go.mod): ${TARGET_PLUGIN_COMMIT}" +fi + +export GOCACHE="${ROLLOUTS_TMP_DIR}/go-cache" +export GOMODCACHE="${ROLLOUTS_TMP_DIR}/go-mod" +mkdir -p "$GOCACHE" "$GOMODCACHE" + +# shellcheck source=/dev/null +source /usr/local/bin/go-cache.sh +go_cache_pull "rollouts-${TARGET_ROLLOUT_MANAGER_COMMIT}" + +enable_rollouts_cluster_scoped + +echo "=== Running cluster-scoped E2E tests ===" +DISABLE_METRICS=true make test-e2e-cluster-scoped 2>&1 | tee "${RESULTS_DIR}/rollout-manager-cluster-scoped.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code + exit_code=0 +fi + +kubectl delete rolloutmanagers --all -n test-rom-ns-1 || true + +enable_rollouts_namespace_scoped + +echo "=== Running namespace-scoped E2E tests ===" +DISABLE_METRICS=true make test-e2e-namespace-scoped 2>&1 | tee "${RESULTS_DIR}/rollout-manager-namespace-scoped.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code + exit_code=0 +fi + +kubectl delete rolloutmanagers --all -n test-rom-ns-1 || true + +# --- 2. Upstream argo-rollouts E2E tests --- + +enable_rollouts_cluster_scoped + +cd "$ROLLOUTS_TMP_DIR/argo-rollouts-manager" + +echo "=== Running upstream argo-rollouts E2E tests ===" +SKIP_RUN_STEP=true hack/run-upstream-argo-rollouts-e2e-tests.sh 2>&1 | tee "${RESULTS_DIR}/argo-rollouts-upstream.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code + exit_code=0 +fi + +# --- 3. rollouts-plugin-trafficrouter-openshift E2E tests --- + +echo "=== Running rollouts OpenShift route plugin E2E tests ===" + +kubectl delete ns argo-rollouts 2>/dev/null || true +kubectl wait --timeout=5m --for=delete namespace/argo-rollouts 2>/dev/null || true +kubectl create ns argo-rollouts +kubectl config set-context --current --namespace=argo-rollouts + +cat <&1 | tee "${RESULTS_DIR}/rollouts-plugin.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code +fi + +# --- Done --- + +go_cache_push "rollouts-${TARGET_ROLLOUT_MANAGER_COMMIT}" + +if [[ $failed != 0 ]]; then + echo "ERROR: One or more rollouts test suites failed" + exit 1 +fi + +echo "All rollouts tests passed" diff --git a/.tekton/test-image/scripts/run-sequential-tests.sh b/.tekton/test-image/scripts/run-sequential-tests.sh new file mode 100644 index 000000000..e9f2751de --- /dev/null +++ b/.tekton/test-image/scripts/run-sequential-tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -x + +# Sequential ginkgo tests for the gitops-operator. +# Env vars expected: TEST_REPO_URL, BRANCH, KUBECONFIG + +export TEST_DIR="${TEST_DIR:-./test/openshift/e2e/ginkgo/sequential}" +export PROCS="${PROCS:-1}" +export TIMEOUT="${TIMEOUT:-240m}" + +SKIP_FILE="/usr/local/bin/skip-sequential.txt" +if [[ -f "$SKIP_FILE" ]]; then + SKIP_PATTERN=$(grep -v '^\s*#' "$SKIP_FILE" | grep -v '^\s*$' | paste -sd '|') + if [[ -n "$SKIP_PATTERN" ]]; then + if [[ -n "${GINKGO_SKIP:-}" ]]; then + export GINKGO_SKIP="${GINKGO_SKIP}|${SKIP_PATTERN}" + else + export GINKGO_SKIP="$SKIP_PATTERN" + fi + fi +fi + +/usr/local/bin/run-e2e-tests.sh +exit $? diff --git a/.tekton/test-image/scripts/run-ui-e2e-tests.sh b/.tekton/test-image/scripts/run-ui-e2e-tests.sh new file mode 100755 index 000000000..1e23bac6c --- /dev/null +++ b/.tekton/test-image/scripts/run-ui-e2e-tests.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +set -u -o pipefail + +# Playwright UI E2E tests for the gitops-operator. +# Runs browser-based tests that verify ArgoCD login via OpenShift SSO. +# +# Unlike the ginkgo-based test scripts, this does NOT exec into run-e2e-tests.sh +# because the test framework is Playwright (Node.js), not Go/ginkgo. +# +# Env vars expected: +# KUBECONFIG - path to cluster kubeconfig +# TEST_REPO_URL - gitops-operator repo URL +# BRANCH - branch/tag containing test/ui-e2e +# Optional: +# CLUSTER_USER - OpenShift username (default: kubeadmin) +# CLUSTER_PASSWORD - OpenShift password (auto-discovered if not set) +# CONSOLE_URL - OpenShift console URL (auto-discovered) +# ARGOCD_URL - ArgoCD server URL (auto-discovered) +# IDP - Identity provider name (default: kube:admin) + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +ROOT_DIR=$(mktemp -d) +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/redhat-developer/gitops-operator.git}" +BRANCH="${BRANCH:-master}" + +# --- Clone test repo --- + +echo "Cloning ${TEST_REPO_URL} @ ${BRANCH}" +git config --global --add safe.directory "*" +git clone --depth 1 --branch "${BRANCH}" "${TEST_REPO_URL}" "${ROOT_DIR}/gitops-operator" 2>&1 + +UI_TEST_DIR="${ROOT_DIR}/gitops-operator/test/ui-e2e" +if [[ ! -d "$UI_TEST_DIR" ]]; then + echo "ERROR: test/ui-e2e directory not found in ${TEST_REPO_URL} @ ${BRANCH}" + exit 1 +fi +cd "${UI_TEST_DIR}" || exit 1 + +# --- Install dependencies --- + +echo "Installing npm dependencies..." +npm ci 2>&1 + +echo "Installing Playwright browser dependencies..." +if command -v dnf &>/dev/null; then + dnf -y install \ + alsa-lib atk at-spi2-atk cups-libs libdrm mesa-libgbm \ + gtk3 nss libXcomposite libXdamage libXrandr pango \ + libxkbcommon libXScrnSaver 2>&1 || true +elif command -v apt-get &>/dev/null; then + npx playwright install-deps chromium 2>&1 || true +fi + +echo "Installing Playwright Chromium..." +npx playwright install chromium 2>&1 + +# --- Discover cluster URLs --- + +if [[ -z "${CONSOLE_URL:-}" ]]; then + CONSOLE_HOST=$(oc get route console -n openshift-console \ + -o jsonpath='{.spec.host}' 2>/dev/null || true) + if [[ -n "$CONSOLE_HOST" ]]; then + CONSOLE_URL="https://${CONSOLE_HOST}" + fi +fi + +if [[ -z "${ARGOCD_URL:-}" ]]; then + ARGOCD_HOST=$(oc get route -n openshift-gitops openshift-gitops-server \ + -o jsonpath='{.spec.host}' 2>/dev/null || true) + if [[ -n "$ARGOCD_HOST" ]]; then + ARGOCD_URL="https://${ARGOCD_HOST}" + fi +fi + +if [[ -z "${ARGOCD_URL:-}" ]]; then + echo "ERROR: Could not discover ArgoCD URL. Set ARGOCD_URL or check the route." + oc get routes -n openshift-gitops 2>/dev/null || true + exit 1 +fi + +if [[ -z "${CONSOLE_URL:-}" ]]; then + echo "WARNING: Could not discover OpenShift Console URL. SSO login tests may fail." +fi + +# --- Get cluster credentials --- + +CLUSTER_USER="${CLUSTER_USER:-kubeadmin}" +if [[ -z "${CLUSTER_PASSWORD:-}" ]]; then + PASS_FILE=$(find /credentials -name "*password" -type f 2>/dev/null | head -1) + if [[ -n "$PASS_FILE" ]]; then + CLUSTER_PASSWORD=$(cat "$PASS_FILE") + echo "Discovered cluster password from ${PASS_FILE}" + fi +fi + +if [[ -z "${CLUSTER_PASSWORD:-}" ]]; then + CLUSTER_PASSWORD=$(oc get secret kubeadmin -n kube-system \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) +fi + +if [[ -z "${CLUSTER_PASSWORD:-}" ]]; then + echo "ERROR: CLUSTER_PASSWORD not set and could not be auto-discovered." + echo "Set CLUSTER_PASSWORD env var, ensure /credentials contains a password file," + echo "or ensure kubeadmin secret exists in kube-system." + exit 1 +fi + +echo "Console URL: ${CONSOLE_URL:-unset}" +echo "ArgoCD URL: ${ARGOCD_URL}" +echo "Cluster user: ${CLUSTER_USER}" + +# --- Handle skip patterns --- + +SKIP_FILE="/usr/local/bin/skip-ui-e2e.txt" +PLAYWRIGHT_EXTRA_ARGS=() +if [[ -f "$SKIP_FILE" ]]; then + SKIP_PATTERN=$(grep -v '^\s*#' "$SKIP_FILE" | grep -v '^\s*$' | paste -sd '|') + if [[ -n "$SKIP_PATTERN" ]]; then + PLAYWRIGHT_EXTRA_ARGS+=(--grep-invert "$SKIP_PATTERN") + echo "Skipping tests matching: ${SKIP_PATTERN}" + fi +fi + +# --- Run Playwright tests --- + +export CONSOLE_URL="${CONSOLE_URL:-}" +export ARGOCD_URL +export CLUSTER_USER +export CLUSTER_PASSWORD +if [[ -z "${IDP:-}" ]]; then + IDP_NAME=$(oc get oauth cluster -o jsonpath='{.spec.identityProviders[0].name}' 2>/dev/null || true) + IDP="${IDP_NAME:-kube:admin}" +fi +export IDP +export CI="konflux" +export PLAYWRIGHT_JUNIT_OUTPUT_NAME="${RESULTS_DIR}/junit-results.xml" + +# Clean stale browser state +rm -f .auth/storageState.json + +echo "Running UI E2E tests..." +npx playwright test \ + --project=chromium \ + --reporter=list,junit \ + "${PLAYWRIGHT_EXTRA_ARGS[@]}" \ + 2>&1 | tee "${RESULTS_DIR}/ui-e2e.log" +TEST_EXIT_CODE=${PIPESTATUS[0]} + +# --- Collect artifacts --- + +for dir in playwright-report test-results; do + if [[ -d "$dir" ]]; then + cp -r "$dir" "${RESULTS_DIR}/" 2>/dev/null || true + fi +done + +if [[ "$TEST_EXIT_CODE" -ne 0 ]]; then + echo "UI E2E tests failed (exit code ${TEST_EXIT_CODE})" + exit 1 +fi + +echo "All UI E2E tests passed." diff --git a/.tekton/test-image/scripts/send-slack-message.py b/.tekton/test-image/scripts/send-slack-message.py new file mode 100644 index 000000000..740ef84ad --- /dev/null +++ b/.tekton/test-image/scripts/send-slack-message.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Send Slack notification with task details for GitOps Catalog E2E tests. + +Environment variables: + SLACK_WEBHOOK_URL - Slack incoming webhook URL + PIPELINE_RUN_NAME - Tekton PipelineRun name + AGGREGATE_STATUS - Overall pipeline status (Succeeded/Failed/etc.) + LOG_URL - Konflux UI link for this pipeline run + QUAY_REPO - OCI repo for log artifacts + QUAY_CREDENTIALS_PATH - Path to .dockerconfigjson for oras + TASK_NAMES - Space-separated task names that have log artifacts +""" +import json +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.request +from datetime import datetime + + +def run_cmd(cmd, timeout=30): + """Run a shell command and return stdout, or None on failure.""" + try: + result = subprocess.run( + cmd, shell=True, capture_output=True, text=True, timeout=timeout + ) + return result.stdout.strip() if result.returncode == 0 else None + except Exception: + return None + + +def get_task_runs(pipeline_run_name): + """Query TaskRuns for timing, status, and result information.""" + raw = run_cmd( + f"oc get taskruns -l tekton.dev/pipelineRun={pipeline_run_name} -o json" + ) + if not raw: + return {} + + try: + data = json.loads(raw) + except json.JSONDecodeError: + return {} + + tasks = {} + for item in data.get("items", []): + labels = item.get("metadata", {}).get("labels", {}) + task_name = labels.get("tekton.dev/pipelineTask", "unknown") + + status = item.get("status", {}) + start_time = status.get("startTime") + completion_time = status.get("completionTime") + + conditions = status.get("conditions", []) + task_status = "Unknown" + if conditions: + reason = conditions[0].get("reason", "") + cond_status = conditions[0].get("status", "") + if cond_status == "True": + task_status = "Succeeded" + elif reason in ("Failed", "TaskRunTimeout"): + task_status = "Failed" + else: + task_status = reason or "Unknown" + + duration = "" + if start_time and completion_time: + try: + start = datetime.fromisoformat(start_time.replace("Z", "+00:00")) + end = datetime.fromisoformat(completion_time.replace("Z", "+00:00")) + total_secs = int((end - start).total_seconds()) + if total_secs >= 3600: + duration = f"{total_secs // 3600}h {(total_secs % 3600) // 60}m {total_secs % 60}s" + elif total_secs >= 60: + duration = f"{total_secs // 60}m {total_secs % 60}s" + else: + duration = f"{total_secs}s" + except Exception: + pass + elif start_time: + duration = "running" + + results = {} + for r in status.get("results", []): + results[r.get("name", "")] = r.get("value", "") + + tasks[task_name] = { + "status": task_status, + "duration": duration, + "results": results, + } + + return tasks + + +def get_failed_task_log_tail(quay_repo, pipeline_run_name, task_name, lines=20): + """Pull a failed task's log artifact and return the tail.""" + ref = f"{quay_repo}:{pipeline_run_name}-task-{task_name}" + tmpdir = f"/tmp/slack-logs-{task_name}" + run_cmd(f"mkdir -p {tmpdir}") + + if run_cmd(f"oras pull --no-tty -o {tmpdir} {ref} 2>/dev/null") is None: + run_cmd(f"rm -rf {tmpdir}") + return None + + log_file = run_cmd(f"find {tmpdir} -name '*.log' -type f 2>/dev/null | head -1") + if not log_file: + run_cmd(f"rm -rf {tmpdir}") + return None + + tail = run_cmd(f"tail -n {lines} {log_file}") + run_cmd(f"rm -rf {tmpdir}") + return tail + + +def build_config_block(task_runs): + """Build a block showing pipeline configuration and actual runtime values.""" + test_script = os.environ.get("TEST_SCRIPT", "") + test_repo_url = os.environ.get("TEST_REPO_URL", "") + test_repo_branch = os.environ.get("TEST_REPO_BRANCH", "") + ocp_requested = os.environ.get("OPENSHIFT_VERSION", "") + operator_channel = os.environ.get("OPERATOR_CHANNEL", "") + fips = os.environ.get("FIPS_ENABLED", "") + + ocp_actual = ( + task_runs.get("provision-cluster", {}).get("results", {}).get("resolvedVersion") + ) + installed_csv = ( + task_runs.get("install-operator", {}).get("results", {}).get("installedCSV") + ) + + lines = [] + if test_script: + lines.append(f"*Test suite:* `{test_script}`") + if test_repo_url: + repo_short = test_repo_url.rstrip("/").rsplit("/", 2)[-2:] + repo_label = "/".join(repo_short).replace(".git", "") + branch_part = f" @ `{test_repo_branch}`" if test_repo_branch else "" + lines.append(f"*Test repo:* `{repo_label}`{branch_part}") + if ocp_requested: + if ocp_actual and ocp_actual != ocp_requested: + lines.append(f"*OpenShift:* `{ocp_requested}` -> `{ocp_actual}`") + else: + lines.append(f"*OpenShift:* `{ocp_requested}`") + if installed_csv: + lines.append(f"*Operator:* `{installed_csv}`") + if operator_channel: + lines.append(f"*Channel:* `{operator_channel}`") + if fips == "true": + lines.append("*FIPS:* enabled") + + if not lines: + return None + + return { + "type": "section", + "text": {"type": "mrkdwn", "text": "\n".join(lines)}, + } + + +def build_blocks( + pipeline_run_name, aggregate_status, log_url, quay_repo, task_runs, loggable_tasks +): + """Build Slack Block Kit blocks for the notification.""" + status_emoji = ( + ":white_check_mark:" if aggregate_status == "Succeeded" else ":x:" + ) + + blocks = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": f"GitOps Catalog E2E: {pipeline_run_name}", + }, + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"{status_emoji} Pipeline finished: *{aggregate_status}*", + }, + }, + ] + + config_block = build_config_block(task_runs) + if config_block: + blocks.append(config_block) + + # Task list with status and timing + if task_runs: + task_order = [ + "parse-metadata", + "build-ginkgo-test-image", + "provision-eaas-space", + "provision-cluster", + "install-operator", + "test-operator", + ] + + lines = [] + failed_tasks = [] + seen = set() + + for name in task_order: + info = task_runs.get(name) + if not info: + continue + seen.add(name) + icon = { + "Succeeded": ":white_check_mark:", + "Failed": ":x:", + }.get(info["status"], ":hourglass_flowing_sand:") + if info["status"] == "Failed": + failed_tasks.append(name) + dur = f" ({info['duration']})" if info["duration"] else "" + lines.append(f"{icon} `{name}`{dur}") + + # Any tasks not in our predefined order + for name, info in sorted(task_runs.items()): + if name in seen: + continue + icon = { + "Succeeded": ":white_check_mark:", + "Failed": ":x:", + }.get(info["status"], ":hourglass_flowing_sand:") + if info["status"] == "Failed": + failed_tasks.append(name) + dur = f" ({info['duration']})" if info["duration"] else "" + lines.append(f"{icon} `{name}`{dur}") + + blocks.append( + { + "type": "section", + "text": {"type": "mrkdwn", "text": "*Tasks:*\n" + "\n".join(lines)}, + } + ) + + # Log tails for failed tasks that have artifacts + for task_name in failed_tasks: + if task_name not in loggable_tasks or not quay_repo: + continue + tail = get_failed_task_log_tail(quay_repo, pipeline_run_name, task_name) + if not tail: + continue + # Slack block text limit is ~3000 chars + if len(tail) > 2800: + tail = tail[-2800:] + blocks.append({"type": "divider"}) + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f":page_facing_up: *`{task_name}` (last 20 lines):*\n```\n{tail}\n```", + }, + } + ) + + # Links + blocks.append({"type": "divider"}) + links_parts = [] + if log_url: + links_parts.append(f":technologist: <{log_url}|View in Konflux UI>") + if quay_repo and pipeline_run_name: + links_parts.append( + f":open_file_folder: `oras pull {quay_repo}:{pipeline_run_name}-logs`" + ) + if links_parts: + blocks.append( + { + "type": "section", + "text": {"type": "mrkdwn", "text": "\n".join(links_parts)}, + } + ) + + return blocks + + +def send_slack_message(webhook_url, blocks, fallback_text): + """Post a Slack message via incoming webhook.""" + msg = {"text": fallback_text, "blocks": blocks} + req = urllib.request.Request( + webhook_url, + data=json.dumps(msg).encode(), + headers={"Content-type": "application/json"}, + method="POST", + ) + try: + resp = urllib.request.urlopen(req, timeout=10) + return resp.read().decode() + except Exception as e: + logging.error(f"Failed to send Slack message: {e}") + return "" + + +def main(): + webhook_url = os.environ.get("SLACK_WEBHOOK_URL", "") + pipeline_run_name = os.environ.get("PIPELINE_RUN_NAME", "") + aggregate_status = os.environ.get("AGGREGATE_STATUS", "Unknown") + log_url = os.environ.get("LOG_URL", "") + quay_repo = os.environ.get("QUAY_REPO", "") + quay_creds = os.environ.get("QUAY_CREDENTIALS_PATH", "") + task_names_str = os.environ.get("TASK_NAMES", "") + + if not webhook_url: + logging.error("SLACK_WEBHOOK_URL is not set") + return 1 + + # Setup oras credentials + if quay_creds and os.path.isfile(quay_creds): + tmpdir = tempfile.mkdtemp() + shutil.copy2(quay_creds, os.path.join(tmpdir, "config.json")) + os.environ["DOCKER_CONFIG"] = tmpdir + + loggable_tasks = set(task_names_str.split()) if task_names_str else set() + task_runs = get_task_runs(pipeline_run_name) + + blocks = build_blocks( + pipeline_run_name, aggregate_status, log_url, quay_repo, task_runs, loggable_tasks + ) + + fallback = f"GitOps Catalog E2E {pipeline_run_name}: {aggregate_status}" + ret = send_slack_message(webhook_url, blocks, fallback) + if ret: + print(ret) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tekton/test-image/scripts/verify-images.sh b/.tekton/test-image/scripts/verify-images.sh new file mode 100644 index 000000000..65c3affb7 --- /dev/null +++ b/.tekton/test-image/scripts/verify-images.sh @@ -0,0 +1,162 @@ +#!/bin/bash +set -euo pipefail + +# Verify that all images referenced by the installed CSV are actually +# available at their mirror locations (from IDMS on the cluster). +# +# Environment variables expected: +# - KUBECONFIG +# - NAMESPACE (operator namespace, e.g. openshift-gitops-operator) +# +# Optional: +# - TARGET_ARCH (e.g. arm64, amd64; auto-detected from cluster if unset) +# - IDMS_FILE (path to images-mirror-set.yaml; falls back to cluster IDMS) + +NAMESPACE="${NAMESPACE:-openshift-gitops-operator}" + +# --- Detect target architecture --- +if [[ -z "${TARGET_ARCH:-}" ]]; then + TARGET_ARCH=$(oc get nodes -o jsonpath='{.items[0].status.nodeInfo.architecture}' 2>/dev/null || echo "amd64") +fi +echo "Target architecture: ${TARGET_ARCH}" + +# --- Get installed CSV name --- +CSV_NAME=$(oc get subscription -n "${NAMESPACE}" -o jsonpath='{.items[0].status.installedCSV}' 2>/dev/null || true) +if [[ -z "$CSV_NAME" ]]; then + echo "ERROR: No installed CSV found in namespace ${NAMESPACE}" + exit 1 +fi +echo "Installed CSV: ${CSV_NAME}" + +# --- Extract related images from CSV --- +RELATED_IMAGES=$(oc get csv "${CSV_NAME}" -n "${NAMESPACE}" \ + -o jsonpath='{range .spec.relatedImages[*]}{.image}{"\n"}{end}' 2>/dev/null) + +if [[ -z "$RELATED_IMAGES" ]]; then + echo "WARNING: No relatedImages found in CSV ${CSV_NAME}" + exit 0 +fi + +IMAGE_COUNT=$(echo "$RELATED_IMAGES" | wc -l) +echo "Found ${IMAGE_COUNT} related images in CSV" + +# --- Build mirror map from IDMS --- +declare -A MIRROR_MAP +if [[ -n "${IDMS_FILE:-}" && -f "${IDMS_FILE}" ]]; then + echo "Loading mirrors from file: ${IDMS_FILE}" + while IFS='|' read -r source mirror; do + MIRROR_MAP["$source"]="$mirror" + done < <(python3 -c " +import yaml, sys +with open('${IDMS_FILE}') as f: + data = yaml.safe_load(f) +for entry in data['spec']['imageDigestMirrors']: + print(entry['source'] + '|' + entry['mirrors'][0]) +") +else + echo "Loading mirrors from cluster IDMS..." + while IFS='|' read -r source mirror; do + [[ -n "$source" ]] && MIRROR_MAP["$source"]="$mirror" + done < <(oc get imagedigestmirrorset -o json 2>/dev/null | python3 -c " +import json, sys +data = json.load(sys.stdin) +for item in data.get('items', []): + for entry in item.get('spec', {}).get('imageDigestMirrors', []): + mirrors = entry.get('mirrors', []) + if mirrors: + print(entry['source'] + '|' + mirrors[0]) +" 2>/dev/null || true) +fi + +if [[ ${#MIRROR_MAP[@]} -eq 0 ]]; then + echo "WARNING: No mirror mappings found (no IDMS on cluster and no IDMS_FILE set)" + echo "Images will be pulled directly from source registries" +fi + +echo "" +echo "Mirror mappings loaded: ${#MIRROR_MAP[@]} entries" + +# --- Set up auth for skopeo --- +AUTH_ARGS="" +if [[ -f "/quay-pull-credentials/.dockerconfigjson" ]]; then + AUTH_ARGS="--authfile=/quay-pull-credentials/.dockerconfigjson" +elif [[ -f "/quay-credentials/.dockerconfigjson" ]]; then + AUTH_ARGS="--authfile=/quay-credentials/.dockerconfigjson" +fi + +# --- Verify each image --- +FAILED=0 +PASSED=0 +SKIPPED=0 + +for IMAGE in $RELATED_IMAGES; do + # Extract registry/repo and digest + REPO="${IMAGE%%@*}" + DIGEST="${IMAGE##*@}" + + # Find mirror for this repo + MIRROR_REPO="" + for SOURCE in "${!MIRROR_MAP[@]}"; do + if [[ "$REPO" == "$SOURCE" ]]; then + MIRROR_REPO="${MIRROR_MAP[$SOURCE]}" + break + fi + done + + if [[ -n "$MIRROR_REPO" ]]; then + CHECK_REF="${MIRROR_REPO}@${DIGEST}" + LABEL="mirror" + else + # No mirror configured — this is a standard Red Hat image (redis, haproxy, etc.) + # not part of the Konflux release. Skip verification. + echo " SKIP [no mirror] ${REPO}" + SKIPPED=$((SKIPPED + 1)) + continue + fi + + # Check if image exists at mirror + if skopeo inspect --raw ${AUTH_ARGS} "docker://${CHECK_REF}" &>/dev/null; then + # Check architecture support + MANIFEST_TYPE=$(skopeo inspect --raw ${AUTH_ARGS} "docker://${CHECK_REF}" 2>/dev/null | python3 -c " +import json, sys +m = json.load(sys.stdin) +mt = m.get('mediaType', m.get('schemaVersion', '')) +if 'manifest.list' in str(mt) or 'image.index' in str(mt): + archs = [p.get('platform', {}).get('architecture', '?') for p in m.get('manifests', [])] + print('multi:' + ','.join(archs)) +else: + print('single') +" 2>/dev/null || echo "unknown") + + if [[ "$MANIFEST_TYPE" == single ]] || [[ "$MANIFEST_TYPE" == unknown ]]; then + echo " OK [${LABEL}] ${REPO} (single-arch manifest)" + PASSED=$((PASSED + 1)) + elif [[ "$MANIFEST_TYPE" == multi:* ]]; then + ARCHS="${MANIFEST_TYPE#multi:}" + if echo "$ARCHS" | grep -q "${TARGET_ARCH}"; then + echo " OK [${LABEL}] ${REPO} (${TARGET_ARCH} in: ${ARCHS})" + PASSED=$((PASSED + 1)) + else + echo " FAIL [${LABEL}] ${REPO} — missing ${TARGET_ARCH} (available: ${ARCHS})" + echo " ref: ${CHECK_REF}" + FAILED=$((FAILED + 1)) + fi + fi + else + echo " FAIL [${LABEL}] ${REPO} — image not found at mirror" + echo " mirror: ${CHECK_REF}" + echo " source: ${IMAGE}" + FAILED=$((FAILED + 1)) + fi +done + +echo "" +echo "==========================================" +echo "Image verification: ${PASSED} OK, ${FAILED} FAILED, ${SKIPPED} SKIPPED (no mirror)" +echo "==========================================" + +if [[ $FAILED -gt 0 ]]; then + echo "ERROR: ${FAILED} image(s) are not available at their expected locations." + echo "The operator will fail to create workload pods for these images." + exit 1 +fi diff --git a/.tekton/test-image/skip-argocd.txt b/.tekton/test-image/skip-argocd.txt new file mode 100644 index 000000000..d2a3f1fc1 --- /dev/null +++ b/.tekton/test-image/skip-argocd.txt @@ -0,0 +1,183 @@ +# Tests to skip when running upstream ArgoCD E2E tests on EaaS HyperShift clusters. +# One test function name per line. Blank lines and lines starting with # are ignored. +# These are combined into a Go -test.skip regex via '|'. +# +# Source: release pipeline run argocd-e2e-tests-120-414 (v1.20 / OCP 4.14) +# Tests are grouped by failure category. + +# ============================================================================ +# Account tests — configmap propagation race +# The server restarts when argocd-cm is patched to add accounts, and tests call +# update-password before the restart finishes. TestCreateAndUseAccount also +# panics with a nil pointer dereference, killing the entire test binary. +# ============================================================================ +TestCreateAndUseAccount +TestCanIGetLogs +TestAccountSessionToken + +# ============================================================================ +# Tests that fail in the release pipeline (65 failures) +# Skipping these to match the release pipeline's baseline. +# ============================================================================ + +# --- CMP (Config Management Plugin) — requires sidecar containers --- +TestCMPDiscoverWithFileName +TestCMPDiscoverWithFindCommandWithEnv +TestCMPDiscoverWithFindGlob +TestCMPDiscoverWithPluginName +TestCMPWithSymlinkFiles +TestCMPWithSymlinkFolder +TestCMPWithSymlinkPartialFiles +TestPreserveFileModeForCMP +TestPruneResourceFromCMP + +# --- Helm repo / OCI — require authenticated registry or TLS --- +TestAddHelmRepoInsecureSkipVerify +TestAddRemoveHelmRepo +TestHelmRepo +TestHelmRepoDiffLocal +TestHelmWithDependencies +TestHelmWithMultipleDependencies +TestOCIImage +TestOCIImageWithOutOfBoundsSymlink +TestOCIWithAuthedOCIHelmRegistryDeps +TestOCIWithOCIHelmRegistryDependencies + +# --- Hydrator — requires authenticated HTTPS git --- +TestHydrateTo +TestHydratorNoOp +TestHydratorWithAuthenticatedRepo +TestHydratorWithDirectory +TestHydratorWithHelm +TestHydratorWithKustomize +TestHydratorWithPlugin +TestSimpleHydrator + +# --- SSH / private repos --- +TestCanAddAppFromPrivateRepoWithCredCfg +TestCustomToolWithSSHGitCreds +TestCustomToolWithSSHGitCredsDisabled +TestGetRepoWithInheritedCreds +TestGitGeneratorPrivateRepoWithTemplatedProjectAndProjectScopedRepo + +# --- Server-side diff — known failures --- +TestServerSideDiffCommand +TestServerSideDiffErrorHandling +TestServerSideDiffWithLocal +TestServerSideDiffWithRevision +TestServerSideDiffWithSyncedApp +TestResourceDiffing +TestHookDiff + +# --- ApplicationSet generators — require external API access --- +TestClusterMatrixGenerator +TestClusterMergeGenerator +TestListMatrixGenerator +TestListMergeGenerator +TestMatrixTerminalMatrixGeneratorSelector +TestMatrixTerminalMergeGeneratorSelector +TestMergeTerminalMergeGeneratorSelector +TestSimplePullRequestGenerator +TestSimplePullRequestGeneratorGoTemplate +TestSimpleSCMProviderGenerator +TestSimpleSCMProviderGeneratorGoTemplate +TestSimpleSCMProviderGeneratorTokenRefStrictKo +TestSimpleSCMProviderGeneratorTokenRefStrictOk +TestSimpleListGeneratorExternalNamespaceNoConflict + +# --- Misc failures --- +TestAddingApp +TestClusterDelete +TestConfigMap +TestDuplicatedClusterResourcesAnnotationTracking +TestDuplicatedResources +TestKubectlMetrics +TestManagedByURLFallbackToCurrentInstance +TestManagedByURLWithAnnotation +TestMaskSecretValues +TestMaskValuesInInvalidSecret +TestNamespacedAppWithSecrets +TestNotificationsHealthcheck + +# --- Disabled in release pipeline via source rename --- +TestSimpleGitFilesPreserveResourcesOnDeletion + +# ============================================================================ +# Tests already skipped in the release pipeline via SkipOnEnv (57 skipped) +# Adding here for completeness — prevents wasted time if env-based skip fails. +# ============================================================================ + +# --- GPG signing --- +TestSyncToSignedBranchWithKnownKey +TestSyncToSignedBranchWithUnknownKey +TestSyncToSignedCommitWithKnownKey +TestSyncToSignedCommitWithoutKnownKey +TestSyncToSignedTagWithKnownKey +TestSyncToSignedTagWithUnknownKey +TestSyncToUnsignedBranch +TestSyncToUnsignedCommit +TestSyncToUnsignedTag +TestSimpleGitDirectoryGeneratorGPGEnabledUnsignedCommits +TestSimpleGitDirectoryGeneratorGPGEnabledWithoutKnownKeys +TestSimpleGitFilesGeneratorGPGEnabledUnsignedCommits +TestSimpleGitFilesGeneratorGPGEnabledWithoutKnownKeys +TestNamespacedSyncToSignedCommitKWKK +TestNamespacedSyncToSignedCommitWKK +TestNamespacedSyncToUnsignedCommit + +# --- Helm OCI (patched with SkipOnEnv in release pipeline) --- +TestHelmOCIRegistry +TestHelmOCIRegistryWithDependencies +TestGitWithHelmOCIRegistryDependencies +TestTemplatesGitWithHelmOCIDependencies +TestTemplatesHelmOCIWithDependencies + +# --- Custom tools (patched with SkipOnEnv in release pipeline) --- +TestCustomToolSyncAndDiffLocal +TestCustomToolWithEnv +TestCustomToolWithGitCreds +TestCustomToolWithGitCredsTemplate +TestKustomizeVersionOverride + +# --- App management --- +TestAppWithSecrets +TestAutomaticallyNamingUnnamedHook +TestSyncWithSkipHook +TestSyncOptionsValidateTrue +TestImmutableChange +TestNamespacedImmutableChange + +# --- Namespace management --- +TestNamespaceAutoCreation +TestNamespaceCreationWithSSA +TestNamespacedNamespaceAutoCreation +TestNamespacedNamespaceAutoCreationWithMetadata +TestNamespacedNamespaceAutoCreationWithMetadataAndNsManifest +TestNamespacedNamespaceAutoCreationWithPreexistingNs + +# --- Logs --- +TestAppLogs +TestGetLogsAllow +TestGetLogsDeny +TestNamespacedAppLogs +TestNamespacedGetLogsAllowNS +TestNamespacedGetLogsDeny + +# --- Resource listing / orphaned --- +TestListResource +TestNamespacedListResource +TestOrphanedResource +TestNamespacedOrphanedResource +TestCRDs + +# --- ApplicationSet progressive sync --- +TestApplicationSetProgressiveSyncStep +TestNoApplicationStatusWhenNoApplications +TestNoApplicationStatusWhenNoSteps +TestProgressiveSyncHealthGating +TestProgressiveSyncMultipleAppsPerStep + +# --- Git submodules --- +TestGitSubmoduleHTTPSSupport +TestGitSubmoduleRemovalSupport +TestGitSubmoduleSSHSupport diff --git a/.tekton/test-image/skip-parallel.txt b/.tekton/test-image/skip-parallel.txt new file mode 100644 index 000000000..b58d7c528 --- /dev/null +++ b/.tekton/test-image/skip-parallel.txt @@ -0,0 +1,12 @@ +# Tests to skip when running parallel e2e tests. +# One regex pattern per line. Blank lines and lines starting with # are ignored. +# These are passed to ginkgo --skip via a combined '|' regex. + +# SSO test expects status "Failed" on OIDC clusters but operator auto-configures Dex successfully (status "Running"). +# Fails intermittently depending on whether the provisioned cluster has OIDC external auth. +# Needs upstream fix in rh-gitops-release-qa/gitops-operator 1-050_validate_sso_test.go +ensures the conditions in status when external Authentication is enabled on clusters + +# Toolchain version snapshot is hardcoded in the test and won't match Konflux-built +# operator images which may carry a newer argocd version than the upstream snapshot. +1-031_validate_toolchain diff --git a/.tekton/test-image/skip-sequential.txt b/.tekton/test-image/skip-sequential.txt new file mode 100644 index 000000000..55cf86344 --- /dev/null +++ b/.tekton/test-image/skip-sequential.txt @@ -0,0 +1,37 @@ +# Tests to skip when running sequential e2e tests on EaaS HyperShift clusters. +# One regex pattern per line. Blank lines and lines starting with # are ignored. +# These are passed to ginkgo --skip via a combined '|' regex. + +# MachineConfig tests require MCO which is not available on HyperShift. +# Application stuck OutOfSync because MachineConfig resources cannot be applied. +1-006_validate_machine_config + +# ArgoCD CLI login fails on ephemeral HyperShift clusters due to route/TLS issues. +1-064_validate_tcp_reset_error + +# Route TLS passthrough verification fails — route does not converge to expected +# state after updating ArgoCD CR on HyperShift. +1-111_validate_default_argocd_route + +# NodeSelector test sets key1:value1 but EaaS HyperShift nodes have no custom labels, +# causing all pods to get stuck in FailedScheduling. +1-071_validate_node_selectors + +# Agent/principal test requires LoadBalancer ingress which is not available on EaaS +# clusters. The argocd-hub-agent-principal deployment is never created. +1-053_validate_argocd_agent_principal_connected + +# Pod-security labels (pod-security.kubernetes.io/enforce: restricted) are not +# applied to the openshift-gitops namespace on HyperShift clusters. +1-110_validate_podsecurity_alerts + +# AllowManagedBy=false is not enforced on HyperShift — the app stays Synced +# instead of going OutOfSync. The AfterEach cleanup then hangs forever waiting +# for the ALLOW_NAMESPACE_MANAGEMENT_IN_NAMESPACE_SCOPED_INSTANCES env var to +# be removed from the operator deployment, consuming all remaining pipeline time. +1-113_validate_namespacemanagement + +# The operator bundles the openshift route plugin as a sidecar (file:/plugins/...), +# but the test expects a GitHub release download URL. This is a packaging difference, +# not a bug. +1-112_validate_rollout_plugin_support diff --git a/.tekton/test-image/skip-ui-e2e.txt b/.tekton/test-image/skip-ui-e2e.txt new file mode 100644 index 000000000..37efa4063 --- /dev/null +++ b/.tekton/test-image/skip-ui-e2e.txt @@ -0,0 +1,3 @@ +# Tests to skip when running UI E2E tests on EaaS HyperShift clusters. +# One regex pattern per line. Blank lines and lines starting with # are ignored. +# These are passed to playwright --grep-invert as a combined '|' regex. diff --git a/containers/argocd/Dockerfile b/containers/argocd/Dockerfile index e9ed4999c..e4fb3c427 100644 --- a/containers/argocd/Dockerfile +++ b/containers/argocd/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2021 Red Hat +#release-1.20 Copyright 2021 Red Hat # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/containers/gitops-operator-bundle/Dockerfile b/containers/gitops-operator-bundle/Dockerfile index 073cf50f8..7c7dac0dd 100644 --- a/containers/gitops-operator-bundle/Dockerfile +++ b/containers/gitops-operator-bundle/Dockerfile @@ -11,6 +11,7 @@ LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.35.0 LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 LABEL operators.operatorframework.io.metrics.project_layout=go.kubebuilder.io/v4 +# Labels for testing. # Labels for testing. LABEL operators.operatorframework.io.test.mediatype.v1=scorecard+v1 LABEL operators.operatorframework.io.test.config.v1=tests/scorecard/ @@ -28,4 +29,4 @@ LABEL \ com.redhat.component="openshift-gitops-operator-bundle-container" \ description="Red Hat OpenShift GitOps Operator Bundle" \ distribution-scope="restricted" \ - io.k8s.description="Red Hat OpenShift GitOps Operator Bundle" \ No newline at end of file + io.k8s.description="Red Hat OpenShift GitOps Operator Bundle"