From 122edf4627d261194dbdbd72e791a46d77f37bbe Mon Sep 17 00:00:00 2001 From: stxkxs <139715017+stxkxs@users.noreply.github.com> Date: Sun, 9 Aug 2026 10:51:07 -0700 Subject: [PATCH] fix: 28 ApplicationSets pinned main while app-of-apps tracked the revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cluster-bootstrap` stamps two sibling annotations on every ArgoCD cluster Secret, `gitops/repo-url` and `gitops/repo-branch`, and the comment above them says appsets template their own source off these so no org is hardcoded in the manifests. That was true of repo-url — 33 files read it. repo-branch had exactly one reference in the whole org: the line that writes it. Meanwhile 28 applied ApplicationSets carried `targetRevision: main` as a literal, directly beneath a repoURL templated off the annotation. So a cluster bootstrapped against any other revision ran app-of-apps at that revision — the Application does honour var.gitops_repo_branch — and every child Application at main. Both report Synced and Healthy: ArgoCD is telling the truth about each Application individually, and the cluster is running two revisions of the catalog at once with nothing anywhere to say so. Two things that makes impossible. Testing a catalog change on a branch: the cluster syncs main's values while app-of-apps displays the branch name, which is exactly the evidence someone would check. And pinning a release: a platform installed from a tag gets a tagged app-of-apps over an unpinned fleet, so "deploy the known-good version" deploys HEAD. Every catalog source now reads the annotation. Chart sources are untouched and must be — their repoURL is a Helm or OCI registry and their targetRevision is a chart version, a different thing sharing a field name. The render fixtures gain the annotation, which is load-bearing rather than housekeeping: the appsets run with goTemplateOptions missingkey=error, so a cluster Secret without it now fails the render instead of silently defaulting. `scripts/check-catalog-revision.py` holds it. It parses each applied ApplicationSet, finds the sources whose repoURL resolves from gitops/repo-url, and asserts their targetRevision resolves from gitops/repo-branch. It also fails when it finds ZERO catalog sources — every addon gets its values files through one, so none means the parse broke rather than the repo being clean. --- .github/workflows/ci.yml | 16 ++ applicationsets/addons-agent-operator.yaml | 4 +- .../addons-ai-platform-kustomize.yaml | 2 +- applicationsets/addons-ai-platform.yaml | 2 +- applicationsets/addons-argo-platform.yaml | 2 +- applicationsets/addons-argo-workflows.yaml | 2 +- .../addons-bootstrap-kustomize.yaml | 2 +- applicationsets/addons-bootstrap.yaml | 2 +- applicationsets/addons-external-dns.yaml | 2 +- applicationsets/addons-karpenter.yaml | 2 +- applicationsets/addons-loki.yaml | 2 +- applicationsets/addons-networking.yaml | 2 +- applicationsets/addons-observability.yaml | 2 +- applicationsets/addons-opencost.yaml | 2 +- applicationsets/addons-operations-helm.yaml | 2 +- .../addons-operations-kustomize.yaml | 2 +- applicationsets/addons-otel-agent.yaml | 2 +- .../addons-otel-gateway-floor.yaml | 2 +- applicationsets/addons-otel-gateway.yaml | 2 +- applicationsets/addons-security.yaml | 2 +- applicationsets/addons-tempo.yaml | 2 +- applicationsets/addons-velero.yaml | 2 +- applicationsets/agent-platform.yaml | 2 +- applicationsets/dashboards.yaml | 2 +- applicationsets/kyverno-policies.yaml | 2 +- applicationsets/opt-in/apps-tenants.yaml | 4 +- applicationsets/opt-in/clusters-appset.yaml | 2 +- applicationsets/portal-tenants.yaml | 2 +- .../argo-workflows-cluster-no-bucket.yaml | 1 + .../argo-workflows-cluster-with-bucket.yaml | 1 + .../rendertest/testdata/cluster-adopt.yaml | 1 + .../rendertest/testdata/cluster-create.yaml | 1 + .../rendertest/testdata/cluster-nolabel.yaml | 1 + .../observability-cluster-no-buckets.yaml | 1 + .../observability-cluster-with-buckets.yaml | 1 + applicationsets/secret-stores.yaml | 2 +- scripts/check-catalog-revision.py | 145 ++++++++++++++++++ 37 files changed, 198 insertions(+), 30 deletions(-) create mode 100755 scripts/check-catalog-revision.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aea1e2a..a883011 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,21 @@ jobs: - name: Every platform CR the catalog declares would be admitted run: ./scripts/check-platform-crs.py --list + catalog-revision: + name: a catalog source reads its revision, never pins one + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Install pyyaml + run: pip install pyyaml + # cluster-bootstrap stamps gitops/repo-branch beside gitops/repo-url. Only + # app-of-apps ever read it, so a cluster bootstrapped against any other + # revision ran app-of-apps there and every child Application on main — two + # revisions of the catalog at once, both reporting Synced and Healthy. + - name: Catalog sources read gitops/repo-branch + run: ./scripts/check-catalog-revision.py --list + kyverno: name: Kyverno policy tests runs-on: ubuntu-latest @@ -610,6 +625,7 @@ jobs: dashboards, serviceaccount-bindings, platform-crs, + catalog-revision, kyverno, fork-safety, helm-render, diff --git a/applicationsets/addons-agent-operator.yaml b/applicationsets/addons-agent-operator.yaml index 5e8dc17..0070dd5 100644 --- a/applicationsets/addons-agent-operator.yaml +++ b/applicationsets/addons-agent-operator.yaml @@ -45,7 +45,7 @@ spec: project: platform sources: # A published chart at a version, not a branch. Rendering from - # `targetRevision: main` meant there was no release boundary at all + # `targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}'` meant there was no release boundary at all # between a merge upstream and every cluster this ApplicationSet syncs — # a chart change reached production the moment it landed, and a chart # could ship a CRD field the operator image it names does not @@ -90,7 +90,7 @@ spec: - $values/addons/ai-platform/operator/values.yaml - $values/addons/ai-platform/operator/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-ai-platform-kustomize.yaml b/applicationsets/addons-ai-platform-kustomize.yaml index 5bad145..2fd19a6 100644 --- a/applicationsets/addons-ai-platform-kustomize.yaml +++ b/applicationsets/addons-ai-platform-kustomize.yaml @@ -46,7 +46,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}/overlays/{{ index .metadata.labels "environment" }}' destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-ai-platform.yaml b/applicationsets/addons-ai-platform.yaml index 5fc6bdb..4d4671e 100644 --- a/applicationsets/addons-ai-platform.yaml +++ b/applicationsets/addons-ai-platform.yaml @@ -76,7 +76,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-argo-platform.yaml b/applicationsets/addons-argo-platform.yaml index f0013da..a4c6fd2 100644 --- a/applicationsets/addons-argo-platform.yaml +++ b/applicationsets/addons-argo-platform.yaml @@ -64,7 +64,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-argo-workflows.yaml b/applicationsets/addons-argo-workflows.yaml index 420e271..4b7dd25 100644 --- a/applicationsets/addons-argo-workflows.yaml +++ b/applicationsets/addons-argo-workflows.yaml @@ -86,7 +86,7 @@ spec: - $values/addons/argo-platform/argo-workflows/values.yaml - $values/addons/argo-platform/argo-workflows/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-bootstrap-kustomize.yaml b/applicationsets/addons-bootstrap-kustomize.yaml index 6745c23..f7cd020 100644 --- a/applicationsets/addons-bootstrap-kustomize.yaml +++ b/applicationsets/addons-bootstrap-kustomize.yaml @@ -44,7 +44,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}/overlays/{{ index .metadata.labels "environment" }}' destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-bootstrap.yaml b/applicationsets/addons-bootstrap.yaml index 9c89da8..55f3d4e 100644 --- a/applicationsets/addons-bootstrap.yaml +++ b/applicationsets/addons-bootstrap.yaml @@ -75,7 +75,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-external-dns.yaml b/applicationsets/addons-external-dns.yaml index 1a9b93b..ac37a4b 100644 --- a/applicationsets/addons-external-dns.yaml +++ b/applicationsets/addons-external-dns.yaml @@ -63,7 +63,7 @@ spec: - $values/addons/networking/external-dns/values.yaml - $values/addons/networking/external-dns/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-karpenter.yaml b/applicationsets/addons-karpenter.yaml index c9fb371..49aa94e 100644 --- a/applicationsets/addons-karpenter.yaml +++ b/applicationsets/addons-karpenter.yaml @@ -71,7 +71,7 @@ spec: - $values/addons/operations/karpenter/values.yaml - $values/addons/operations/karpenter/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-loki.yaml b/applicationsets/addons-loki.yaml index 09ab943..1a0fba9 100644 --- a/applicationsets/addons-loki.yaml +++ b/applicationsets/addons-loki.yaml @@ -86,7 +86,7 @@ spec: - $values/addons/observability/loki/values.yaml - $values/addons/observability/loki/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-networking.yaml b/applicationsets/addons-networking.yaml index 3b4f8b1..969529f 100644 --- a/applicationsets/addons-networking.yaml +++ b/applicationsets/addons-networking.yaml @@ -65,7 +65,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-observability.yaml b/applicationsets/addons-observability.yaml index d99742d..eb6f47f 100644 --- a/applicationsets/addons-observability.yaml +++ b/applicationsets/addons-observability.yaml @@ -73,7 +73,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-opencost.yaml b/applicationsets/addons-opencost.yaml index afae3aa..6cecd16 100644 --- a/applicationsets/addons-opencost.yaml +++ b/applicationsets/addons-opencost.yaml @@ -63,7 +63,7 @@ spec: region: "{{ index .metadata.labels \"region\" }}" host: "aps-workspaces.{{ index .metadata.labels \"region\" }}.amazonaws.com" - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-operations-helm.yaml b/applicationsets/addons-operations-helm.yaml index 6aa028c..a14488e 100644 --- a/applicationsets/addons-operations-helm.yaml +++ b/applicationsets/addons-operations-helm.yaml @@ -82,7 +82,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-operations-kustomize.yaml b/applicationsets/addons-operations-kustomize.yaml index dcae869..60a4574 100644 --- a/applicationsets/addons-operations-kustomize.yaml +++ b/applicationsets/addons-operations-kustomize.yaml @@ -42,7 +42,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}/overlays/{{ index .metadata.labels "environment" }}' # The overlay renders this environment's primary-cluster identity; patch # the EC2NodeClass per cluster from the cluster_name label so co-located diff --git a/applicationsets/addons-otel-agent.yaml b/applicationsets/addons-otel-agent.yaml index c931c8d..eb5657d 100644 --- a/applicationsets/addons-otel-agent.yaml +++ b/applicationsets/addons-otel-agent.yaml @@ -47,7 +47,7 @@ spec: - $values/addons/observability/otel-agent/values.yaml - $values/addons/observability/otel-agent/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-otel-gateway-floor.yaml b/applicationsets/addons-otel-gateway-floor.yaml index 2617210..8ee97fa 100644 --- a/applicationsets/addons-otel-gateway-floor.yaml +++ b/applicationsets/addons-otel-gateway-floor.yaml @@ -59,7 +59,7 @@ spec: - $values/addons/observability/otel-gateway-floor/values.yaml - $values/addons/observability/otel-gateway-floor/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-otel-gateway.yaml b/applicationsets/addons-otel-gateway.yaml index cb8992e..98f6499 100644 --- a/applicationsets/addons-otel-gateway.yaml +++ b/applicationsets/addons-otel-gateway.yaml @@ -63,7 +63,7 @@ spec: - $values/addons/observability/otel-gateway/values.yaml - $values/addons/observability/otel-gateway/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-security.yaml b/applicationsets/addons-security.yaml index afe2346..0646d7c 100644 --- a/applicationsets/addons-security.yaml +++ b/applicationsets/addons-security.yaml @@ -70,7 +70,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-tempo.yaml b/applicationsets/addons-tempo.yaml index 2a0664b..329c7d6 100644 --- a/applicationsets/addons-tempo.yaml +++ b/applicationsets/addons-tempo.yaml @@ -65,7 +65,7 @@ spec: - $values/addons/observability/tempo/values.yaml - $values/addons/observability/tempo/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/addons-velero.yaml b/applicationsets/addons-velero.yaml index efe4237..2be42f4 100644 --- a/applicationsets/addons-velero.yaml +++ b/applicationsets/addons-velero.yaml @@ -75,7 +75,7 @@ spec: - $values/addons/operations/velero/values.yaml - $values/addons/operations/velero/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/agent-platform.yaml b/applicationsets/agent-platform.yaml index c026808..0719b2d 100644 --- a/applicationsets/agent-platform.yaml +++ b/applicationsets/agent-platform.yaml @@ -55,7 +55,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}/overlays/{{ index .metadata.labels "environment" }}' destination: server: https://kubernetes.default.svc diff --git a/applicationsets/dashboards.yaml b/applicationsets/dashboards.yaml index 6557ecc..f9a12f7 100644 --- a/applicationsets/dashboards.yaml +++ b/applicationsets/dashboards.yaml @@ -45,7 +45,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}/base' # The Grafana CR's workspace URL is per-cluster and can't be templated # from a Secret, so inject it here from the monitoring/grafana-url diff --git a/applicationsets/kyverno-policies.yaml b/applicationsets/kyverno-policies.yaml index 48ce04d..3fb3dd0 100644 --- a/applicationsets/kyverno-policies.yaml +++ b/applicationsets/kyverno-policies.yaml @@ -55,7 +55,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}/overlays/{{ index .metadata.labels "environment" }}' destination: server: https://kubernetes.default.svc diff --git a/applicationsets/opt-in/apps-tenants.yaml b/applicationsets/opt-in/apps-tenants.yaml index 8167fc9..7993c28 100644 --- a/applicationsets/opt-in/apps-tenants.yaml +++ b/applicationsets/opt-in/apps-tenants.yaml @@ -55,7 +55,7 @@ spec: project: '{{ .app }}' sources: - repoURL: '{{ .repoURL }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}' helm: # Tenant charts default grafanaDashboard.enabled true, which renders a @@ -81,7 +81,7 @@ spec: - $values/{{ .path }}/values.yaml - $values/{{ .path }}/values-{{ index .metadata.labels "environment" }}.yaml - repoURL: '{{ .repoURL }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' ref: values destination: server: https://kubernetes.default.svc diff --git a/applicationsets/opt-in/clusters-appset.yaml b/applicationsets/opt-in/clusters-appset.yaml index 4fde75e..01ef8e1 100644 --- a/applicationsets/opt-in/clusters-appset.yaml +++ b/applicationsets/opt-in/clusters-appset.yaml @@ -70,7 +70,7 @@ spec: project: platform source: repoURL: git@github.com:nanohype/clusters.git - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path.path }}' directory: include: '{{ .path.filename }}' diff --git a/applicationsets/portal-tenants.yaml b/applicationsets/portal-tenants.yaml index 28d7b23..9613b59 100644 --- a/applicationsets/portal-tenants.yaml +++ b/applicationsets/portal-tenants.yaml @@ -68,7 +68,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/tenants-repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path.path }}' directory: recurse: true diff --git a/applicationsets/rendertest/testdata/argo-workflows-cluster-no-bucket.yaml b/applicationsets/rendertest/testdata/argo-workflows-cluster-no-bucket.yaml index c4b0949..703de7d 100644 --- a/applicationsets/rendertest/testdata/argo-workflows-cluster-no-bucket.yaml +++ b/applicationsets/rendertest/testdata/argo-workflows-cluster-no-bucket.yaml @@ -16,4 +16,5 @@ metadata: region: us-west-2 annotations: gitops/repo-url: https://github.com/example/gitops + gitops/repo-branch: main type: Opaque diff --git a/applicationsets/rendertest/testdata/argo-workflows-cluster-with-bucket.yaml b/applicationsets/rendertest/testdata/argo-workflows-cluster-with-bucket.yaml index 5197269..8aa4cfc 100644 --- a/applicationsets/rendertest/testdata/argo-workflows-cluster-with-bucket.yaml +++ b/applicationsets/rendertest/testdata/argo-workflows-cluster-with-bucket.yaml @@ -15,5 +15,6 @@ metadata: region: us-west-2 annotations: gitops/repo-url: https://github.com/example/gitops + gitops/repo-branch: main argo-workflows/artifact-bucket: alpha-111111111111-us-west-2-argo-workflows type: Opaque diff --git a/applicationsets/rendertest/testdata/cluster-adopt.yaml b/applicationsets/rendertest/testdata/cluster-adopt.yaml index d6890a0..cce0ba5 100644 --- a/applicationsets/rendertest/testdata/cluster-adopt.yaml +++ b/applicationsets/rendertest/testdata/cluster-adopt.yaml @@ -14,6 +14,7 @@ metadata: network_mode: adopt annotations: gitops/repo-url: https://github.com/example/gitops + gitops/repo-branch: main network/private-subnet-ids: subnet-priv-a,subnet-priv-b,subnet-priv-c network/public-subnet-ids: subnet-pub-a,subnet-pub-b,subnet-pub-c type: Opaque diff --git a/applicationsets/rendertest/testdata/cluster-create.yaml b/applicationsets/rendertest/testdata/cluster-create.yaml index 4559dbc..336b609 100644 --- a/applicationsets/rendertest/testdata/cluster-create.yaml +++ b/applicationsets/rendertest/testdata/cluster-create.yaml @@ -13,4 +13,5 @@ metadata: network_mode: create annotations: gitops/repo-url: https://github.com/example/gitops + gitops/repo-branch: main type: Opaque diff --git a/applicationsets/rendertest/testdata/cluster-nolabel.yaml b/applicationsets/rendertest/testdata/cluster-nolabel.yaml index 4c4e958..b8dad76 100644 --- a/applicationsets/rendertest/testdata/cluster-nolabel.yaml +++ b/applicationsets/rendertest/testdata/cluster-nolabel.yaml @@ -13,4 +13,5 @@ metadata: cluster_name: charlie annotations: gitops/repo-url: https://github.com/example/gitops + gitops/repo-branch: main type: Opaque diff --git a/applicationsets/rendertest/testdata/observability-cluster-no-buckets.yaml b/applicationsets/rendertest/testdata/observability-cluster-no-buckets.yaml index 959de26..15e87b0 100644 --- a/applicationsets/rendertest/testdata/observability-cluster-no-buckets.yaml +++ b/applicationsets/rendertest/testdata/observability-cluster-no-buckets.yaml @@ -14,4 +14,5 @@ metadata: region: us-west-2 annotations: gitops/repo-url: https://github.com/example/gitops + gitops/repo-branch: main type: Opaque diff --git a/applicationsets/rendertest/testdata/observability-cluster-with-buckets.yaml b/applicationsets/rendertest/testdata/observability-cluster-with-buckets.yaml index 87399e7..e235797 100644 --- a/applicationsets/rendertest/testdata/observability-cluster-with-buckets.yaml +++ b/applicationsets/rendertest/testdata/observability-cluster-with-buckets.yaml @@ -15,6 +15,7 @@ metadata: region: us-west-2 annotations: gitops/repo-url: https://github.com/example/gitops + gitops/repo-branch: main observability/loki-bucket: alpha-111111111111-us-west-2-loki observability/tempo-bucket: alpha-111111111111-us-west-2-tempo type: Opaque diff --git a/applicationsets/secret-stores.yaml b/applicationsets/secret-stores.yaml index 32097db..799a45e 100644 --- a/applicationsets/secret-stores.yaml +++ b/applicationsets/secret-stores.yaml @@ -61,7 +61,7 @@ spec: project: platform source: repoURL: '{{ index .metadata.annotations "gitops/repo-url" }}' - targetRevision: main + targetRevision: '{{ index .metadata.annotations "gitops/repo-branch" }}' path: '{{ .path }}' # The AMP endpoints Secrets Manager secret is cluster-scoped so co-located # sibling clusters don't collide; patch remoteRef.key per cluster. The diff --git a/scripts/check-catalog-revision.py b/scripts/check-catalog-revision.py new file mode 100755 index 0000000..3f3d37c --- /dev/null +++ b/scripts/check-catalog-revision.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""A catalog source must read its revision from the cluster, not hardcode one. + +WHY THIS EXISTS + +`cluster-bootstrap` stamps two sibling annotations on every ArgoCD cluster +Secret: `gitops/repo-url` and `gitops/repo-branch`. The comment above them says +appsets template their own source off these so no org is hardcoded in the +manifests. That was true of `repo-url` — 33 files read it — and false of +`repo-branch`, which had exactly one reference in the entire org: the line that +writes it. + +Meanwhile 28 applied ApplicationSets carried `targetRevision: main` as a literal, +directly beneath a `repoURL` templated off the annotation. + +So a cluster bootstrapped against any other revision ran `app-of-apps` at that +revision — the Application honours `var.gitops_repo_branch` — and every child +Application at `main`. **Both report Synced and Healthy.** ArgoCD is telling the +truth about each Application individually, and the cluster is running two +revisions of the catalog at once with nothing anywhere to say so. + +Two things that makes impossible. Testing a catalog change on a branch: the +cluster syncs `main`'s values while `app-of-apps` displays the branch name, which +is exactly the evidence someone would check. And pinning a release: a platform +installed from a tag gets a tagged app-of-apps over an unpinned fleet, so +"deploy the known-good version" quietly deploys HEAD. + +WHAT THIS CHECKS + +For every APPLIED ApplicationSet (top level only — app-of-apps does not recurse), +every source whose `repoURL` resolves from the `gitops/repo-url` annotation must +take its `targetRevision` from the `gitops/repo-branch` annotation. + +Chart sources are untouched and must be: their `repoURL` is a Helm or OCI +registry and their `targetRevision` is a chart version, which is a different +thing that happens to share a field name. + + scripts/check-catalog-revision.py + scripts/check-catalog-revision.py --list +""" + +from __future__ import annotations + +import argparse +import pathlib +import sys + +try: + import yaml +except ImportError: # pragma: no cover + sys.exit("PyYAML required: pip install pyyaml") + +ROOT = pathlib.Path(__file__).resolve().parent.parent +APPSETS = ROOT / "applicationsets" + +URL_ANNOTATION = 'gitops/repo-url' +REV_ANNOTATION = 'gitops/repo-branch' + + +def sources_of(doc): + """Every source block an ApplicationSet template declares.""" + spec = ((doc.get("spec") or {}).get("template") or {}).get("spec") or {} + out = list(spec.get("sources") or []) + if isinstance(spec.get("source"), dict): + out.append(spec["source"]) + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--list", action="store_true", help="print every catalog source found") + args = ap.parse_args() + + if not APPSETS.is_dir(): + print(f"no applicationsets/ under {ROOT}") + return 0 + + # Non-recursive, matching check-hardcoded-org.py: app-of-apps applies only the + # top level, so only the top level can put a cluster on two revisions. + files = sorted(p for p in APPSETS.glob("*.y*ml") if p.is_file()) + + problems: list[str] = [] + found = 0 + for path in files: + try: + docs = list(yaml.safe_load_all(path.read_text())) + except yaml.YAMLError as e: + problems.append(f"{path.relative_to(ROOT)}: not loadable as YAML ({e})") + continue + for doc in docs: + if not isinstance(doc, dict) or doc.get("kind") != "ApplicationSet": + continue + for src in sources_of(doc): + if not isinstance(src, dict): + continue + url = str(src.get("repoURL", "")) + if URL_ANNOTATION not in url: + continue # a chart registry, not this catalog + found += 1 + rev = str(src.get("targetRevision", "")) + rel = path.relative_to(ROOT) + if args.list: + print(f" {rel}: targetRevision {rev!r}") + if REV_ANNOTATION not in rev: + problems.append( + f"{rel}: a source reading {URL_ANNOTATION} pins " + f"targetRevision: {rev!r} instead of reading {REV_ANNOTATION}" + ) + + if found == 0: + # A gate that finds nothing to check has stopped checking. The catalog's + # own sources are how every addon gets its values files; zero of them means + # the parse is wrong, not that the repo is clean. + print( + f"error: no source in applicationsets/ reads the {URL_ANNOTATION} annotation. " + "This gate found nothing to validate, which is a broken gate rather than a " + "clean repo.", + file=sys.stderr, + ) + return 2 + + if problems: + print( + "\napplied ApplicationSets pin a catalog revision instead of reading it:\n", + file=sys.stderr, + ) + for p in problems: + print(f" - {p}", file=sys.stderr) + print( + f"\n Each must become:\n" + f" targetRevision: '{{{{ index .metadata.annotations \"{REV_ANNOTATION}\" }}}}'\n" + f"\n cluster-bootstrap stamps that annotation beside gitops/repo-url on every\n" + f" cluster Secret. Without it, app-of-apps honours the revision it was\n" + f" bootstrapped with and every child Application syncs main — two revisions of\n" + f" the catalog on one cluster, both reporting Synced and Healthy.", + file=sys.stderr, + ) + return 1 + + print(f"\nok: all {found} catalog source(s) read their revision from {REV_ANNOTATION}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())