diff --git a/.github/workflows/chart-provenance.yml b/.github/workflows/chart-provenance.yml new file mode 100644 index 0000000..dba66c4 --- /dev/null +++ b/.github/workflows/chart-provenance.yml @@ -0,0 +1,50 @@ +name: chart provenance + +# The live half of the chart-provenance check: is every chart this catalog pins +# still the chart it was pinned for, upstream, right now? +# +# It is a separate workflow rather than a job in ci.yml on purpose, and there +# are two reasons. +# +# The verdict is not a function of this commit. A maintainer can deprecate a +# chart, or hand it to a different organisation, at any moment. Run in the merge +# path, that turns a pull request red for a reason the pull request did not +# cause — the same trap `mirror-check freshness` and `schemas:freshness` are +# kept off the blocking path to avoid. +# +# And the merge gate in ci.yml refuses any workflow containing a job it does not +# watch, while counting a skipped dependency as a failure. A job that only runs +# on a schedule cannot satisfy both, so it does not belong in that file. The +# offline half — every pin has a record, every record a pin — does run there, +# because that IS a function of the tree. +on: + schedule: + # Mondays, ahead of the working week and after the weekend's upstream releases. + - cron: "0 13 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + live: + name: pinned charts still are what they were + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5 + + - name: Install PyYAML + run: pip install --quiet pyyaml + + # Fails on a `deprecated: true`, and on a chart whose description no + # longer matches what was recorded. The second is the one worth having: + # a chart can be re-scoped to a different product, or forked away to a + # different maintainer, without anyone ever setting the deprecated flag, + # and then the pin keeps resolving and the currency signal keeps saying + # "current" for something that is no longer the same software. + - name: Compare every pinned chart against its record + run: ./scripts/check-chart-deprecation.py --live diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7214cd..8d1ec8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -335,6 +335,16 @@ jobs: - name: Sync-wave ordering gate run: ./scripts/check-sync-waves.py + # Offline half of the chart-provenance gate: every pinned chart has a + # record and every record a pin. Deliberately does not touch the network — + # whether a chart went deprecated overnight is not a function of this + # commit, and gating merges on it would redden pull requests that changed + # nothing. chart-provenance.yml asks that question on a schedule. + - name: Chart provenance gate + run: | + ./scripts/check-chart-deprecation.py + ./scripts/check-chart-deprecation.py --self-test + # ── Appset render gate ─────────────────────────────────────────────── # The Karpenter EC2NodeClass patch carries if/range control flow inside a # `patch: |-` string block. Every other gate treats that string as opaque — diff --git a/applicationsets/addons-loki.yaml b/applicationsets/addons-loki.yaml index 45ae679..3d7cd6a 100644 --- a/applicationsets/addons-loki.yaml +++ b/applicationsets/addons-loki.yaml @@ -38,9 +38,15 @@ spec: spec: project: platform sources: - - repoURL: https://grafana.github.io/helm-charts + # The OSS Loki chart moved to grafana-community, forked at 6.55.0. The + # chart still published at grafana/helm-charts is now maintained for + # Grafana Enterprise Logs only — its description says so and its 7.x + # line is the GEL line, but it carries no `deprecated` flag, so nothing + # would ever have reported that the pin had changed product underneath + # it. This is the OSS lineage, continuous from the fork point. + - repoURL: https://grafana-community.github.io/helm-charts chart: loki - targetRevision: 7.2.0 + targetRevision: 18.7.5 helm: releaseName: loki # S3 log storage, injected only on clusters carrying the observability/loki-bucket diff --git a/applicationsets/addons-tempo.yaml b/applicationsets/addons-tempo.yaml index edaca8d..80891ce 100644 --- a/applicationsets/addons-tempo.yaml +++ b/applicationsets/addons-tempo.yaml @@ -35,9 +35,15 @@ spec: spec: project: platform sources: - - repoURL: https://grafana.github.io/helm-charts + # grafana/helm-charts deprecated this chart and handed it to + # grafana-community, which carries the full history rather than + # restarting: same chart, same single-binary topology, renumbered at the + # fork (1.24.4 -> 2.x). Grafana's own published chart names this as the + # destination. tempo-distributed is NOT the successor — it was deprecated + # in the same move and lives in the community repo too. + - repoURL: https://grafana-community.github.io/helm-charts chart: tempo - targetRevision: 1.24.4 + targetRevision: 2.2.3 helm: releaseName: tempo # S3 trace storage, injected only on clusters carrying the observability/tempo-bucket diff --git a/scripts/chart-provenance.json b/scripts/chart-provenance.json new file mode 100644 index 0000000..6464294 --- /dev/null +++ b/scripts/chart-provenance.json @@ -0,0 +1,155 @@ +{ + "_README": "What each pinned chart says it is, recorded so that a change is visible. check-chart-deprecation.py compares upstream against this on a schedule; the blocking gate only checks that every pin has a record and every record a pin. A description change means the chart redescribed itself \u2014 read the upstream notes before running --sync, because that is the signal a chart has changed product or maintainer without ever setting a deprecated flag.", + "charts": { + "ai-gateway-crds-helm": { + "repo": "oci://docker.io/envoyproxy/ai-gateway-crds-helm", + "description": "The Helm chart for Envoy AI Gateway CRD", + "deprecated": false + }, + "ai-gateway-helm": { + "repo": "oci://docker.io/envoyproxy/ai-gateway-helm", + "description": "The Helm chart for Envoy AI Gateway", + "deprecated": false + }, + "argo-events": { + "repo": "https://argoproj.github.io/argo-helm", + "description": "A Helm chart for Argo Events, the event-driven workflow automation framework", + "deprecated": false + }, + "argo-rollouts": { + "repo": "https://argoproj.github.io/argo-helm", + "description": "A Helm chart for Argo Rollouts", + "deprecated": false + }, + "argo-workflows": { + "repo": "https://argoproj.github.io/argo-helm", + "description": "A Helm chart for Argo Workflows", + "deprecated": false + }, + "aws-load-balancer-controller": { + "repo": "https://aws.github.io/eks-charts", + "description": "AWS Load Balancer Controller Helm chart for Kubernetes", + "deprecated": false + }, + "cert-manager": { + "repo": "https://charts.jetstack.io", + "description": "A Helm chart for cert-manager", + "deprecated": false + }, + "cilium": { + "repo": "https://helm.cilium.io", + "description": "eBPF-based Networking, Security, and Observability", + "deprecated": false + }, + "descheduler": { + "repo": "https://kubernetes-sigs.github.io/descheduler/", + "description": "Descheduler for Kubernetes is used to rebalance clusters by evicting pods that can potentially be scheduled on better nodes. In the current implementation, descheduler does not schedule replacement of evicted pods but relies on the default scheduler for that.", + "deprecated": false + }, + "external-dns": { + "repo": "https://kubernetes-sigs.github.io/external-dns/", + "description": "ExternalDNS synchronizes exposed Kubernetes Services and Ingresses with DNS providers.", + "deprecated": false + }, + "external-secrets": { + "repo": "https://charts.external-secrets.io", + "description": "External secrets management for Kubernetes", + "deprecated": false + }, + "falco": { + "repo": "https://falcosecurity.github.io/charts", + "description": "Falco", + "deprecated": false + }, + "gateway-helm": { + "repo": "oci://docker.io/envoyproxy/gateway-helm", + "description": "The Helm chart for Envoy Gateway", + "deprecated": false + }, + "goldilocks": { + "repo": "https://charts.fairwinds.com/stable", + "description": "A Helm chart for running Fairwinds Goldilocks. See https://github.com/FairwindsOps/goldilocks", + "deprecated": false + }, + "grafana-operator": { + "repo": "https://grafana.github.io/helm-charts", + "description": "Helm chart for the Grafana Operator", + "deprecated": false + }, + "karpenter": { + "repo": "oci://public.ecr.aws/karpenter/karpenter", + "description": "A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes.", + "deprecated": false + }, + "keda": { + "repo": "https://kedacore.github.io/charts", + "description": "Event-based autoscaler for workloads on Kubernetes", + "deprecated": false + }, + "kube-state-metrics": { + "repo": "https://prometheus-community.github.io/helm-charts", + "description": "Install kube-state-metrics to generate and expose cluster-level metrics", + "deprecated": false + }, + "kyverno": { + "repo": "https://kyverno.github.io/kyverno", + "description": "Kubernetes Native Policy Management", + "deprecated": false + }, + "loki": { + "repo": "https://grafana-community.github.io/helm-charts", + "description": "Helm chart for Grafana Loki supporting monolithic, simple scalable, and microservices modes.", + "deprecated": false + }, + "metrics-server": { + "repo": "https://kubernetes-sigs.github.io/metrics-server/", + "description": "Metrics Server is a scalable, efficient source of container resource metrics for Kubernetes built-in autoscaling pipelines.", + "deprecated": false + }, + "opencost": { + "repo": "https://opencost.github.io/opencost-helm-chart", + "description": "OpenCost and OpenCost UI", + "deprecated": false + }, + "opentelemetry-collector": { + "repo": "https://open-telemetry.github.io/opentelemetry-helm-charts", + "description": "OpenTelemetry Collector Helm chart for Kubernetes", + "deprecated": false + }, + "operator": { + "repo": "oci://ghcr.io/nanohype/eks-agent-platform/charts/operator", + "description": "eks-agent-platform \u2014 Kubernetes operator (CRDs + Deployment + RBAC).", + "deprecated": false + }, + "prometheus-operator-crds": { + "repo": "https://prometheus-community.github.io/helm-charts", + "description": "A Helm chart that collects custom resource definitions (CRDs) from the Prometheus Operator, allowing for seamless integration with GitOps tools", + "deprecated": false + }, + "reloader": { + "repo": "https://stakater.github.io/stakater-charts", + "description": "Reloader chart that runs on kubernetes", + "deprecated": false + }, + "tempo": { + "repo": "https://grafana-community.github.io/helm-charts", + "description": "Grafana Tempo Single Binary Mode", + "deprecated": false + }, + "trivy-operator": { + "repo": "https://aquasecurity.github.io/helm-charts", + "description": "Keeps security report resources updated", + "deprecated": false + }, + "velero": { + "repo": "https://vmware-tanzu.github.io/helm-charts", + "description": "A Helm chart for velero", + "deprecated": false + }, + "vpa": { + "repo": "https://charts.fairwinds.com/stable", + "description": "A Helm chart for Kubernetes Vertical Pod Autoscaler", + "deprecated": false + } + } +} diff --git a/scripts/check-chart-deprecation.py b/scripts/check-chart-deprecation.py new file mode 100755 index 0000000..e550eea --- /dev/null +++ b/scripts/check-chart-deprecation.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Every chart this catalog pins is still the chart it was pinned for. + + python3 scripts/check-chart-deprecation.py # blocking gate, offline + python3 scripts/check-chart-deprecation.py --live # scheduled, hits the registries + python3 scripts/check-chart-deprecation.py --sync # rewrite the records from upstream + python3 scripts/check-chart-deprecation.py --self-test + +Two different questions, and only one of them is a function of this commit. + +A chart can be deprecated, handed to a different maintainer, or re-scoped to a +different product at any moment, with no change here. Asking that at merge time +would turn an unrelated pull request red because someone upstream pushed +overnight — the same reason mirror-check asks "has upstream moved?" on a +schedule rather than in the gate. So the work splits: + + default (offline, BLOCKING) — every pinned chart has a provenance record and + every record names a chart still pinned. A function of the tree, so the + verdict cannot change without a commit. + + --live (network, SCHEDULED) — fetch each pinned chart and compare it against + its record: a `deprecated: true`, or a description that no longer matches. + +The description comparison is the part that earns its keep. `deprecated: true` +is easy and loud. The failure that prompted this was quiet: the OSS Loki chart +moved to grafana-community, and the chart still published at the original +repository was re-scoped to Grafana Enterprise Logs. No deprecation flag was +ever set on it. The pin resolved, the chart installed, CI stayed green, and +Renovate kept offering patches — a currency signal reporting "current" for a +chart that had changed product underneath it. The description was the only +field that moved. +""" + +from __future__ import annotations + +import json +import pathlib +import subprocess +import sys + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parent.parent +APPSETS = ROOT / "applicationsets" +# Beside the checker, not beside the pins. `applicationsets/` is scanned by +# kubeconform, which reads every file in it as a manifest and rejects one with +# no `kind` — the record would have had to be exempted from a schema gate to +# live next to what it describes, and weakening a gate to make room for a new +# file is the wrong trade. +RECORDS = ROOT / "scripts" / "chart-provenance.json" + + +def die(msg: str) -> None: + print(f"chart-provenance: {msg}", file=sys.stderr) + sys.exit(1) + + +def walk(node): + if isinstance(node, dict): + yield node + for v in node.values(): + yield from walk(v) + elif isinstance(node, list): + for v in node: + yield from walk(v) + + +def pins() -> dict[str, dict]: + """{chart: {repo, version, source}} for every literal helm pin in the catalog. + + Two shapes carry one: an ApplicationSet source (`repoURL` + `chart` + + `targetRevision`) and a list-generator element (`chartRepo` + `chart` + + `chartVersion`). Templated pins are skipped — their value is not in the tree. + """ + found: dict[str, dict] = {} + for path in sorted(APPSETS.glob("*.yaml")): + for doc in yaml.safe_load_all(path.read_text()): + for node in walk(doc or {}): + chart = node.get("chart") + version = node.get("chartVersion") or node.get("targetRevision") + repo = node.get("repoURL") or node.get("chartRepo") + if not (isinstance(chart, str) and isinstance(version, str) and isinstance(repo, str)): + continue + if "{{" in chart or "{{" in version or "{{" in repo: + continue + if not (repo.startswith("http://") or repo.startswith("https://") or repo.startswith("oci://")): + continue + prior = found.get(chart) + if prior and (prior["repo"], prior["version"]) != (repo, version): + die( + f"{chart} is pinned twice and they disagree: " + f"{prior['version']} from {prior['repo']} ({prior['source']}) " + f"vs {version} from {repo} ({path.name})" + ) + found[chart] = {"repo": repo, "version": version, "source": path.name} + if not found: + die("read no chart pins out of applicationsets/ — the parser and the catalog disagree") + return found + + +def load_records() -> dict: + if not RECORDS.exists(): + die(f"{RECORDS.relative_to(ROOT)} does not exist. Run --sync to create it.") + return json.loads(RECORDS.read_text()).get("charts", {}) + + +def fetch(chart: str, repo: str, version: str) -> dict: + """Chart.yaml as upstream currently publishes it.""" + if repo.startswith("oci://"): + cmd = ["helm", "show", "chart", repo, "--version", version] + else: + cmd = ["helm", "show", "chart", "--repo", repo, chart, "--version", version] + out = subprocess.run(cmd, capture_output=True, text=True, timeout=120) + if out.returncode != 0: + return {"_error": (out.stderr or out.stdout).strip().splitlines()[-1][:200]} + return yaml.safe_load(out.stdout) or {} + + +# ---------------------------------------------------------------- offline gate + +def check_offline(live: dict, recorded: dict) -> int: + problems = [] + + for chart, pin in sorted(live.items()): + rec = recorded.get(chart) + if rec is None: + problems.append( + f"{chart} is pinned ({pin['version']}, {pin['source']}) with no provenance record. " + f"Nothing would notice if that chart were deprecated or re-scoped. Run --sync." + ) + continue + if rec.get("repo") != pin["repo"]: + problems.append( + f"{chart} is pinned from {pin['repo']} but recorded against {rec.get('repo')}. " + f"A repository change is a change of maintainer — re-record it deliberately." + ) + if not rec.get("description"): + problems.append(f"{chart} has a provenance record with no description to compare against.") + if rec.get("deprecated") is True: + problems.append( + f"{chart} is recorded as deprecated upstream and is still pinned. " + f"Either migrate it or record why it stays." + ) + + for chart in sorted(set(recorded) - set(live)): + problems.append(f"{chart} has a provenance record but is no longer pinned — drop the record.") + + if problems: + print(f"FAIL {len(problems)} problem(s):") + for p in problems: + print(f" {p}") + return 1 + print(f"OK {len(live)} chart pin(s), each with a provenance record, none recorded deprecated.") + return 0 + + +# ------------------------------------------------------------------ live check + +def check_live() -> int: + live = pins() + recorded = load_records() + problems, ok = [], 0 + + for chart, pin in sorted(live.items()): + meta = fetch(chart, pin["repo"], pin["version"]) + if "_error" in meta: + problems.append(f"{chart}: could not read upstream metadata — {meta['_error']}") + continue + rec = recorded.get(chart, {}) + if meta.get("deprecated") is True: + problems.append( + f"{chart} {pin['version']} is marked deprecated by upstream ({pin['repo']})." + ) + desc = (meta.get("description") or "").strip() + was = (rec.get("description") or "").strip() + if was and desc != was: + problems.append( + f"{chart} changed what it says it is.\n" + f" recorded: {was}\n" + f" upstream: {desc}\n" + f" A chart that redescribes itself may have changed product or maintainer. " + f"Read the upstream notes, then --sync if it is still the chart you want." + ) + if not problems or problems[-1].split()[0] != chart: + ok += 1 + + if problems: + print(f"FAIL {len(problems)} problem(s) across {len(live)} pinned chart(s):") + for p in problems: + print(f" {p}") + return 1 + print(f"OK all {len(live)} pinned chart(s) match their record and none is deprecated.") + return 0 + + +# ------------------------------------------------------------------------ sync + +def sync() -> int: + live = pins() + charts = {} + for chart, pin in sorted(live.items()): + meta = fetch(chart, pin["repo"], pin["version"]) + if "_error" in meta: + die(f"{chart}: {meta['_error']}") + charts[chart] = { + "repo": pin["repo"], + "description": (meta.get("description") or "").strip(), + "deprecated": bool(meta.get("deprecated", False)), + } + print(f" recorded {chart:32} {pin['version']:12} deprecated={charts[chart]['deprecated']}") + RECORDS.write_text( + json.dumps( + { + "_README": ( + "What each pinned chart says it is, recorded so that a change is visible. " + "check-chart-deprecation.py compares upstream against this on a schedule; the " + "blocking gate only checks that every pin has a record and every record a pin. " + "A description change means the chart redescribed itself — read the upstream " + "notes before running --sync, because that is the signal a chart has changed " + "product or maintainer without ever setting a deprecated flag." + ), + "charts": charts, + }, + indent=2, + ) + + "\n" + ) + print(f"\nwrote {RECORDS.relative_to(ROOT)} ({len(charts)} charts)") + return 0 + + +# ------------------------------------------------------------------- self-test + +def self_test() -> int: + """Break the offline gate's inputs and confirm each break is rejected.""" + import contextlib + import io + + real_pins = pins() + real_records = load_records() + + def run(p, r): + with contextlib.redirect_stdout(io.StringIO()): + return check_offline(p, r) + + breaks = [] + # a pinned chart with no record + p = dict(real_pins); p["ghost-chart"] = {"repo": "https://example.invalid", "version": "1.0.0", "source": "x.yaml"} + breaks.append(("a pinned chart with no provenance record", p, real_records)) + # a record for a chart nobody pins + r = dict(real_records); r["retired-chart"] = {"repo": "https://example.invalid", "description": "x", "deprecated": False} + breaks.append(("a record for a chart no longer pinned", real_pins, r)) + # the recorded repo disagrees with the pin + name = sorted(real_pins)[0] + r2 = json.loads(json.dumps(real_records)); r2[name]["repo"] = "https://somewhere.else.invalid" + breaks.append(("the recorded repository differs from the pin", real_pins, r2)) + # a record marked deprecated but still pinned + r3 = json.loads(json.dumps(real_records)); r3[name]["deprecated"] = True + breaks.append(("a chart recorded deprecated but still pinned", real_pins, r3)) + # a record with no description to compare + r4 = json.loads(json.dumps(real_records)); r4[name]["description"] = "" + breaks.append(("a record with no description", real_pins, r4)) + + failures = [] + for label, p, r in breaks: + if run(p, r) == 0: + failures.append(label) + print(f" ACCEPTED {label} <-- not caught") + else: + print(f" rejected {label}") + + if run(real_pins, real_records) != 0: + failures.append("the real catalog does not pass") + print(" ACCEPTED (control) the shipped catalog is rejected") + else: + print(" passed (control) the shipped catalog") + + if failures: + print(f"\nFAIL {len(failures)} break(s) not caught.") + return 1 + print(f"\nOK all {len(breaks)} breaks rejected, and the shipped catalog passes.") + return 0 + + +def main() -> int: + if "--self-test" in sys.argv: + return self_test() + if "--sync" in sys.argv: + return sync() + if "--live" in sys.argv: + return check_live() + return check_offline(pins(), load_records()) + + +if __name__ == "__main__": + sys.exit(main())