From cdb804cf2c25bbeeaa9ebd6ed15d26cb1b587885 Mon Sep 17 00:00:00 2001 From: Awosdot Date: Mon, 24 Aug 2026 14:16:36 +0100 Subject: [PATCH] fix(k8s): validate kustomize build output in CI, remove dead components key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pdb.yaml was already wired into kustomization.yaml's resources (an earlier commit fixed that half), but three things from #322 were still open: - The empty `components:` key (comments, no items) is removed. Kustomize components compose partial patches into a base; k8s/scheduler/ ships its own Namespace and RBAC, so it's a full standalone base, not a component — listing it there would have been structurally wrong even populated. It's documented instead as a separate Kustomization applied on its own. - scripts/check-k8s-manifests.py now runs its Ingress-backend, PDB-selector, and NetworkPolicy-admission checks against actual `kustomize build k8s/` output instead of manually re-parsed source files, so a selector broken by the namespace transform, a patch, or a future component would still be caught. Verified by reintroducing both defects it targets (an unwired pdb.yaml, a PDB selector matching no workload) and confirming each fails. - CI never ran `kustomize build` or a schema validator, so a rendering error was invisible until `kubectl apply` hit it in a real cluster. Adds a k8s-manifests job to ci.yml that installs kustomize and kubeconform (SHA256-pinned downloads) and runs `kustomize build | kubeconform -strict` against both k8s/ and k8s/scheduler/, plus the Python script above. The community CRD schema catalog is wired in alongside kubeconform's built-in schemas, since k8s/ renders Argo Rollouts CRDs (Rollout, AnalysisTemplate) that kubeconform doesn't recognize out of the box. Also fixes a stale docs/deployment-mainnet.md reference to k8s/ingress.yaml, which no longer exists — Ingress is now defined inline in backend.yaml and frontend.yaml, with no TLS block, matching what's actually there. Verified locally: `kustomize build k8s/` renders all 4 PDBs with selectors matching real workloads, kubeconform passes 37/37 resources for k8s/ and 10/10 for k8s/scheduler/, and check-k8s-manifests.py passes end-to-end against the built output. --- .github/workflows/ci.yml | 68 ++++++++++++++++++++++++++++++++-- docs/deployment-mainnet.md | 2 +- k8s/kustomization.yaml | 22 +++++++---- scripts/check-k8s-manifests.py | 32 ++++++++++++---- 4 files changed, 104 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef8f5455..75743e9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,9 +35,71 @@ jobs: # the function-ordering regression that broke nightly backups for months, # which `bash -n` cannot catch, and needs no database. - run: bash scripts/tests/test-backup-restore.sh - # Cluster-free consistency checks over k8s/: kustomization coverage, - # Ingress backends resolving to real Services, PDB selectors matching a - # workload, and default-deny paired with allow rules. + + k8s-manifests: + name: Kubernetes Manifests (kustomize + kubeconform) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pyyaml + + - name: Install kustomize + env: + KUSTOMIZE_VERSION: "5.8.1" + KUSTOMIZE_SHA256: "029a7f0f4e1932c52a0476cf02a0fd855c0bb85694b82c338fc648dcb53a819d" + run: | + set -euo pipefail + curl --fail --proto '=https' --tlsv1.2 -sSL \ + "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \ + -o kustomize.tar.gz + echo "${KUSTOMIZE_SHA256} kustomize.tar.gz" | sha256sum -c - + tar -xzf kustomize.tar.gz kustomize + sudo mv kustomize /usr/local/bin/kustomize + sudo chmod +x /usr/local/bin/kustomize + rm kustomize.tar.gz + + - name: Install kubeconform + env: + KUBECONFORM_VERSION: "0.8.0" + KUBECONFORM_SHA256: "9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883" + run: | + set -euo pipefail + curl --fail --proto '=https' --tlsv1.2 -sSL \ + "https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" \ + -o kubeconform.tar.gz + echo "${KUBECONFORM_SHA256} kubeconform.tar.gz" | sha256sum -c - + tar -xzf kubeconform.tar.gz kubeconform + sudo mv kubeconform /usr/local/bin/kubeconform + sudo chmod +x /usr/local/bin/kubeconform + rm kubeconform.tar.gz + + # kubeconform's bundled schemas only cover core/built-in Kubernetes + # kinds. k8s/ also renders Argo Rollouts CRDs (Rollout, AnalysisTemplate) + # via backend.yaml/frontend.yaml/ml-workloads/, which need the + # community CRD schema catalog or `-strict` fails them as unrecognized. + - name: Build and schema-validate k8s/ + run: | + kustomize build k8s/ | kubeconform \ + -strict -summary \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' + + - name: Build and schema-validate k8s/scheduler/ + run: | + kustomize build k8s/scheduler/ | kubeconform \ + -strict -summary \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' + + # Deeper checks kubeconform can't do: every k8s/ manifest is wired into + # kustomization.yaml, every Ingress backend names a real Service, every + # PodDisruptionBudget selector matches a real workload (asserted + # against this same `kustomize build k8s/` output above), and a + # default-deny NetworkPolicy is paired with an explicit allow. - run: python3 scripts/check-k8s-manifests.py frontend: diff --git a/docs/deployment-mainnet.md b/docs/deployment-mainnet.md index 66de2125..02736d79 100644 --- a/docs/deployment-mainnet.md +++ b/docs/deployment-mainnet.md @@ -271,7 +271,7 @@ The Mainnet overlay turns TLS on and wires cert-manager so the certificate is is | `certManager.acme.email` | empty | Let's Encrypt account contact — set at deploy time | | `certManager.acme.server` | `https://acme-v02.api.letsencrypt.org/directory` | Production ACME. Use the staging directory first (below). | -`k8s/ingress.yaml` is the local/dev manifest (`greenpay.local` + a `tls` block pointing at `greenpay-tls`). It is not applied to Mainnet; populate that Secret with mkcert or a self-signed cert if you need HTTPS on a local cluster. +The local/dev Ingresses (`greenpay.local`, defined inline in `k8s/backend.yaml` and `k8s/frontend.yaml`) carry no `tls` block and are not applied to Mainnet — HTTPS there is a Helm-only concern, provisioned as described above. #### Provision the certificate diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 97d59188..49f83afe 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -1,3 +1,13 @@ +# Local/dev testnet manifests. helm/greenpay is the authoritative path for +# real environments (testnet and Mainnet, see docs/deployment-mainnet.md) — +# this kustomization is for local/dev clusters, and additionally carries the +# ML-workload and custom-scheduler pieces that have no Helm templates yet. +# +# k8s/scheduler/ is a separate, independent Kustomization for the optional +# ML-aware scheduler — it ships its own Namespace and RBAC, so it isn't a +# Kustomize `component` (those compose partial patches into a base; this is +# a full standalone base). Apply it on its own, after this one: +# kubectl apply -k k8s/scheduler/ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -19,12 +29,8 @@ resources: - ml-workloads/ml-inference.yaml - ml-workloads/ml-training.yaml - network-policy.yaml - # PodDisruptionBudgets. Selectors verified against the Rollout pod labels in - # backend.yaml, frontend.yaml and ml-workloads/. Present in the tree but - # never referenced here, so kustomize never created them. + # PodDisruptionBudgets. Selectors verified (in CI, against the actual + # `kustomize build` output — see scripts/check-k8s-manifests.py) to match + # the Rollout/Deployment pod labels in backend.yaml, frontend.yaml and + # ml-workloads/. - pdb.yaml - -components: - # Deploy the custom scheduler infrastructure. - # Apply separately with: kubectl apply -k k8s/scheduler/ - # Listed here for documentation; exclude from base apply if not yet deployed. diff --git a/scripts/check-k8s-manifests.py b/scripts/check-k8s-manifests.py index 79a6103e..18c698e7 100755 --- a/scripts/check-k8s-manifests.py +++ b/scripts/check-k8s-manifests.py @@ -14,8 +14,16 @@ 4. If a namespace-wide default-deny ingress policy exists, some other policy must admit traffic to the frontend and backend, otherwise applying k8s/ blackholes the application. + +Checks 2-4 run against the actual `kustomize build` output rather than the +raw source files, so a selector or label broken by the namespace transform, +a patch, or a future `components:` composition is still caught — checking +source files alone would miss anything that only exists post-build. Check 1 +is inherently about source-tree wiring (kustomization.yaml correctness), so +it stays on the raw files. Requires the `kustomize` binary on PATH. """ import os +import subprocess import sys import yaml @@ -24,9 +32,21 @@ problems = [] -def load_all(path): - with open(path, encoding="utf-8") as fh: - return [d for d in yaml.safe_load_all(fh) if isinstance(d, dict)] +def kustomize_build(path): + try: + result = subprocess.run( + ["kustomize", "build", path], + capture_output=True, + text=True, + check=True, + ) + except FileNotFoundError: + print("error: the `kustomize` binary is not installed or not on PATH", file=sys.stderr) + sys.exit(2) + except subprocess.CalledProcessError as e: + print(f"error: `kustomize build {path}` failed:\n{e.stderr}", file=sys.stderr) + sys.exit(2) + return [d for d in yaml.safe_load_all(result.stdout) if isinstance(d, dict)] with open(os.path.join(K8S, "kustomization.yaml"), encoding="utf-8") as fh: @@ -42,11 +62,7 @@ def load_all(path): if name.endswith(".yaml") and name != "kustomization.yaml" and name not in referenced: problems.append(f"k8s/{name} is not referenced by kustomization.yaml") -docs = [] -for ref in referenced: - path = os.path.join(K8S, ref) - if os.path.exists(path): - docs.extend(load_all(path)) +docs = kustomize_build(K8S) services = {d["metadata"]["name"] for d in docs if d.get("kind") == "Service"}