fix(k8s): validate kustomize build output in CI, remove dead components key

.github/workflows/ci.yml

@@ -41,9 +41,71 @@ jobs:
       # the function-ordering regression that broke nightly backups for months,
       # which `bash -n` cannot catch, and needs no database.
       - run: bash scripts/tests/test-backup-restore.sh
-      # Cluster-free consistency checks over k8s/: kustomization coverage,
-      # Ingress backends resolving to real Services, PDB selectors matching a
-      # workload, and default-deny paired with allow rules.
+
+  k8s-manifests:
+    name: Kubernetes Manifests (kustomize + kubeconform)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install pyyaml
+
+      - name: Install kustomize
+        env:
+          KUSTOMIZE_VERSION: "5.8.1"
+          KUSTOMIZE_SHA256: "029a7f0f4e1932c52a0476cf02a0fd855c0bb85694b82c338fc648dcb53a819d"
+        run: |
+          set -euo pipefail
+          curl --fail --proto '=https' --tlsv1.2 -sSL \
+            "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \
+            -o kustomize.tar.gz
+          echo "${KUSTOMIZE_SHA256}  kustomize.tar.gz" | sha256sum -c -
+          tar -xzf kustomize.tar.gz kustomize
+          sudo mv kustomize /usr/local/bin/kustomize
+          sudo chmod +x /usr/local/bin/kustomize
+          rm kustomize.tar.gz
+
+      - name: Install kubeconform
+        env:
+          KUBECONFORM_VERSION: "0.8.0"
+          KUBECONFORM_SHA256: "9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883"
+        run: |
+          set -euo pipefail
+          curl --fail --proto '=https' --tlsv1.2 -sSL \
+            "https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" \
+            -o kubeconform.tar.gz
+          echo "${KUBECONFORM_SHA256}  kubeconform.tar.gz" | sha256sum -c -
+          tar -xzf kubeconform.tar.gz kubeconform
+          sudo mv kubeconform /usr/local/bin/kubeconform
+          sudo chmod +x /usr/local/bin/kubeconform
+          rm kubeconform.tar.gz
+
+      # kubeconform's bundled schemas only cover core/built-in Kubernetes
+      # kinds. k8s/ also renders Argo Rollouts CRDs (Rollout, AnalysisTemplate)
+      # via backend.yaml/frontend.yaml/ml-workloads/, which need the
+      # community CRD schema catalog or `-strict` fails them as unrecognized.
+      - name: Build and schema-validate k8s/
+        run: |
+          kustomize build k8s/ | kubeconform \
+            -strict -summary \
+            -schema-location default \
+            -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json'
+
+      - name: Build and schema-validate k8s/scheduler/
+        run: |
+          kustomize build k8s/scheduler/ | kubeconform \
+            -strict -summary \
+            -schema-location default \
+            -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json'
+
+      # Deeper checks kubeconform can't do: every k8s/ manifest is wired into
+      # kustomization.yaml, every Ingress backend names a real Service, every
+      # PodDisruptionBudget selector matches a real workload (asserted
+      # against this same `kustomize build k8s/` output above), and a
+      # default-deny NetworkPolicy is paired with an explicit allow.
       - run: python3 scripts/check-k8s-manifests.py
 
   frontend:

docs/deployment-mainnet.md

@@ -271,7 +271,7 @@ The Mainnet overlay turns TLS on and wires cert-manager so the certificate is is
 | `certManager.acme.email` | empty | Let's Encrypt account contact — set at deploy time |
 | `certManager.acme.server` | `https://acme-v02.api.letsencrypt.org/directory` | Production ACME. Use the staging directory first (below). |
 
-`k8s/ingress.yaml` is the local/dev manifest (`greenpay.local` + a `tls` block pointing at `greenpay-tls`). It is not applied to Mainnet; populate that Secret with mkcert or a self-signed cert if you need HTTPS on a local cluster.
+The local/dev Ingresses (`greenpay.local`, defined inline in `k8s/backend.yaml` and `k8s/frontend.yaml`) carry no `tls` block and are not applied to Mainnet — HTTPS there is a Helm-only concern, provisioned as described above.
 
 #### Provision the certificate
 

k8s/kustomization.yaml

@@ -1,3 +1,13 @@
+# Local/dev testnet manifests. helm/greenpay is the authoritative path for
+# real environments (testnet and Mainnet, see docs/deployment-mainnet.md) —
+# this kustomization is for local/dev clusters, and additionally carries the
+# ML-workload and custom-scheduler pieces that have no Helm templates yet.
+#
+# k8s/scheduler/ is a separate, independent Kustomization for the optional
+# ML-aware scheduler — it ships its own Namespace and RBAC, so it isn't a
+# Kustomize `component` (those compose partial patches into a base; this is
+# a full standalone base). Apply it on its own, after this one:
+#   kubectl apply -k k8s/scheduler/
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
@@ -24,12 +34,8 @@ resources:
   - ml-workloads/ml-inference.yaml
   - ml-workloads/ml-training.yaml
   - network-policy.yaml
-  # PodDisruptionBudgets. Selectors verified against the Rollout pod labels in
-  # backend.yaml, frontend.yaml and ml-workloads/. Present in the tree but
-  # never referenced here, so kustomize never created them.
+  # PodDisruptionBudgets. Selectors verified (in CI, against the actual
+  # `kustomize build` output — see scripts/check-k8s-manifests.py) to match
+  # the Rollout/Deployment pod labels in backend.yaml, frontend.yaml and
+  # ml-workloads/.
   - pdb.yaml
-
-components:
-  # Deploy the custom scheduler infrastructure.
-  # Apply separately with: kubectl apply -k k8s/scheduler/
-  # Listed here for documentation; exclude from base apply if not yet deployed.

scripts/check-k8s-manifests.py

@@ -14,8 +14,16 @@
  4. If a namespace-wide default-deny ingress policy exists, some other policy
     must admit traffic to the frontend and backend, otherwise applying k8s/
     blackholes the application.
+
+Checks 2-4 run against the actual `kustomize build` output rather than the
+raw source files, so a selector or label broken by the namespace transform,
+a patch, or a future `components:` composition is still caught — checking
+source files alone would miss anything that only exists post-build. Check 1
+is inherently about source-tree wiring (kustomization.yaml correctness), so
+it stays on the raw files. Requires the `kustomize` binary on PATH.
 """
 import os
+import subprocess
 import sys
 
 import yaml
@@ -24,9 +32,21 @@
 problems = []
 
 
-def load_all(path):
-    with open(path, encoding="utf-8") as fh:
-        return [d for d in yaml.safe_load_all(fh) if isinstance(d, dict)]
+def kustomize_build(path):
+    try:
+        result = subprocess.run(
+            ["kustomize", "build", path],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except FileNotFoundError:
+        print("error: the `kustomize` binary is not installed or not on PATH", file=sys.stderr)
+        sys.exit(2)
+    except subprocess.CalledProcessError as e:
+        print(f"error: `kustomize build {path}` failed:\n{e.stderr}", file=sys.stderr)
+        sys.exit(2)
+    return [d for d in yaml.safe_load_all(result.stdout) if isinstance(d, dict)]
 
 
 with open(os.path.join(K8S, "kustomization.yaml"), encoding="utf-8") as fh:
@@ -42,11 +62,7 @@ def load_all(path):
     if name.endswith(".yaml") and name != "kustomization.yaml" and name not in referenced:
         problems.append(f"k8s/{name} is not referenced by kustomization.yaml")
 
-docs = []
-for ref in referenced:
-    path = os.path.join(K8S, ref)
-    if os.path.exists(path):
-        docs.extend(load_all(path))
+docs = kustomize_build(K8S)
 
 services = {d["metadata"]["name"] for d in docs if d.get("kind") == "Service"}