From b79a9637fe431f43e07ee66424edf91e8bddbcd8 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 16:14:57 -0700
Subject: [PATCH 1/5] Update (base update)

[ghstack-poisoned]
---
 osdc/clusters.yaml                            |  24 ++-
 .../scripts/python/phases.py                  |   5 +-
 .../scripts/python/test_run.py                |   2 +-
 .../workflows/build-image.yaml                |  97 ++++++++--
 .../workflows/integration-test.yaml.tpl       |   6 +-
 osdc/modules/buildkit/README.md               |  70 +++++++
 osdc/modules/buildkit/deploy.sh               |  57 +++++-
 .../kubernetes/base/drain-configmap.yaml      |  21 ++
 .../buildkit/kubernetes/base/haproxy.yaml     |  12 +-
 .../kubernetes/base/kustomization.yaml        |   2 +
 .../kubernetes/base/poddisruptionbudget.yaml  |  25 +++
 .../scripts/python/generate_buildkit.py       | 180 ++++++++++++++++--
 .../scripts/python/test_generate_buildkit.py  | 129 +++++++++++++
 osdc/modules/keda/deploy.sh                   |  34 ++++
 14 files changed, 618 insertions(+), 46 deletions(-)
 create mode 100644 osdc/modules/buildkit/README.md
 create mode 100644 osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml
 create mode 100644 osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml
 create mode 100755 osdc/modules/keda/deploy.sh

diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml
index 44aa9a75..bd618668 100644
--- a/osdc/clusters.yaml
+++ b/osdc/clusters.yaml
@@ -83,6 +83,10 @@ defaults:
     cpu_disruption_budget: "20%"
   buildkit:
     replicas_per_arch: 12
+    autoscaling:
+      enabled: false
+  keda:
+    chart_version: "2.16.1"
   monitoring:
     grafana_cloud_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom/push"
     grafana_cloud_read_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom"
@@ -162,8 +166,14 @@ clusters:
       github_secret_name: pytorch-arc-staging
       runner_name_prefix: "c-mt-"
     buildkit:
-      replicas_per_arch: 2
       arm64_instance_type: m7gd.16xlarge
+      arm64_pods_per_node: 4
+      autoscaling:
+        enabled: true
+        amd64_min: 2  # 1x m6id.24xlarge (2 pods/node)
+        amd64_max: 8
+        arm64_min: 4  # 1x m7gd.16xlarge (4 pods/node)
+        arm64_max: 8
     pypi_cache:
       replicas: 1
     modules:
@@ -171,6 +181,7 @@ clusters:
       - arc
       - nodepools
       - arc-runners
+      - keda
       - buildkit
       - pypi-cache
       - cache-enforcer
@@ -283,11 +294,17 @@ clusters:
       min_node_age_seconds: 900
     buildkit:
       amd64_instance_type: m6id.24xlarge
-      amd64_replicas: 32
       amd64_pods_per_node: 2
       arm64_instance_type: m7gd.16xlarge
-      arm64_replicas: 8
       arm64_pods_per_node: 4
+      autoscaling:
+        enabled: true
+        amd64_min: 2   # 1x m6id.24xlarge (2 pods/node)
+        amd64_max: 360  # ~90d peak ≈180, x2 for headroom
+        arm64_min: 4   # 1x m7gd.16xlarge (4 pods/node)
+        arm64_max: 30  # ~90d peak ≈15, x2 for headroom
+        amd64_fallback: 32  # if KEDA can't read metrics, hold the proven fixed pool
+        arm64_fallback: 8
     arc-runners:
       github_config_url: "https://github.com/pytorch"
       github_secret_name: pytorch-arc-cbr-production
@@ -313,6 +330,7 @@ clusters:
       - arc-runners
       - arc-runners-b200
       - arc-runners-h100
+      - keda
       - buildkit
       - pypi-cache
       - cache-enforcer
diff --git a/osdc/integration-tests/scripts/python/phases.py b/osdc/integration-tests/scripts/python/phases.py
index 27ddebf6..7a21f204 100644
--- a/osdc/integration-tests/scripts/python/phases.py
+++ b/osdc/integration-tests/scripts/python/phases.py
@@ -276,11 +276,12 @@ def prepare_pr(
     # Write integration test workflow
     (workflows_dir / "integration-test.yaml").write_text(workflow_content)
 
-    # Copy build-image reusable workflow
+    # Copy the reusable BuildKit workflow (connectivity + autoscaling scale jobs).
+    # The scale job builds an inline Dockerfile, so it needs no copied context.
     build_wf_src = upstream_dir / "integration-tests" / "workflows" / "build-image.yaml"
     (workflows_dir / "build-image.yaml").write_text(build_wf_src.read_text())
 
-    # Copy test Dockerfile
+    # Copy test Dockerfile (connectivity test context)
     docker_dir = canary_path / "docker" / "test-buildkit"
     docker_dir.mkdir(parents=True, exist_ok=True)
     dockerfile_src = upstream_dir / "integration-tests" / "docker" / "test-buildkit" / "Dockerfile"
diff --git a/osdc/integration-tests/scripts/python/test_run.py b/osdc/integration-tests/scripts/python/test_run.py
index ff3a7bcb..58e6ab48 100644
--- a/osdc/integration-tests/scripts/python/test_run.py
+++ b/osdc/integration-tests/scripts/python/test_run.py
@@ -112,7 +112,7 @@ def workflow_template(tmp_path):
     )
     (wf_dir / "integration-test.yaml.tpl").write_text(template)
 
-    # Also create build-image.yaml and Dockerfile for prepare_pr
+    # Also create reusable workflow and Dockerfile for prepare_pr
     (wf_dir / "build-image.yaml").write_text("name: build-image\n")
     docker_dir = upstream / "integration-tests" / "docker" / "test-buildkit"
     docker_dir.mkdir(parents=True)
diff --git a/osdc/integration-tests/workflows/build-image.yaml b/osdc/integration-tests/workflows/build-image.yaml
index b523d03d..df36cfc8 100644
--- a/osdc/integration-tests/workflows/build-image.yaml
+++ b/osdc/integration-tests/workflows/build-image.yaml
@@ -1,6 +1,7 @@
-# Reusable workflow: Build a test image via OSDC BuildKit
-# Called by integration-test.yaml to validate BuildKit connectivity.
-# Uses buildctl directly — no Docker daemon required.
+# Reusable workflow: exercise OSDC BuildKit for one arch.
+# Called by integration-test.yaml. Two jobs:
+#   build  — single buildctl build (validates connectivity; buildctl route)
+#   scale  — burst of docker buildx builds (validates autoscaling; prod client)
 name: Build Test Image
 
 on:
@@ -38,21 +39,95 @@ jobs:
 
       - name: Build test image via BuildKit
         run: |
+          set -eu
           echo "=== BuildKit ${{ inputs.arch }} connectivity test ==="
           ENDPOINT="tcp://buildkitd-${{ inputs.arch }}.buildkit:1234"
           echo "Connecting to: $ENDPOINT"
 
-          buildctl --addr "$ENDPOINT" build \
-            --frontend dockerfile.v0 \
-            --local context=docker/test-buildkit \
-            --local dockerfile=docker/test-buildkit \
-            --output type=image,name=ghcr.io/${{ github.repository }}:integration-test-${{ inputs.arch }}-${{ github.sha }},push=false
-
-          echo "PASS: BuildKit ${{ inputs.arch }} built successfully"
-          echo "Endpoint: $ENDPOINT"
+          # The buildkit client dials with gRPC's ~20s connect timeout, so a busy
+          # / cold pool drops the connection fast (no HAProxy queue holds it).
+          # Retry long enough to outlast a peer's ~10 min build when the pool is
+          # over-subscribed (9 builds > 8 pods): ~45 x (≈5s fail + 15s) ≈ 15 min.
+          for attempt in $(seq 1 45); do
+            if buildctl --addr "$ENDPOINT" build \
+                --frontend dockerfile.v0 \
+                --local context=docker/test-buildkit \
+                --local dockerfile=docker/test-buildkit \
+                --output type=image,name=ghcr.io/${{ github.repository }}:integration-test-${{ inputs.arch }}-${{ github.sha }},push=false; then
+              echo "PASS: BuildKit ${{ inputs.arch }} built on attempt ${attempt} ($ENDPOINT)"
+              exit 0
+            fi
+            echo "attempt ${attempt}/45 failed (BuildKit cold/queued); retrying in 15s..."
+            sleep 15
+          done
+          echo "FAIL: BuildKit ${{ inputs.arch }} build failed after retries" >&2
+          exit 1
 
       - name: Verify BuildKit endpoint info
         run: |
           ENDPOINT="tcp://buildkitd-${{ inputs.arch }}.buildkit:1234"
           buildctl --addr "$ENDPOINT" debug info || echo "WARN: debug info not available"
           echo "PASS: BuildKit ${{ inputs.arch }} endpoint is responsive"
+
+  scale:
+    # 8 parallel docker buildx builds (the prod client), each holding a BuildKit
+    # slot (server maxconn=1) ~10 min via a sleep. The warm baseline is below the
+    # burst, so they finish within timeout-minutes only if KEDA scales the pool
+    # up; otherwise the back of the burst serializes and the job times out — i.e.
+    # this FAILS if autoscaling does not happen.
+    #
+    # Runs concurrently with `build`, so 9 builds contend for a max-8 pool: the
+    # odd one out has no pod until a peer's ~10 min build finishes, exercising
+    # the over-subscription wait (the retry below must outlast that).
+    runs-on: ${{ inputs.runner_label }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        replica: [1, 2, 3, 4, 5, 6, 7, 8]
+    container:
+      image: ghcr.io/actions/actions-runner:latest
+    steps:
+      - name: Set up Docker Buildx (remote, no bootstrap)
+        shell: bash
+        # NOT docker/setup-buildx-action: it runs `buildx inspect --bootstrap`,
+        # whose ~20s connect timeout fails at setup during a cold scale-up.
+        # `create` (no --bootstrap) just registers the builder; the build step
+        # retries to wait out scale-up.
+        run: |
+          set -ex
+          docker buildx create \
+            --name osdc-remote \
+            --driver remote \
+            --use \
+            "tcp://buildkitd-${{ inputs.arch }}.buildkit:1234"
+
+      - name: Occupy a BuildKit slot (~10 min) to drive autoscaling
+        shell: bash
+        run: |
+          set -eu
+          cat > Dockerfile.scale <<'EOF'
+          FROM alpine:3.21
+          ARG CACHEBUST
+          RUN echo "osdc buildkit scale-test ${CACHEBUST}" && sleep 600
+          EOF
+          # buildx boots the builder with a hardcoded ~20s connect timeout (gRPC
+          # MinConnectTimeout), so retry to wait out cold scale-up and, when the
+          # pool is over-subscribed, a peer's ~10 min build; the repeated attempts
+          # also keep KEDA's load signal alive. ~45 x (≈5s fail + 15s) ≈ 15 min,
+          # within the 30-min job timeout (still fails if scale-up never happens).
+          for attempt in $(seq 1 45); do
+            if docker buildx build \
+                --platform "linux/${{ inputs.arch }}" \
+                --build-arg "CACHEBUST=${{ inputs.arch }}-${{ matrix.replica }}-${{ github.run_id }}" \
+                --no-cache \
+                --output type=cacheonly \
+                -f Dockerfile.scale .; then
+              echo "build succeeded on attempt ${attempt}"
+              exit 0
+            fi
+            echo "attempt ${attempt}/45 failed (BuildKit cold/queued); retrying in 15s..."
+            sleep 15
+          done
+          echo "build failed after retries" >&2
+          exit 1
diff --git a/osdc/integration-tests/workflows/integration-test.yaml.tpl b/osdc/integration-tests/workflows/integration-test.yaml.tpl
index c9563480..bb453f0d 100644
--- a/osdc/integration-tests/workflows/integration-test.yaml.tpl
+++ b/osdc/integration-tests/workflows/integration-test.yaml.tpl
@@ -1502,13 +1502,15 @@ jobs:
   # END_B200
 
   # ── BuildKit Tests ────────────────────────────────────────────────────
-  build-amd64:
+  # Each call runs a buildctl connectivity build + an 8-wide docker buildx burst
+  # (fails if KEDA does not scale the pool up).
+  buildkit-amd64:
     uses: ./.github/workflows/build-image.yaml
     with:
       arch: amd64
       runner_label: {{PREFIX}}l-x86iamx-8-32
 
-  build-arm64:
+  buildkit-arm64:
     uses: ./.github/workflows/build-image.yaml
     with:
       arch: arm64
diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
new file mode 100644
index 00000000..6e4e30eb
--- /dev/null
+++ b/osdc/modules/buildkit/README.md
@@ -0,0 +1,70 @@
+# BuildKit module
+
+Remote BuildKit build service: per-arch `buildkitd` Deployments behind an HAProxy
+LB, on dedicated Karpenter NodePools. Clients build with
+`buildctl --addr tcp://buildkitd-<arch>.buildkit:1234`.
+
+Sizing is per-arch in `clusters.yaml` (`buildkit.{amd64,arm64}_{replicas,pods_per_node}`,
+`*_instance_type`); pod CPU/mem is computed by `scripts/python/generate_buildkit.py`.
+
+## Autoscaling (optional, `buildkit.autoscaling.enabled`)
+
+Absorb bursts of concurrent builds without overloading existing pods, and scale
+back to a small warm baseline when idle.
+
+- **One build per pod** — HAProxy `server maxconn 1` (matches buildkitd
+  `max-parallelism = 1`) so a build never stacks on a busy pod. When every pod is
+  busy the LB has no slot, so the client must **retry the connect** (see below)
+  until a pod frees or the pool scales up.
+- **In-cluster scale signal** — KEDA `ScaledObject` per arch, `metrics-api`
+  scraping the LB's own metrics (`haproxy_backend_current_sessions`) — no external
+  metrics backend.
+- **Warm baseline** — `amd64_min` / `arm64_min` keep ≥1 node per arch up so the
+  common case gets a free warm pod immediately. `*_max` caps the burst; NodePool
+  limits are sized to `*_max`.
+- **Safe scale-down** — `preStop` drain (waits until the pod's `:1234` is idle)
+  + long `terminationGracePeriodSeconds` + PDB, so a build is never killed
+  mid-flight. Scale-down removes an arbitrary pod, which may be mid-build; the
+  drain holds termination until that build finishes, but
+  `terminationGracePeriodSeconds` is a hard SIGKILL cap, so it must outlast the
+  longest possible build. It's set to **8100s (135m) = 120m** (the max time a
+  docker build may run, matching HAProxy `timeout server`) **+ ~15m** of
+  headroom for the drain's idle-detection polling. A build that starts just
+  before drain still completes; the cap only fires as a backstop if a pod never
+  drains.
+  The **PDB** (`maxUnavailable: 1` per arch) bounds *voluntary* disruptions —
+  node consolidation and manual `kubectl drain` — to one builder per arch at a
+  time, so those go through the preStop drain one pod at a time instead of
+  evicting several in-flight builds at once. (KEDA scale-down deletes pods
+  directly rather than via the eviction API, so it isn't PDB-gated — the drain +
+  grace cap above is what protects that path.)
+
+## Clients must retry the connect
+
+Build clients (both `docker buildx` and `buildctl`) use the `moby/buildkit` Go
+client, which dials with gRPC's default **~20s `MinConnectTimeout`** and
+**fail-fast** RPCs — there is no client-side flag to make it wait longer. During
+a burst, a build whose connection finds no free pod (`maxconn 1`) is dropped by
+the client after ~20s, well before KEDA/Karpenter can add a pod (minutes). An
+HAProxy-side `timeout queue` does **not** help: the client gives up at 20s
+regardless, so queueing on the LB is pointless (and was removed).
+
+So the **client must retry the build** on connection failures until a pod is
+free or the pool has scaled up; the repeated attempts also keep the autoscaler's
+load signal alive. PyTorch's `.ci/docker/build.sh` does this when
+`REMOTE_BUILDKIT` is set, and the workflow creates the remote builder *without*
+`--bootstrap` (the `docker buildx inspect --bootstrap` health check hits the same
+20s gate at setup). This was confirmed on the staging cluster.
+
+## HAProxy config changes roll the LB
+
+HAProxy renders its config only at container start, and nothing else restarts
+the `buildkitd-lb` pod, so a bare ConfigMap update (`maxconn`, timeouts,
+backends) would silently not take effect. `deploy.sh` stamps the LB pod template
+with a `checksum/config` annotation = a hash of `haproxy.yaml`; when the config
+changes the hash changes, which rolls the Deployment so the new pod picks up the
+new config. An unchanged config keeps the same hash, so routine deploys don't
+churn the LB. (The buildkitd worker pods do **not** yet have this, so a
+`buildkitd.toml` / `drain.sh` change needs a manual rollout to take effect.)
+
+Requires the `keda` module deployed before `buildkit` (provides the CRDs).
diff --git a/osdc/modules/buildkit/deploy.sh b/osdc/modules/buildkit/deploy.sh
index c7be6d39..eb08b727 100755
--- a/osdc/modules/buildkit/deploy.sh
+++ b/osdc/modules/buildkit/deploy.sh
@@ -34,22 +34,43 @@ AMD64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_replicas "$REPLICAS")
 ARM64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_replicas "$REPLICAS")
 AMD64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_pods_per_node "$PODS_PER_NODE")
 ARM64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_pods_per_node "$PODS_PER_NODE")
+AUTOSCALING=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.enabled false)
 
 GENERATED_DIR="$MODULE_DIR/generated"
 
 # --- Generate manifests ---
 
 echo "Generating BuildKit manifests..."
-uv run "$MODULE_DIR/scripts/python/generate_buildkit.py" \
-  --arm64-instance-type "$ARM64_INSTANCE" \
-  --amd64-instance-type "$AMD64_INSTANCE" \
-  --replicas "$REPLICAS" \
-  --pods-per-node "$PODS_PER_NODE" \
-  --amd64-replicas "$AMD64_REPLICAS" \
-  --arm64-replicas "$ARM64_REPLICAS" \
-  --amd64-pods-per-node "$AMD64_PODS_PER_NODE" \
-  --arm64-pods-per-node "$ARM64_PODS_PER_NODE" \
+GEN_ARGS=(
+  --arm64-instance-type "$ARM64_INSTANCE"
+  --amd64-instance-type "$AMD64_INSTANCE"
+  --replicas "$REPLICAS"
+  --pods-per-node "$PODS_PER_NODE"
+  --amd64-replicas "$AMD64_REPLICAS"
+  --arm64-replicas "$ARM64_REPLICAS"
+  --amd64-pods-per-node "$AMD64_PODS_PER_NODE"
+  --arm64-pods-per-node "$ARM64_PODS_PER_NODE"
   --output-dir "$GENERATED_DIR"
+)
+if [[ "${AUTOSCALING,,}" == "true" ]]; then
+  AMD64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_min 2)
+  AMD64_MAX=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_max 8)
+  ARM64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_min 4)
+  ARM64_MAX=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_max 8)
+  # Fallback replicas if KEDA can't read the scale metric (0 = no fallback).
+  AMD64_FALLBACK=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_fallback 0)
+  ARM64_FALLBACK=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_fallback 0)
+  GEN_ARGS+=(
+    --autoscaling
+    --amd64-min "$AMD64_MIN"
+    --amd64-max "$AMD64_MAX"
+    --arm64-min "$ARM64_MIN"
+    --arm64-max "$ARM64_MAX"
+    --amd64-fallback "$AMD64_FALLBACK"
+    --arm64-fallback "$ARM64_FALLBACK"
+  )
+fi
+uv run "$MODULE_DIR/scripts/python/generate_buildkit.py" "${GEN_ARGS[@]}"
 
 # --- Apply NodePools (with cluster name substitution) ---
 
@@ -59,7 +80,15 @@ sed "s/CLUSTER_NAME_PLACEHOLDER/$CNAME/g" "$GENERATED_DIR/nodepools.yaml" | kube
 # --- Apply static k8s resources ---
 
 echo "Applying BuildKit static manifests..."
-kubectl_apply_if_changed -k "$MODULE_DIR/kubernetes/base/"
+# Stamp the buildkitd-lb pod template with a hash of haproxy.yaml. HAProxy reads
+# its config only at container start, and nothing else restarts the LB, so
+# without this a ConfigMap change (maxconn, timeouts, backends) would silently
+# not take effect until the pod happened to be recreated. Changing the hash
+# rolls the Deployment whenever the config changes.
+HAPROXY_SUM=$(shasum -a 256 "$MODULE_DIR/kubernetes/base/haproxy.yaml" | cut -c1-12)
+kubectl kustomize "$MODULE_DIR/kubernetes/base/" \
+  | sed "s/__HAPROXY_CFG_CHECKSUM__/$HAPROXY_SUM/" \
+  | kubectl_apply_if_changed -f -
 
 # --- Apply generated Deployments (only if changed) ---
 
@@ -93,5 +122,13 @@ else
   kubectl rollout status deployment/buildkitd-amd64 -n buildkit --timeout=15m
 fi
 
+# --- KEDA autoscaling (optional) ---
+# Scales on the in-cluster buildkit LB metrics; no external metrics backend.
+
+if [[ "${AUTOSCALING,,}" == "true" ]]; then
+  echo "Applying KEDA autoscaling manifests..."
+  kubectl_apply_if_changed -f "$GENERATED_DIR/autoscaling.yaml"
+fi
+
 echo "BuildKit deployed."
 kubectl get pods -n buildkit -o wide
diff --git a/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml b/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml
new file mode 100644
index 00000000..89395ae9
--- /dev/null
+++ b/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: buildkitd-drain
+  namespace: buildkit
+data:
+  # preStop drain: block termination until no in-flight build remains. A build
+  # keeps an ESTABLISHED inbound connection on :1234 for its whole duration;
+  # require two consecutive idle polls so a transient health check can't be
+  # mistaken for "done". terminationGracePeriodSeconds caps the total wait.
+  drain.sh: |
+    #!/bin/sh
+    idle=0
+    while [ "$idle" -lt 2 ]; do
+      if netstat -tn 2>/dev/null | awk '$NF=="ESTABLISHED" && $4 ~ /:1234$/{f=1} END{exit !f}'; then
+        idle=0
+      else
+        idle=$((idle + 1))
+      fi
+      sleep 15
+    done
diff --git a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
index 3bfd4eb2..d37eeca9 100644
--- a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
+++ b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
@@ -71,15 +71,15 @@ data:
     # resolution warnings.
     backend bk_arm64
       balance leastconn
-      server-template arm 10 buildkitd-arm64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6
+      server-template arm 10 buildkitd-arm64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1
 
     backend bk_amd64
       balance leastconn
-      server-template amd 10 buildkitd-amd64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6
+      server-template amd 10 buildkitd-amd64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1
 
     backend bk_all
       balance leastconn
-      server-template all 20 buildkitd-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6
+      server-template all 20 buildkitd-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1
 
 ---
 apiVersion: apps/v1
@@ -100,6 +100,12 @@ spec:
     metadata:
       labels:
         app: buildkitd-lb
+      annotations:
+        # HAProxy renders its config once at container start, so a ConfigMap
+        # change has no effect until this pod restarts. deploy.sh fills this in
+        # with a hash of haproxy.yaml, so the Deployment rolls automatically
+        # whenever the config changes (nothing else restarts the LB).
+        checksum/config: "__HAPROXY_CFG_CHECKSUM__"
     spec:
       # Runs on base-infra nodes (CriticalAddonsOnly taint)
       tolerations:
diff --git a/osdc/modules/buildkit/kubernetes/base/kustomization.yaml b/osdc/modules/buildkit/kubernetes/base/kustomization.yaml
index 9cbffb38..32572408 100644
--- a/osdc/modules/buildkit/kubernetes/base/kustomization.yaml
+++ b/osdc/modules/buildkit/kubernetes/base/kustomization.yaml
@@ -5,6 +5,8 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - configmap.yaml
+  - drain-configmap.yaml
   - haproxy.yaml
   - service.yaml
   - networkpolicy.yaml
+  - poddisruptionbudget.yaml
diff --git a/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml b/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml
new file mode 100644
index 00000000..b717bb64
--- /dev/null
+++ b/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml
@@ -0,0 +1,25 @@
+# Cap voluntary disruptions (node consolidation, drains) to one builder per arch
+# at a time so evictions go through the preStop drain instead of killing builds.
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: buildkitd-arm64
+  namespace: buildkit
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: buildkitd
+      arch: arm64
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: buildkitd-amd64
+  namespace: buildkit
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: buildkitd
+      arch: amd64
diff --git a/osdc/modules/buildkit/scripts/python/generate_buildkit.py b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
index e7468418..c9bce324 100644
--- a/osdc/modules/buildkit/scripts/python/generate_buildkit.py
+++ b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
@@ -36,6 +36,12 @@
 RED = "\033[0;31m"
 NC = "\033[0m"
 
+# Sentinel for optional template lines. An optional fragment is either its YAML
+# lines or this sentinel; lines equal to it are dropped when a block is assembled
+# (see _deployment_block). This lets every fragment sit on its own line in the
+# templates below instead of being concatenated onto an adjacent line.
+_OMIT = "<<omit>>"
+
 
 def log_info(msg):
     print(f"{GREEN}\u2192{NC} {msg}")
@@ -103,6 +109,7 @@ def generate_deployment_yaml(
     arm64_replicas: int | None = None,
     amd64_pods_per_node: int | None = None,
     arm64_pods_per_node: int | None = None,
+    autoscaling: bool = False,
 ) -> str:
     """Generate the combined Deployment YAML for both architectures.
 
@@ -118,6 +125,37 @@ def generate_deployment_yaml(
     arm64_res = compute_pod_resources(arm64_instance, arm64_pods_per_node)
     amd64_res = compute_pod_resources(amd64_instance, amd64_pods_per_node)
 
+    # When KEDA owns the replica count, omit `replicas` and add a preStop drain
+    # that holds the pod open until its in-flight build finishes. Each fragment is
+    # either its YAML or _OMIT; _deployment_block drops _OMIT lines (matched after
+    # stripping), so single lines carry their indent in the template (e.g.
+    # `      {grace_line}`) while multi-line blocks self-indent. (`replicas_line`
+    # is per-arch — computed below.)
+    grace_line = "terminationGracePeriodSeconds: 8100" if autoscaling else _OMIT
+    lifecycle_block = (
+        """          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "/opt/drain/drain.sh"]"""
+        if autoscaling
+        else _OMIT
+    )
+    drain_mount = (
+        """            - name: drain
+              mountPath: /opt/drain
+              readOnly: true"""
+        if autoscaling
+        else _OMIT
+    )
+    drain_volume = (
+        """        - name: drain
+          configMap:
+            name: buildkitd-drain
+            defaultMode: 0555"""
+        if autoscaling
+        else _OMIT
+    )
+
     log_info(
         f"arm64 ({arm64_instance}): {arm64_res['cpu']} vCPU, {arm64_res['memory_gi']}Gi per pod "
         f"(allocatable: {arm64_res['allocatable_cpu_m']}m CPU, {arm64_res['allocatable_mem_mi']}Mi mem)"
@@ -128,7 +166,8 @@ def generate_deployment_yaml(
     )
 
     def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_node):
-        return f"""apiVersion: apps/v1
+        replicas_line = _OMIT if autoscaling else f"replicas: {replicas}"
+        block = f"""apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: buildkitd-{arch}
@@ -139,7 +178,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
     app.kubernetes.io/name: buildkitd
     app.kubernetes.io/component: build-service
 spec:
-  replicas: {replicas}
+  {replicas_line}
   strategy:
     type: RollingUpdate
     rollingUpdate:
@@ -155,6 +194,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
         app: buildkitd
         arch: {arch}
     spec:
+      {grace_line}
       nodeSelector:
         workload-type: buildkit
         instance-type: "{instance_type}"
@@ -209,6 +249,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
 
           securityContext:
             privileged: true
+{lifecycle_block}
 
           readinessProbe:
             exec:
@@ -238,6 +279,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
             - name: git-cache
               mountPath: /opt/git-cache
               readOnly: true
+{drain_mount}
 
       volumes:
         - name: config
@@ -252,7 +294,9 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
         - name: git-cache
           hostPath:
             path: /mnt/k8s-disks/0/git-cache
-            type: DirectoryOrCreate"""
+            type: DirectoryOrCreate
+{drain_volume}"""
+        return "\n".join(line for line in block.splitlines() if line.strip() != _OMIT)
 
     arm64_block = _deployment_block(
         "arm64", arm64_instance, arm64_res["cpu"], arm64_res["memory_gi"], arm64_replicas, arm64_pods_per_node
@@ -453,6 +497,76 @@ def _nodepool_block(arch, instance_type, cpu_limit, memory_limit_gi):
     return arm64_block + "\n\n---\n" + amd64_block + "\n"
 
 
+def generate_autoscaling_yaml(
+    amd64_min: int,
+    amd64_max: int,
+    arm64_min: int,
+    arm64_max: int,
+    amd64_fallback: int = 0,
+    arm64_fallback: int = 0,
+) -> str:
+    """Generate per-arch KEDA ScaledObjects.
+
+    Each arch scales on its HAProxy backend's active build count
+    (haproxy_backend_current_sessions), scraped in-cluster from the buildkit LB
+    metrics endpoint — no external metrics backend. With server maxconn=1, the LB
+    queues bursts while KEDA/Karpenter bring up pods; minReplicaCount keeps a warm
+    baseline so the common case has a free pod immediately.
+    """
+
+    metrics_url = "http://buildkitd-lb-metrics.buildkit.svc.cluster.local:9404/metrics"
+
+    def _scaledobject(arch, backend, min_replicas, max_replicas, fallback):
+        # If KEDA can't read the scale metric (e.g. LB metrics endpoint down),
+        # hold the proven fixed pool instead of letting the HPA freeze.
+        fallback_yaml = ""
+        if fallback:
+            fallback_yaml = f"\n  fallback:\n    failureThreshold: 3\n    replicas: {fallback}"
+        return f"""apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: buildkitd-{arch}
+  namespace: buildkit
+spec:
+  scaleTargetRef:
+    name: buildkitd-{arch}
+  minReplicaCount: {min_replicas}
+  maxReplicaCount: {max_replicas}{fallback_yaml}
+  cooldownPeriod: 1200
+  advanced:
+    horizontalPodAutoscalerConfig:
+      behavior:
+        scaleDown:
+          # No-flap: hold a pod ~20 min after idle (reuses its NVMe layer cache),
+          # then shed at most max(10 pods, 20%) per 20 min.
+          stabilizationWindowSeconds: 1200
+          policies:
+            - type: Pods
+              value: 10
+              periodSeconds: 1200
+            - type: Percent
+              value: 20
+              periodSeconds: 1200
+          selectPolicy: Max
+  triggers:
+    - type: metrics-api
+      metadata:
+        url: "{metrics_url}"
+        format: "prometheus"
+        valueLocation: 'haproxy_backend_current_sessions{{proxy="{backend}"}}'
+        targetValue: "1"
+"""
+
+    header = "# KEDA autoscaling — auto-generated by generate_buildkit.py. Do not edit by hand.\n"
+    return (
+        header
+        + "\n"
+        + _scaledobject("amd64", "bk_amd64", amd64_min, amd64_max, amd64_fallback)
+        + "\n---\n"
+        + _scaledobject("arm64", "bk_arm64", arm64_min, arm64_max, arm64_fallback)
+    )
+
+
 def main():
     parser = argparse.ArgumentParser(description="Generate BuildKit Deployment and NodePool YAMLs")
     parser.add_argument("--arm64-instance-type", required=True, help="ARM64 instance type (e.g., m8gd.24xlarge)")
@@ -464,8 +578,19 @@ def main():
     parser.add_argument("--amd64-pods-per-node", type=int, default=None, help="Override amd64 pods per node")
     parser.add_argument("--arm64-pods-per-node", type=int, default=None, help="Override arm64 pods per node")
     parser.add_argument("--output-dir", required=True, help="Output directory for generated YAMLs")
+    parser.add_argument("--autoscaling", action="store_true", help="Generate KEDA autoscaling manifests")
+    parser.add_argument("--amd64-min", type=int, default=0, help="KEDA minReplicaCount for amd64")
+    parser.add_argument("--amd64-max", type=int, default=0, help="KEDA maxReplicaCount for amd64")
+    parser.add_argument("--arm64-min", type=int, default=0, help="KEDA minReplicaCount for arm64")
+    parser.add_argument("--arm64-max", type=int, default=0, help="KEDA maxReplicaCount for arm64")
+    parser.add_argument("--amd64-fallback", type=int, default=0, help="KEDA fallback replicas for amd64 (0=off)")
+    parser.add_argument("--arm64-fallback", type=int, default=0, help="KEDA fallback replicas for arm64 (0=off)")
     args = parser.parse_args()
 
+    if args.autoscaling and not (args.amd64_max and args.arm64_max):
+        log_error("--autoscaling requires --amd64-max and --arm64-max")
+        return 1
+
     # Validate instance types
     for it in [args.arm64_instance_type, args.amd64_instance_type]:
         if it not in INSTANCE_SPECS:
@@ -498,26 +623,53 @@ def main():
         arm64_replicas=args.arm64_replicas,
         amd64_pods_per_node=args.amd64_pods_per_node,
         arm64_pods_per_node=args.arm64_pods_per_node,
+        autoscaling=args.autoscaling,
     )
     deployment_path = output_dir / "deployment.yaml"
     deployment_path.write_text(deployment_yaml)
     log_info(f"Wrote {deployment_path}")
 
-    # Generate nodepools
-    nodepools_yaml = generate_nodepools_yaml(
-        args.arm64_instance_type,
-        args.amd64_instance_type,
-        args.replicas,
-        args.pods_per_node,
-        amd64_replicas=args.amd64_replicas,
-        arm64_replicas=args.arm64_replicas,
-        amd64_pods_per_node=args.amd64_pods_per_node,
-        arm64_pods_per_node=args.arm64_pods_per_node,
-    )
+    # Size NodePool limits for the peak: the per-arch autoscaling ceiling
+    # (amd64_max / arm64_max) when enabled, otherwise the per-arch replica counts.
+    if args.autoscaling:
+        nodepools_yaml = generate_nodepools_yaml(
+            args.arm64_instance_type,
+            args.amd64_instance_type,
+            args.replicas,
+            args.pods_per_node,
+            amd64_replicas=args.amd64_max,
+            arm64_replicas=args.arm64_max,
+            amd64_pods_per_node=args.amd64_pods_per_node,
+            arm64_pods_per_node=args.arm64_pods_per_node,
+        )
+    else:
+        nodepools_yaml = generate_nodepools_yaml(
+            args.arm64_instance_type,
+            args.amd64_instance_type,
+            args.replicas,
+            args.pods_per_node,
+            amd64_replicas=args.amd64_replicas,
+            arm64_replicas=args.arm64_replicas,
+            amd64_pods_per_node=args.amd64_pods_per_node,
+            arm64_pods_per_node=args.arm64_pods_per_node,
+        )
     nodepools_path = output_dir / "nodepools.yaml"
     nodepools_path.write_text(nodepools_yaml)
     log_info(f"Wrote {nodepools_path}")
 
+    if args.autoscaling:
+        autoscaling_yaml = generate_autoscaling_yaml(
+            args.amd64_min,
+            args.amd64_max,
+            args.arm64_min,
+            args.arm64_max,
+            amd64_fallback=args.amd64_fallback,
+            arm64_fallback=args.arm64_fallback,
+        )
+        autoscaling_path = output_dir / "autoscaling.yaml"
+        autoscaling_path.write_text(autoscaling_yaml)
+        log_info(f"Wrote {autoscaling_path}")
+
     return 0
 
 
diff --git a/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py b/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py
index 387a283d..8a01a751 100644
--- a/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py
+++ b/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py
@@ -12,6 +12,7 @@
     DAEMONSET_OVERHEAD_MEM_MI,
     MARGIN,
     compute_pod_resources,
+    generate_autoscaling_yaml,
     generate_deployment_yaml,
     generate_nodepools_yaml,
 )
@@ -341,6 +342,76 @@ def test_nodepool_limits_scale_per_arch(self):
         assert np["buildkit-arm64"]["spec"]["limits"]["cpu"] == "256"
 
 
+# ============================================================================
+# autoscaling
+# ============================================================================
+
+
+class TestAutoscalingDeployment:
+    """When autoscaling=True the Deployment yields control of replicas to KEDA
+    and gains a preStop drain so scale-down never kills an in-flight build."""
+
+    def test_omits_replicas(self):
+        output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2, autoscaling=True)
+        for d in parse_all_yaml(output):
+            assert "replicas" not in d["spec"]
+
+    def test_keeps_replicas_when_disabled(self):
+        output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2)
+        for d in parse_all_yaml(output):
+            assert d["spec"]["replicas"] == 4
+
+    def test_drain_prestop_and_grace(self):
+        output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2, autoscaling=True)
+        for d in parse_all_yaml(output):
+            spec = d["spec"]["template"]["spec"]
+            assert spec["terminationGracePeriodSeconds"] == 8100
+            container = spec["containers"][0]
+            assert container["lifecycle"]["preStop"]["exec"]["command"] == ["/bin/sh", "/opt/drain/drain.sh"]
+            assert "drain" in {vm["name"] for vm in container["volumeMounts"]}
+            assert "drain" in {v["name"] for v in spec["volumes"]}
+
+
+class TestGenerateAutoscalingYaml:
+    """Tests for generate_autoscaling_yaml — in-cluster KEDA ScaledObjects."""
+
+    def _docs(self):
+        output = generate_autoscaling_yaml(2, 8, 4, 8)
+        return parse_all_yaml(output)
+
+    def test_only_scaledobjects_no_external_auth(self):
+        # In-cluster signal → no TriggerAuthentication / Grafana secret needed.
+        kinds = sorted(d["kind"] for d in self._docs())
+        assert kinds == ["ScaledObject", "ScaledObject"]
+
+    def test_per_arch_min_max(self):
+        scaled = {d["metadata"]["name"]: d for d in self._docs() if d["kind"] == "ScaledObject"}
+        assert set(scaled) == {"buildkitd-amd64", "buildkitd-arm64"}
+        assert scaled["buildkitd-amd64"]["spec"]["minReplicaCount"] == 2
+        assert scaled["buildkitd-amd64"]["spec"]["maxReplicaCount"] == 8
+        assert scaled["buildkitd-arm64"]["spec"]["minReplicaCount"] == 4
+        assert scaled["buildkitd-arm64"]["spec"]["maxReplicaCount"] == 8
+
+    def test_in_cluster_metrics_api_trigger(self):
+        scaled = {d["metadata"]["name"]: d for d in self._docs() if d["kind"] == "ScaledObject"}
+        for name, backend in [("buildkitd-amd64", "bk_amd64"), ("buildkitd-arm64", "bk_arm64")]:
+            trig = scaled[name]["spec"]["triggers"][0]
+            assert trig["type"] == "metrics-api"
+            assert "buildkitd-lb-metrics.buildkit" in trig["metadata"]["url"]
+            assert f'proxy="{backend}"' in trig["metadata"]["valueLocation"]
+
+    def test_no_fallback_by_default(self):
+        scaled = {d["metadata"]["name"]: d for d in self._docs() if d["kind"] == "ScaledObject"}
+        assert "fallback" not in scaled["buildkitd-amd64"]["spec"]
+        assert "fallback" not in scaled["buildkitd-arm64"]["spec"]
+
+    def test_fallback_replicas_when_set(self):
+        output = generate_autoscaling_yaml(2, 360, 4, 30, amd64_fallback=32, arm64_fallback=8)
+        scaled = {d["metadata"]["name"]: d for d in parse_all_yaml(output) if d["kind"] == "ScaledObject"}
+        assert scaled["buildkitd-amd64"]["spec"]["fallback"] == {"failureThreshold": 3, "replicas": 32}
+        assert scaled["buildkitd-arm64"]["spec"]["fallback"] == {"failureThreshold": 3, "replicas": 8}
+
+
 # ============================================================================
 # generate_nodepools_yaml
 # ============================================================================
@@ -520,6 +591,64 @@ def test_deployment_yaml_parseable(self, tmp_path):
         docs = parse_all_yaml(deployment_text)
         assert len(docs) == 2
 
+    def test_autoscaling_writes_manifests(self, tmp_path):
+        output_dir = tmp_path / "output"
+
+        import generate_buildkit
+
+        test_args = [
+            "generate_buildkit.py",
+            "--arm64-instance-type",
+            "m8gd.24xlarge",
+            "--amd64-instance-type",
+            "m6id.24xlarge",
+            "--replicas",
+            "1",
+            "--pods-per-node",
+            "2",
+            "--output-dir",
+            str(output_dir),
+            "--autoscaling",
+            "--amd64-min",
+            "2",
+            "--amd64-max",
+            "8",
+            "--arm64-min",
+            "4",
+            "--arm64-max",
+            "8",
+        ]
+        with patch.object(sys, "argv", test_args):
+            result = generate_buildkit.main()
+
+        assert result == 0
+        autoscaling = parse_all_yaml((output_dir / "autoscaling.yaml").read_text())
+        assert sorted(d["kind"] for d in autoscaling) == ["ScaledObject", "ScaledObject"]
+
+    def test_autoscaling_requires_params(self, tmp_path):
+        output_dir = tmp_path / "output"
+
+        import generate_buildkit
+
+        test_args = [
+            "generate_buildkit.py",
+            "--arm64-instance-type",
+            "m8gd.24xlarge",
+            "--amd64-instance-type",
+            "m6id.24xlarge",
+            "--replicas",
+            "1",
+            "--pods-per-node",
+            "2",
+            "--output-dir",
+            str(output_dir),
+            "--autoscaling",
+        ]
+        with patch.object(sys, "argv", test_args):
+            result = generate_buildkit.main()
+
+        assert result == 1
+
     def test_unknown_instance_type_fails(self, tmp_path):
         output_dir = tmp_path / "output"
         test_args = [
diff --git a/osdc/modules/keda/deploy.sh b/osdc/modules/keda/deploy.sh
new file mode 100755
index 00000000..59f422ad
--- /dev/null
+++ b/osdc/modules/keda/deploy.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+#
+# KEDA module deploy script.
+# Called by: just deploy-module <cluster> keda
+# Args: $1=cluster-id  $2=cluster-name  $3=region
+#
+# Installs the KEDA operator (provides the ScaledObject/TriggerAuthentication
+# CRDs that the buildkit module uses to autoscale builders).
+
+CLUSTER="$1"
+MODULE_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="${OSDC_ROOT:-$(cd "$MODULE_DIR/../.." && pwd)}"
+UPSTREAM_ROOT="${OSDC_UPSTREAM:-$REPO_ROOT}"
+# shellcheck source=/dev/null
+source "$UPSTREAM_ROOT/scripts/mise-activate.sh"
+# shellcheck source=/dev/null
+source "$UPSTREAM_ROOT/scripts/helm-upgrade.sh"
+CFG="$UPSTREAM_ROOT/scripts/cluster-config.py"
+
+NAMESPACE="keda"
+CHART_VERSION=$(uv run "$CFG" "$CLUSTER" keda.chart_version 2.16.1)
+
+helm repo add kedacore https://kedacore.github.io/charts >/dev/null 2>&1 || true
+helm repo update kedacore >/dev/null 2>&1 || true
+
+helm_upgrade_if_changed keda "$NAMESPACE" \
+  --create-namespace \
+  --version "$CHART_VERSION" \
+  --timeout 5m \
+  --wait \
+  kedacore/keda
+
+echo "KEDA deployed (chart $CHART_VERSION)."

From ae31e6a4969b85069037253506b723fbdeabbee8 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 16:14:57 -0700
Subject: [PATCH 2/5] Update

[ghstack-poisoned]
---
 osdc/modules/keda/deploy.sh                   |  1 +
 osdc/modules/keda/helm/values.yaml            |  6 ++++
 .../kubernetes/monitors/kustomization.yaml    |  1 +
 .../monitors/servicemonitors/keda.yaml        | 28 +++++++++++++++++++
 .../monitoring/tests/smoke/test_monitoring.py |  2 ++
 5 files changed, 38 insertions(+)
 create mode 100644 osdc/modules/keda/helm/values.yaml
 create mode 100644 osdc/modules/monitoring/kubernetes/monitors/servicemonitors/keda.yaml

diff --git a/osdc/modules/keda/deploy.sh b/osdc/modules/keda/deploy.sh
index 59f422ad..e74d3d9f 100755
--- a/osdc/modules/keda/deploy.sh
+++ b/osdc/modules/keda/deploy.sh
@@ -27,6 +27,7 @@ helm repo update kedacore >/dev/null 2>&1 || true
 helm_upgrade_if_changed keda "$NAMESPACE" \
   --create-namespace \
   --version "$CHART_VERSION" \
+  -f "$MODULE_DIR/helm/values.yaml" \
   --timeout 5m \
   --wait \
   kedacore/keda
diff --git a/osdc/modules/keda/helm/values.yaml b/osdc/modules/keda/helm/values.yaml
new file mode 100644
index 00000000..8a9fcf15
--- /dev/null
+++ b/osdc/modules/keda/helm/values.yaml
@@ -0,0 +1,6 @@
+# Expose the KEDA operator's Prometheus metrics (keda_scaler_* /
+# keda_scaledobject_*) on :8080. The ServiceMonitor that scrapes it lives in the
+# monitoring module, so it applies after the monitoring.coreos.com CRDs exist.
+prometheus:
+  operator:
+    enabled: true
diff --git a/osdc/modules/monitoring/kubernetes/monitors/kustomization.yaml b/osdc/modules/monitoring/kubernetes/monitors/kustomization.yaml
index 4b81b416..c6a8a70a 100644
--- a/osdc/modules/monitoring/kubernetes/monitors/kustomization.yaml
+++ b/osdc/modules/monitoring/kubernetes/monitors/kustomization.yaml
@@ -13,6 +13,7 @@ resources:
   - servicemonitors/git-cache-central.yaml
   - servicemonitors/harbor.yaml
   - servicemonitors/karpenter.yaml
+  - servicemonitors/keda.yaml
   - servicemonitors/node-compactor.yaml
   - servicemonitors/pushgateway.yaml
   - servicemonitors/pypi-cache.yaml
diff --git a/osdc/modules/monitoring/kubernetes/monitors/servicemonitors/keda.yaml b/osdc/modules/monitoring/kubernetes/monitors/servicemonitors/keda.yaml
new file mode 100644
index 00000000..7200fc48
--- /dev/null
+++ b/osdc/modules/monitoring/kubernetes/monitors/servicemonitors/keda.yaml
@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: keda
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: osdc-monitoring
+spec:
+  namespaceSelector:
+    matchNames:
+      - keda
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: keda-operator
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 60s
+      metricRelabelings:
+        # Keep only KEDA's own metrics (scaler/scaledobject values, errors,
+        # activity, latency) — this also drops the endpoint's go_/process_/
+        # controller-runtime series. Then drop histogram buckets.
+        - action: keep
+          sourceLabels: [__name__]
+          regex: "keda_.*"
+        - action: drop
+          sourceLabels: [__name__]
+          regex: ".*_bucket"
diff --git a/osdc/modules/monitoring/tests/smoke/test_monitoring.py b/osdc/modules/monitoring/tests/smoke/test_monitoring.py
index c4d50fdc..36c813b1 100644
--- a/osdc/modules/monitoring/tests/smoke/test_monitoring.py
+++ b/osdc/modules/monitoring/tests/smoke/test_monitoring.py
@@ -29,6 +29,7 @@
     "buildkit-haproxy",
     "harbor",
     "karpenter",
+    "keda",
     "node-compactor",
     "git-cache-central",
     "pypi-cache",
@@ -291,6 +292,7 @@ def test_metrics_arriving(self, resolve_config) -> None:
     "buildkit": ("buildkitd-pods", "buildkit"),
     "buildkit-haproxy": ("buildkitd-lb-metrics", "buildkit"),
     "karpenter": ("karpenter", "karpenter"),
+    "keda": ("keda-operator", "keda"),
     "node-compactor": ("node-compactor", None),
     "git-cache-central": ("git-cache-central-metrics", None),
     # arc-controller: skipped — ARC controller metrics Service varies by chart version

From 4fe83b8a12b939cf0ed197e155fe9dd01cf75e0b Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 16:23:15 -0700
Subject: [PATCH 3/5] Update (base update)

[ghstack-poisoned]
---
 osdc/modules/buildkit/README.md | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
index 6e4e30eb..b38c1511 100644
--- a/osdc/modules/buildkit/README.md
+++ b/osdc/modules/buildkit/README.md
@@ -4,8 +4,9 @@ Remote BuildKit build service: per-arch `buildkitd` Deployments behind an HAProx
 LB, on dedicated Karpenter NodePools. Clients build with
 `buildctl --addr tcp://buildkitd-<arch>.buildkit:1234`.
 
-Sizing is per-arch in `clusters.yaml` (`buildkit.{amd64,arm64}_{replicas,pods_per_node}`,
-`*_instance_type`); pod CPU/mem is computed by `scripts/python/generate_buildkit.py`.
+Sizing is per-arch in `clusters.yaml` (`buildkit.{amd64,arm64}_*` instance type,
+pods-per-node, and autoscaling `*_min` / `*_max`); pod CPU/mem is computed by
+`scripts/python/generate_buildkit.py`.
 
 ## Autoscaling (optional, `buildkit.autoscaling.enabled`)
 
@@ -18,10 +19,15 @@ back to a small warm baseline when idle.
   until a pod frees or the pool scales up.
 - **In-cluster scale signal** — KEDA `ScaledObject` per arch, `metrics-api`
   scraping the LB's own metrics (`haproxy_backend_current_sessions`) — no external
-  metrics backend.
+  metrics backend. If KEDA can't read the metric, a `fallback` (`*_fallback`,
+  e.g. 32/8 on prod) holds the proven fixed pool instead of freezing the count.
 - **Warm baseline** — `amd64_min` / `arm64_min` keep ≥1 node per arch up so the
   common case gets a free warm pod immediately. `*_max` caps the burst; NodePool
   limits are sized to `*_max`.
+- **No-flap scale-down** — KEDA holds a pod ~20 min after it goes idle
+  (`stabilizationWindowSeconds: 1200`), then sheds at most `max(10 pods, 20%)`
+  per 20 min, so a follow-up build reuses the pod's warm decompressed NVMe layer
+  cache. Node churn is left to Karpenter.
 - **Safe scale-down** — `preStop` drain (waits until the pod's `:1234` is idle)
   + long `terminationGracePeriodSeconds` + PDB, so a build is never killed
   mid-flight. Scale-down removes an arbitrary pod, which may be mid-build; the
@@ -67,4 +73,6 @@ new config. An unchanged config keeps the same hash, so routine deploys don't
 churn the LB. (The buildkitd worker pods do **not** yet have this, so a
 `buildkitd.toml` / `drain.sh` change needs a manual rollout to take effect.)
 
-Requires the `keda` module deployed before `buildkit` (provides the CRDs).
+Requires the `keda` module deployed before `buildkit` (provides the CRDs). The
+`monitoring` module scrapes the KEDA operator's metrics and ships
+buildkit-autoscaling alerts (scaler / fallback errors, queue backlog).

From 0ab12e06d06573901ac57994fdc0163a51e1e969 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Thu, 11 Jun 2026 09:19:56 -0700
Subject: [PATCH 4/5] Update (base update)

[ghstack-poisoned]
---
 osdc/clusters.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml
index bd618668..05f2ea4d 100644
--- a/osdc/clusters.yaml
+++ b/osdc/clusters.yaml
@@ -299,9 +299,9 @@ clusters:
       arm64_pods_per_node: 4
       autoscaling:
         enabled: true
-        amd64_min: 2   # 1x m6id.24xlarge (2 pods/node)
+        amd64_min: 32  # warm baseline = proven fixed pool (16x m6id.24xlarge)
         amd64_max: 360  # ~90d peak ≈180, x2 for headroom
-        arm64_min: 4   # 1x m7gd.16xlarge (4 pods/node)
+        arm64_min: 8   # warm baseline = proven fixed pool (2x m7gd.16xlarge)
         arm64_max: 30  # ~90d peak ≈15, x2 for headroom
         amd64_fallback: 32  # if KEDA can't read metrics, hold the proven fixed pool
         arm64_fallback: 8

From f3c853809bd7894ad5e348185b699cda09c67aae Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Fri, 12 Jun 2026 02:20:41 -0700
Subject: [PATCH 5/5] Update

[ghstack-poisoned]
---
 osdc/modules/buildkit/deploy.sh    | 7 ++++---
 osdc/modules/keda/helm/values.yaml | 7 +++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/osdc/modules/buildkit/deploy.sh b/osdc/modules/buildkit/deploy.sh
index eb08b727..3f26c9b8 100755
--- a/osdc/modules/buildkit/deploy.sh
+++ b/osdc/modules/buildkit/deploy.sh
@@ -34,7 +34,8 @@ AMD64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_replicas "$REPLICAS")
 ARM64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_replicas "$REPLICAS")
 AMD64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_pods_per_node "$PODS_PER_NODE")
 ARM64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_pods_per_node "$PODS_PER_NODE")
-AUTOSCALING=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.enabled false)
+# Lowercase via tr (not ${VAR,,}) — deploy.sh runs under macOS bash 3.2 too.
+AUTOSCALING=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.enabled false | tr '[:upper:]' '[:lower:]')
 
 GENERATED_DIR="$MODULE_DIR/generated"
 
@@ -52,7 +53,7 @@ GEN_ARGS=(
   --arm64-pods-per-node "$ARM64_PODS_PER_NODE"
   --output-dir "$GENERATED_DIR"
 )
-if [[ "${AUTOSCALING,,}" == "true" ]]; then
+if [[ "$AUTOSCALING" == "true" ]]; then
   AMD64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_min 2)
   AMD64_MAX=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_max 8)
   ARM64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_min 4)
@@ -125,7 +126,7 @@ fi
 # --- KEDA autoscaling (optional) ---
 # Scales on the in-cluster buildkit LB metrics; no external metrics backend.
 
-if [[ "${AUTOSCALING,,}" == "true" ]]; then
+if [[ "$AUTOSCALING" == "true" ]]; then
   echo "Applying KEDA autoscaling manifests..."
   kubectl_apply_if_changed -f "$GENERATED_DIR/autoscaling.yaml"
 fi
diff --git a/osdc/modules/keda/helm/values.yaml b/osdc/modules/keda/helm/values.yaml
index 8a9fcf15..282ca179 100644
--- a/osdc/modules/keda/helm/values.yaml
+++ b/osdc/modules/keda/helm/values.yaml
@@ -1,3 +1,10 @@
+# Schedule on the base-infra nodes (tainted CriticalAddonsOnly); every other node
+# is reserved by workload taints, so without this the keda pods stay Pending and
+# the install's --wait times out. Applies to operator, metrics server, webhooks.
+tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
+
 # Expose the KEDA operator's Prometheus metrics (keda_scaler_* /
 # keda_scaledobject_*) on :8080. The ServiceMonitor that scrapes it lives in the
 # monitoring module, so it applies after the monitoring.coreos.com CRDs exist.