From 43a6b39805a6d948288a3fcf8771c9cb0e17457a Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 01:52:12 -0700
Subject: [PATCH 1/7] Update

[ghstack-poisoned]
---
 osdc/clusters.yaml                            |  13 +-
 osdc/modules/buildkit/README.md               |  32 ++++
 osdc/modules/buildkit/deploy.sh               |  42 ++++-
 .../kubernetes/base/drain-configmap.yaml      |  21 +++
 .../buildkit/kubernetes/base/haproxy.yaml     |  10 +-
 .../kubernetes/base/kustomization.yaml        |   2 +
 .../kubernetes/base/poddisruptionbudget.yaml  |  25 +++
 .../scripts/python/generate_buildkit.py       | 157 ++++++++++++++++--
 .../scripts/python/test_generate_buildkit.py  | 118 +++++++++++++
 osdc/modules/keda/deploy.sh                   |  34 ++++
 10 files changed, 424 insertions(+), 30 deletions(-)
 create mode 100644 osdc/modules/buildkit/README.md
 create mode 100644 osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml
 create mode 100644 osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml
 create mode 100755 osdc/modules/keda/deploy.sh

diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml
index 44aa9a75..94d59eaa 100644
--- a/osdc/clusters.yaml
+++ b/osdc/clusters.yaml
@@ -83,6 +83,10 @@ defaults:
     cpu_disruption_budget: "20%"
   buildkit:
     replicas_per_arch: 12
+    autoscaling:
+      enabled: false
+  keda:
+    chart_version: "2.16.1"
   monitoring:
     grafana_cloud_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom/push"
     grafana_cloud_read_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom"
@@ -162,8 +166,14 @@ clusters:
       github_secret_name: pytorch-arc-staging
       runner_name_prefix: "c-mt-"
     buildkit:
-      replicas_per_arch: 2
       arm64_instance_type: m7gd.16xlarge
+      arm64_pods_per_node: 4
+      autoscaling:
+        enabled: true
+        amd64_min: 2  # 1x m6id.24xlarge (2 pods/node)
+        amd64_max: 8
+        arm64_min: 4  # 1x m7gd.16xlarge (4 pods/node)
+        arm64_max: 8
     pypi_cache:
       replicas: 1
     modules:
@@ -171,6 +181,7 @@ clusters:
       - arc
       - nodepools
       - arc-runners
+      - keda
       - buildkit
       - pypi-cache
       - cache-enforcer
diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
new file mode 100644
index 00000000..e17fafdf
--- /dev/null
+++ b/osdc/modules/buildkit/README.md
@@ -0,0 +1,32 @@
+# BuildKit module
+
+Remote BuildKit build service: per-arch `buildkitd` Deployments behind an HAProxy
+LB, on dedicated Karpenter NodePools. Clients build with
+`buildctl --addr tcp://buildkitd-<arch>.buildkit:1234`.
+
+Sizing is per-arch in `clusters.yaml` (`buildkit.{amd64,arm64}_{replicas,pods_per_node}`,
+`*_instance_type`); pod CPU/mem is computed by `scripts/python/generate_buildkit.py`.
+
+## Autoscaling (optional, `buildkit.autoscaling.enabled`)
+
+Absorb bursts of concurrent builds without overloading existing pods, and scale
+back to a small warm baseline when idle.
+
+- **One build per pod** — HAProxy `server maxconn 1` (matches buildkitd
+  `max-parallelism = 1`). Excess builds **queue** in HAProxy (`timeout queue`)
+  instead of stacking on a busy pod; as new pods register (DNS), queued builds
+  flow onto them, so scaled-up pods never sit idle.
+- **In-cluster scale signal** — KEDA `ScaledObject` per arch, `metrics-api`
+  scraping the LB's own metrics (`haproxy_backend_current_sessions`) — no external
+  metrics backend.
+- **Warm baseline** — `amd64_min` / `arm64_min` keep ≥1 node per arch up so the
+  common case gets a free warm pod immediately. `*_max` caps the burst; NodePool
+  limits are sized to `*_max`.
+- **Safe scale-down** — `preStop` drain (waits until the pod's `:1234` is idle)
+  + long `terminationGracePeriodSeconds` + PDB, so a build is never killed
+  mid-flight.
+
+Build clients should retry the connect so a build can wait for a pod from a cold
+or queued pool.
+
+Requires the `keda` module deployed before `buildkit` (provides the CRDs).
diff --git a/osdc/modules/buildkit/deploy.sh b/osdc/modules/buildkit/deploy.sh
index c7be6d39..ece804ad 100755
--- a/osdc/modules/buildkit/deploy.sh
+++ b/osdc/modules/buildkit/deploy.sh
@@ -34,22 +34,38 @@ AMD64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_replicas "$REPLICAS")
 ARM64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_replicas "$REPLICAS")
 AMD64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_pods_per_node "$PODS_PER_NODE")
 ARM64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_pods_per_node "$PODS_PER_NODE")
+AUTOSCALING=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.enabled false)
 
 GENERATED_DIR="$MODULE_DIR/generated"
 
 # --- Generate manifests ---
 
 echo "Generating BuildKit manifests..."
-uv run "$MODULE_DIR/scripts/python/generate_buildkit.py" \
-  --arm64-instance-type "$ARM64_INSTANCE" \
-  --amd64-instance-type "$AMD64_INSTANCE" \
-  --replicas "$REPLICAS" \
-  --pods-per-node "$PODS_PER_NODE" \
-  --amd64-replicas "$AMD64_REPLICAS" \
-  --arm64-replicas "$ARM64_REPLICAS" \
-  --amd64-pods-per-node "$AMD64_PODS_PER_NODE" \
-  --arm64-pods-per-node "$ARM64_PODS_PER_NODE" \
+GEN_ARGS=(
+  --arm64-instance-type "$ARM64_INSTANCE"
+  --amd64-instance-type "$AMD64_INSTANCE"
+  --replicas "$REPLICAS"
+  --pods-per-node "$PODS_PER_NODE"
+  --amd64-replicas "$AMD64_REPLICAS"
+  --arm64-replicas "$ARM64_REPLICAS"
+  --amd64-pods-per-node "$AMD64_PODS_PER_NODE"
+  --arm64-pods-per-node "$ARM64_PODS_PER_NODE"
   --output-dir "$GENERATED_DIR"
+)
+if [[ "${AUTOSCALING,,}" == "true" ]]; then
+  AMD64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_min 2)
+  AMD64_MAX=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_max 8)
+  ARM64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_min 4)
+  ARM64_MAX=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_max 8)
+  GEN_ARGS+=(
+    --autoscaling
+    --amd64-min "$AMD64_MIN"
+    --amd64-max "$AMD64_MAX"
+    --arm64-min "$ARM64_MIN"
+    --arm64-max "$ARM64_MAX"
+  )
+fi
+uv run "$MODULE_DIR/scripts/python/generate_buildkit.py" "${GEN_ARGS[@]}"
 
 # --- Apply NodePools (with cluster name substitution) ---
 
@@ -93,5 +109,13 @@ else
   kubectl rollout status deployment/buildkitd-amd64 -n buildkit --timeout=15m
 fi
 
+# --- KEDA autoscaling (optional) ---
+# Scales on the in-cluster buildkit LB metrics; no external metrics backend.
+
+if [[ "${AUTOSCALING,,}" == "true" ]]; then
+  echo "Applying KEDA autoscaling manifests..."
+  kubectl_apply_if_changed -f "$GENERATED_DIR/autoscaling.yaml"
+fi
+
 echo "BuildKit deployed."
 kubectl get pods -n buildkit -o wide
diff --git a/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml b/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml
new file mode 100644
index 00000000..89395ae9
--- /dev/null
+++ b/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: buildkitd-drain
+  namespace: buildkit
+data:
+  # preStop drain: block termination until no in-flight build remains. A build
+  # keeps an ESTABLISHED inbound connection on :1234 for its whole duration;
+  # require two consecutive idle polls so a transient health check can't be
+  # mistaken for "done". terminationGracePeriodSeconds caps the total wait.
+  drain.sh: |
+    #!/bin/sh
+    idle=0
+    while [ "$idle" -lt 2 ]; do
+      if netstat -tn 2>/dev/null | awk '$NF=="ESTABLISHED" && $4 ~ /:1234$/{f=1} END{exit !f}'; then
+        idle=0
+      else
+        idle=$((idle + 1))
+      fi
+      sleep 15
+    done
diff --git a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
index 3bfd4eb2..a52cece7 100644
--- a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
+++ b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
@@ -24,6 +24,10 @@ data:
       timeout connect 5s
       timeout client 120m
       timeout server 120m
+      # Queue a build (server maxconn=1, below) while KEDA/Karpenter add pods,
+      # instead of stacking it on a busy pod. The runner also retries the
+      # connect, so this is an upper bound, not the only safety net.
+      timeout queue 10m
       log global
       option tcplog
 
@@ -71,15 +75,15 @@ data:
     # resolution warnings.
     backend bk_arm64
       balance leastconn
-      server-template arm 10 buildkitd-arm64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6
+      server-template arm 10 buildkitd-arm64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1
 
     backend bk_amd64
       balance leastconn
-      server-template amd 10 buildkitd-amd64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6
+      server-template amd 10 buildkitd-amd64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1
 
     backend bk_all
       balance leastconn
-      server-template all 20 buildkitd-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6
+      server-template all 20 buildkitd-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1
 
 ---
 apiVersion: apps/v1
diff --git a/osdc/modules/buildkit/kubernetes/base/kustomization.yaml b/osdc/modules/buildkit/kubernetes/base/kustomization.yaml
index 9cbffb38..32572408 100644
--- a/osdc/modules/buildkit/kubernetes/base/kustomization.yaml
+++ b/osdc/modules/buildkit/kubernetes/base/kustomization.yaml
@@ -5,6 +5,8 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - configmap.yaml
+  - drain-configmap.yaml
   - haproxy.yaml
   - service.yaml
   - networkpolicy.yaml
+  - poddisruptionbudget.yaml
diff --git a/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml b/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml
new file mode 100644
index 00000000..b717bb64
--- /dev/null
+++ b/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml
@@ -0,0 +1,25 @@
+# Cap voluntary disruptions (node consolidation, drains) to one builder per arch
+# at a time so evictions go through the preStop drain instead of killing builds.
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: buildkitd-arm64
+  namespace: buildkit
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: buildkitd
+      arch: arm64
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: buildkitd-amd64
+  namespace: buildkit
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: buildkitd
+      arch: amd64
diff --git a/osdc/modules/buildkit/scripts/python/generate_buildkit.py b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
index e7468418..830b8998 100644
--- a/osdc/modules/buildkit/scripts/python/generate_buildkit.py
+++ b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
@@ -103,6 +103,7 @@ def generate_deployment_yaml(
     arm64_replicas: int | None = None,
     amd64_pods_per_node: int | None = None,
     arm64_pods_per_node: int | None = None,
+    autoscaling: bool = False,
 ) -> str:
     """Generate the combined Deployment YAML for both architectures.
 
@@ -118,6 +119,37 @@ def generate_deployment_yaml(
     arm64_res = compute_pod_resources(arm64_instance, arm64_pods_per_node)
     amd64_res = compute_pod_resources(amd64_instance, amd64_pods_per_node)
 
+    # When KEDA owns the replica count, omit `replicas` and add a preStop drain
+    # that holds the pod open until its in-flight build finishes. `replicas_line`
+    # is computed per-arch inside _deployment_block (below).
+    grace_line = "      terminationGracePeriodSeconds: 8100\n" if autoscaling else ""
+    lifecycle_block = (
+        """
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "/opt/drain/drain.sh"]"""
+        if autoscaling
+        else ""
+    )
+    drain_mount = (
+        """
+            - name: drain
+              mountPath: /opt/drain
+              readOnly: true"""
+        if autoscaling
+        else ""
+    )
+    drain_volume = (
+        """
+        - name: drain
+          configMap:
+            name: buildkitd-drain
+            defaultMode: 0555"""
+        if autoscaling
+        else ""
+    )
+
     log_info(
         f"arm64 ({arm64_instance}): {arm64_res['cpu']} vCPU, {arm64_res['memory_gi']}Gi per pod "
         f"(allocatable: {arm64_res['allocatable_cpu_m']}m CPU, {arm64_res['allocatable_mem_mi']}Mi mem)"
@@ -128,6 +160,7 @@ def generate_deployment_yaml(
     )
 
     def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_node):
+        replicas_line = "" if autoscaling else f"  replicas: {replicas}\n"
         return f"""apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -139,8 +172,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
     app.kubernetes.io/name: buildkitd
     app.kubernetes.io/component: build-service
 spec:
-  replicas: {replicas}
-  strategy:
+{replicas_line}  strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 0
@@ -155,7 +187,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
         app: buildkitd
         arch: {arch}
     spec:
-      nodeSelector:
+{grace_line}      nodeSelector:
         workload-type: buildkit
         instance-type: "{instance_type}"
 
@@ -208,7 +240,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
                   fieldPath: metadata.name
 
           securityContext:
-            privileged: true
+            privileged: true{lifecycle_block}
 
           readinessProbe:
             exec:
@@ -237,7 +269,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
               subPathExpr: $(POD_NAME)
             - name: git-cache
               mountPath: /opt/git-cache
-              readOnly: true
+              readOnly: true{drain_mount}
 
       volumes:
         - name: config
@@ -252,7 +284,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
         - name: git-cache
           hostPath:
             path: /mnt/k8s-disks/0/git-cache
-            type: DirectoryOrCreate"""
+            type: DirectoryOrCreate{drain_volume}"""
 
     arm64_block = _deployment_block(
         "arm64", arm64_instance, arm64_res["cpu"], arm64_res["memory_gi"], arm64_replicas, arm64_pods_per_node
@@ -453,6 +485,63 @@ def _nodepool_block(arch, instance_type, cpu_limit, memory_limit_gi):
     return arm64_block + "\n\n---\n" + amd64_block + "\n"
 
 
+def generate_autoscaling_yaml(
+    amd64_min: int,
+    amd64_max: int,
+    arm64_min: int,
+    arm64_max: int,
+) -> str:
+    """Generate per-arch KEDA ScaledObjects.
+
+    Each arch scales on its HAProxy backend's active build count
+    (haproxy_backend_current_sessions), scraped in-cluster from the buildkit LB
+    metrics endpoint — no external metrics backend. With server maxconn=1, the LB
+    queues bursts while KEDA/Karpenter bring up pods; minReplicaCount keeps a warm
+    baseline so the common case has a free pod immediately.
+    """
+
+    metrics_url = "http://buildkitd-lb-metrics.buildkit.svc.cluster.local:9404/metrics"
+
+    def _scaledobject(arch, backend, min_replicas, max_replicas):
+        return f"""apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: buildkitd-{arch}
+  namespace: buildkit
+spec:
+  scaleTargetRef:
+    name: buildkitd-{arch}
+  minReplicaCount: {min_replicas}
+  maxReplicaCount: {max_replicas}
+  cooldownPeriod: 600
+  advanced:
+    horizontalPodAutoscalerConfig:
+      behavior:
+        scaleDown:
+          stabilizationWindowSeconds: 600
+          policies:
+            - type: Pods
+              value: 1
+              periodSeconds: 120
+  triggers:
+    - type: metrics-api
+      metadata:
+        url: "{metrics_url}"
+        format: "prometheus"
+        valueLocation: 'haproxy_backend_current_sessions{{proxy="{backend}"}}'
+        targetValue: "1"
+"""
+
+    header = "# KEDA autoscaling — auto-generated by generate_buildkit.py. Do not edit by hand.\n"
+    return (
+        header
+        + "\n"
+        + _scaledobject("amd64", "bk_amd64", amd64_min, amd64_max)
+        + "\n---\n"
+        + _scaledobject("arm64", "bk_arm64", arm64_min, arm64_max)
+    )
+
+
 def main():
     parser = argparse.ArgumentParser(description="Generate BuildKit Deployment and NodePool YAMLs")
     parser.add_argument("--arm64-instance-type", required=True, help="ARM64 instance type (e.g., m8gd.24xlarge)")
@@ -464,8 +553,17 @@ def main():
     parser.add_argument("--amd64-pods-per-node", type=int, default=None, help="Override amd64 pods per node")
     parser.add_argument("--arm64-pods-per-node", type=int, default=None, help="Override arm64 pods per node")
     parser.add_argument("--output-dir", required=True, help="Output directory for generated YAMLs")
+    parser.add_argument("--autoscaling", action="store_true", help="Generate KEDA autoscaling manifests")
+    parser.add_argument("--amd64-min", type=int, default=0, help="KEDA minReplicaCount for amd64")
+    parser.add_argument("--amd64-max", type=int, default=0, help="KEDA maxReplicaCount for amd64")
+    parser.add_argument("--arm64-min", type=int, default=0, help="KEDA minReplicaCount for arm64")
+    parser.add_argument("--arm64-max", type=int, default=0, help="KEDA maxReplicaCount for arm64")
     args = parser.parse_args()
 
+    if args.autoscaling and not (args.amd64_max and args.arm64_max):
+        log_error("--autoscaling requires --amd64-max and --arm64-max")
+        return 1
+
     # Validate instance types
     for it in [args.arm64_instance_type, args.amd64_instance_type]:
         if it not in INSTANCE_SPECS:
@@ -498,26 +596,51 @@ def main():
         arm64_replicas=args.arm64_replicas,
         amd64_pods_per_node=args.amd64_pods_per_node,
         arm64_pods_per_node=args.arm64_pods_per_node,
+        autoscaling=args.autoscaling,
     )
     deployment_path = output_dir / "deployment.yaml"
     deployment_path.write_text(deployment_yaml)
     log_info(f"Wrote {deployment_path}")
 
-    # Generate nodepools
-    nodepools_yaml = generate_nodepools_yaml(
-        args.arm64_instance_type,
-        args.amd64_instance_type,
-        args.replicas,
-        args.pods_per_node,
-        amd64_replicas=args.amd64_replicas,
-        arm64_replicas=args.arm64_replicas,
-        amd64_pods_per_node=args.amd64_pods_per_node,
-        arm64_pods_per_node=args.arm64_pods_per_node,
-    )
+    # Size NodePool limits for the peak: the per-arch autoscaling ceiling
+    # (amd64_max / arm64_max) when enabled, otherwise the per-arch replica counts.
+    if args.autoscaling:
+        nodepools_yaml = generate_nodepools_yaml(
+            args.arm64_instance_type,
+            args.amd64_instance_type,
+            args.replicas,
+            args.pods_per_node,
+            amd64_replicas=args.amd64_max,
+            arm64_replicas=args.arm64_max,
+            amd64_pods_per_node=args.amd64_pods_per_node,
+            arm64_pods_per_node=args.arm64_pods_per_node,
+        )
+    else:
+        nodepools_yaml = generate_nodepools_yaml(
+            args.arm64_instance_type,
+            args.amd64_instance_type,
+            args.replicas,
+            args.pods_per_node,
+            amd64_replicas=args.amd64_replicas,
+            arm64_replicas=args.arm64_replicas,
+            amd64_pods_per_node=args.amd64_pods_per_node,
+            arm64_pods_per_node=args.arm64_pods_per_node,
+        )
     nodepools_path = output_dir / "nodepools.yaml"
     nodepools_path.write_text(nodepools_yaml)
     log_info(f"Wrote {nodepools_path}")
 
+    if args.autoscaling:
+        autoscaling_yaml = generate_autoscaling_yaml(
+            args.amd64_min,
+            args.amd64_max,
+            args.arm64_min,
+            args.arm64_max,
+        )
+        autoscaling_path = output_dir / "autoscaling.yaml"
+        autoscaling_path.write_text(autoscaling_yaml)
+        log_info(f"Wrote {autoscaling_path}")
+
     return 0
 
 
diff --git a/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py b/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py
index 387a283d..c373c9c0 100644
--- a/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py
+++ b/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py
@@ -12,6 +12,7 @@
     DAEMONSET_OVERHEAD_MEM_MI,
     MARGIN,
     compute_pod_resources,
+    generate_autoscaling_yaml,
     generate_deployment_yaml,
     generate_nodepools_yaml,
 )
@@ -341,6 +342,65 @@ def test_nodepool_limits_scale_per_arch(self):
         assert np["buildkit-arm64"]["spec"]["limits"]["cpu"] == "256"
 
 
+# ============================================================================
+# autoscaling
+# ============================================================================
+
+
+class TestAutoscalingDeployment:
+    """When autoscaling=True the Deployment yields control of replicas to KEDA
+    and gains a preStop drain so scale-down never kills an in-flight build."""
+
+    def test_omits_replicas(self):
+        output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2, autoscaling=True)
+        for d in parse_all_yaml(output):
+            assert "replicas" not in d["spec"]
+
+    def test_keeps_replicas_when_disabled(self):
+        output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2)
+        for d in parse_all_yaml(output):
+            assert d["spec"]["replicas"] == 4
+
+    def test_drain_prestop_and_grace(self):
+        output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2, autoscaling=True)
+        for d in parse_all_yaml(output):
+            spec = d["spec"]["template"]["spec"]
+            assert spec["terminationGracePeriodSeconds"] == 8100
+            container = spec["containers"][0]
+            assert container["lifecycle"]["preStop"]["exec"]["command"] == ["/bin/sh", "/opt/drain/drain.sh"]
+            assert "drain" in {vm["name"] for vm in container["volumeMounts"]}
+            assert "drain" in {v["name"] for v in spec["volumes"]}
+
+
+class TestGenerateAutoscalingYaml:
+    """Tests for generate_autoscaling_yaml — in-cluster KEDA ScaledObjects."""
+
+    def _docs(self):
+        output = generate_autoscaling_yaml(2, 8, 4, 8)
+        return parse_all_yaml(output)
+
+    def test_only_scaledobjects_no_external_auth(self):
+        # In-cluster signal → no TriggerAuthentication / Grafana secret needed.
+        kinds = sorted(d["kind"] for d in self._docs())
+        assert kinds == ["ScaledObject", "ScaledObject"]
+
+    def test_per_arch_min_max(self):
+        scaled = {d["metadata"]["name"]: d for d in self._docs() if d["kind"] == "ScaledObject"}
+        assert set(scaled) == {"buildkitd-amd64", "buildkitd-arm64"}
+        assert scaled["buildkitd-amd64"]["spec"]["minReplicaCount"] == 2
+        assert scaled["buildkitd-amd64"]["spec"]["maxReplicaCount"] == 8
+        assert scaled["buildkitd-arm64"]["spec"]["minReplicaCount"] == 4
+        assert scaled["buildkitd-arm64"]["spec"]["maxReplicaCount"] == 8
+
+    def test_in_cluster_metrics_api_trigger(self):
+        scaled = {d["metadata"]["name"]: d for d in self._docs() if d["kind"] == "ScaledObject"}
+        for name, backend in [("buildkitd-amd64", "bk_amd64"), ("buildkitd-arm64", "bk_arm64")]:
+            trig = scaled[name]["spec"]["triggers"][0]
+            assert trig["type"] == "metrics-api"
+            assert "buildkitd-lb-metrics.buildkit" in trig["metadata"]["url"]
+            assert f'proxy="{backend}"' in trig["metadata"]["valueLocation"]
+
+
 # ============================================================================
 # generate_nodepools_yaml
 # ============================================================================
@@ -520,6 +580,64 @@ def test_deployment_yaml_parseable(self, tmp_path):
         docs = parse_all_yaml(deployment_text)
         assert len(docs) == 2
 
+    def test_autoscaling_writes_manifests(self, tmp_path):
+        output_dir = tmp_path / "output"
+
+        import generate_buildkit
+
+        test_args = [
+            "generate_buildkit.py",
+            "--arm64-instance-type",
+            "m8gd.24xlarge",
+            "--amd64-instance-type",
+            "m6id.24xlarge",
+            "--replicas",
+            "1",
+            "--pods-per-node",
+            "2",
+            "--output-dir",
+            str(output_dir),
+            "--autoscaling",
+            "--amd64-min",
+            "2",
+            "--amd64-max",
+            "8",
+            "--arm64-min",
+            "4",
+            "--arm64-max",
+            "8",
+        ]
+        with patch.object(sys, "argv", test_args):
+            result = generate_buildkit.main()
+
+        assert result == 0
+        autoscaling = parse_all_yaml((output_dir / "autoscaling.yaml").read_text())
+        assert sorted(d["kind"] for d in autoscaling) == ["ScaledObject", "ScaledObject"]
+
+    def test_autoscaling_requires_params(self, tmp_path):
+        output_dir = tmp_path / "output"
+
+        import generate_buildkit
+
+        test_args = [
+            "generate_buildkit.py",
+            "--arm64-instance-type",
+            "m8gd.24xlarge",
+            "--amd64-instance-type",
+            "m6id.24xlarge",
+            "--replicas",
+            "1",
+            "--pods-per-node",
+            "2",
+            "--output-dir",
+            str(output_dir),
+            "--autoscaling",
+        ]
+        with patch.object(sys, "argv", test_args):
+            result = generate_buildkit.main()
+
+        assert result == 1
+
     def test_unknown_instance_type_fails(self, tmp_path):
         output_dir = tmp_path / "output"
         test_args = [
diff --git a/osdc/modules/keda/deploy.sh b/osdc/modules/keda/deploy.sh
new file mode 100755
index 00000000..59f422ad
--- /dev/null
+++ b/osdc/modules/keda/deploy.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+#
+# KEDA module deploy script.
+# Called by: just deploy-module <cluster> keda
+# Args: $1=cluster-id  $2=cluster-name  $3=region
+#
+# Installs the KEDA operator (provides the ScaledObject/TriggerAuthentication
+# CRDs that the buildkit module uses to autoscale builders).
+
+CLUSTER="$1"
+MODULE_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="${OSDC_ROOT:-$(cd "$MODULE_DIR/../.." && pwd)}"
+UPSTREAM_ROOT="${OSDC_UPSTREAM:-$REPO_ROOT}"
+# shellcheck source=/dev/null
+source "$UPSTREAM_ROOT/scripts/mise-activate.sh"
+# shellcheck source=/dev/null
+source "$UPSTREAM_ROOT/scripts/helm-upgrade.sh"
+CFG="$UPSTREAM_ROOT/scripts/cluster-config.py"
+
+NAMESPACE="keda"
+CHART_VERSION=$(uv run "$CFG" "$CLUSTER" keda.chart_version 2.16.1)
+
+helm repo add kedacore https://kedacore.github.io/charts >/dev/null 2>&1 || true
+helm repo update kedacore >/dev/null 2>&1 || true
+
+helm_upgrade_if_changed keda "$NAMESPACE" \
+  --create-namespace \
+  --version "$CHART_VERSION" \
+  --timeout 5m \
+  --wait \
+  kedacore/keda
+
+echo "KEDA deployed (chart $CHART_VERSION)."

From 86bcea3caabb8bd04f7838ae7027bf37c4914080 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 02:07:33 -0700
Subject: [PATCH 2/7] Update

[ghstack-poisoned]
---
 osdc/modules/buildkit/README.md               | 15 +++++-
 .../scripts/python/generate_buildkit.py       | 48 +++++++++++--------
 2 files changed, 43 insertions(+), 20 deletions(-)

diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
index e17fafdf..37d64f9c 100644
--- a/osdc/modules/buildkit/README.md
+++ b/osdc/modules/buildkit/README.md
@@ -24,7 +24,20 @@ back to a small warm baseline when idle.
   limits are sized to `*_max`.
 - **Safe scale-down** — `preStop` drain (waits until the pod's `:1234` is idle)
   + long `terminationGracePeriodSeconds` + PDB, so a build is never killed
-  mid-flight.
+  mid-flight. Scale-down removes an arbitrary pod, which may be mid-build; the
+  drain holds termination until that build finishes, but
+  `terminationGracePeriodSeconds` is a hard SIGKILL cap, so it must outlast the
+  longest possible build. It's set to **8100s (135m) = 120m** (the max time a
+  docker build may run, matching HAProxy `timeout server`) **+ ~15m** of
+  headroom for the drain's idle-detection polling. A build that starts just
+  before drain still completes; the cap only fires as a backstop if a pod never
+  drains.
+  The **PDB** (`maxUnavailable: 1` per arch) bounds *voluntary* disruptions —
+  node consolidation and manual `kubectl drain` — to one builder per arch at a
+  time, so those go through the preStop drain one pod at a time instead of
+  evicting several in-flight builds at once. (KEDA scale-down deletes pods
+  directly rather than via the eviction API, so it isn't PDB-gated — the drain +
+  grace cap above is what protects that path.)
 
 Build clients should retry the connect so a build can wait for a pod from a cold
 or queued pool.
diff --git a/osdc/modules/buildkit/scripts/python/generate_buildkit.py b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
index 830b8998..35f00fd0 100644
--- a/osdc/modules/buildkit/scripts/python/generate_buildkit.py
+++ b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
@@ -36,6 +36,12 @@
 RED = "\033[0;31m"
 NC = "\033[0m"
 
+# Sentinel for optional template lines. An optional fragment is either its YAML
+# lines or this sentinel; lines equal to it are dropped when a block is assembled
+# (see _deployment_block). This lets every fragment sit on its own line in the
+# templates below instead of being concatenated onto an adjacent line.
+_OMIT = "<<omit>>"
+
 
 def log_info(msg):
     print(f"{GREEN}\u2192{NC} {msg}")
@@ -120,34 +126,32 @@ def generate_deployment_yaml(
     amd64_res = compute_pod_resources(amd64_instance, amd64_pods_per_node)
 
     # When KEDA owns the replica count, omit `replicas` and add a preStop drain
-    # that holds the pod open until its in-flight build finishes. `replicas_line`
-    # is computed per-arch inside _deployment_block (below).
-    grace_line = "      terminationGracePeriodSeconds: 8100\n" if autoscaling else ""
+    # that holds the pod open until its in-flight build finishes. Each fragment is
+    # either its YAML lines or _OMIT (dropped at assembly), so it sits on its own
+    # line in the template. (`replicas_line` is per-arch — computed below.)
+    grace_line = "      terminationGracePeriodSeconds: 8100" if autoscaling else _OMIT
     lifecycle_block = (
-        """
-          lifecycle:
+        """          lifecycle:
             preStop:
               exec:
                 command: ["/bin/sh", "/opt/drain/drain.sh"]"""
         if autoscaling
-        else ""
+        else _OMIT
     )
     drain_mount = (
-        """
-            - name: drain
+        """            - name: drain
               mountPath: /opt/drain
               readOnly: true"""
         if autoscaling
-        else ""
+        else _OMIT
     )
     drain_volume = (
-        """
-        - name: drain
+        """        - name: drain
           configMap:
             name: buildkitd-drain
             defaultMode: 0555"""
         if autoscaling
-        else ""
+        else _OMIT
     )
 
     log_info(
@@ -160,8 +164,8 @@ def generate_deployment_yaml(
     )
 
     def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_node):
-        replicas_line = "" if autoscaling else f"  replicas: {replicas}\n"
-        return f"""apiVersion: apps/v1
+        replicas_line = _OMIT if autoscaling else f"  replicas: {replicas}"
+        block = f"""apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: buildkitd-{arch}
@@ -172,7 +176,8 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
     app.kubernetes.io/name: buildkitd
     app.kubernetes.io/component: build-service
 spec:
-{replicas_line}  strategy:
+{replicas_line}
+  strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 0
@@ -187,7 +192,8 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
         app: buildkitd
         arch: {arch}
     spec:
-{grace_line}      nodeSelector:
+{grace_line}
+      nodeSelector:
         workload-type: buildkit
         instance-type: "{instance_type}"
 
@@ -240,7 +246,8 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
                   fieldPath: metadata.name
 
           securityContext:
-            privileged: true{lifecycle_block}
+            privileged: true
+{lifecycle_block}
 
           readinessProbe:
             exec:
@@ -269,7 +276,8 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
               subPathExpr: $(POD_NAME)
             - name: git-cache
               mountPath: /opt/git-cache
-              readOnly: true{drain_mount}
+              readOnly: true
+{drain_mount}
 
       volumes:
         - name: config
@@ -284,7 +292,9 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
         - name: git-cache
           hostPath:
             path: /mnt/k8s-disks/0/git-cache
-            type: DirectoryOrCreate{drain_volume}"""
+            type: DirectoryOrCreate
+{drain_volume}"""
+        return "\n".join(line for line in block.splitlines() if line != _OMIT)
 
     arm64_block = _deployment_block(
         "arm64", arm64_instance, arm64_res["cpu"], arm64_res["memory_gi"], arm64_replicas, arm64_pods_per_node

From 9dc83fcdbac9832e55c1b7c2b5b7db8033ac9870 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 02:51:44 -0700
Subject: [PATCH 3/7] Update

[ghstack-poisoned]
---
 .../buildkit/kubernetes/base/haproxy.yaml        |  6 +++---
 .../buildkit/scripts/python/generate_buildkit.py | 16 +++++++++-------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
index a52cece7..45c630d6 100644
--- a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
+++ b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
@@ -25,9 +25,9 @@ data:
       timeout client 120m
       timeout server 120m
       # Queue a build (server maxconn=1, below) while KEDA/Karpenter add pods,
-      # instead of stacking it on a busy pod. The runner also retries the
-      # connect, so this is an upper bound, not the only safety net.
-      timeout queue 10m
+      # instead of stacking it on a busy pod. Keep <= 120m, the max time a
+      # docker build is allowed to run.
+      timeout queue 60m
       log global
       option tcplog
 
diff --git a/osdc/modules/buildkit/scripts/python/generate_buildkit.py b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
index 35f00fd0..b39a09a2 100644
--- a/osdc/modules/buildkit/scripts/python/generate_buildkit.py
+++ b/osdc/modules/buildkit/scripts/python/generate_buildkit.py
@@ -127,9 +127,11 @@ def generate_deployment_yaml(
 
     # When KEDA owns the replica count, omit `replicas` and add a preStop drain
     # that holds the pod open until its in-flight build finishes. Each fragment is
-    # either its YAML lines or _OMIT (dropped at assembly), so it sits on its own
-    # line in the template. (`replicas_line` is per-arch — computed below.)
-    grace_line = "      terminationGracePeriodSeconds: 8100" if autoscaling else _OMIT
+    # either its YAML or _OMIT; _deployment_block drops _OMIT lines (matched after
+    # stripping), so single lines carry their indent in the template (e.g.
+    # `      {grace_line}`) while multi-line blocks self-indent. (`replicas_line`
+    # is per-arch — computed below.)
+    grace_line = "terminationGracePeriodSeconds: 8100" if autoscaling else _OMIT
     lifecycle_block = (
         """          lifecycle:
             preStop:
@@ -164,7 +166,7 @@ def generate_deployment_yaml(
     )
 
     def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_node):
-        replicas_line = _OMIT if autoscaling else f"  replicas: {replicas}"
+        replicas_line = _OMIT if autoscaling else f"replicas: {replicas}"
         block = f"""apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -176,7 +178,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
     app.kubernetes.io/name: buildkitd
     app.kubernetes.io/component: build-service
 spec:
-{replicas_line}
+  {replicas_line}
   strategy:
     type: RollingUpdate
     rollingUpdate:
@@ -192,7 +194,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
         app: buildkitd
         arch: {arch}
     spec:
-{grace_line}
+      {grace_line}
       nodeSelector:
         workload-type: buildkit
         instance-type: "{instance_type}"
@@ -294,7 +296,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no
             path: /mnt/k8s-disks/0/git-cache
             type: DirectoryOrCreate
 {drain_volume}"""
-        return "\n".join(line for line in block.splitlines() if line != _OMIT)
+        return "\n".join(line for line in block.splitlines() if line.strip() != _OMIT)
 
     arm64_block = _deployment_block(
         "arm64", arm64_instance, arm64_res["cpu"], arm64_res["memory_gi"], arm64_replicas, arm64_pods_per_node

From 16e85204069de1ea4eda21afe265628ef9723db3 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 03:52:28 -0700
Subject: [PATCH 4/7] Update

[ghstack-poisoned]
---
 osdc/modules/buildkit/README.md                    | 11 +++++++++++
 osdc/modules/buildkit/deploy.sh                    | 10 +++++++++-
 osdc/modules/buildkit/kubernetes/base/haproxy.yaml |  6 ++++++
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
index 37d64f9c..8f63639b 100644
--- a/osdc/modules/buildkit/README.md
+++ b/osdc/modules/buildkit/README.md
@@ -42,4 +42,15 @@ back to a small warm baseline when idle.
 Build clients should retry the connect so a build can wait for a pod from a cold
 or queued pool.
 
+## HAProxy config changes roll the LB
+
+HAProxy renders its config only at container start, and nothing else restarts
+the `buildkitd-lb` pod, so a bare ConfigMap update (`maxconn`, timeouts,
+backends) would silently not take effect. `deploy.sh` stamps the LB pod template
+with a `checksum/config` annotation = a hash of `haproxy.yaml`; when the config
+changes the hash changes, which rolls the Deployment so the new pod picks up the
+new config. An unchanged config keeps the same hash, so routine deploys don't
+churn the LB. (The buildkitd worker pods do **not** yet have this, so a
+`buildkitd.toml` / `drain.sh` change needs a manual rollout to take effect.)
+
 Requires the `keda` module deployed before `buildkit` (provides the CRDs).
diff --git a/osdc/modules/buildkit/deploy.sh b/osdc/modules/buildkit/deploy.sh
index ece804ad..f97d0f42 100755
--- a/osdc/modules/buildkit/deploy.sh
+++ b/osdc/modules/buildkit/deploy.sh
@@ -75,7 +75,15 @@ sed "s/CLUSTER_NAME_PLACEHOLDER/$CNAME/g" "$GENERATED_DIR/nodepools.yaml" | kube
 # --- Apply static k8s resources ---
 
 echo "Applying BuildKit static manifests..."
-kubectl_apply_if_changed -k "$MODULE_DIR/kubernetes/base/"
+# Stamp the buildkitd-lb pod template with a hash of haproxy.yaml. HAProxy reads
+# its config only at container start, and nothing else restarts the LB, so
+# without this a ConfigMap change (maxconn, timeouts, backends) would silently
+# not take effect until the pod happened to be recreated. Changing the hash
+# rolls the Deployment whenever the config changes.
+HAPROXY_SUM=$(shasum -a 256 "$MODULE_DIR/kubernetes/base/haproxy.yaml" | cut -c1-12)
+kubectl kustomize "$MODULE_DIR/kubernetes/base/" \
+  | sed "s/__HAPROXY_CFG_CHECKSUM__/$HAPROXY_SUM/" \
+  | kubectl_apply_if_changed -f -
 
 # --- Apply generated Deployments (only if changed) ---
 
diff --git a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
index 45c630d6..5b5f2f7f 100644
--- a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
+++ b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
@@ -104,6 +104,12 @@ spec:
     metadata:
       labels:
         app: buildkitd-lb
+      annotations:
+        # HAProxy renders its config once at container start, so a ConfigMap
+        # change has no effect until this pod restarts. deploy.sh fills this in
+        # with a hash of haproxy.yaml, so the Deployment rolls automatically
+        # whenever the config changes (nothing else restarts the LB).
+        checksum/config: "__HAPROXY_CFG_CHECKSUM__"
     spec:
       # Runs on base-infra nodes (CriticalAddonsOnly taint)
       tolerations:

From 3caba61557f5770d58413f3ccd4868589bec9f7e Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 10:28:59 -0700
Subject: [PATCH 5/7] Update

[ghstack-poisoned]
---
 osdc/modules/buildkit/README.md | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
index 8f63639b..a98ee0ac 100644
--- a/osdc/modules/buildkit/README.md
+++ b/osdc/modules/buildkit/README.md
@@ -39,8 +39,28 @@ back to a small warm baseline when idle.
   directly rather than via the eviction API, so it isn't PDB-gated — the drain +
   grace cap above is what protects that path.)
 
-Build clients should retry the connect so a build can wait for a pod from a cold
-or queued pool.
+## Build with `buildctl`, not `docker buildx`
+
+Clients must reach the pool with **`buildctl`** (`buildctl --addr
+tcp://buildkitd-<arch>.buildkit:1234 build ...`), not `docker buildx` against a
+remote builder.
+
+The autoscaling design relies on a *patient* client: during a burst the build's
+connection sits in HAProxy's queue (above) for the minutes it takes KEDA +
+Karpenter to add a pod, and that pending connection is also what keeps the
+scale-up signal alive. `buildctl` does exactly this — its build call waits in
+the queue up to `timeout queue` with no separate connect deadline.
+
+`docker buildx` does **not**: before solving it "boots" the remote builder with
+a **hardcoded ~20s connect timeout** (`[internal] waiting for connection`), which
+is not configurable and far shorter than a cold scale-up. Under a burst the
+connection is still queued at 20s, buildx aborts with `context deadline
+exceeded`, and — because the connection then drops — the scale-up signal
+disappears before KEDA can act, so the pool never grows and every queued build
+fails. (`docker/setup-buildx-action` hits the same gate via `inspect
+--bootstrap`; removing it doesn't help because `docker buildx build` re-runs the
+same boot.) This was confirmed on the staging cluster. So PyTorch's
+`.ci/docker/build.sh` uses `buildctl` whenever `REMOTE_BUILDKIT` is set.
 
 ## HAProxy config changes roll the LB
 

From 229998c931712d58ae3a8910502814d142d4c714 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 11:44:15 -0700
Subject: [PATCH 6/7] Update

[ghstack-poisoned]
---
 osdc/modules/buildkit/README.md               | 40 ++++++++-----------
 .../buildkit/kubernetes/base/haproxy.yaml     |  4 --
 2 files changed, 17 insertions(+), 27 deletions(-)

diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
index a98ee0ac..6e4e30eb 100644
--- a/osdc/modules/buildkit/README.md
+++ b/osdc/modules/buildkit/README.md
@@ -13,9 +13,9 @@ Absorb bursts of concurrent builds without overloading existing pods, and scale
 back to a small warm baseline when idle.
 
 - **One build per pod** — HAProxy `server maxconn 1` (matches buildkitd
-  `max-parallelism = 1`). Excess builds **queue** in HAProxy (`timeout queue`)
-  instead of stacking on a busy pod; as new pods register (DNS), queued builds
-  flow onto them, so scaled-up pods never sit idle.
+  `max-parallelism = 1`) so a build never stacks on a busy pod. When every pod is
+  busy the LB has no slot, so the client must **retry the connect** (see below)
+  until a pod frees or the pool scales up.
 - **In-cluster scale signal** — KEDA `ScaledObject` per arch, `metrics-api`
   scraping the LB's own metrics (`haproxy_backend_current_sessions`) — no external
   metrics backend.
@@ -39,28 +39,22 @@ back to a small warm baseline when idle.
   directly rather than via the eviction API, so it isn't PDB-gated — the drain +
   grace cap above is what protects that path.)
 
-## Build with `buildctl`, not `docker buildx`
+## Clients must retry the connect
 
-Clients must reach the pool with **`buildctl`** (`buildctl --addr
-tcp://buildkitd-<arch>.buildkit:1234 build ...`), not `docker buildx` against a
-remote builder.
+Build clients (both `docker buildx` and `buildctl`) use the `moby/buildkit` Go
+client, which dials with gRPC's default **~20s `MinConnectTimeout`** and
+**fail-fast** RPCs — there is no client-side flag to make it wait longer. During
+a burst, a build whose connection finds no free pod (`maxconn 1`) is dropped by
+the client after ~20s, well before KEDA/Karpenter can add a pod (minutes). An
+HAProxy-side `timeout queue` does **not** help: the client gives up at 20s
+regardless, so queueing on the LB is pointless (and was removed).
 
-The autoscaling design relies on a *patient* client: during a burst the build's
-connection sits in HAProxy's queue (above) for the minutes it takes KEDA +
-Karpenter to add a pod, and that pending connection is also what keeps the
-scale-up signal alive. `buildctl` does exactly this — its build call waits in
-the queue up to `timeout queue` with no separate connect deadline.
-
-`docker buildx` does **not**: before solving it "boots" the remote builder with
-a **hardcoded ~20s connect timeout** (`[internal] waiting for connection`), which
-is not configurable and far shorter than a cold scale-up. Under a burst the
-connection is still queued at 20s, buildx aborts with `context deadline
-exceeded`, and — because the connection then drops — the scale-up signal
-disappears before KEDA can act, so the pool never grows and every queued build
-fails. (`docker/setup-buildx-action` hits the same gate via `inspect
---bootstrap`; removing it doesn't help because `docker buildx build` re-runs the
-same boot.) This was confirmed on the staging cluster. So PyTorch's
-`.ci/docker/build.sh` uses `buildctl` whenever `REMOTE_BUILDKIT` is set.
+So the **client must retry the build** on connection failures until a pod is
+free or the pool has scaled up; the repeated attempts also keep the autoscaler's
+load signal alive. PyTorch's `.ci/docker/build.sh` does this when
+`REMOTE_BUILDKIT` is set, and the workflow creates the remote builder *without*
+`--bootstrap` (the `docker buildx inspect --bootstrap` health check hits the same
+20s gate at setup). This was confirmed on the staging cluster.
 
 ## HAProxy config changes roll the LB
 
diff --git a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
index 5b5f2f7f..d37eeca9 100644
--- a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
+++ b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml
@@ -24,10 +24,6 @@ data:
       timeout connect 5s
       timeout client 120m
       timeout server 120m
-      # Queue a build (server maxconn=1, below) while KEDA/Karpenter add pods,
-      # instead of stacking it on a busy pod. Keep <= 120m, the max time a
-      # docker build is allowed to run.
-      timeout queue 60m
       log global
       option tcplog
 

From d6e63b6739f409f919226ed5919a9ad9ccd0ae52 Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Wed, 10 Jun 2026 16:23:12 -0700
Subject: [PATCH 7/7] Update

[ghstack-poisoned]
---
 osdc/modules/buildkit/README.md | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md
index 6e4e30eb..b38c1511 100644
--- a/osdc/modules/buildkit/README.md
+++ b/osdc/modules/buildkit/README.md
@@ -4,8 +4,9 @@ Remote BuildKit build service: per-arch `buildkitd` Deployments behind an HAProx
 LB, on dedicated Karpenter NodePools. Clients build with
 `buildctl --addr tcp://buildkitd-<arch>.buildkit:1234`.
 
-Sizing is per-arch in `clusters.yaml` (`buildkit.{amd64,arm64}_{replicas,pods_per_node}`,
-`*_instance_type`); pod CPU/mem is computed by `scripts/python/generate_buildkit.py`.
+Sizing is per-arch in `clusters.yaml` (`buildkit.{amd64,arm64}_*` instance type,
+pods-per-node, and autoscaling `*_min` / `*_max`); pod CPU/mem is computed by
+`scripts/python/generate_buildkit.py`.
 
 ## Autoscaling (optional, `buildkit.autoscaling.enabled`)
 
@@ -18,10 +19,15 @@ back to a small warm baseline when idle.
   until a pod frees or the pool scales up.
 - **In-cluster scale signal** — KEDA `ScaledObject` per arch, `metrics-api`
   scraping the LB's own metrics (`haproxy_backend_current_sessions`) — no external
-  metrics backend.
+  metrics backend. If KEDA can't read the metric, a `fallback` (`*_fallback`,
+  e.g. 32/8 on prod) holds the proven fixed pool instead of freezing the count.
 - **Warm baseline** — `amd64_min` / `arm64_min` keep ≥1 node per arch up so the
   common case gets a free warm pod immediately. `*_max` caps the burst; NodePool
   limits are sized to `*_max`.
+- **No-flap scale-down** — KEDA holds a pod ~20 min after it goes idle
+  (`stabilizationWindowSeconds: 1200`), then sheds at most `max(10 pods, 20%)`
+  per 20 min, so a follow-up build reuses the pod's warm decompressed NVMe layer
+  cache. Node churn is left to Karpenter.
 - **Safe scale-down** — `preStop` drain (waits until the pod's `:1234` is idle)
   + long `terminationGracePeriodSeconds` + PDB, so a build is never killed
   mid-flight. Scale-down removes an arbitrary pod, which may be mid-build; the
@@ -67,4 +73,6 @@ new config. An unchanged config keeps the same hash, so routine deploys don't
 churn the LB. (The buildkitd worker pods do **not** yet have this, so a
 `buildkitd.toml` / `drain.sh` change needs a manual rollout to take effect.)
 
-Requires the `keda` module deployed before `buildkit` (provides the CRDs).
+Requires the `keda` module deployed before `buildkit` (provides the CRDs). The
+`monitoring` module scrapes the KEDA operator's metrics and ships
+buildkit-autoscaling alerts (scaler / fallback errors, queue backlog).