diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml index 44aa9a75..f6e15a08 100644 --- a/osdc/clusters.yaml +++ b/osdc/clusters.yaml @@ -83,6 +83,10 @@ defaults: cpu_disruption_budget: "20%" buildkit: replicas_per_arch: 12 + autoscaling: + enabled: false + keda: + chart_version: "2.16.1" monitoring: grafana_cloud_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom/push" grafana_cloud_read_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom" @@ -162,8 +166,14 @@ clusters: github_secret_name: pytorch-arc-staging runner_name_prefix: "c-mt-" buildkit: - replicas_per_arch: 2 arm64_instance_type: m7gd.16xlarge + arm64_pods_per_node: 4 + autoscaling: + enabled: true + amd64_min: 2 # 1x m6id.24xlarge (2 pods/node) + amd64_max: 8 + arm64_min: 4 # 1x m7gd.16xlarge (4 pods/node) + arm64_max: 8 pypi_cache: replicas: 1 modules: @@ -171,6 +181,7 @@ clusters: - arc - nodepools - arc-runners + - keda - buildkit - pypi-cache - cache-enforcer @@ -283,11 +294,15 @@ clusters: min_node_age_seconds: 900 buildkit: amd64_instance_type: m6id.24xlarge - amd64_replicas: 32 amd64_pods_per_node: 2 arm64_instance_type: m7gd.16xlarge - arm64_replicas: 8 arm64_pods_per_node: 4 + autoscaling: + enabled: true + amd64_min: 2 # 1x m6id.24xlarge (2 pods/node) + amd64_max: 128 # 14d peak 105; headroom to spare above it + arm64_min: 4 # 1x m7gd.16xlarge (4 pods/node) + arm64_max: 16 # 14d peak 8, likely capped by fixed pool; headroom arc-runners: github_config_url: "https://github.com/pytorch" github_secret_name: pytorch-arc-cbr-production diff --git a/osdc/integration-tests/docker/test-buildkit-scale/Dockerfile b/osdc/integration-tests/docker/test-buildkit-scale/Dockerfile new file mode 100644 index 00000000..e317eb61 --- /dev/null +++ b/osdc/integration-tests/docker/test-buildkit-scale/Dockerfile @@ -0,0 +1,6 @@ +# Holds one BuildKit slot (~10 min) so a burst of parallel builds exceeds the +# warm baseline and forces KEDA scale-up. CACHEBUST differs per build so each +# actually runs (no layer reuse). +FROM alpine:3.21 +ARG CACHEBUST +RUN echo "osdc buildkit scale-test ${CACHEBUST}" && sleep 600 diff --git a/osdc/integration-tests/scripts/python/phases.py b/osdc/integration-tests/scripts/python/phases.py index 27ddebf6..17654000 100644 --- a/osdc/integration-tests/scripts/python/phases.py +++ b/osdc/integration-tests/scripts/python/phases.py @@ -276,15 +276,17 @@ def prepare_pr( # Write integration test workflow (workflows_dir / "integration-test.yaml").write_text(workflow_content) - # Copy build-image reusable workflow - build_wf_src = upstream_dir / "integration-tests" / "workflows" / "build-image.yaml" - (workflows_dir / "build-image.yaml").write_text(build_wf_src.read_text()) - - # Copy test Dockerfile - docker_dir = canary_path / "docker" / "test-buildkit" - docker_dir.mkdir(parents=True, exist_ok=True) - dockerfile_src = upstream_dir / "integration-tests" / "docker" / "test-buildkit" / "Dockerfile" - (docker_dir / "Dockerfile").write_text(dockerfile_src.read_text()) + # Copy reusable BuildKit workflows + wf_root = upstream_dir / "integration-tests" / "workflows" + for wf in ("build-image.yaml", "build-image-scale.yaml"): + (workflows_dir / wf).write_text((wf_root / wf).read_text()) + + # Copy test Dockerfiles (connectivity + autoscaling scale test) + docker_root = upstream_dir / "integration-tests" / "docker" + for name in ("test-buildkit", "test-buildkit-scale"): + dst = canary_path / "docker" / name + dst.mkdir(parents=True, exist_ok=True) + (dst / "Dockerfile").write_text((docker_root / name / "Dockerfile").read_text()) # Commit run_cmd(["git", "add", "-A"], cwd=canary_path) diff --git a/osdc/integration-tests/scripts/python/test_run.py b/osdc/integration-tests/scripts/python/test_run.py index ff3a7bcb..dc4d5188 100644 --- a/osdc/integration-tests/scripts/python/test_run.py +++ b/osdc/integration-tests/scripts/python/test_run.py @@ -112,11 +112,13 @@ def workflow_template(tmp_path): ) (wf_dir / "integration-test.yaml.tpl").write_text(template) - # Also create build-image.yaml and Dockerfile for prepare_pr + # Also create reusable workflows and Dockerfiles for prepare_pr (wf_dir / "build-image.yaml").write_text("name: build-image\n") - docker_dir = upstream / "integration-tests" / "docker" / "test-buildkit" - docker_dir.mkdir(parents=True) - (docker_dir / "Dockerfile").write_text("FROM alpine\n") + (wf_dir / "build-image-scale.yaml").write_text("name: build-image-scale\n") + for name in ("test-buildkit", "test-buildkit-scale"): + docker_dir = upstream / "integration-tests" / "docker" / name + docker_dir.mkdir(parents=True) + (docker_dir / "Dockerfile").write_text("FROM alpine\n") return upstream diff --git a/osdc/integration-tests/workflows/build-image-scale.yaml b/osdc/integration-tests/workflows/build-image-scale.yaml new file mode 100644 index 00000000..b485dc5a --- /dev/null +++ b/osdc/integration-tests/workflows/build-image-scale.yaml @@ -0,0 +1,59 @@ +# Reusable workflow: BuildKit autoscaling scale test. +# Launches a burst of 8 parallel builds against one arch's remote BuildKit, each +# holding a slot (server maxconn=1) for ~10 min via a sleep. The warm baseline +# (amd64_min / arm64_min) is below the burst, so the builds finish within +# timeout-minutes only if KEDA scales the pool up. Without scale-up they +# serialize through the baseline pods and the back of the queue times out — i.e. +# this job FAILS when autoscaling does not happen. +name: BuildKit Scale Test + +on: + workflow_call: + inputs: + arch: + description: "Target architecture (amd64 or arm64)" + required: true + type: string + runner_label: + description: "Runner label to use (includes cluster prefix)" + required: false + type: string + default: "l-x86iavx512-2-4" + +jobs: + scale: + # Runs on x86 — BuildKit is a *remote* builder; arch selects the endpoint. + # timeout-minutes is the gate: scaled-up ~18 min, serialized ~43 min. + runs-on: ${{ inputs.runner_label }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + replica: [1, 2, 3, 4, 5, 6, 7, 8] + container: + image: ghcr.io/actions/actions-runner:latest + steps: + - name: Install buildctl + run: | + BUILDKIT_VERSION="v0.29.0" + mkdir -p "$HOME/.local/bin" + curl -sSL "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz" \ + | tar xz --strip-components=1 -C "$HOME/.local/bin" bin/buildctl + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + "$HOME/.local/bin/buildctl" --version + + - name: Checkout + uses: actions/checkout@v4 + + - name: Occupy a BuildKit slot (~10 min) to drive autoscaling + run: | + ENDPOINT="tcp://buildkitd-${{ inputs.arch }}.buildkit:1234" + echo "Build ${{ matrix.replica }} -> $ENDPOINT" + buildctl --addr "$ENDPOINT" build \ + --frontend dockerfile.v0 \ + --local context=docker/test-buildkit-scale \ + --local dockerfile=docker/test-buildkit-scale \ + --opt build-arg:CACHEBUST=${{ inputs.arch }}-${{ matrix.replica }}-${{ github.run_id }} \ + --no-cache \ + --output type=cacheonly + echo "PASS: build ${{ matrix.replica }} finished within timeout" diff --git a/osdc/integration-tests/workflows/integration-test.yaml.tpl b/osdc/integration-tests/workflows/integration-test.yaml.tpl index c9563480..7d4fb3aa 100644 --- a/osdc/integration-tests/workflows/integration-test.yaml.tpl +++ b/osdc/integration-tests/workflows/integration-test.yaml.tpl @@ -1514,6 +1514,20 @@ jobs: arch: arm64 runner_label: {{PREFIX}}l-x86iamx-8-32 + # ── BuildKit Autoscaling Scale Test ─────────────────────────────────── + # Bursts 8 parallel builds per arch; fails if KEDA does not scale the pool up. + buildkit-scale-amd64: + uses: ./.github/workflows/build-image-scale.yaml + with: + arch: amd64 + runner_label: {{PREFIX}}l-x86iamx-8-32 + + buildkit-scale-arm64: + uses: ./.github/workflows/build-image-scale.yaml + with: + arch: arm64 + runner_label: {{PREFIX}}l-x86iamx-8-32 + # ── Harbor Cache Test ───────────────────────────────────────────────── test-harbor: runs-on: {{PREFIX}}l-x86iamx-8-32 diff --git a/osdc/modules/buildkit/README.md b/osdc/modules/buildkit/README.md new file mode 100644 index 00000000..21fd44dd --- /dev/null +++ b/osdc/modules/buildkit/README.md @@ -0,0 +1,40 @@ +# BuildKit module + +Remote BuildKit build service: per-arch `buildkitd` Deployments behind an HAProxy +LB, on dedicated Karpenter NodePools. Clients build with +`buildctl --addr tcp://buildkitd-.buildkit:1234`. + +Sizing is per-arch in `clusters.yaml` (`buildkit.{amd64,arm64}_{replicas,pods_per_node}`, +`*_instance_type`); pod CPU/mem is computed by `scripts/python/generate_buildkit.py`. + +## Autoscaling (optional, `buildkit.autoscaling.enabled`) + +Absorb bursts of concurrent builds without overloading existing pods, and scale +back to a small warm baseline when idle. + +- **One build per pod** — HAProxy `server maxconn 1` (matches buildkitd + `max-parallelism = 1`). Excess builds **queue** in HAProxy instead of stacking + on a busy pod; as new pods register (DNS), queued builds flow onto them, so + scaled-up pods never sit idle. `timeout queue` must stay set and large enough + to outlast a node-provision cycle — if omitted it falls back to `timeout + connect` (5s), which would abort queued builds before pods scale up. +- **In-cluster scale signal** — KEDA `ScaledObject` per arch, `metrics-api` + scraping the LB's own metrics (`haproxy_backend_current_sessions`) — no external + metrics backend. +- **Warm baseline** — `amd64_min` / `arm64_min` keep ≥1 node per arch up so the + common case gets a free warm pod immediately. `*_max` caps the burst; NodePool + limits are sized to `*_max`. +- **Safe scale-down** — `preStop` drain (waits until the pod's `:1234` is idle) + + long `terminationGracePeriodSeconds` + PDB, so a build is never killed + mid-flight. +- **Node consolidation** — the NodePool uses Karpenter `consolidationPolicy: + WhenEmpty`: a node is reclaimed only once it has no buildkitd pod, never by + evicting a running build to bin-pack. So after a burst, scattered survivor + pods can leave nodes half-full (more nodes than the cold baseline) until they + drain naturally — a deliberate trade of some idle node cost for zero build + disruption. + +Build clients should retry the connect so a build can wait for a pod from a cold +or queued pool. + +Requires the `keda` module deployed before `buildkit` (provides the CRDs). diff --git a/osdc/modules/buildkit/deploy.sh b/osdc/modules/buildkit/deploy.sh index c7be6d39..d7c1fa72 100755 --- a/osdc/modules/buildkit/deploy.sh +++ b/osdc/modules/buildkit/deploy.sh @@ -34,22 +34,38 @@ AMD64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_replicas "$REPLICAS") ARM64_REPLICAS=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_replicas "$REPLICAS") AMD64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.amd64_pods_per_node "$PODS_PER_NODE") ARM64_PODS_PER_NODE=$(uv run "$CFG" "$CLUSTER" buildkit.arm64_pods_per_node "$PODS_PER_NODE") +AUTOSCALING=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.enabled false | tr '[:upper:]' '[:lower:]') GENERATED_DIR="$MODULE_DIR/generated" # --- Generate manifests --- echo "Generating BuildKit manifests..." -uv run "$MODULE_DIR/scripts/python/generate_buildkit.py" \ - --arm64-instance-type "$ARM64_INSTANCE" \ - --amd64-instance-type "$AMD64_INSTANCE" \ - --replicas "$REPLICAS" \ - --pods-per-node "$PODS_PER_NODE" \ - --amd64-replicas "$AMD64_REPLICAS" \ - --arm64-replicas "$ARM64_REPLICAS" \ - --amd64-pods-per-node "$AMD64_PODS_PER_NODE" \ - --arm64-pods-per-node "$ARM64_PODS_PER_NODE" \ +GEN_ARGS=( + --arm64-instance-type "$ARM64_INSTANCE" + --amd64-instance-type "$AMD64_INSTANCE" + --replicas "$REPLICAS" + --pods-per-node "$PODS_PER_NODE" + --amd64-replicas "$AMD64_REPLICAS" + --arm64-replicas "$ARM64_REPLICAS" + --amd64-pods-per-node "$AMD64_PODS_PER_NODE" + --arm64-pods-per-node "$ARM64_PODS_PER_NODE" --output-dir "$GENERATED_DIR" +) +if [[ "$AUTOSCALING" == "true" ]]; then + AMD64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_min 2) + AMD64_MAX=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.amd64_max 8) + ARM64_MIN=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_min 4) + ARM64_MAX=$(uv run "$CFG" "$CLUSTER" buildkit.autoscaling.arm64_max 8) + GEN_ARGS+=( + --autoscaling + --amd64-min "$AMD64_MIN" + --amd64-max "$AMD64_MAX" + --arm64-min "$ARM64_MIN" + --arm64-max "$ARM64_MAX" + ) +fi +uv run "$MODULE_DIR/scripts/python/generate_buildkit.py" "${GEN_ARGS[@]}" # --- Apply NodePools (with cluster name substitution) --- @@ -93,5 +109,13 @@ else kubectl rollout status deployment/buildkitd-amd64 -n buildkit --timeout=15m fi +# --- KEDA autoscaling (optional) --- +# Scales on the in-cluster buildkit LB metrics; no external metrics backend. + +if [[ "$AUTOSCALING" == "true" ]]; then + echo "Applying KEDA autoscaling manifests..." + kubectl_apply_if_changed -f "$GENERATED_DIR/autoscaling.yaml" +fi + echo "BuildKit deployed." kubectl get pods -n buildkit -o wide diff --git a/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml b/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml new file mode 100644 index 00000000..89395ae9 --- /dev/null +++ b/osdc/modules/buildkit/kubernetes/base/drain-configmap.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: buildkitd-drain + namespace: buildkit +data: + # preStop drain: block termination until no in-flight build remains. A build + # keeps an ESTABLISHED inbound connection on :1234 for its whole duration; + # require two consecutive idle polls so a transient health check can't be + # mistaken for "done". terminationGracePeriodSeconds caps the total wait. + drain.sh: | + #!/bin/sh + idle=0 + while [ "$idle" -lt 2 ]; do + if netstat -tn 2>/dev/null | awk '$NF=="ESTABLISHED" && $4 ~ /:1234$/{f=1} END{exit !f}'; then + idle=0 + else + idle=$((idle + 1)) + fi + sleep 15 + done diff --git a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml index 3bfd4eb2..cd0b25ee 100644 --- a/osdc/modules/buildkit/kubernetes/base/haproxy.yaml +++ b/osdc/modules/buildkit/kubernetes/base/haproxy.yaml @@ -24,6 +24,9 @@ data: timeout connect 5s timeout client 120m timeout server 120m + # Queue builds (server maxconn=1) while pods scale up. Keep <= 120m, the max + # time a docker build is allowed to run. + timeout queue 60m log global option tcplog @@ -71,15 +74,15 @@ data: # resolution warnings. backend bk_arm64 balance leastconn - server-template arm 10 buildkitd-arm64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 + server-template arm 10 buildkitd-arm64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1 backend bk_amd64 balance leastconn - server-template amd 10 buildkitd-amd64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 + server-template amd 10 buildkitd-amd64-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1 backend bk_all balance leastconn - server-template all 20 buildkitd-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 + server-template all 20 buildkitd-pods.buildkit.svc.cluster.local:1234 check resolvers k8s init-addr none resolve-prefer ipv6 maxconn 1 --- apiVersion: apps/v1 diff --git a/osdc/modules/buildkit/kubernetes/base/kustomization.yaml b/osdc/modules/buildkit/kubernetes/base/kustomization.yaml index 9cbffb38..32572408 100644 --- a/osdc/modules/buildkit/kubernetes/base/kustomization.yaml +++ b/osdc/modules/buildkit/kubernetes/base/kustomization.yaml @@ -5,6 +5,8 @@ kind: Kustomization resources: - namespace.yaml - configmap.yaml + - drain-configmap.yaml - haproxy.yaml - service.yaml - networkpolicy.yaml + - poddisruptionbudget.yaml diff --git a/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml b/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml new file mode 100644 index 00000000..b717bb64 --- /dev/null +++ b/osdc/modules/buildkit/kubernetes/base/poddisruptionbudget.yaml @@ -0,0 +1,25 @@ +# Cap voluntary disruptions (node consolidation, drains) to one builder per arch +# at a time so evictions go through the preStop drain instead of killing builds. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: buildkitd-arm64 + namespace: buildkit +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: buildkitd + arch: arm64 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: buildkitd-amd64 + namespace: buildkit +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: buildkitd + arch: amd64 diff --git a/osdc/modules/buildkit/scripts/python/generate_buildkit.py b/osdc/modules/buildkit/scripts/python/generate_buildkit.py index e7468418..788dd9e6 100644 --- a/osdc/modules/buildkit/scripts/python/generate_buildkit.py +++ b/osdc/modules/buildkit/scripts/python/generate_buildkit.py @@ -103,6 +103,7 @@ def generate_deployment_yaml( arm64_replicas: int | None = None, amd64_pods_per_node: int | None = None, arm64_pods_per_node: int | None = None, + autoscaling: bool = False, ) -> str: """Generate the combined Deployment YAML for both architectures. @@ -118,6 +119,35 @@ def generate_deployment_yaml( arm64_res = compute_pod_resources(arm64_instance, arm64_pods_per_node) amd64_res = compute_pod_resources(amd64_instance, amd64_pods_per_node) + # Under autoscaling KEDA owns replicas: omit `replicas`, add a preStop drain. + grace_line = " terminationGracePeriodSeconds: 8100\n" if autoscaling else "" + lifecycle_block = ( + """ + lifecycle: + preStop: + exec: + command: ["/bin/sh", "/opt/drain/drain.sh"]""" + if autoscaling + else "" + ) + drain_mount = ( + """ + - name: drain + mountPath: /opt/drain + readOnly: true""" + if autoscaling + else "" + ) + drain_volume = ( + """ + - name: drain + configMap: + name: buildkitd-drain + defaultMode: 0555""" + if autoscaling + else "" + ) + log_info( f"arm64 ({arm64_instance}): {arm64_res['cpu']} vCPU, {arm64_res['memory_gi']}Gi per pod " f"(allocatable: {arm64_res['allocatable_cpu_m']}m CPU, {arm64_res['allocatable_mem_mi']}Mi mem)" @@ -128,6 +158,7 @@ def generate_deployment_yaml( ) def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_node): + replicas_line = "" if autoscaling else f" replicas: {replicas}\n" return f"""apiVersion: apps/v1 kind: Deployment metadata: @@ -139,8 +170,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no app.kubernetes.io/name: buildkitd app.kubernetes.io/component: build-service spec: - replicas: {replicas} - strategy: +{replicas_line} strategy: type: RollingUpdate rollingUpdate: maxSurge: 0 @@ -155,7 +185,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no app: buildkitd arch: {arch} spec: - nodeSelector: +{grace_line} nodeSelector: workload-type: buildkit instance-type: "{instance_type}" @@ -208,7 +238,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no fieldPath: metadata.name securityContext: - privileged: true + privileged: true{lifecycle_block} readinessProbe: exec: @@ -237,7 +267,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no subPathExpr: $(POD_NAME) - name: git-cache mountPath: /opt/git-cache - readOnly: true + readOnly: true{drain_mount} volumes: - name: config @@ -252,7 +282,7 @@ def _deployment_block(arch, instance_type, cpu, memory_gi, replicas, pods_per_no - name: git-cache hostPath: path: /mnt/k8s-disks/0/git-cache - type: DirectoryOrCreate""" + type: DirectoryOrCreate{drain_volume}""" arm64_block = _deployment_block( "arm64", arm64_instance, arm64_res["cpu"], arm64_res["memory_gi"], arm64_replicas, arm64_pods_per_node @@ -453,6 +483,63 @@ def _nodepool_block(arch, instance_type, cpu_limit, memory_limit_gi): return arm64_block + "\n\n---\n" + amd64_block + "\n" +def generate_autoscaling_yaml( + amd64_min: int, + amd64_max: int, + arm64_min: int, + arm64_max: int, +) -> str: + """Generate per-arch KEDA ScaledObjects. + + Each arch scales on its HAProxy backend's active build count + (haproxy_backend_current_sessions), scraped in-cluster from the buildkit LB + metrics endpoint — no external metrics backend. With server maxconn=1, the LB + queues bursts while KEDA/Karpenter bring up pods; minReplicaCount keeps a warm + baseline so the common case has a free pod immediately. + """ + + metrics_url = "http://buildkitd-lb-metrics.buildkit.svc.cluster.local:9404/metrics" + + def _scaledobject(arch, backend, min_replicas, max_replicas): + return f"""apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: buildkitd-{arch} + namespace: buildkit +spec: + scaleTargetRef: + name: buildkitd-{arch} + minReplicaCount: {min_replicas} + maxReplicaCount: {max_replicas} + cooldownPeriod: 600 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 600 + policies: + - type: Pods + value: 1 + periodSeconds: 120 + triggers: + - type: metrics-api + metadata: + url: "{metrics_url}" + format: "prometheus" + valueLocation: 'haproxy_backend_current_sessions{{proxy="{backend}"}}' + targetValue: "1" +""" + + header = "# KEDA autoscaling — auto-generated by generate_buildkit.py. Do not edit by hand.\n" + return ( + header + + "\n" + + _scaledobject("amd64", "bk_amd64", amd64_min, amd64_max) + + "\n---\n" + + _scaledobject("arm64", "bk_arm64", arm64_min, arm64_max) + ) + + def main(): parser = argparse.ArgumentParser(description="Generate BuildKit Deployment and NodePool YAMLs") parser.add_argument("--arm64-instance-type", required=True, help="ARM64 instance type (e.g., m8gd.24xlarge)") @@ -464,8 +551,17 @@ def main(): parser.add_argument("--amd64-pods-per-node", type=int, default=None, help="Override amd64 pods per node") parser.add_argument("--arm64-pods-per-node", type=int, default=None, help="Override arm64 pods per node") parser.add_argument("--output-dir", required=True, help="Output directory for generated YAMLs") + parser.add_argument("--autoscaling", action="store_true", help="Generate KEDA autoscaling manifests") + parser.add_argument("--amd64-min", type=int, default=0, help="KEDA minReplicaCount for amd64") + parser.add_argument("--amd64-max", type=int, default=0, help="KEDA maxReplicaCount for amd64") + parser.add_argument("--arm64-min", type=int, default=0, help="KEDA minReplicaCount for arm64") + parser.add_argument("--arm64-max", type=int, default=0, help="KEDA maxReplicaCount for arm64") args = parser.parse_args() + if args.autoscaling and not (args.amd64_max and args.arm64_max): + log_error("--autoscaling requires --amd64-max and --arm64-max") + return 1 + # Validate instance types for it in [args.arm64_instance_type, args.amd64_instance_type]: if it not in INSTANCE_SPECS: @@ -498,26 +594,51 @@ def main(): arm64_replicas=args.arm64_replicas, amd64_pods_per_node=args.amd64_pods_per_node, arm64_pods_per_node=args.arm64_pods_per_node, + autoscaling=args.autoscaling, ) deployment_path = output_dir / "deployment.yaml" deployment_path.write_text(deployment_yaml) log_info(f"Wrote {deployment_path}") - # Generate nodepools - nodepools_yaml = generate_nodepools_yaml( - args.arm64_instance_type, - args.amd64_instance_type, - args.replicas, - args.pods_per_node, - amd64_replicas=args.amd64_replicas, - arm64_replicas=args.arm64_replicas, - amd64_pods_per_node=args.amd64_pods_per_node, - arm64_pods_per_node=args.arm64_pods_per_node, - ) + # Size NodePool limits for the peak: the per-arch autoscaling ceiling + # (amd64_max / arm64_max) when enabled, otherwise the per-arch replica counts. + if args.autoscaling: + nodepools_yaml = generate_nodepools_yaml( + args.arm64_instance_type, + args.amd64_instance_type, + args.replicas, + args.pods_per_node, + amd64_replicas=args.amd64_max, + arm64_replicas=args.arm64_max, + amd64_pods_per_node=args.amd64_pods_per_node, + arm64_pods_per_node=args.arm64_pods_per_node, + ) + else: + nodepools_yaml = generate_nodepools_yaml( + args.arm64_instance_type, + args.amd64_instance_type, + args.replicas, + args.pods_per_node, + amd64_replicas=args.amd64_replicas, + arm64_replicas=args.arm64_replicas, + amd64_pods_per_node=args.amd64_pods_per_node, + arm64_pods_per_node=args.arm64_pods_per_node, + ) nodepools_path = output_dir / "nodepools.yaml" nodepools_path.write_text(nodepools_yaml) log_info(f"Wrote {nodepools_path}") + if args.autoscaling: + autoscaling_yaml = generate_autoscaling_yaml( + args.amd64_min, + args.amd64_max, + args.arm64_min, + args.arm64_max, + ) + autoscaling_path = output_dir / "autoscaling.yaml" + autoscaling_path.write_text(autoscaling_yaml) + log_info(f"Wrote {autoscaling_path}") + return 0 diff --git a/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py b/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py index 387a283d..c373c9c0 100644 --- a/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py +++ b/osdc/modules/buildkit/scripts/python/test_generate_buildkit.py @@ -12,6 +12,7 @@ DAEMONSET_OVERHEAD_MEM_MI, MARGIN, compute_pod_resources, + generate_autoscaling_yaml, generate_deployment_yaml, generate_nodepools_yaml, ) @@ -341,6 +342,65 @@ def test_nodepool_limits_scale_per_arch(self): assert np["buildkit-arm64"]["spec"]["limits"]["cpu"] == "256" +# ============================================================================ +# autoscaling +# ============================================================================ + + +class TestAutoscalingDeployment: + """When autoscaling=True the Deployment yields control of replicas to KEDA + and gains a preStop drain so scale-down never kills an in-flight build.""" + + def test_omits_replicas(self): + output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2, autoscaling=True) + for d in parse_all_yaml(output): + assert "replicas" not in d["spec"] + + def test_keeps_replicas_when_disabled(self): + output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2) + for d in parse_all_yaml(output): + assert d["spec"]["replicas"] == 4 + + def test_drain_prestop_and_grace(self): + output = generate_deployment_yaml("m8gd.24xlarge", "m6id.24xlarge", 4, 2, autoscaling=True) + for d in parse_all_yaml(output): + spec = d["spec"]["template"]["spec"] + assert spec["terminationGracePeriodSeconds"] == 8100 + container = spec["containers"][0] + assert container["lifecycle"]["preStop"]["exec"]["command"] == ["/bin/sh", "/opt/drain/drain.sh"] + assert "drain" in {vm["name"] for vm in container["volumeMounts"]} + assert "drain" in {v["name"] for v in spec["volumes"]} + + +class TestGenerateAutoscalingYaml: + """Tests for generate_autoscaling_yaml — in-cluster KEDA ScaledObjects.""" + + def _docs(self): + output = generate_autoscaling_yaml(2, 8, 4, 8) + return parse_all_yaml(output) + + def test_only_scaledobjects_no_external_auth(self): + # In-cluster signal → no TriggerAuthentication / Grafana secret needed. + kinds = sorted(d["kind"] for d in self._docs()) + assert kinds == ["ScaledObject", "ScaledObject"] + + def test_per_arch_min_max(self): + scaled = {d["metadata"]["name"]: d for d in self._docs() if d["kind"] == "ScaledObject"} + assert set(scaled) == {"buildkitd-amd64", "buildkitd-arm64"} + assert scaled["buildkitd-amd64"]["spec"]["minReplicaCount"] == 2 + assert scaled["buildkitd-amd64"]["spec"]["maxReplicaCount"] == 8 + assert scaled["buildkitd-arm64"]["spec"]["minReplicaCount"] == 4 + assert scaled["buildkitd-arm64"]["spec"]["maxReplicaCount"] == 8 + + def test_in_cluster_metrics_api_trigger(self): + scaled = {d["metadata"]["name"]: d for d in self._docs() if d["kind"] == "ScaledObject"} + for name, backend in [("buildkitd-amd64", "bk_amd64"), ("buildkitd-arm64", "bk_arm64")]: + trig = scaled[name]["spec"]["triggers"][0] + assert trig["type"] == "metrics-api" + assert "buildkitd-lb-metrics.buildkit" in trig["metadata"]["url"] + assert f'proxy="{backend}"' in trig["metadata"]["valueLocation"] + + # ============================================================================ # generate_nodepools_yaml # ============================================================================ @@ -520,6 +580,64 @@ def test_deployment_yaml_parseable(self, tmp_path): docs = parse_all_yaml(deployment_text) assert len(docs) == 2 + def test_autoscaling_writes_manifests(self, tmp_path): + output_dir = tmp_path / "output" + + import generate_buildkit + + test_args = [ + "generate_buildkit.py", + "--arm64-instance-type", + "m8gd.24xlarge", + "--amd64-instance-type", + "m6id.24xlarge", + "--replicas", + "1", + "--pods-per-node", + "2", + "--output-dir", + str(output_dir), + "--autoscaling", + "--amd64-min", + "2", + "--amd64-max", + "8", + "--arm64-min", + "4", + "--arm64-max", + "8", + ] + with patch.object(sys, "argv", test_args): + result = generate_buildkit.main() + + assert result == 0 + autoscaling = parse_all_yaml((output_dir / "autoscaling.yaml").read_text()) + assert sorted(d["kind"] for d in autoscaling) == ["ScaledObject", "ScaledObject"] + + def test_autoscaling_requires_params(self, tmp_path): + output_dir = tmp_path / "output" + + import generate_buildkit + + test_args = [ + "generate_buildkit.py", + "--arm64-instance-type", + "m8gd.24xlarge", + "--amd64-instance-type", + "m6id.24xlarge", + "--replicas", + "1", + "--pods-per-node", + "2", + "--output-dir", + str(output_dir), + "--autoscaling", + ] + with patch.object(sys, "argv", test_args): + result = generate_buildkit.main() + + assert result == 1 + def test_unknown_instance_type_fails(self, tmp_path): output_dir = tmp_path / "output" test_args = [ diff --git a/osdc/modules/keda/deploy.sh b/osdc/modules/keda/deploy.sh new file mode 100755 index 00000000..af16c6ab --- /dev/null +++ b/osdc/modules/keda/deploy.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +# +# KEDA module deploy script. +# Called by: just deploy-module keda +# Args: $1=cluster-id $2=cluster-name $3=region +# +# Installs the KEDA operator (provides the ScaledObject/TriggerAuthentication +# CRDs that the buildkit module uses to autoscale builders). + +CLUSTER="$1" +MODULE_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="${OSDC_ROOT:-$(cd "$MODULE_DIR/../.." && pwd)}" +UPSTREAM_ROOT="${OSDC_UPSTREAM:-$REPO_ROOT}" +# shellcheck source=/dev/null +source "$UPSTREAM_ROOT/scripts/mise-activate.sh" +# shellcheck source=/dev/null +source "$UPSTREAM_ROOT/scripts/helm-upgrade.sh" +CFG="$UPSTREAM_ROOT/scripts/cluster-config.py" + +NAMESPACE="keda" +CHART_VERSION=$(uv run "$CFG" "$CLUSTER" keda.chart_version 2.16.1) + +helm repo add kedacore https://kedacore.github.io/charts >/dev/null 2>&1 || true +helm repo update kedacore >/dev/null 2>&1 || true + +helm_upgrade_if_changed keda "$NAMESPACE" \ + --create-namespace \ + --version "$CHART_VERSION" \ + -f "$MODULE_DIR/helm/values.yaml" \ + --timeout 10m \ + --wait \ + kedacore/keda + +echo "KEDA deployed (chart $CHART_VERSION)." diff --git a/osdc/modules/keda/helm/values.yaml b/osdc/modules/keda/helm/values.yaml new file mode 100644 index 00000000..2d151e9f --- /dev/null +++ b/osdc/modules/keda/helm/values.yaml @@ -0,0 +1,4 @@ +# Schedule on base-infra nodes; all other nodes are reserved by taints. +tolerations: + - key: CriticalAddonsOnly + operator: Exists