diff --git a/benchmarking/README.md b/benchmarking/README.md index 7251e3c383..bfcf385372 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -26,10 +26,48 @@ image, then deploys the Locust workers: Useful flags: -* `--worker-count N` — number of `WorkerPool` replicas (default 1). +* `--worker-count N` — total number of `WorkerPool` replicas (default 1). +* `--worker-pools LIST` — comma-separated `name:weight[:nodeSelectorKey=value]` + entries. See [Multiple worker pools](#multiple-worker-pools). * `--skip-build` — reuse the existing `:latest` locust image (skip the `docker build && docker push` step). +### Multiple worker pools + +By default the stack creates one `WorkerPool`, and the scheduler may place an +actor on any of its workers. `--worker-pools` creates one pool per entry +instead, splits `--worker-count` between them by weight, and pins each actor to +a single pool for its whole life: + +```bash +./benchmarking/deploy_locust.sh --deploy --worker-count 100 \ + --worker-pools 'n4d:1:cloud.google.com/machine-family=n4d,c4:1:cloud.google.com/machine-family=c4' +``` + +That run puts 50 workers on `n4d` nodes and 50 on `c4`, and sends half the +actors to each. + +Pinning is a correctness requirement once the pools differ in machine type, not +a tuning knob. A suspended actor's memory snapshot records the CPU features the +guest saw, and nothing masks them to a common baseline on resume, so an actor +that moves between CPU models fails to restore. Pinning is also how a run +measures one machine type against another in the same test. + +A pool name is a class of interchangeable workers, not one `WorkerPool`: it +reaches the scheduler as a `pool=` label that every worker in the pool +inherits, so several pools may share a value when an actor can freely move +between them. What a value must never span is workers a snapshot cannot move +between. The key is `pool` and not `cpu-class` because CPU compatibility is +only today's reason to separate workers. + +The pool list reaches the actors through the boomer workers, which set it as +each actor's `worker_selector`; `deploy_locust.sh` forwards the same list to +both halves so they cannot drift. Passing `--worker-pools` to +`benchmarking/workloads/deploy.sh` alone creates the pools but leaves the +actors unpinned. + +### Teardown + To tear everything down (locust then workloads, in reverse order): ```bash @@ -44,8 +82,9 @@ convenience: ./hack/install-ate.sh --delete-benchmarks ``` -The installer accepts `--benchmark-worker-count N` (default `1`). -`--skip-build` is only available when invoking +The installer accepts `--benchmark-worker-count N` (default `1`) and +`--benchmark-worker-pools LIST`, which it forwards to +`benchmarking/deploy_locust.sh`. `--skip-build` is only available when invoking `benchmarking/deploy_locust.sh` directly. ## Running Tests diff --git a/benchmarking/deploy_locust.sh b/benchmarking/deploy_locust.sh index 36dbc55845..59c65391ee 100755 --- a/benchmarking/deploy_locust.sh +++ b/benchmarking/deploy_locust.sh @@ -26,6 +26,10 @@ ROOT="$(git rev-parse --show-toplevel)" BENCHMARKING_DIR="${ROOT}/benchmarking" WORKER_COUNT=1 +# Forwarded to both halves of the stack, which must agree on the pools: one +# creates them, the other tells the boomer workers which to use. Empty keeps +# the single unpinned pool. +WORKER_POOLS="" SANDBOX_CLASS=gvisor SKIP_BUILD=0 OTLP_ENDPOINT="" @@ -39,7 +43,11 @@ usage() { echo "Options:" echo " --deploy Deploy workloads, build/push locust image, then deploy locust" echo " --delete Delete locust and then workloads" - echo " --worker-count N Number of WorkerPool replicas (default: 1)" + echo " --worker-count N Total number of WorkerPool replicas across all pools (default: 1)" + echo " --worker-pools LIST Comma-separated name:weight[:nodeSelectorKey=value] entries." + echo " One WorkerPool per entry, --worker-count split between them by" + echo " weight, each actor pinned to one pool. Default: one pool," + echo " actors unpinned. See benchmarking/README.md." echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." echo " microvm requires hack/install-microvm-deps.sh --install to have run." echo " --otlp-endpoint URL Forwarded to workloads/deploy.sh. The address to which an" @@ -56,6 +64,22 @@ usage() { echo " scripts this wrapper invokes." } +# boomer_worker_pools drops the optional node selector from each WORKER_POOLS +# entry: it says where a pool's worker pods run, which the boomer workers have +# no use for and reject as a third field. +boomer_worker_pools() { + local entry name weight rest out="" + local IFS=, + for entry in ${WORKER_POOLS}; do + [[ -z "${entry}" ]] && continue + name="${entry%%:*}" + rest="${entry#*:}" + weight="${rest%%:*}" + out+="${out:+,}${name}:${weight}" + done + printf '%s' "${out}" +} + if [[ "$#" -eq 0 ]]; then usage exit 1 @@ -68,6 +92,8 @@ while [[ "$#" -gt 0 ]]; do --delete) action="delete" ;; --worker-count) shift; WORKER_COUNT="$1" ;; --worker-count=*) WORKER_COUNT="${1#*=}" ;; + --worker-pools) shift; WORKER_POOLS="$1" ;; + --worker-pools=*) WORKER_POOLS="${1#*=}" ;; --sandbox-class) shift; SANDBOX_CLASS="$1" ;; --sandbox-class=*) SANDBOX_CLASS="${1#*=}" ;; --otlp-endpoint) shift; OTLP_ENDPOINT="$1" ;; @@ -101,11 +127,14 @@ if [[ -n "${WAIT_TIMEOUT_SECS}" ]] && ! [[ "${WAIT_TIMEOUT_SECS}" =~ ^[0-9]+$ ]] fi if [[ "${action}" == "deploy" ]]; then - echo "=== Deploying benchmark workloads (worker_count=${WORKER_COUNT}, sandbox_class=${SANDBOX_CLASS}) ===" + echo "=== Deploying benchmark workloads (worker_count=${WORKER_COUNT}, worker_pools=${WORKER_POOLS:-none}, sandbox_class=${SANDBOX_CLASS}) ===" # An empty OTLP_ENDPOINT must not become an empty --otlp-endpoint argument, # which would overwrite the default in workloads/deploy.sh with an empty # string and send the actor telemetry nowhere. workload_args=(--deploy --worker-count "${WORKER_COUNT}" --sandbox-class "${SANDBOX_CLASS}") + if [[ -n "${WORKER_POOLS}" ]]; then + workload_args+=(--worker-pools "${WORKER_POOLS}") + fi if [[ -n "${OTLP_ENDPOINT}" ]]; then workload_args+=(--otlp-endpoint "${OTLP_ENDPOINT}") fi @@ -128,14 +157,24 @@ if [[ "${action}" == "deploy" ]]; then echo echo "=== Deploying locust ===" - "${BENCHMARKING_DIR}/locust/deploy.sh" --deploy + locust_args=(--deploy) + if [[ -n "${WORKER_POOLS}" ]]; then + locust_args+=(--worker-pools "$(boomer_worker_pools)") + fi + "${BENCHMARKING_DIR}/locust/deploy.sh" "${locust_args[@]}" elif [[ "${action}" == "delete" ]]; then echo "=== Deleting locust ===" "${BENCHMARKING_DIR}/locust/deploy.sh" --delete echo echo "=== Deleting benchmark workloads ===" - "${BENCHMARKING_DIR}/workloads/deploy.sh" --delete + # workloads/deploy.sh renders one manifest per pool to delete it, so the + # teardown needs the same list the deploy ran with. + workload_args=(--delete) + if [[ -n "${WORKER_POOLS}" ]]; then + workload_args+=(--worker-pools "${WORKER_POOLS}") + fi + "${BENCHMARKING_DIR}/workloads/deploy.sh" "${workload_args[@]}" else usage exit 1 diff --git a/benchmarking/locust/deploy.sh b/benchmarking/locust/deploy.sh index 37df81415d..765a735636 100755 --- a/benchmarking/locust/deploy.sh +++ b/benchmarking/locust/deploy.sh @@ -35,6 +35,10 @@ MANIFEST="${SCRIPT_DIR}/manifests/locust.yaml" # Substituted into the boomer container's --user-class argument and the master's -f. BENCHMARK_USER_CLASS=glutton +# Substituted into the boomer container's --worker-pools argument. Empty leaves +# the actors unpinned. +BENCHMARK_WORKER_POOLS="" + usage() { echo "Usage: $0 [options]" echo "" @@ -42,6 +46,8 @@ usage() { echo " --deploy Deploy the locust workers" echo " --delete Delete the locust workers" echo " --user-class NAME Locust user class, lowercase; runs tests/NAME.py (default: glutton)" + echo " --worker-pools LIST Comma-separated name:weight entries pinning each actor to one" + echo " WorkerPool. Use the names and weights the pools were created with." echo " -h|--help Show this help message" } @@ -51,7 +57,7 @@ deploy() { # benchmarking/monitoring.yaml is otherwise optional. echo "Ensuring benchmarking namespace exists..." kubectl create namespace benchmarking --dry-run=client -o yaml | kubectl apply -f - - echo "Deploying Locust load (PROJECT_ID=${PROJECT_ID}, user_class=${BENCHMARK_USER_CLASS})..." + echo "Deploying Locust load (PROJECT_ID=${PROJECT_ID}, user_class=${BENCHMARK_USER_CLASS}, worker_pools=${BENCHMARK_WORKER_POOLS:-none})..." envsubst < "${MANIFEST}" | kubectl apply -f - } @@ -72,6 +78,8 @@ while [[ "$#" -gt 0 ]]; do --delete) action="delete" ;; --user-class) shift; BENCHMARK_USER_CLASS="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" ;; --user-class=*) BENCHMARK_USER_CLASS="$(printf '%s' "${1#*=}" | tr '[:upper:]' '[:lower:]')" ;; + --worker-pools) shift; BENCHMARK_WORKER_POOLS="$1" ;; + --worker-pools=*) BENCHMARK_WORKER_POOLS="${1#*=}" ;; -h|--help) usage; exit 0 ;; *) echo "Error: Unknown option: $1" >&2 @@ -87,6 +95,7 @@ if [[ ! -f "${SCRIPT_DIR}/tests/${BENCHMARK_USER_CLASS}.py" ]]; then exit 1 fi export BENCHMARK_USER_CLASS +export BENCHMARK_WORKER_POOLS if [[ "${action}" == "deploy" ]]; then deploy diff --git a/benchmarking/locust/manifests/locust.yaml b/benchmarking/locust/manifests/locust.yaml index 36374e61b4..476bb7a256 100644 --- a/benchmarking/locust/manifests/locust.yaml +++ b/benchmarking/locust/manifests/locust.yaml @@ -129,6 +129,10 @@ spec: # benchmarking/locust/common/boomer_config.py on the master. - "--master-web-port=8089" - "--user-class=${BENCHMARK_USER_CLASS}" + # Pins each actor to one WorkerPool, as name:weight[,...] matching the + # pools benchmarking/workloads/deploy.sh created. Empty leaves actors + # unpinned. See benchmarking/README.md#multiple-worker-pools. + - "--worker-pools=${BENCHMARK_WORKER_POOLS}" env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 diff --git a/benchmarking/locust/runner.py b/benchmarking/locust/runner.py index 1996e91d0d..d0a8f65824 100644 --- a/benchmarking/locust/runner.py +++ b/benchmarking/locust/runner.py @@ -108,6 +108,16 @@ def parse_args() -> argparse.Namespace: "default of 1." ), ) + p.add_argument( + "--worker-pools", + default=None, + help=( + "Comma-separated name:weight entries pinning each actor to one " + "WorkerPool, forwarded to boomer as --worker-pools. Use the names " + "and weights the pools were created with. Omit to leave actors " + "unpinned." + ), + ) args, extra = p.parse_known_args() args.locust_extra = extra return args @@ -309,6 +319,8 @@ def run_test(args: argparse.Namespace, csv_prefix: Path, logs: TextIO, traces: T boomer_cmd += ["--config-json", cfg_json] if args.actors_per_user is not None: boomer_cmd += ["--actors-per-user", str(args.actors_per_user)] + if args.worker_pools: + boomer_cmd += ["--worker-pools", args.worker_pools] # Read the endpoint again at each spawn message. Thus a value that # changes while the run continues, such as the sample rate of a load # shape, reaches boomer at the change. boomer's --master-host default diff --git a/benchmarking/workloads/deploy.sh b/benchmarking/workloads/deploy.sh index 0fa2c4c6c0..d93ed8ccdf 100755 --- a/benchmarking/workloads/deploy.sh +++ b/benchmarking/workloads/deploy.sh @@ -30,19 +30,27 @@ if [[ -z "${BUCKET_NAME:-}" ]]; then fi MANIFEST_DIR="benchmarking/workloads/manifests" -POOL_MANIFEST="${MANIFEST_DIR}/workloads.yaml.tmpl" +NAMESPACE_MANIFEST="${MANIFEST_DIR}/workloads.yaml.tmpl" +WORKER_POOL_MANIFEST="${MANIFEST_DIR}/workerpool.yaml.tmpl" # The benchmark ActorTemplates: -template.yaml.tmpl each, created # through the ate API in the benchmark-workloads atespace. WORKLOAD_TEMPLATES # overrides the default set — the usermem and kernelmem templates (for the # matching locust tests) are not deployed by default. read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full}" -if [[ ! -f "${POOL_MANIFEST}" ]]; then - echo "Error: ${POOL_MANIFEST} not found in $(pwd)" >&2 - exit 1 -fi +for manifest in "${NAMESPACE_MANIFEST}" "${WORKER_POOL_MANIFEST}"; do + if [[ ! -f "${manifest}" ]]; then + echo "Error: ${manifest} not found in $(pwd)" >&2 + exit 1 + fi +done WORKER_COUNT=1 +# Worker pools to create, as name:weight[:nodeSelectorKey=value] entries. Empty +# means the single pool named benchmark-ateom that this script has always +# created. Use the same weights the boomer workers run with, so each pool gets +# the actors its workers can hold. +WORKER_POOLS="" SANDBOX_CLASS="gvisor" # Actor memory limit (ActorTemplate resources.limits.memory). The default # is the smallest size microvm admits (128Mi VMM reserve + 128Mi guest floor), @@ -63,7 +71,12 @@ usage() { echo "Options:" echo " --deploy Substitute env vars and deploy workloads to the cluster using ko apply" echo " --delete Substitute env vars and delete workloads from the cluster" - echo " --worker-count N Number of WorkerPool replicas (default: 1)" + echo " --worker-count N Total number of WorkerPool replicas across all pools (default: 1)" + echo " --worker-pools LIST Comma-separated name:weight[:nodeSelectorKey=value] entries." + echo " One WorkerPool per entry, labelled pool=, with" + echo " --worker-count split between them by weight. Pass the same" + echo " name:weight list to the boomer workers (--worker-pools)." + echo " Default: a single pool named benchmark-ateom." echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)." echo " microvm requires hack/install-microvm-deps.sh --install to have run." echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi," @@ -128,6 +141,9 @@ substitute() { esac sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ -e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \ + -e "s|\${POOL_NAME}|${POOL_NAME:-}|g" \ + -e "s|\${POOL_WORKERPOOL_NAME}|${POOL_WORKERPOOL_NAME:-}|g" \ + -e "s|\${POOL_WORKER_COUNT}|${POOL_WORKER_COUNT:-}|g" \ -e "s|\${SANDBOX_CLASS}|${SANDBOX_CLASS}|g" \ -e "s|\${SANDBOX_CLASS_ENUM}|${sandbox_class_enum}|g" \ -e "s|\${SANDBOX_CONFIG_NAME}|${sandbox_config_name}|g" \ @@ -136,6 +152,126 @@ substitute() { "${manifest}" } +# Parsed form of WORKER_POOLS, filled by parse_worker_pools. Index i of each +# array describes the same pool. +POOL_NAMES=() +POOL_WEIGHTS=() +POOL_NODE_SELECTORS=() +POOL_WORKER_COUNTS=() + +# parse_worker_pools reads WORKER_POOLS into the POOL_* arrays and splits +# WORKER_COUNT between the pools by weight. An empty WORKER_POOLS yields the +# historical single pool, so a caller that does not know about pools keeps the +# layout it had. +parse_worker_pools() { + POOL_NAMES=() + POOL_WEIGHTS=() + POOL_NODE_SELECTORS=() + POOL_WORKER_COUNTS=() + + if [[ -z "${WORKER_POOLS}" ]]; then + POOL_NAMES=("benchmark-ateom") + POOL_WEIGHTS=(1) + POOL_NODE_SELECTORS=("") + POOL_WORKER_COUNTS=("${WORKER_COUNT}") + return 0 + fi + + local entry name weight selector rest + local IFS=, + for entry in ${WORKER_POOLS}; do + [[ -z "${entry}" ]] && continue + name="${entry%%:*}" + rest="${entry#*:}" + if [[ "${rest}" == "${entry}" ]]; then + echo "Error: worker pool '${entry}': want name:weight[:nodeSelectorKey=value]" >&2 + exit 1 + fi + weight="${rest%%:*}" + # A third field is optional; without it rest still holds just the weight. + if [[ "${rest}" == *:* ]]; then + selector="${rest#*:}" + else + selector="" + fi + if [[ -z "${name}" ]]; then + echo "Error: worker pool '${entry}': name must not be empty" >&2 + exit 1 + fi + if ! [[ "${weight}" =~ ^[0-9]+$ ]] || (( weight == 0 )); then + echo "Error: worker pool '${entry}': weight must be a positive integer" >&2 + exit 1 + fi + if [[ -n "${selector}" && "${selector}" != *=* ]]; then + echo "Error: worker pool '${entry}': node selector must be key=value" >&2 + exit 1 + fi + POOL_NAMES+=("${name}") + POOL_WEIGHTS+=("${weight}") + POOL_NODE_SELECTORS+=("${selector}") + done + + if (( ${#POOL_NAMES[@]} == 0 )); then + echo "Error: --worker-pools is set but names no pool" >&2 + exit 1 + fi + + local total=0 weight + for weight in "${POOL_WEIGHTS[@]}"; do + total=$((total + weight)) + done + + # Round each pool but the last to nearest and give the last the remainder, so + # the counts sum to WORKER_COUNT exactly. A share that rounds to zero still + # gets one worker: a pool with none can never serve the actors pinned to it. + local i assigned=0 count + for (( i = 0; i < ${#POOL_NAMES[@]} - 1; i++ )); do + count=$(( (WORKER_COUNT * POOL_WEIGHTS[i] + total / 2) / total )) + (( count < 1 )) && count=1 + POOL_WORKER_COUNTS+=("${count}") + assigned=$((assigned + count)) + done + count=$((WORKER_COUNT - assigned)) + if (( count < 1 )); then + echo "Error: --worker-count ${WORKER_COUNT} is too small to give every one of ${#POOL_NAMES[@]} pools a worker" >&2 + exit 1 + fi + POOL_WORKER_COUNTS+=("${count}") +} + +# render_worker_pool writes the WorkerPool manifest for pool index $1. The node +# selector is appended rather than templated: it is an optional nested block, +# which a line-oriented placeholder cannot express without leaving a dangling +# `nodeSelector:` behind when it is unset. +render_worker_pool() { + local idx="$1" + local POOL_NAME="${POOL_NAMES[idx]}" + local POOL_WORKER_COUNT="${POOL_WORKER_COUNTS[idx]}" + local POOL_WORKERPOOL_NAME + POOL_WORKERPOOL_NAME="$(worker_pool_deployment_name "${idx}")" + + substitute "${WORKER_POOL_MANIFEST}" + + local selector="${POOL_NODE_SELECTORS[idx]}" + if [[ -n "${selector}" ]]; then + # Split on the first = only, so a value may contain one. + printf ' template:\n nodeSelector:\n %s: %s\n' "${selector%%=*}" "${selector#*=}" + fi +} + +# worker_pool_deployment_name is both the WorkerPool name and the name of the +# Deployment ate-controller derives from it, which is what deploy waits on. +worker_pool_deployment_name() { + local idx="$1" + if [[ -z "${WORKER_POOLS}" ]]; then + # Unchanged from the single-pool layout, so existing tooling that waits on + # deployment/benchmark-ateom keeps working. + echo "benchmark-ateom" + return + fi + echo "benchmark-ateom-${POOL_NAMES[idx]}" +} + # wait_actortemplate_ready polls a substrate ActorTemplate resource until its # golden snapshot exists (the substrate counterpart of `kubectl wait # --for=condition=Ready actortemplate/...`). Fails fast when the template @@ -183,13 +319,24 @@ wait_templates_ready() { deploy() { resolve_otlp_endpoint - echo "Deploying workloads (worker_count=${WORKER_COUNT}, actor_memory=${ACTOR_MEMORY}, otlp_endpoint=${OTLP_ENDPOINT})..." - substitute "${POOL_MANIFEST}" | hack/run-tool.sh ko apply -f - - echo "Waiting for worker pool to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." - kubectl wait --for=create deployment/benchmark-ateom \ - --namespace=benchmark-workloads --timeout="${WAIT_TIMEOUT_SECS}s" - kubectl rollout status deployment/benchmark-ateom \ - --namespace=benchmark-workloads --timeout="${WAIT_TIMEOUT_SECS}s" + parse_worker_pools + echo "Deploying workloads (worker_count=${WORKER_COUNT}, pools=${#POOL_NAMES[@]}, actor_memory=${ACTOR_MEMORY}, otlp_endpoint=${OTLP_ENDPOINT})..." + substitute "${NAMESPACE_MANIFEST}" | kubectl apply -f - + + local idx deployment + for idx in "${!POOL_NAMES[@]}"; do + echo " pool ${POOL_NAMES[idx]}: ${POOL_WORKER_COUNTS[idx]} worker(s)${POOL_NODE_SELECTORS[idx]:+ on ${POOL_NODE_SELECTORS[idx]}}" + render_worker_pool "${idx}" | hack/run-tool.sh ko apply -f - + done + + echo "Waiting for worker pools to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..." + for idx in "${!POOL_NAMES[@]}"; do + deployment="$(worker_pool_deployment_name "${idx}")" + kubectl wait --for=create "deployment/${deployment}" \ + --namespace=benchmark-workloads --timeout="${WAIT_TIMEOUT_SECS}s" + kubectl rollout status "deployment/${deployment}" \ + --namespace=benchmark-workloads --timeout="${WAIT_TIMEOUT_SECS}s" + done # The store enforces that a template's atespace exists at create time. run_kubectl_ate create atespace benchmark-workloads >/dev/null 2>&1 \ @@ -223,9 +370,15 @@ delete() { done run_kubectl_ate delete atespace benchmark-workloads >/dev/null 2>&1 \ || echo "atespace benchmark-workloads not deleted (may not exist or is not empty)" - # The pool manifest contains ko:// image references; route through - # `ko delete` so they get resolved before kubectl sees them. - substitute "${POOL_MANIFEST}" | hack/run-tool.sh ko delete --ignore-not-found -f - + # The WorkerPool manifests contain ko:// image references; route through + # `ko delete` so they get resolved before kubectl sees them. The namespace + # goes last, after the pools it holds. + parse_worker_pools + local idx + for idx in "${!POOL_NAMES[@]}"; do + render_worker_pool "${idx}" | hack/run-tool.sh ko delete --ignore-not-found -f - + done + substitute "${NAMESPACE_MANIFEST}" | kubectl delete --ignore-not-found -f - } if [[ "$#" -eq 0 ]]; then @@ -249,6 +402,13 @@ while [[ "$#" -gt 0 ]]; do --worker-count=*) WORKER_COUNT="${1#*=}" ;; + --worker-pools) + shift + WORKER_POOLS="$1" + ;; + --worker-pools=*) + WORKER_POOLS="${1#*=}" + ;; --sandbox-class) shift SANDBOX_CLASS="$1" diff --git a/benchmarking/workloads/manifests/workerpool.yaml.tmpl b/benchmarking/workloads/manifests/workerpool.yaml.tmpl new file mode 100644 index 0000000000..ba0e667e85 --- /dev/null +++ b/benchmarking/workloads/manifests/workerpool.yaml.tmpl @@ -0,0 +1,42 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# One worker pool for the benchmark workloads. workloads/deploy.sh renders this +# once per --worker-pools entry, so a run can spread its actors over several +# node shapes. A WorkerPool's labels become the labels of every Worker it owns, +# which is what the scheduler matches, and it ANDs the two below: +# +# workload every benchmark pool carries it, and the ActorTemplates' +# workerSelector matches it, so a benchmark actor never lands on a +# worker outside these pools. +# pool names a class of interchangeable workers. Boomer sets it as an +# actor's own worker_selector, which keeps that actor in one class. +# See internal/benchmarking/boomer/userclass.PoolLabelKey. +# +# Keeping an actor in one class is a correctness requirement, not tuning, once +# the classes differ in machine type: a micro-VM snapshot carries the CPU +# features the guest observed, and nothing masks them to a common baseline on +# resume, so an actor that moves between CPU models fails to restore. +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: ${POOL_WORKERPOOL_NAME} + namespace: benchmark-workloads + labels: + workload: benchmark-ateom + pool: ${POOL_NAME} +spec: + replicas: ${POOL_WORKER_COUNT} + sandboxClass: ${SANDBOX_CLASS} + workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-${SANDBOX_CLASS} diff --git a/benchmarking/workloads/manifests/workloads.yaml.tmpl b/benchmarking/workloads/manifests/workloads.yaml.tmpl index 2a1e099960..6df6049854 100644 --- a/benchmarking/workloads/manifests/workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/workloads.yaml.tmpl @@ -12,26 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Worker pool for the benchmark workloads. The ActorTemplates are not CRDs: -# they live in the *-template.yaml.tmpl files next to this one and are -# created through the ate API by workloads/deploy.sh with +# Namespace for the benchmark workloads. The worker pools live in +# workerpool.yaml.tmpl, which deploy.sh renders once per pool. The +# ActorTemplates are not CRDs: they live in the *-template.yaml.tmpl files next +# to this one and are created through the ate API by workloads/deploy.sh with # `kubectl ate create actor-template`. apiVersion: v1 kind: Namespace metadata: name: benchmark-workloads - ---- - -apiVersion: ate.dev/v1alpha1 -kind: WorkerPool -metadata: - name: benchmark-ateom - namespace: benchmark-workloads - labels: - workload: benchmark-ateom -spec: - replicas: ${WORKER_COUNT} - sandboxClass: ${SANDBOX_CLASS} - workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-${SANDBOX_CLASS} diff --git a/cmd/benchmarking/boomer-worker/main.go b/cmd/benchmarking/boomer-worker/main.go index 1214eb4707..68c4720857 100644 --- a/cmd/benchmarking/boomer-worker/main.go +++ b/cmd/benchmarking/boomer-worker/main.go @@ -51,6 +51,7 @@ func main() { userClass = flag.String("user-class", "glutton", fmt.Sprintf("Locust user class to run, lowercase; one of %s.", strings.Join(userclass.Names(), "|"))) actorsPerUser = flag.Int("actors-per-user", 1, "Number of actors each user (VU) creates and cycles through in round-robin: on iteration i, the user targets actor i%actors-per-user. Startup creates all actors; shutdown hibernates+deletes them.") httpMaxIdleConnsPerHost = flag.Int("http-max-idle-conns-per-host", 10000, "Idle HTTP connections the router client keeps per host. Set it to at least the number of users this worker runs, so each VU reuses its connection to the router across wakes instead of opening a new one per request.") + workerPools = flag.String("worker-pools", "", "Comma-separated name:weight list, e.g. \"n4:528,n4d:1056\", spreading actors over several worker pools. Each actor draws one pool at creation, weighted by these values, and stays there via Actor.worker_selector, which the scheduler ANDs with the ActorTemplate's own selector. A name is matched against the worker's \"pool\" label. Weights are usually the pools' vCPU counts. Empty leaves actors unpinned.") ) // boomer.Run will call flag.Parse() if we haven't yet; calling here so // our flag-derived values are usable before that. @@ -147,6 +148,24 @@ func main() { slog.Duration("poll_interval", *configPollInterval)) } + // Parsed up front: a bad spec otherwise surfaces as every actor failing to + // schedule, which is far harder to read than a startup error. + parsedPools, err := userclass.ParsePools(*workerPools) + if err != nil { + slog.Error("fatal: invalid --worker-pools", slog.String("err", err.Error())) + os.Exit(1) + } + pools, err := userclass.NewPoolPicker(parsedPools) + if err != nil { + slog.Error("fatal: invalid --worker-pools", slog.String("err", err.Error())) + os.Exit(1) + } + if pools != nil { + slog.Info("spreading actors over worker pools", + slog.String("label_key", userclass.PoolLabelKey), + slog.String("pools", strings.Join(pools.Names(), ","))) + } + cfg := &userclass.Config{ APIStub: apiStub, HTTPClient: httpClient, @@ -154,6 +173,7 @@ func main() { Atespace: *atespace, Dyn: dyn, ActorsPerUser: *actorsPerUser, + Pools: pools, } entry, ok := userclass.Lookup(class) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 278467d1cf..f6aa8d70da 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -150,7 +150,10 @@ function usage() { echo "" echo " --deploy-benchmarks Deploy workloads + locust load test stack" echo " --delete-benchmarks Delete the locust stack and workloads" - echo " --benchmark-worker-count N Number of WorkerPool replicas (default: 1)" + echo " --benchmark-worker-count N Total number of WorkerPool replicas (default: 1)" + echo " --benchmark-worker-pools LIST Comma-separated name:weight[:nodeSelectorKey=value]" + echo " entries, forwarded to benchmarking/deploy_locust.sh:" + echo " one WorkerPool each, one pool per actor." echo " --benchmark-sandbox-class CLASS Sandbox runtime for the benchmark WorkerPool: gvisor | microvm (default: gvisor)." echo " microvm requires hack/install-microvm-deps.sh --install to have run." echo " --benchmark-actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi," @@ -1394,7 +1397,7 @@ delete_atenet() { } deploy_benchmarks() { - log_step "deploy_benchmarks (worker_count=${BENCHMARK_WORKER_COUNT}, sandbox_class=${BENCHMARK_SANDBOX_CLASS})" + log_step "deploy_benchmarks (worker_count=${BENCHMARK_WORKER_COUNT}, worker_pools=${BENCHMARK_WORKER_POOLS:-none}, sandbox_class=${BENCHMARK_SANDBOX_CLASS})" # The microvm SandboxConfig lives outside --deploy-ate-system's default set # (which only installs gvisor-default); the workloads deploy references it # by name and would fail if we skipped this. @@ -1405,6 +1408,9 @@ deploy_benchmarks() { local benchmark_args=(--deploy --worker-count "${BENCHMARK_WORKER_COUNT}" --sandbox-class "${BENCHMARK_SANDBOX_CLASS}") + if [[ -n "${BENCHMARK_WORKER_POOLS}" ]]; then + benchmark_args+=(--worker-pools "${BENCHMARK_WORKER_POOLS}") + fi if [[ -n "${ATE_OTLP_ENDPOINT:-}" ]]; then benchmark_args+=(--otlp-endpoint "${ATE_OTLP_ENDPOINT}") fi @@ -1416,7 +1422,13 @@ deploy_benchmarks() { delete_benchmarks() { log_step "delete_benchmarks (sandbox_class=${BENCHMARK_SANDBOX_CLASS})" - "${ROOT}/benchmarking/deploy_locust.sh" --delete + # The teardown renders one manifest per pool, so it needs the same list the + # deploy ran with to find them all. + local benchmark_args=(--delete) + if [[ -n "${BENCHMARK_WORKER_POOLS}" ]]; then + benchmark_args+=(--worker-pools "${BENCHMARK_WORKER_POOLS}") + fi + "${ROOT}/benchmarking/deploy_locust.sh" "${benchmark_args[@]}" # only tear down the microvm SandboxConfig if the caller opted into microvm. if [[ "${BENCHMARK_SANDBOX_CLASS}" == "microvm" ]]; then "${ROOT}/hack/install-microvm-deps.sh" --delete @@ -1457,6 +1469,9 @@ done # workstation will not have loaded. SETUP_CSI="${SETUP_CSI:-none}" BENCHMARK_WORKER_COUNT=1 +# Empty keeps the single unpinned pool that benchmarking/deploy_locust.sh +# creates by default. +BENCHMARK_WORKER_POOLS="" BENCHMARK_SANDBOX_CLASS=gvisor # Empty keeps the default in benchmarking/workloads/deploy.sh (256Mi). BENCHMARK_ACTOR_MEMORY="" @@ -1530,6 +1545,16 @@ for ((i = 0; i < ${#prescan_args[@]}; i++)); do --benchmark-worker-count=*) BENCHMARK_WORKER_COUNT="${prescan_args[i]#*=}" ;; + --benchmark-worker-pools) + if (( i + 1 >= ${#prescan_args[@]} )); then + echo "Error: --benchmark-worker-pools requires name:weight[:nodeSelectorKey=value] entries" >&2 + exit 1 + fi + BENCHMARK_WORKER_POOLS="${prescan_args[$((i + 1))]}" + ;; + --benchmark-worker-pools=*) + BENCHMARK_WORKER_POOLS="${prescan_args[i]#*=}" + ;; --benchmark-sandbox-class) if (( i + 1 >= ${#prescan_args[@]} )); then echo "Error: --benchmark-sandbox-class requires gvisor or microvm" >&2 @@ -1672,6 +1697,8 @@ while [[ "$#" -gt 0 ]]; do # dispatch loop's `*)` unknown-option branch doesn't reject it. --benchmark-worker-count) shift ;; --benchmark-worker-count=*) ;; + --benchmark-worker-pools) shift ;; + --benchmark-worker-pools=*) ;; --benchmark-sandbox-class) shift ;; --benchmark-sandbox-class=*) ;; --benchmark-actor-memory) shift ;; diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index ff9e072f8a..6d5a2cd1b8 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -127,6 +127,7 @@ func (r *durDirRuntime) startUser(ctx context.Context, dynCfg dynconfig.Config) actorName: "sb-" + uuid.NewString(), templateName: tmpl, userClass: durDirUserClass, + pool: r.cfg.Pools.Pick(), } bmetrics.UpdateUsers(durDirUserClass, 1) if err := u.ensureAtespace(ctx); err != nil { @@ -162,6 +163,9 @@ type durDirUser struct { userClass string expectedDigest string expectedSize int64 + // pool pins this actor to one worker pool for its whole life (see + // userclass.PoolPicker). Empty means no per-actor constraint. + pool string } func (u *durDirUser) ref() *ateapipb.ObjectRef { @@ -188,12 +192,16 @@ func (u *durDirUser) ensureAtespace(ctx context.Context) error { } func (u *durDirUser) create(ctx context.Context) error { + actor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: u.cfg.Atespace, Name: u.actorName}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: templateAtespace, Name: u.templateName}, + } + // ANDed with the template's workerSelector by the scheduler, so this only + // narrows the actor to one pool. Nil when no pools are configured. + actor.WorkerSelector = u.cfg.Pools.SelectorFor(u.pool) return u.tracedCall(ctx, "CreateActor", func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.CreateActor(callCtx, &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: u.cfg.Atespace, Name: u.actorName}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: templateAtespace, Name: u.templateName}, - }, + Actor: actor, }, grpc.Trailer(tr)) return err }) diff --git a/internal/benchmarking/boomer/glutton/fixture_test.go b/internal/benchmarking/boomer/glutton/fixture_test.go index c20b4b7ddd..61dc6b84a6 100644 --- a/internal/benchmarking/boomer/glutton/fixture_test.go +++ b/internal/benchmarking/boomer/glutton/fixture_test.go @@ -31,6 +31,7 @@ type fakeControlClient struct { ateapipb.ControlClient mu sync.Mutex calls []string + createRequests []*ateapipb.CreateActorRequest deleteRequests []*ateapipb.DeleteActorRequest // resumeErrs is returned by successive ResumeActor calls, in order, until // it is drained; every call after that succeeds. @@ -48,6 +49,7 @@ func (f *fakeControlClient) CreateActor(ctx context.Context, in *ateapipb.Create f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, "CreateActor") + f.createRequests = append(f.createRequests, in) return &ateapipb.Actor{}, nil } @@ -91,6 +93,12 @@ func (f *fakeControlClient) recordedCalls() []string { return append([]string(nil), f.calls...) } +func (f *fakeControlClient) recordedCreateRequests() []*ateapipb.CreateActorRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*ateapipb.CreateActorRequest(nil), f.createRequests...) +} + func (f *fakeControlClient) recordedDeleteRequests() []*ateapipb.DeleteActorRequest { f.mu.Lock() defer f.mu.Unlock() diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index 608beb5208..b3c9834293 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -199,6 +199,9 @@ func (r *taskRuntime) startUser(ctx context.Context) (*gluttonUser, error) { cfg: r.cfg, actorName: "sb-" + uuid.NewString(), firstResume: true, + // Drawn per actor rather than per VU so a VU with several + // actors still spreads them over the pools. + pool: r.cfg.Pools.Pick(), } // Ensuring the atespace is idempotent (swallows AlreadyExists), so // doing it once per VU is enough — subsequent actors would just make @@ -301,6 +304,10 @@ type gluttonActor struct { // rehabilitates a crashed actor, so retrying would just fail forever. // The VU's other actors are unaffected. crashed bool + // pool is the worker pool this actor is pinned to, drawn once in + // startUser and never redrawn (see userclass.PoolPicker). Empty means no + // per-actor constraint. + pool string } func (u *gluttonActor) ref() *ateapipb.ObjectRef { @@ -331,12 +338,16 @@ func (u *gluttonActor) ensureAtespace(ctx context.Context) error { } func (u *gluttonActor) create(ctx context.Context) error { + actor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: u.cfg.Atespace, Name: u.actorName}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: templateAtespace, Name: templateName}, + } + // ANDed with the template's workerSelector by the scheduler, so this only + // narrows the actor to one pool. Nil when no pools are configured. + actor.WorkerSelector = u.cfg.Pools.SelectorFor(u.pool) return u.tracedCall(ctx, "CreateActor", func(callCtx context.Context, tr *metadata.MD) error { _, err := u.cfg.APIStub.CreateActor(callCtx, &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: u.cfg.Atespace, Name: u.actorName}, - ActorTemplate: &ateapipb.ObjectRef{Atespace: templateAtespace, Name: templateName}, - }, + Actor: actor, }, grpc.Trailer(tr)) return err }) diff --git a/internal/benchmarking/boomer/glutton/pools_wiring_test.go b/internal/benchmarking/boomer/glutton/pools_wiring_test.go new file mode 100644 index 0000000000..658b5db82c --- /dev/null +++ b/internal/benchmarking/boomer/glutton/pools_wiring_test.go @@ -0,0 +1,248 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glutton + +import ( + "context" + "maps" + "testing" + + "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" + "github.com/agent-substrate/substrate/internal/benchmarking/glutton/fake" +) + +// These tests cover the wiring from Config.Pools to the worker_selector on the +// wire. userclass/pools_test.go covers the picker itself; what can break here +// instead is an actor that never gets a selector, or one that redraws its pool +// and so asks to resume a snapshot on the wrong CPU. + +// picker builds a PoolPicker over equally weighted names, failing the test +// rather than returning an error, since a malformed spec here is a test bug. +func picker(t *testing.T, names ...string) *userclass.PoolPicker { + t.Helper() + pools := make([]userclass.Pool, 0, len(names)) + for _, name := range names { + pools = append(pools, userclass.Pool{Name: name, Weight: 1}) + } + p, err := userclass.NewPoolPicker(pools) + if err != nil { + t.Fatalf("NewPoolPicker(%v) = %v", names, err) + } + return p +} + +// createdSelectors is the match_labels of every CreateActor the fake saw, with +// nil for a request that carried no selector. +func createdSelectors(f *fakeControlClient) []map[string]string { + reqs := f.recordedCreateRequests() + out := make([]map[string]string, 0, len(reqs)) + for _, req := range reqs { + out = append(out, req.GetActor().GetWorkerSelector().GetMatchLabels()) + } + return out +} + +func TestGluttonCreateSetsWorkerSelector(t *testing.T) { + tests := []struct { + name string + pools *userclass.PoolPicker + pool string + want map[string]string + wantNo bool // want no selector at all + }{{ + // The single-pool default: placement is left to the ActorTemplate's + // own workerSelector, exactly as before pools existed. + name: "no pools configured", + pools: nil, + pool: "", + wantNo: true, + }, { + name: "pool configured", + pools: picker(t, "n4"), + pool: "n4", + want: map[string]string{userclass.PoolLabelKey: "n4"}, + }} + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fakeCtrl := &fakeControlClient{} + cfg := newTestConfig(t, &fake.Server{}, &userclass.Config{ + APIStub: fakeCtrl, + Atespace: "bench-test", + Pools: tc.pools, + }) + u := &gluttonActor{cfg: cfg, actorName: "sb-test", pool: tc.pool} + + if err := u.create(context.Background()); err != nil { + t.Fatalf("create() = %v", err) + } + + got := createdSelectors(fakeCtrl) + if len(got) != 1 { + t.Fatalf("CreateActor calls = %d, want 1", len(got)) + } + if tc.wantNo { + if got[0] != nil { + t.Errorf("worker_selector = %v, want none", got[0]) + } + return + } + if !maps.Equal(got[0], tc.want) { + t.Errorf("worker_selector = %v, want %v", got[0], tc.want) + } + }) + } +} + +// An actor's pool is drawn once and held. Were a future change to move the +// draw into create, a recreated actor could ask for a different pool and then +// fail to restore its snapshot on a different CPU model. +func TestGluttonCreateKeepsTheSamePoolAcrossCalls(t *testing.T) { + fakeCtrl := &fakeControlClient{} + cfg := newTestConfig(t, &fake.Server{}, &userclass.Config{ + APIStub: fakeCtrl, + Atespace: "bench-test", + // Many equally weighted pools, so a redraw would almost certainly + // pick a different one. + Pools: picker(t, "a", "b", "c", "d", "e", "f", "g", "h"), + }) + u := &gluttonActor{cfg: cfg, actorName: "sb-test", pool: "d"} + + for i := 0; i < 20; i++ { + if err := u.create(context.Background()); err != nil { + t.Fatalf("create() #%d = %v", i, err) + } + } + + want := map[string]string{userclass.PoolLabelKey: "d"} + for i, got := range createdSelectors(fakeCtrl) { + if !maps.Equal(got, want) { + t.Fatalf("create #%d worker_selector = %v, want %v", i, got, want) + } + } +} + +// startUser draws per actor, not per VU, so a VU holding several actors +// spreads them. Every actor must still land on a configured pool. +func TestStartUserPinsEveryActorToAConfiguredPool(t *testing.T) { + const actorsPerUser = 60 + names := []string{"n4", "c4", "c3"} + + fakeCtrl := &fakeControlClient{} + cfg := newTestConfig(t, &fake.Server{}, &userclass.Config{ + APIStub: fakeCtrl, + Atespace: "bench-test", + ActorsPerUser: actorsPerUser, + Pools: picker(t, names...), + }) + + rt := &taskRuntime{cfg: cfg} + if _, err := rt.startUser(context.Background()); err != nil { + t.Fatalf("startUser() = %v", err) + } + + selectors := createdSelectors(fakeCtrl) + if len(selectors) != actorsPerUser { + t.Fatalf("CreateActor calls = %d, want %d", len(selectors), actorsPerUser) + } + + valid := make(map[string]bool, len(names)) + for _, name := range names { + valid[name] = true + } + seen := make(map[string]int, len(names)) + for i, sel := range selectors { + if len(sel) != 1 { + t.Fatalf("actor %d worker_selector = %v, want exactly one label", i, sel) + } + value, ok := sel[userclass.PoolLabelKey] + if !ok { + t.Fatalf("actor %d worker_selector = %v, want key %q", i, sel, userclass.PoolLabelKey) + } + if !valid[value] { + t.Fatalf("actor %d pinned to unknown pool %q, want one of %v", i, value, names) + } + seen[value]++ + } + + // With 60 actors over 3 equal pools, a pool missing entirely means the + // draw is not spreading. P(some pool empty) < 3*(2/3)^60, far below any + // flake threshold. + for _, name := range names { + if seen[name] == 0 { + t.Errorf("pool %q got no actors; distribution = %v", name, seen) + } + } +} + +func TestDurDirCreateSetsWorkerSelector(t *testing.T) { + fakeCtrl := &fakeControlClient{} + cfg := newTestConfig(t, &fake.Server{}, &userclass.Config{ + APIStub: fakeCtrl, + Atespace: "bench-test", + Pools: picker(t, "n4", "c4"), + }) + u := &durDirUser{ + cfg: cfg, + actorName: "duractor", + templateName: defaultDurTemplate, + userClass: durDirUserClass, + pool: "c4", + } + + if err := u.create(context.Background()); err != nil { + t.Fatalf("create() = %v", err) + } + + got := createdSelectors(fakeCtrl) + want := map[string]string{userclass.PoolLabelKey: "c4"} + if len(got) != 1 || !maps.Equal(got[0], want) { + t.Errorf("worker_selector = %v, want [%v]", got, want) + } +} + +// The template reference must survive the create() refactor that hoisted the +// Actor literal into a local to attach the selector. +func TestCreateStillCarriesTemplateAndName(t *testing.T) { + fakeCtrl := &fakeControlClient{} + cfg := newTestConfig(t, &fake.Server{}, &userclass.Config{ + APIStub: fakeCtrl, + Atespace: "bench-test", + Pools: picker(t, "n4"), + }) + u := &gluttonActor{cfg: cfg, actorName: "sb-test", pool: "n4"} + + if err := u.create(context.Background()); err != nil { + t.Fatalf("create() = %v", err) + } + + reqs := fakeCtrl.recordedCreateRequests() + if len(reqs) != 1 { + t.Fatalf("CreateActor calls = %d, want 1", len(reqs)) + } + actor := reqs[0].GetActor() + if got := actor.GetMetadata().GetName(); got != "sb-test" { + t.Errorf("actor name = %q, want %q", got, "sb-test") + } + if got := actor.GetMetadata().GetAtespace(); got != "bench-test" { + t.Errorf("actor atespace = %q, want %q", got, "bench-test") + } + if got := actor.GetActorTemplate().GetName(); got != templateName { + t.Errorf("actor template = %q, want %q", got, templateName) + } + if got := actor.GetActorTemplate().GetAtespace(); got != templateAtespace { + t.Errorf("template atespace = %q, want %q", got, templateAtespace) + } +} diff --git a/internal/benchmarking/boomer/userclass/config.go b/internal/benchmarking/boomer/userclass/config.go index 04ded7e0f1..096373edc9 100644 --- a/internal/benchmarking/boomer/userclass/config.go +++ b/internal/benchmarking/boomer/userclass/config.go @@ -46,4 +46,8 @@ type Config struct { // that honor this knob; classes that don't honor it treat every VU as // owning exactly one actor. ActorsPerUser int + // Pools spreads new actors over several worker pools, weighted by + // capacity, by setting Actor.worker_selector at create time. Nil leaves + // placement to the ActorTemplate's own workerSelector. + Pools *PoolPicker } diff --git a/internal/benchmarking/boomer/userclass/pools.go b/internal/benchmarking/boomer/userclass/pools.go new file mode 100644 index 0000000000..9e60f1b01c --- /dev/null +++ b/internal/benchmarking/boomer/userclass/pools.go @@ -0,0 +1,174 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package userclass + +import ( + "fmt" + "math/rand/v2" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// PoolLabelKey is the worker label a pool is selected by. A WorkerPool's +// metadata.labels become the labels of every Worker it owns, and the scheduler +// matches Actor.worker_selector against those (see +// cmd/atecontroller/internal/workersync and +// cmd/ateapi/internal/scheduling.Applies). +// +// A value names a class of interchangeable workers, so several WorkerPools may +// share one; what must not share one is workers a snapshot cannot move +// between. The key is generic because CPU compatibility is only today's reason +// to separate them. +const PoolLabelKey = "pool" + +// maxLabelValueLen is the Kubernetes limit a selector value must fit in. The +// API server rejects a longer one; checking here reports the offending pool +// at startup instead of failing every CreateActor mid-run. +const maxLabelValueLen = 63 + +// labelValueRE is the Kubernetes label value grammar. +var labelValueRE = regexp.MustCompile(`^[a-zA-Z0-9]([-_.a-zA-Z0-9]*[a-zA-Z0-9])?$`) + +// Pool is one placement target: the label value that identifies a set of +// interchangeable workers, and the share of new actors that set should +// receive. +type Pool struct { + // Name is the value the pool label must equal for a worker to belong to + // this pool. + Name string + // Weight is this pool's share of new actors, relative to the sum of all + // weights. Callers normally pass the pool's vCPU count, so actors land in + // proportion to the capacity that has to run them. + Weight int +} + +// PoolPicker assigns actors to worker pools, weighted by Pool.Weight. +// +// Call Pick once, when the actor name is minted, and keep the result for the +// actor's whole life, including across a recreate after a failed resume: a +// micro-VM snapshot records the CPU features the guest observed, and nothing +// masks them to a common baseline on resume, so an actor that moves between +// CPU models fails to restore. +// +// A nil *PoolPicker is usable and picks nothing, which leaves placement to +// the ActorTemplate's own workerSelector. That is the single-pool default. +type PoolPicker struct { + names []string + // cumulative[i] is the total weight of pools 0..i, so one draw below + // total picks a pool in a single scan. + cumulative []int + total int +} + +// NewPoolPicker builds a picker over pools. No pools yields a nil picker, so +// the caller can pass the result through unconditionally. +func NewPoolPicker(pools []Pool) (*PoolPicker, error) { + if len(pools) == 0 { + return nil, nil + } + + p := &PoolPicker{ + names: make([]string, 0, len(pools)), + cumulative: make([]int, 0, len(pools)), + } + seen := make(map[string]bool, len(pools)) + for _, pool := range pools { + switch { + case pool.Name == "": + return nil, fmt.Errorf("pool name must not be empty") + case len(pool.Name) > maxLabelValueLen: + return nil, fmt.Errorf("pool %q: name is longer than the %d-character Kubernetes label value limit", pool.Name, maxLabelValueLen) + case !labelValueRE.MatchString(pool.Name): + return nil, fmt.Errorf("pool %q: name is not a valid Kubernetes label value", pool.Name) + case seen[pool.Name]: + return nil, fmt.Errorf("pool %q: duplicate name", pool.Name) + case pool.Weight <= 0: + return nil, fmt.Errorf("pool %q: weight must be positive, got %d", pool.Name, pool.Weight) + } + seen[pool.Name] = true + p.total += pool.Weight + p.names = append(p.names, pool.Name) + p.cumulative = append(p.cumulative, p.total) + } + return p, nil +} + +// ParsePools reads a "name:weight,name:weight" list, the spelling the +// boomer-worker --worker-pools flag takes. An empty spec yields no pools. +func ParsePools(spec string) ([]Pool, error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return nil, nil + } + + var pools []Pool + for _, entry := range strings.Split(spec, ",") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + name, weightStr, ok := strings.Cut(entry, ":") + if !ok { + return nil, fmt.Errorf("pool %q: want name:weight", entry) + } + weight, err := strconv.Atoi(strings.TrimSpace(weightStr)) + if err != nil { + return nil, fmt.Errorf("pool %q: weight %q is not an integer", entry, weightStr) + } + pools = append(pools, Pool{Name: strings.TrimSpace(name), Weight: weight}) + } + return pools, nil +} + +// Pick draws a pool name, weighted by the pools' weights. A nil picker +// returns the empty string, which means "no per-actor constraint". +// +// Safe for concurrent use: every VU goroutine mints its actors independently. +func (p *PoolPicker) Pick() string { + if p == nil || p.total <= 0 { + return "" + } + r := rand.IntN(p.total) + for i, c := range p.cumulative { + if r < c { + return p.names[i] + } + } + // Unreachable while r < total == the last cumulative entry; kept so a + // future change to the draw cannot silently return "" and scatter actors. + return p.names[len(p.names)-1] +} + +// SelectorFor turns a pool name from Pick into the Actor.worker_selector the +// scheduler ANDs with the ActorTemplate's own selector. An empty name, or a +// nil picker, yields nil: the template's selector then decides alone. +func (p *PoolPicker) SelectorFor(name string) *ateapipb.Selector { + if p == nil || name == "" { + return nil + } + return &ateapipb.Selector{MatchLabels: map[string]string{PoolLabelKey: name}} +} + +// Names lists the pool names in the order they were configured, for logging. +func (p *PoolPicker) Names() []string { + if p == nil { + return nil + } + return slices.Clone(p.names) +} diff --git a/internal/benchmarking/boomer/userclass/pools_test.go b/internal/benchmarking/boomer/userclass/pools_test.go new file mode 100644 index 0000000000..a3e8d1cb17 --- /dev/null +++ b/internal/benchmarking/boomer/userclass/pools_test.go @@ -0,0 +1,239 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package userclass + +import ( + "math" + "strings" + "testing" +) + +func TestParsePools(t *testing.T) { + for _, tc := range []struct { + name string + spec string + want []Pool + wantErr string + }{ + {name: "empty", spec: ""}, + {name: "blanks only", spec: " "}, + { + name: "single", + spec: "n4:528", + want: []Pool{{Name: "n4", Weight: 528}}, + }, + { + name: "several with spaces", + spec: " n4:528 , n4d:1056 ", + want: []Pool{{Name: "n4", Weight: 528}, {Name: "n4d", Weight: 1056}}, + }, + { + name: "trailing comma is ignored", + spec: "n4:528,", + want: []Pool{{Name: "n4", Weight: 528}}, + }, + { + name: "missing weight", + spec: "n4", + wantErr: "want name:weight", + }, + { + name: "weight is not a number", + spec: "n4:many", + wantErr: "not an integer", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := ParsePools(tc.spec) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ParsePools(%q) error = %v, want one containing %q", tc.spec, err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("ParsePools(%q) = %v", tc.spec, err) + } + if len(got) != len(tc.want) { + t.Fatalf("ParsePools(%q) = %+v, want %+v", tc.spec, got, tc.want) + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("ParsePools(%q)[%d] = %+v, want %+v", tc.spec, i, got[i], tc.want[i]) + } + } + }) + } +} + +func TestNewPoolPickerRejects(t *testing.T) { + for _, tc := range []struct { + name string + pools []Pool + wantErr string + }{ + { + name: "empty name", + pools: []Pool{{Name: "", Weight: 1}}, + wantErr: "must not be empty", + }, + { + name: "name is not a label value", + pools: []Pool{{Name: "not a label", Weight: 1}}, + wantErr: "not a valid Kubernetes label value", + }, + { + name: "name too long", + pools: []Pool{{Name: strings.Repeat("a", maxLabelValueLen+1), Weight: 1}}, + wantErr: "label value limit", + }, + { + name: "duplicate name", + pools: []Pool{{Name: "n4", Weight: 1}, {Name: "n4", Weight: 2}}, + wantErr: "duplicate name", + }, + { + name: "zero weight", + pools: []Pool{{Name: "n4", Weight: 0}}, + wantErr: "weight must be positive", + }, + { + name: "negative weight", + pools: []Pool{{Name: "n4", Weight: -1}}, + wantErr: "weight must be positive", + }, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := NewPoolPicker(tc.pools); err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("NewPoolPicker(%+v) error = %v, want one containing %q", tc.pools, err, tc.wantErr) + } + }) + } +} + +// A nil picker is the single-pool default and every method must tolerate it: +// the user classes call through unconditionally rather than branching. +func TestNilPickerPicksNothing(t *testing.T) { + var p *PoolPicker + if got := p.Pick(); got != "" { + t.Errorf("nil.Pick() = %q, want empty", got) + } + if got := p.SelectorFor("n4"); got != nil { + t.Errorf("nil.SelectorFor() = %v, want nil", got) + } + if got := p.Names(); got != nil { + t.Errorf("nil.Names() = %v, want nil", got) + } +} + +func TestNewPoolPickerNoPoolsIsNil(t *testing.T) { + p, err := NewPoolPicker(nil) + if err != nil { + t.Fatalf("NewPoolPicker(nil) = %v", err) + } + if p != nil { + t.Fatalf("NewPoolPicker(nil) = %+v, want nil picker", p) + } +} + +func TestSelectorFor(t *testing.T) { + p, err := NewPoolPicker([]Pool{{Name: "n4", Weight: 1}}) + if err != nil { + t.Fatalf("NewPoolPicker() = %v", err) + } + sel := p.SelectorFor("n4") + if sel == nil { + t.Fatal("SelectorFor(\"n4\") = nil, want a selector") + } + if got := sel.GetMatchLabels()[PoolLabelKey]; got != "n4" { + t.Errorf("selector[%q] = %q, want %q", PoolLabelKey, got, "n4") + } + // An actor that drew no pool must not be constrained, even when the + // worker is running with pools configured. + if got := p.SelectorFor(""); got != nil { + t.Errorf("SelectorFor(\"\") = %v, want nil", got) + } +} + +// Actors must land in proportion to the weights, which is what makes the +// weights usable as vCPU counts: a pool with twice the capacity has to take +// twice the actors, or the smaller pool saturates first and caps the run. +func TestPickIsWeighted(t *testing.T) { + p, err := NewPoolPicker([]Pool{ + {Name: "small", Weight: 1}, + {Name: "medium", Weight: 3}, + {Name: "large", Weight: 6}, + }) + if err != nil { + t.Fatalf("NewPoolPicker() = %v", err) + } + + const draws = 200_000 + counts := map[string]int{} + for range draws { + counts[p.Pick()]++ + } + + // 3 percentage points is far outside the sampling noise of 200k draws + // (sigma is under 0.12pp for every share here) and far inside any real + // weighting bug, which would be off by tens of points. + const tolerance = 0.03 + for name, wantShare := range map[string]float64{"small": 0.1, "medium": 0.3, "large": 0.6} { + gotShare := float64(counts[name]) / draws + if math.Abs(gotShare-wantShare) > tolerance { + t.Errorf("pool %q got %.4f of %d draws, want %.2f (+/- %.2f)", name, gotShare, draws, wantShare, tolerance) + } + } + if len(counts) != 3 { + t.Errorf("drew %d distinct pools, want 3: %v", len(counts), counts) + } +} + +// Every configured pool has to be reachable. A cumulative-weight off-by-one +// would strand the first or last pool while the shares still look plausible. +func TestPickReachesEveryPool(t *testing.T) { + p, err := NewPoolPicker([]Pool{ + {Name: "first", Weight: 1}, + {Name: "middle", Weight: 1}, + {Name: "last", Weight: 1}, + }) + if err != nil { + t.Fatalf("NewPoolPicker() = %v", err) + } + seen := map[string]bool{} + for range 1000 { + seen[p.Pick()] = true + } + for _, want := range []string{"first", "middle", "last"} { + if !seen[want] { + t.Errorf("pool %q never drawn in 1000 picks", want) + } + } + if seen[""] { + t.Error("Pick() returned the empty pool name") + } +} + +func TestNamesIsACopy(t *testing.T) { + p, err := NewPoolPicker([]Pool{{Name: "n4", Weight: 1}}) + if err != nil { + t.Fatalf("NewPoolPicker() = %v", err) + } + names := p.Names() + names[0] = "mutated" + if got := p.Names()[0]; got != "n4" { + t.Errorf("Names() exposed internal state: got %q after caller mutation, want %q", got, "n4") + } +}