benchmarking: spread actors over several worker pools

benchmarking/README.md

@@ -26,10 +26,48 @@ image, then deploys the Locust workers:
 
 Useful flags:
 
-* `--worker-count N` — number of `WorkerPool` replicas (default 1).
+* `--worker-count N` — total number of `WorkerPool` replicas (default 1).
+* `--worker-pools LIST` — comma-separated `name:weight[:nodeSelectorKey=value]`
+  entries. See [Multiple worker pools](#multiple-worker-pools).
 * `--skip-build` — reuse the existing `:latest` locust image (skip the
   `docker build && docker push` step).
 
+### Multiple worker pools
+
+By default the stack creates one `WorkerPool`, and the scheduler may place an
+actor on any of its workers. `--worker-pools` creates one pool per entry
+instead, splits `--worker-count` between them by weight, and pins each actor to
+a single pool for its whole life:
+
+```bash
+./benchmarking/deploy_locust.sh --deploy --worker-count 100 \
+  --worker-pools 'n4d:1:cloud.google.com/machine-family=n4d,c4:1:cloud.google.com/machine-family=c4'
+```
+
+That run puts 50 workers on `n4d` nodes and 50 on `c4`, and sends half the
+actors to each.
+
+Pinning is a correctness requirement once the pools differ in machine type, not
+a tuning knob. A suspended actor's memory snapshot records the CPU features the
+guest saw, and nothing masks them to a common baseline on resume, so an actor
+that moves between CPU models fails to restore. Pinning is also how a run
+measures one machine type against another in the same test.
+
+A pool name is a class of interchangeable workers, not one `WorkerPool`: it
+reaches the scheduler as a `pool=<name>` label that every worker in the pool
+inherits, so several pools may share a value when an actor can freely move
+between them. What a value must never span is workers a snapshot cannot move
+between. The key is `pool` and not `cpu-class` because CPU compatibility is
+only today's reason to separate workers.
+
+The pool list reaches the actors through the boomer workers, which set it as
+each actor's `worker_selector`; `deploy_locust.sh` forwards the same list to
+both halves so they cannot drift. Passing `--worker-pools` to
+`benchmarking/workloads/deploy.sh` alone creates the pools but leaves the
+actors unpinned.
+
+### Teardown
+
 To tear everything down (locust then workloads, in reverse order):
 
 ```bash
@@ -44,8 +82,9 @@ convenience:
 ./hack/install-ate.sh --delete-benchmarks
 ```
 
-The installer accepts `--benchmark-worker-count N` (default `1`).
-`--skip-build` is only available when invoking
+The installer accepts `--benchmark-worker-count N` (default `1`) and
+`--benchmark-worker-pools LIST`, which it forwards to
+`benchmarking/deploy_locust.sh`. `--skip-build` is only available when invoking
 `benchmarking/deploy_locust.sh` directly.
 
 ## Running Tests

benchmarking/deploy_locust.sh

@@ -26,6 +26,10 @@ ROOT="$(git rev-parse --show-toplevel)"
 BENCHMARKING_DIR="${ROOT}/benchmarking"
 
 WORKER_COUNT=1
+# Forwarded to both halves of the stack, which must agree on the pools: one
+# creates them, the other tells the boomer workers which to use. Empty keeps
+# the single unpinned pool.
+WORKER_POOLS=""
 SANDBOX_CLASS=gvisor
 SKIP_BUILD=0
 OTLP_ENDPOINT=""
@@ -39,7 +43,11 @@ usage() {
   echo "Options:"
   echo "  --deploy                Deploy workloads, build/push locust image, then deploy locust"
   echo "  --delete                Delete locust and then workloads"
-  echo "  --worker-count N        Number of WorkerPool replicas (default: 1)"
+  echo "  --worker-count N        Total number of WorkerPool replicas across all pools (default: 1)"
+  echo "  --worker-pools LIST     Comma-separated name:weight[:nodeSelectorKey=value] entries."
+  echo "                          One WorkerPool per entry, --worker-count split between them by"
+  echo "                          weight, each actor pinned to one pool. Default: one pool,"
+  echo "                          actors unpinned. See benchmarking/README.md."
   echo "  --sandbox-class CLASS   Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)."
   echo "                          microvm requires hack/install-microvm-deps.sh --install to have run."
   echo "  --otlp-endpoint URL     Forwarded to workloads/deploy.sh. The address to which an"
@@ -56,6 +64,22 @@ usage() {
   echo "  scripts this wrapper invokes."
 }
 
+# boomer_worker_pools drops the optional node selector from each WORKER_POOLS
+# entry: it says where a pool's worker pods run, which the boomer workers have
+# no use for and reject as a third field.
+boomer_worker_pools() {
+  local entry name weight rest out=""
+  local IFS=,
+  for entry in ${WORKER_POOLS}; do
+    [[ -z "${entry}" ]] && continue
+    name="${entry%%:*}"
+    rest="${entry#*:}"
+    weight="${rest%%:*}"
+    out+="${out:+,}${name}:${weight}"
+  done
+  printf '%s' "${out}"
+}
+
 if [[ "$#" -eq 0 ]]; then
   usage
   exit 1
@@ -68,6 +92,8 @@ while [[ "$#" -gt 0 ]]; do
     --delete) action="delete" ;;
     --worker-count) shift; WORKER_COUNT="$1" ;;
     --worker-count=*) WORKER_COUNT="${1#*=}" ;;
+    --worker-pools) shift; WORKER_POOLS="$1" ;;
+    --worker-pools=*) WORKER_POOLS="${1#*=}" ;;
     --sandbox-class) shift; SANDBOX_CLASS="$1" ;;
     --sandbox-class=*) SANDBOX_CLASS="${1#*=}" ;;
     --otlp-endpoint) shift; OTLP_ENDPOINT="$1" ;;
@@ -101,11 +127,14 @@ if [[ -n "${WAIT_TIMEOUT_SECS}" ]] && ! [[ "${WAIT_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]
 fi
 
 if [[ "${action}" == "deploy" ]]; then
-  echo "=== Deploying benchmark workloads (worker_count=${WORKER_COUNT}, sandbox_class=${SANDBOX_CLASS}) ==="
+  echo "=== Deploying benchmark workloads (worker_count=${WORKER_COUNT}, worker_pools=${WORKER_POOLS:-none}, sandbox_class=${SANDBOX_CLASS}) ==="
   # An empty OTLP_ENDPOINT must not become an empty --otlp-endpoint argument,
   # which would overwrite the default in workloads/deploy.sh with an empty
   # string and send the actor telemetry nowhere.
   workload_args=(--deploy --worker-count "${WORKER_COUNT}" --sandbox-class "${SANDBOX_CLASS}")
+  if [[ -n "${WORKER_POOLS}" ]]; then
+    workload_args+=(--worker-pools "${WORKER_POOLS}")
+  fi
   if [[ -n "${OTLP_ENDPOINT}" ]]; then
     workload_args+=(--otlp-endpoint "${OTLP_ENDPOINT}")
   fi
@@ -128,14 +157,24 @@ if [[ "${action}" == "deploy" ]]; then
 
   echo
   echo "=== Deploying locust ==="
-  "${BENCHMARKING_DIR}/locust/deploy.sh" --deploy
+  locust_args=(--deploy)
+  if [[ -n "${WORKER_POOLS}" ]]; then
+    locust_args+=(--worker-pools "$(boomer_worker_pools)")
+  fi
+  "${BENCHMARKING_DIR}/locust/deploy.sh" "${locust_args[@]}"
 elif [[ "${action}" == "delete" ]]; then
   echo "=== Deleting locust ==="
   "${BENCHMARKING_DIR}/locust/deploy.sh" --delete
 
   echo
   echo "=== Deleting benchmark workloads ==="
-  "${BENCHMARKING_DIR}/workloads/deploy.sh" --delete
+  # workloads/deploy.sh renders one manifest per pool to delete it, so the
+  # teardown needs the same list the deploy ran with.
+  workload_args=(--delete)
+  if [[ -n "${WORKER_POOLS}" ]]; then
+    workload_args+=(--worker-pools "${WORKER_POOLS}")
+  fi
+  "${BENCHMARKING_DIR}/workloads/deploy.sh" "${workload_args[@]}"
 else
   usage
   exit 1

benchmarking/locust/deploy.sh

@@ -35,13 +35,19 @@ MANIFEST="${SCRIPT_DIR}/manifests/locust.yaml"
 # Substituted into the boomer container's --user-class argument and the master's -f.
 BENCHMARK_USER_CLASS=glutton
 
+# Substituted into the boomer container's --worker-pools argument. Empty leaves
+# the actors unpinned.
+BENCHMARK_WORKER_POOLS=""
+
 usage() {
   echo "Usage: $0 [options]"
   echo ""
   echo "Options:"
   echo "  --deploy           Deploy the locust workers"
   echo "  --delete           Delete the locust workers"
   echo "  --user-class NAME  Locust user class, lowercase; runs tests/NAME.py (default: glutton)"
+  echo "  --worker-pools LIST  Comma-separated name:weight entries pinning each actor to one"
+  echo "                     WorkerPool. Use the names and weights the pools were created with."
   echo "  -h|--help          Show this help message"
 }
 
@@ -51,7 +57,7 @@ deploy() {
   # benchmarking/monitoring.yaml is otherwise optional.
   echo "Ensuring benchmarking namespace exists..."
   kubectl create namespace benchmarking --dry-run=client -o yaml | kubectl apply -f -
-  echo "Deploying Locust load (PROJECT_ID=${PROJECT_ID}, user_class=${BENCHMARK_USER_CLASS})..."
+  echo "Deploying Locust load (PROJECT_ID=${PROJECT_ID}, user_class=${BENCHMARK_USER_CLASS}, worker_pools=${BENCHMARK_WORKER_POOLS:-none})..."
   envsubst < "${MANIFEST}" | kubectl apply -f -
 }
 
@@ -72,6 +78,8 @@ while [[ "$#" -gt 0 ]]; do
     --delete) action="delete" ;;
     --user-class) shift; BENCHMARK_USER_CLASS="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" ;;
     --user-class=*) BENCHMARK_USER_CLASS="$(printf '%s' "${1#*=}" | tr '[:upper:]' '[:lower:]')" ;;
+    --worker-pools) shift; BENCHMARK_WORKER_POOLS="$1" ;;
+    --worker-pools=*) BENCHMARK_WORKER_POOLS="${1#*=}" ;;
     -h|--help) usage; exit 0 ;;
     *)
       echo "Error: Unknown option: $1" >&2
@@ -87,6 +95,7 @@ if [[ ! -f "${SCRIPT_DIR}/tests/${BENCHMARK_USER_CLASS}.py" ]]; then
   exit 1
 fi
 export BENCHMARK_USER_CLASS
+export BENCHMARK_WORKER_POOLS
 
 if [[ "${action}" == "deploy" ]]; then
   deploy

benchmarking/locust/manifests/locust.yaml

@@ -129,6 +129,10 @@ spec:
         # benchmarking/locust/common/boomer_config.py on the master.
         - "--master-web-port=8089"
         - "--user-class=${BENCHMARK_USER_CLASS}"
+        # Pins each actor to one WorkerPool, as name:weight[,...] matching the
+        # pools benchmarking/workloads/deploy.sh created. Empty leaves actors
+        # unpinned. See benchmarking/README.md#multiple-worker-pools.
+        - "--worker-pools=${BENCHMARK_WORKER_POOLS}"
         env:
         - name: OTEL_EXPORTER_OTLP_ENDPOINT
           value: opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317

benchmarking/locust/runner.py

@@ -108,6 +108,16 @@ def parse_args() -> argparse.Namespace:
             "default of 1."
         ),
     )
+    p.add_argument(
+        "--worker-pools",
+        default=None,
+        help=(
+            "Comma-separated name:weight entries pinning each actor to one "
+            "WorkerPool, forwarded to boomer as --worker-pools. Use the names "
+            "and weights the pools were created with. Omit to leave actors "
+            "unpinned."
+        ),
+    )
     args, extra = p.parse_known_args()
     args.locust_extra = extra
     return args
@@ -309,6 +319,8 @@ def run_test(args: argparse.Namespace, csv_prefix: Path, logs: TextIO, traces: T
             boomer_cmd += ["--config-json", cfg_json]
         if args.actors_per_user is not None:
             boomer_cmd += ["--actors-per-user", str(args.actors_per_user)]
+        if args.worker_pools:
+            boomer_cmd += ["--worker-pools", args.worker_pools]
         # Read the endpoint again at each spawn message. Thus a value that
         # changes while the run continues, such as the sample rate of a load
         # shape, reaches boomer at the change. boomer's --master-host default