From 0e9690b914773979f6097aa4d6a8b7c168bbbac2 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:50:45 -0700 Subject: [PATCH 01/25] fix(swebench-service): let srun find its config and honour proxy policy The Pyxis runtime builds each container step's environment from an explicit allow-list, and two variables that srun and enroot genuinely need were missing. Both failures are invisible in the run: they happen inside a subprocess whose only report is the generic "Pyxis infrastructure failure before the command completed". SLURM_CONF. Without it the child srun falls back to /etc/slurm/slurm.conf. On a configless site that file does not exist and srun aborts with "Could not establish a configuration source"; on a multi-cluster site it exists but is a different cluster's file whose plugins are not installed locally, and srun aborts with "failed to initialize cli_filter plugin". Either way every step dies before a container is created. The remaining SLURM_* variables stay out of the allow-list deliberately: inheriting SLURM_JOB_ID / SLURM_STEP_ID is exactly what breaks a nested srun, which is why the allow-list exists. Proxy policy. enroot performs the registry pull inside the step, so it needs the same proxy configuration as the caller. A site that pins a container-cache proxy system-wide will 403 the CONNECT for any registry outside that cache's allow-list, and every per-instance image import fails with "curl: (56) CONNECT tunnel failed, response 403" -- including the SWE-bench task images this runtime is built to pull. Verified on a GB200 cluster whose enroot pins a container cache: before the change no sweb.eval.arm64 image could be imported from any node; after it, the image imports and the container starts. Tests: `test_pyxis_srun_environment_forwards_config_and_proxy_policy` asserts each of the seven newly allowed variables reaches the step (all seven fail against the previous allow-list), and `test_pyxis_srun_environment_withholds_inherited_step_identity` pins the other half of the contract -- SLURM step identity is still withheld -- so a later "just forward all SLURM_*" cannot pass unnoticed. The service README documents the allow-list and why those two entries are on it. --- .../evaluation/swebench_service/README.md | 10 ++++++ .../swebench_service/pyxis_environment.py | 18 ++++++++++ .../swebench_service/test_runner.py | 34 +++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index e6ddf8ac8..cd11904b4 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -54,6 +54,16 @@ registry requires authentication. Launch the service on the compute node inside active one-node Slurm allocation. The runtime requires `SLURM_JOB_ID` and `SLURMD_NODENAME` and assumes the node is exclusive to the user. +Each `srun` step is given an explicit allow-list of environment variables rather +than the service's whole environment, so that inherited `SLURM_JOB_ID` / +`SLURM_STEP_ID` cannot corrupt a nested `srun`. Two entries on that list are load +bearing on real clusters: `SLURM_CONF`, without which the child `srun` falls back +to `/etc/slurm/slurm.conf` and aborts on a configless or multi-cluster site; and +the proxy variables (`http_proxy`, `https_proxy`, `no_proxy` and their uppercase +forms), which Enroot needs because it performs the registry pull inside the step. +Credentials such as `OPENAI_API_KEY`, `HF_TOKEN` and the service auth token are +never forwarded. + During generation, the service still uses mini-swe-agent for the agent loop and model requests, but replaces its Docker environment with `PyxisEnvironment`. Every trajectory receives a named, writable Pyxis container. Each tool call becomes an diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py index 141c1ccba..7381b9d95 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py @@ -29,6 +29,24 @@ "LC_ALL", "TMPDIR", "XDG_RUNTIME_DIR", + # Proxy policy must reach enroot, which performs the registry pull inside + # the step. Clusters that pin a container-cache proxy system-wide 403 any + # registry outside its allow-list, and without no_proxy every per-instance + # image import fails with "CONNECT tunnel failed, response 403". + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + # srun locates its own configuration through SLURM_CONF. Dropping it makes + # the child fall back to /etc/slurm/slurm.conf, which on a configless or + # multi-cluster site is either absent ("Could not establish a configuration + # source") or a different file whose plugins are not installed + # ("failed to initialize cli_filter plugin"). Every step then fails before + # the container is ever created. The remaining SLURM_* variables stay out: + # inheriting SLURM_JOB_ID / SLURM_STEP_ID is what breaks a nested srun. + "SLURM_CONF", ) _STEP_STATUS = "/tmp/.mlperf_srun_status" _STEP_SCRIPT = r"""set +e diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index f6c92e135..67a4090bb 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -835,6 +835,40 @@ def test_pyxis_srun_environment_does_not_forward_credentials(monkeypatch): assert "HF_TOKEN" not in environment +@pytest.mark.parametrize( + "name", + [ + # srun locates its own configuration through SLURM_CONF; without it a + # configless or multi-cluster site aborts every step before a container + # is created. + "SLURM_CONF", + # enroot performs the registry pull inside the step and needs the same + # proxy policy as the caller. + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + ], +) +def test_pyxis_srun_environment_forwards_config_and_proxy_policy(monkeypatch, name): + monkeypatch.setenv(name, "value") + + assert safe_srun_env().get(name) == "value" + + +@pytest.mark.parametrize( + "name", + ["SLURM_JOB_ID", "SLURM_STEP_ID", "SLURM_NTASKS", "SLURM_NNODES", "SLURM_PROCID"], +) +def test_pyxis_srun_environment_withholds_inherited_step_identity(monkeypatch, name): + """Only SLURM_CONF is forwarded; step identity would break a nested srun.""" + monkeypatch.setenv(name, "inherited") + + assert name not in safe_srun_env() + + def test_pyxis_environment_reuses_named_writable_container( monkeypatch, tmp_path, caplog ): From 61ff0341a67559ddb941d170a73fc88f151c5391 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 14:09:19 -0700 Subject: [PATCH 02/25] fix(swebench-service): let enroot's temp and config overrides reach the step Pyxis creates the container *inside* the `srun` step, so Enroot reads `ENROOT_TEMP_PATH` and `ENROOT_CONFIG_PATH` there and not in the service process. Neither was on the step environment allow-list, so both were silently dropped. The consequence is not a failed step, which is why it took so long to see: an operator points `ENROOT_TEMP_PATH` at a large device precisely so that unpacking a ~2.5 GB rootfs with ~16.8k hardlinks does not compete for space with the unpacked rootfs itself, the override never arrives, and the create-time temp lands back on the very device it was meant to spare. On a 20-node run this is how `/raid` reached 4.1 GB free of 527 GB, after which every subsequent container creation failed for want of space -- reported as an ordinary infrastructure failure with no mention of the setting that was discarded. This is the same class as the proxy variables already on the list and for the same structural reason: work that looks like it happens in the service actually happens in the step, and configuration that does not cross that boundary is configuration that does nothing. No other `SLURM_*` variable is added; inheriting `SLURM_JOB_ID` / `SLURM_STEP_ID` is what breaks a nested `srun` and is why the allow-list exists. Kept deliberately separate from the `SLURM_CONF` + proxy commit rather than folded into it. That commit is also PR #452 upstream; if #452 merges on its own and the stack drops its commit, this fix has to survive that, and it only does if it stands alone. Tests: the existing allow-list parametrisation gains both variables (each fails against the previous list), and `test_pyxis_srun_environment_withholds_inherited_step_identity` continues to pin the other half of the contract. The service README documents the allow-list as a table with the reason each load-bearing entry is on it. --- .../evaluation/swebench_service/README.md | 16 ++++++++++------ .../swebench_service/pyxis_environment.py | 6 ++++++ .../evaluation/swebench_service/test_runner.py | 3 +++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index cd11904b4..8c56bc0c8 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -56,13 +56,17 @@ active one-node Slurm allocation. The runtime requires `SLURM_JOB_ID` and Each `srun` step is given an explicit allow-list of environment variables rather than the service's whole environment, so that inherited `SLURM_JOB_ID` / -`SLURM_STEP_ID` cannot corrupt a nested `srun`. Two entries on that list are load -bearing on real clusters: `SLURM_CONF`, without which the child `srun` falls back -to `/etc/slurm/slurm.conf` and aborts on a configless or multi-cluster site; and -the proxy variables (`http_proxy`, `https_proxy`, `no_proxy` and their uppercase -forms), which Enroot needs because it performs the registry pull inside the step. +`SLURM_STEP_ID` cannot corrupt a nested `srun`. Several entries on that list are +load bearing on real clusters: + +| Variable | Why it must reach the step | +| --- | --- | +| `SLURM_CONF` | Without it the child `srun` falls back to `/etc/slurm/slurm.conf` and aborts on a configless or multi-cluster site. | +| `http_proxy`, `https_proxy`, `no_proxy` (+ uppercase) | Enroot performs the registry pull inside the step and needs the caller's proxy policy. | +| `ENROOT_TEMP_PATH`, `ENROOT_CONFIG_PATH` | Enroot creates the container inside the step. Dropping these discards the operator's override, so the multi-gigabyte create-time temp lands back on the device holding the unpacked rootfs. | + Credentials such as `OPENAI_API_KEY`, `HF_TOKEN` and the service auth token are -never forwarded. +never forwarded, and no other `SLURM_*` variable is. During generation, the service still uses mini-swe-agent for the agent loop and model requests, but replaces its Docker environment with `PyxisEnvironment`. Every diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py index 7381b9d95..263e7f5b3 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py @@ -47,6 +47,12 @@ # the container is ever created. The remaining SLURM_* variables stay out: # inheriting SLURM_JOB_ID / SLURM_STEP_ID is what breaks a nested srun. "SLURM_CONF", + # Enroot reads these when Pyxis creates the container, which happens inside + # the step. Dropping them silently discards the operator's override, so the + # ~2.5 GB create-time temp lands back on whichever device holds the unpacked + # rootfs -- exactly the device the override existed to protect. + "ENROOT_TEMP_PATH", + "ENROOT_CONFIG_PATH", ) _STEP_STATUS = "/tmp/.mlperf_srun_status" _STEP_SCRIPT = r"""set +e diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 67a4090bb..b1f807b39 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -850,6 +850,9 @@ def test_pyxis_srun_environment_does_not_forward_credentials(monkeypatch): "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + # enroot creates the container inside the step and reads these there. + "ENROOT_TEMP_PATH", + "ENROOT_CONFIG_PATH", ], ) def test_pyxis_srun_environment_forwards_config_and_proxy_policy(monkeypatch, name): From a0050da585204bc99ba3872be3aa11c2083b07cd Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 12:51:11 -0700 Subject: [PATCH 03/25] fix(swebench-service): remove the Pyxis container that was actually created `PyxisEnvironment.cleanup()` never reclaimed anything. Pyxis namespaces named containers by the allocation, so `--container-name=X` inside job `N` is the Enroot container `pyxis_N_X`. cleanup() asked for `pyxis_X`, which does not exist. `enroot remove` exited non-zero, and `check=False` with `capture_output=True` discarded both the status and the message, so the failure was invisible. Symptom: nothing is reclaimed for the life of an allocation. Measured on a 20-node run -- 199 trajectory rootfs coexisting on one node and `/raid` down to 4.1 GB free of 527 GB, after which every subsequent container creation failed for want of space. It is also the origin of the "scancel doesn't reap enroot containers" folklore: `scancel` genuinely does not remove Enroot containers, but the containers here were never asked to go away in the first place, so the blame landed on SLURM. Two changes: * `enroot_container_name(job_id, name)` builds the name Pyxis actually created, and cleanup() uses it. * A non-zero `enroot remove` is logged with its stderr instead of being swallowed, so the next time this path breaks it says so. Tests: `test_pyxis_cleanup_removes_the_container_pyxis_actually_created` models an Enroot container set and asserts the created container is the one removed (it fails against the old name, which removes nothing); `test_pyxis_cleanup_reports_a_removal_that_did_not_happen` asserts the warning; `test_enroot_container_name_is_namespaced_by_job` pins the naming rule. The existing container-reuse test asserted the unnamespaced form and is corrected. --- .../evaluation/swebench_service/README.md | 9 ++ .../swebench_service/pyxis_environment.py | 33 +++++-- .../swebench_service/test_runner.py | 93 ++++++++++++++++++- 3 files changed, 128 insertions(+), 7 deletions(-) diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index 8c56bc0c8..f599fefda 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -84,6 +84,15 @@ unresolved task; an `srun`, Enroot, or container-start failure is an infrastruct error that fails the run. The service then aggregates the per-instance reports and removes its named Pyxis containers. +Pyxis namespaces a named container by its allocation: `--container-name=X` inside +job `N` is the Enroot container `pyxis_N_X`. `PyxisEnvironment.cleanup()` removes +that name and logs a warning if the removal does not succeed. Each trajectory's +rootfs is on the order of gigabytes and is only reclaimed by this call, so a +removal that quietly fails fills the node's Enroot data path for the rest of the +allocation. `scancel` does not reclaim them either -- ending the job does not +remove Enroot containers -- which is why the removal has to be both correctly +named and audible. + The benchmark client submits a run to this service only in `ACC` or `BOTH` mode; the default `PERF` mode skips external evaluation. diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py index 263e7f5b3..ddef3a7db 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py @@ -173,6 +173,17 @@ def run_srun_step( return result +def enroot_container_name(job_id: str, container_name: str) -> str: + """The Enroot container name Pyxis derives from ``--container-name``. + + Pyxis namespaces every named container by the allocation it belongs to, so + ``--container-name=X`` inside job ``N`` becomes the Enroot container + ``pyxis_N_X``. Anything that later addresses the container by name -- + ``enroot list``, ``enroot remove`` -- has to use the same form. + """ + return f"pyxis_{job_id}_{container_name}" + + def resolve_image(image_registry: str, instance_id: str) -> str: if Path(instance_id).name != instance_id or instance_id in {".", ".."}: raise RunnerError(f"invalid SWE-bench instance ID: {instance_id}") @@ -307,12 +318,12 @@ def cleanup(self) -> None: return self._cleaned = True try: - if os.environ.get("SLURM_JOB_ID", "").strip(): + job_id = os.environ.get("SLURM_JOB_ID", "").strip() + if job_id: + container = enroot_container_name(job_id, self.name) try: - subprocess.run( - build_srun_command( - argv=["enroot", "remove", "-f", f"pyxis_{self.name}"] - ), + completed = subprocess.run( + build_srun_command(argv=["enroot", "remove", "-f", container]), check=False, capture_output=True, text=True, @@ -322,9 +333,19 @@ def cleanup(self) -> None: except (OSError, RunnerError, subprocess.SubprocessError): logger.warning( "Could not remove Pyxis container %s", - self.name, + container, exc_info=True, ) + else: + if completed.returncode != 0: + # Never silent: an unreclaimed rootfs is ~2.5 GB and + # they accumulate for the whole allocation. + logger.warning( + "enroot remove %s exited %s: %s", + container, + completed.returncode, + (completed.stderr or completed.stdout or "").strip()[-500:], + ) finally: self._tmp.cleanup() diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index b1f807b39..3955aacfe 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -22,6 +22,7 @@ from inference_endpoint.evaluation.swebench_service.swebench_service.pyxis_environment import ( PyxisEnvironment, build_srun_command, + enroot_container_name, resolve_image, safe_srun_env, ) @@ -926,7 +927,7 @@ def fake_run(command, **kwargs): "enroot", "remove", "-f", - f"pyxis_{container_name}", + f"pyxis_1738605_{container_name}", ] assert first["returncode"] == second["returncode"] == 0 assert "Executing Pyxis command: touch state" in caplog.text @@ -1081,6 +1082,96 @@ def fake_run(command, **kwargs): environment.cleanup() +def test_enroot_container_name_is_namespaced_by_job(): + assert enroot_container_name("1738605", "mswe_run-1_abcd1234") == ( + "pyxis_1738605_mswe_run-1_abcd1234" + ) + + +def test_pyxis_cleanup_removes_the_container_pyxis_actually_created( + monkeypatch, tmp_path +): + """The removal must name ``pyxis__``, not ``pyxis_``. + + Addressing the wrong name made every removal a no-op, so no rootfs was ever + reclaimed for the life of an allocation. + """ + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + existing = {"pyxis_1738605_placeholder"} + removed: list[str] = [] + + def fake_run(command, **kwargs): + if command[-4:-1] == ["enroot", "remove", "-f"]: + target = command[-1] + if target not in existing: + return subprocess.CompletedProcess( + command, 1, stdout="", stderr=f"[ERROR] No such container: {target}" + ) + existing.discard(target) + removed.append(target) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + name = next( + ( + argument.split("=", 1)[1] + for argument in command + if argument.startswith("--container-name=") + ), + None, + ) + if name is not None: + existing.add(f"pyxis_1738605_{name}") + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + image = tmp_path / "task.sqsh" + image.touch() + environment = PyxisEnvironment(image=image, run_id="run-1") + + environment.cleanup() + + assert removed == [f"pyxis_1738605_{environment.name}"] + assert existing == {"pyxis_1738605_placeholder"} + + +def test_pyxis_cleanup_reports_a_removal_that_did_not_happen( + monkeypatch, tmp_path, caplog +): + """A non-zero ``enroot remove`` must be logged, not swallowed. + + ``check=False`` plus ``capture_output=True`` discarded the only evidence + that nothing was being reclaimed. + """ + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + + def fake_run(command, **kwargs): + if command[-4:-1] == ["enroot", "remove", "-f"]: + return subprocess.CompletedProcess( + command, 1, stdout="", stderr="[ERROR] No such container\n" + ) + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + image = tmp_path / "task.sqsh" + image.touch() + environment = PyxisEnvironment(image=image, run_id="run-1") + + with caplog.at_level( + logging.WARNING, + logger=( + "inference_endpoint.evaluation.swebench_service.swebench_service" + ".pyxis_environment" + ), + ): + environment.cleanup() + + assert "enroot remove" in caplog.text + assert "No such container" in caplog.text + + def test_pyxis_cleanup_is_best_effort_outside_allocation(monkeypatch, tmp_path): monkeypatch.setenv("SLURM_JOB_ID", "1738605") monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") From c3cbdb8bf11ac789f285c92ad4fe832598195b40 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:36:25 -0700 Subject: [PATCH 04/25] feat(swe-bench): content-addressed unit plan + durable mkdir-atomic work queue Adds the two foundations of the distributed SWE-bench harness: - units.py: shards an instance-id list into immutable, content-addressed units. The sha256 digest covers the ordered id list, so a plan cannot be silently reused across a different run, instance list, or ordering. - queue.py: a filesystem work queue whose claim is a bare os.mkdir (never makedirs(exist_ok=True), which hands a unit to every caller). available() is plan - claims - results, so deleting a result alone does NOT requeue a unit; requeue() is the only supported path and removes the result, the claim and the attempt records together. Env faults are ledgered separately from counted attempts, and abandoning a unit publishes a terminal result AND releases the claim so claims/ and results/ never disagree. --- .../swe_bench_distributed/__init__.py | 30 ++ .../evaluation/swe_bench_distributed/queue.py | 495 ++++++++++++++++++ .../evaluation/swe_bench_distributed/units.py | 187 +++++++ .../test_units_and_queue.py | 246 +++++++++ 4 files changed, 958 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/queue.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/units.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py new file mode 100644 index 000000000..2042c3d55 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Distributed SWE-bench execution across a fleet of SWE-bench services. + +The single-service :class:`~inference_endpoint.evaluation.swe_bench_scorer.SWEBenchScorer` +issues one run covering every instance. This package shards the instance list +into units, dispatches units across several services concurrently, classifies +infrastructure damage separately from genuine model failures, and refuses to +emit an accuracy number unless every planned instance id is accounted for +exactly once. +""" + +from .queue import ( + ClaimError, + UnitOutcome, + UnitResult, + WorkQueue, +) +from .units import Unit, UnitPlan, plan_units + +__all__ = [ + "ClaimError", + "Unit", + "UnitOutcome", + "UnitPlan", + "UnitResult", + "WorkQueue", + "plan_units", +] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/queue.py b/src/inference_endpoint/evaluation/swe_bench_distributed/queue.py new file mode 100644 index 000000000..3feddae63 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/queue.py @@ -0,0 +1,495 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Durable work queue for distributed SWE-bench units. + +The queue is a directory tree so that a client crash costs nothing but the +in-flight units, and so that the merge gate reads durable records rather than +process memory. + +Layout under ``root``:: + + units.json immutable plan (see units.py) + claims//owner json owner record, written temp+rename + claims//hb heartbeat, mtime only + results/.json terminal record (succeeded OR abandoned) + failed/..json one record per *counted* attempt + failed/env/.*.json environment faults, NOT counted + failed/artifacts/... evidence snapshot taken before a retry + +Two invariants are load-bearing and are enforced here rather than by +convention: + +1. ``claim()`` is ``os.mkdir`` and nothing else. ``mkdir`` on an existing + directory fails atomically with ``EEXIST`` on every filesystem we run on, + including Lustre, so exactly one of N racing callers wins. ``makedirs(..., + exist_ok=True)`` would hand the unit to every caller. +2. ``requeue()`` is the only way to make a terminal unit runnable again, and it + removes the result, the claim tombstone *and* the counted attempt records + together. Deleting a result file by hand does not requeue a unit -- the + claim tombstone still hides it -- and that misunderstanding has cost real + campaign time. +""" + +from __future__ import annotations + +import errno +import logging +import os +import shutil +import socket +import time +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from typing import Any + +import msgspec + +from .units import PLAN_FILENAME, UnitPlan, read_plan + +logger = logging.getLogger(__name__) + +_OWNER = "owner" +_HEARTBEAT = "hb" +_CLAIM_CONTENTS = frozenset({_OWNER, _HEARTBEAT}) + +# Small files that explain a failure. Snapshotted before a retry reuses the +# unit's run directory: a unit that fails and then succeeds otherwise leaves +# only the success's artifacts, and a post-mortem then reads the wrong run. +EVIDENCE_FILES = ( + "status.json", + "swe_bench_results.json", + "preds.json", + "swe_bench_service_status.json", +) +EVIDENCE_LOG_TAIL_BYTES = 200_000 +EVIDENCE_LOGS = ("swe_bench_agent.log", "swe_bench_eval.log") + + +class ClaimError(RuntimeError): + """A claim operation could not be performed.""" + + +class UnitOutcome(StrEnum): + """How an attempt at a unit ended. + + ``ENV_FAULT`` is deliberately separate from ``FAILED``: a broken service, an + unreachable endpoint or a refused gate is a property of the *worker*, not of + the unit. Charging it to the unit's attempt budget abandons perfectly good + units because they happened to land on a sick host. + """ + + SUCCEEDED = "succeeded" + INFRA = "infra" + FAILED = "failed" + ENV_FAULT = "env_fault" + + +#: Outcomes that consume one of the unit's ``max_attempts``. +COUNTED_OUTCOMES = frozenset({UnitOutcome.INFRA, UnitOutcome.FAILED}) + + +@dataclass(slots=True) +class UnitResult: + """A terminal or attempt record for one unit.""" + + unit_id: str + run_id: str + plan_digest: str + outcome: UnitOutcome + accounted_instance_ids: tuple[str, ...] = () + resolved_instance_ids: tuple[str, ...] = () + infra_error_count: int = 0 + genuine_error_count: int = 0 + error_kinds: dict[str, int] = field(default_factory=dict) + service_url: str | None = None + endpoint_fingerprint: str | None = None + service_run_id: str | None = None + attempt: int = 0 + abandoned: bool = False + duration_s: float = 0.0 + detail: str | None = None + finished_at: float = field(default_factory=time.time) + + def to_dict(self) -> dict[str, Any]: + return { + "unit_id": self.unit_id, + "run_id": self.run_id, + "plan_digest": self.plan_digest, + "outcome": self.outcome.value, + "accounted_instance_ids": list(self.accounted_instance_ids), + "resolved_instance_ids": list(self.resolved_instance_ids), + "infra_error_count": self.infra_error_count, + "genuine_error_count": self.genuine_error_count, + "error_kinds": dict(self.error_kinds), + "service_url": self.service_url, + "endpoint_fingerprint": self.endpoint_fingerprint, + "service_run_id": self.service_run_id, + "attempt": self.attempt, + "abandoned": self.abandoned, + "duration_s": self.duration_s, + "detail": self.detail, + "finished_at": self.finished_at, + } + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> UnitResult: + return cls( + unit_id=str(raw["unit_id"]), + run_id=str(raw["run_id"]), + plan_digest=str(raw["plan_digest"]), + outcome=UnitOutcome(str(raw["outcome"])), + accounted_instance_ids=tuple( + str(x) for x in raw.get("accounted_instance_ids") or () + ), + resolved_instance_ids=tuple( + str(x) for x in raw.get("resolved_instance_ids") or () + ), + infra_error_count=int(raw.get("infra_error_count") or 0), + genuine_error_count=int(raw.get("genuine_error_count") or 0), + error_kinds=dict(raw.get("error_kinds") or {}), + service_url=raw.get("service_url"), + endpoint_fingerprint=raw.get("endpoint_fingerprint"), + service_run_id=raw.get("service_run_id"), + attempt=int(raw.get("attempt") or 0), + abandoned=bool(raw.get("abandoned")), + duration_s=float(raw.get("duration_s") or 0.0), + detail=raw.get("detail"), + finished_at=float(raw.get("finished_at") or 0.0), + ) + + +@dataclass(slots=True) +class OwnerRecord: + unit_id: str + host: str + pid: int + boot_id: str + plan_digest: str + claimed_at: float + endpoint_fingerprint: str | None = None + slurm_job_id: str | None = None + slurm_step_id: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "unit_id": self.unit_id, + "host": self.host, + "pid": self.pid, + "boot_id": self.boot_id, + "plan_digest": self.plan_digest, + "claimed_at": self.claimed_at, + "endpoint_fingerprint": self.endpoint_fingerprint, + "slurm_job_id": self.slurm_job_id, + "slurm_step_id": self.slurm_step_id, + } + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> OwnerRecord: + return cls( + unit_id=str(raw["unit_id"]), + host=str(raw.get("host") or ""), + pid=int(raw.get("pid") or 0), + boot_id=str(raw.get("boot_id") or ""), + plan_digest=str(raw.get("plan_digest") or ""), + claimed_at=float(raw.get("claimed_at") or 0.0), + endpoint_fingerprint=raw.get("endpoint_fingerprint"), + slurm_job_id=raw.get("slurm_job_id"), + slurm_step_id=raw.get("slurm_step_id"), + ) + + +def boot_id() -> str: + """Identify this boot of this host. + + A pid alone is not proof of liveness: after a reboot the same pid can belong + to something else entirely, and the reaper would then conclude a dead owner + is alive and leave its unit blocked forever. + """ + try: + return Path("/proc/sys/kernel/random/boot_id").read_text().strip() + except OSError: + try: + return str(int(time.time() - time.monotonic())) + except (OSError, ValueError): # pragma: no cover - defensive + return "unknown" + + +def _atomic_write_json(path: Path, payload: dict[str, Any]) -> None: + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + tmp.write_bytes(msgspec.json.encode(payload)) + tmp.replace(path) + + +class WorkQueue: + """Filesystem-backed queue over an immutable :class:`UnitPlan`.""" + + def __init__(self, root: os.PathLike[str] | str, plan: UnitPlan) -> None: + self.root = Path(root) + self.plan = plan + self.claims_dir = self.root / "claims" + self.results_dir = self.root / "results" + self.failed_dir = self.root / "failed" + self.env_failed_dir = self.failed_dir / "env" + self.artifacts_dir = self.failed_dir / "artifacts" + for directory in ( + self.root, + self.claims_dir, + self.results_dir, + self.failed_dir, + self.env_failed_dir, + self.artifacts_dir, + ): + directory.mkdir(parents=True, exist_ok=True) + self.plan.write(self.root) + self._boot_id = boot_id() + + # ------------------------------------------------------------------ open -- + + @classmethod + def open(cls, root: os.PathLike[str] | str) -> WorkQueue: + """Reopen an existing queue, reading its plan from disk.""" + root_path = Path(root) + return cls(root_path, read_plan(root_path / PLAN_FILENAME)) + + # ----------------------------------------------------------- inspection -- + + def claimed_unit_ids(self) -> set[str]: + try: + return {entry.name for entry in self.claims_dir.iterdir() if entry.is_dir()} + except FileNotFoundError: # pragma: no cover - created in __init__ + return set() + + def completed_unit_ids(self) -> set[str]: + return {path.stem for path in self.results_dir.glob("*.json")} + + def available_unit_ids(self) -> list[str]: + """Units that are neither claimed nor terminal, in plan order. + + Subtracting *both* claims and results is what makes a hand-deleted + result file a no-op: the claim tombstone still hides the unit. Use + :meth:`requeue`. + """ + taken = self.claimed_unit_ids() | self.completed_unit_ids() + return [unit_id for unit_id in self.plan.unit_ids if unit_id not in taken] + + def attempts(self, unit_id: str) -> int: + """Number of *counted* attempts recorded for a unit.""" + return len(list(self.failed_dir.glob(f"{unit_id}.*.json"))) + + def owner(self, unit_id: str) -> OwnerRecord | None: + path = self.claims_dir / unit_id / _OWNER + try: + raw = msgspec.json.decode(path.read_bytes(), type=dict) + except (OSError, msgspec.DecodeError): + return None + try: + return OwnerRecord.from_dict(raw) + except (KeyError, TypeError, ValueError): + return None + + def heartbeat_age(self, unit_id: str, *, now: float | None = None) -> float | None: + path = self.claims_dir / unit_id / _HEARTBEAT + try: + mtime = path.stat().st_mtime + except OSError: + return None + return (time.time() if now is None else now) - mtime + + def result(self, unit_id: str) -> UnitResult | None: + path = self.results_dir / f"{unit_id}.json" + try: + raw = msgspec.json.decode(path.read_bytes(), type=dict) + except (OSError, msgspec.DecodeError): + return None + try: + return UnitResult.from_dict(raw) + except (KeyError, TypeError, ValueError): + return None + + def results(self) -> dict[str, UnitResult]: + found: dict[str, UnitResult] = {} + for path in sorted(self.results_dir.glob("*.json")): + result = self.result(path.stem) + if result is not None: + found[path.stem] = result + return found + + # ---------------------------------------------------------------- claim -- + + def claim( + self, + unit_id: str, + *, + endpoint_fingerprint: str | None = None, + ) -> OwnerRecord | None: + """Take exclusive ownership of ``unit_id``; ``None`` if someone else has it.""" + if unit_id not in self.plan.unit_ids: + raise ClaimError( + f"{unit_id!r} is not in the plan for run {self.plan.run_id}" + ) + claim_dir = self.claims_dir / unit_id + try: + # THE RACE IS DECIDED HERE. mkdir, never makedirs/exist_ok. + os.mkdir(claim_dir) + except FileExistsError: + return None + except OSError as exc: + if exc.errno == errno.EEXIST: # pragma: no cover - platform variance + return None + raise ClaimError(f"could not claim {unit_id}: {exc}") from exc + + record = OwnerRecord( + unit_id=unit_id, + host=socket.gethostname(), + pid=os.getpid(), + boot_id=self._boot_id, + plan_digest=self.plan.digest, + claimed_at=time.time(), + endpoint_fingerprint=endpoint_fingerprint, + slurm_job_id=os.environ.get("SLURM_JOB_ID") or None, + slurm_step_id=os.environ.get("SLURM_STEP_ID") or None, + ) + # Sole owner from here, but still temp+rename so the reaper never reads + # a half-written owner record and calls it malformed. + _atomic_write_json(claim_dir / _OWNER, record.to_dict()) + (claim_dir / _HEARTBEAT).touch() + return record + + def beat(self, unit_id: str) -> None: + path = self.claims_dir / unit_id / _HEARTBEAT + try: + path.touch() + except OSError: + logger.debug("could not refresh heartbeat for %s", unit_id, exc_info=True) + + def release(self, unit_id: str) -> bool: + """Hand a claimed unit back to the queue. + + Removes the claim *directory*. Removing only the ``owner`` file leaves an + ownerless directory, which still hides the unit and merely relabels the + problem. + """ + claim_dir = self.claims_dir / unit_id + if not claim_dir.exists(): + return False + shutil.rmtree(claim_dir, ignore_errors=True) + return True + + def is_pure_bookkeeping(self, unit_id: str) -> bool: + """True when a claim directory holds only ``owner``/``hb``.""" + claim_dir = self.claims_dir / unit_id + try: + contents = {entry.name for entry in claim_dir.iterdir()} + except OSError: + return False + return not (contents - _CLAIM_CONTENTS) + + # -------------------------------------------------------------- publish -- + + def publish(self, result: UnitResult) -> None: + """Record a terminal result and release the claim. + + Releasing here is not optional. The abandon path once published a result + but kept the claim directory, so ``claims/`` and ``results/`` disagreed + for the rest of the campaign and every reaper pass had a phantom to + reason about. Releasing is safe because :meth:`available_unit_ids` + subtracts results as well as claims. + """ + self._check_digest(result) + _atomic_write_json( + self.results_dir / f"{result.unit_id}.json", result.to_dict() + ) + self.release(result.unit_id) + + def record_attempt(self, result: UnitResult) -> int: + """Record a non-terminal attempt. Returns the counted-attempt total. + + ``ENV_FAULT`` attempts are written to ``failed/env/`` and do not + increment the counter. + """ + self._check_digest(result) + if result.outcome is UnitOutcome.ENV_FAULT: + path = self.env_failed_dir / f"{result.unit_id}.{time.time_ns()}.json" + _atomic_write_json(path, result.to_dict()) + return self.attempts(result.unit_id) + count = self.attempts(result.unit_id) + 1 + result.attempt = count + _atomic_write_json( + self.failed_dir / f"{result.unit_id}.{count}.json", result.to_dict() + ) + return count + + def snapshot_evidence(self, unit_id: str, source_dir: Path, attempt: int) -> Path: + """Copy the small files that explain a failure before a retry overwrites them.""" + target = self.artifacts_dir / f"{unit_id}.attempt{attempt}" + target.mkdir(parents=True, exist_ok=True) + for name in EVIDENCE_FILES: + candidate = source_dir / name + if candidate.is_file(): + try: + shutil.copy2(candidate, target / name) + except OSError: + logger.debug("could not snapshot %s", candidate, exc_info=True) + for name in EVIDENCE_LOGS: + candidate = source_dir / name + if not candidate.is_file(): + continue + try: + with candidate.open("rb") as handle: + handle.seek(0, os.SEEK_END) + size = handle.tell() + handle.seek(max(0, size - EVIDENCE_LOG_TAIL_BYTES)) + (target / f"{name}.tail").write_bytes(handle.read()) + except OSError: + logger.debug("could not snapshot %s", candidate, exc_info=True) + return target + + def abandon(self, result: UnitResult) -> None: + """Publish a terminal, explicitly-abandoned result.""" + result.abandoned = True + self.publish(result) + + # -------------------------------------------------------------- requeue -- + + def requeue(self, unit_id: str) -> dict[str, list[str]]: + """Make a unit runnable again. The *only* supported way. + + Removes, together: the terminal result, the claim tombstone, and every + counted attempt record. Removing any subset leaves the unit invisible or + already out of attempts, which is how "I deleted the result, why is it + not rerunning?" happens. + """ + if unit_id not in self.plan.unit_ids: + raise ClaimError( + f"{unit_id!r} is not in the plan for run {self.plan.run_id}" + ) + removed: dict[str, list[str]] = {"results": [], "claims": [], "attempts": []} + result_path = self.results_dir / f"{unit_id}.json" + if result_path.exists(): + result_path.unlink() + removed["results"].append(str(result_path)) + claim_dir = self.claims_dir / unit_id + if claim_dir.exists(): + shutil.rmtree(claim_dir, ignore_errors=True) + removed["claims"].append(str(claim_dir)) + for path in sorted(self.failed_dir.glob(f"{unit_id}.*.json")): + path.unlink() + removed["attempts"].append(str(path)) + return removed + + # ---------------------------------------------------------------- utils -- + + def _check_digest(self, result: UnitResult) -> None: + if result.plan_digest != self.plan.digest: + raise ClaimError( + f"refusing to record {result.unit_id}: plan digest " + f"{result.plan_digest[:12]} does not match this queue's " + f"{self.plan.digest[:12]}" + ) + if result.run_id != self.plan.run_id: + raise ClaimError( + f"refusing to record {result.unit_id}: run id {result.run_id!r} " + f"does not match this queue's {self.plan.run_id!r}" + ) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/units.py b/src/inference_endpoint/evaluation/swe_bench_distributed/units.py new file mode 100644 index 000000000..9a163a037 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/units.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shard plan: the immutable binding between units and instance ids. + +The plan is content-addressed. Every unit result carries the plan digest, and +the merge gate refuses to combine results whose digest differs from the plan +being merged. That is what makes it impossible to accidentally merge results +from a different run, a different instance list, or a different ordering into +one accuracy number. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import msgspec + +PLAN_FILENAME = "units.json" + + +class PlanError(ValueError): + """The requested shard plan cannot be built, or a plan file is invalid.""" + + +@dataclass(frozen=True, slots=True) +class Unit: + """One dispatchable shard of a run.""" + + unit_id: str + run_id: str + shard: int + instance_ids: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "unit_id": self.unit_id, + "run_id": self.run_id, + "shard": self.shard, + "instance_ids": list(self.instance_ids), + } + + +@dataclass(frozen=True, slots=True) +class UnitPlan: + """The full, immutable set of units for one run id.""" + + run_id: str + shard_size: int + digest: str + units: tuple[Unit, ...] + + @property + def instance_ids(self) -> tuple[str, ...]: + return tuple( + instance_id for unit in self.units for instance_id in unit.instance_ids + ) + + def unit(self, unit_id: str) -> Unit: + for candidate in self.units: + if candidate.unit_id == unit_id: + return candidate + raise KeyError(unit_id) + + @property + def unit_ids(self) -> tuple[str, ...]: + return tuple(unit.unit_id for unit in self.units) + + def to_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "shard_size": self.shard_size, + "digest": self.digest, + "units": [unit.to_dict() for unit in self.units], + } + + def write(self, directory: Path) -> Path: + """Write the plan once. Rewriting an existing, different plan is an error.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / PLAN_FILENAME + if path.exists(): + existing = read_plan(path) + if existing.digest != self.digest or existing.run_id != self.run_id: + raise PlanError( + f"refusing to overwrite plan at {path}: existing run_id=" + f"{existing.run_id!r} digest={existing.digest[:12]} differs from " + f"new run_id={self.run_id!r} digest={self.digest[:12]}" + ) + return path + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_bytes(msgspec.json.encode(self.to_dict())) + tmp.replace(path) + return path + + +def plan_digest(run_id: str, instance_ids: list[str] | tuple[str, ...]) -> str: + """Digest over the run id and the ordered instance list. + + Order is included deliberately: two plans over the same ids in a different + order produce different shards, so they are different plans. + """ + hasher = hashlib.sha256() + hasher.update(run_id.encode()) + hasher.update(b"\0") + for instance_id in instance_ids: + hasher.update(instance_id.encode()) + hasher.update(b"\n") + return hasher.hexdigest() + + +def plan_units( + run_id: str, + instance_ids: list[str] | tuple[str, ...], + *, + shard_size: int = 10, +) -> UnitPlan: + """Split ``instance_ids`` into fixed-size shards, in order. + + The final shard is short when the count is not a multiple of ``shard_size``; + it is never padded and never merged into its neighbour, because the merge + gate compares id sets and a padded shard would claim ids it never ran. + """ + if not run_id or "/" in run_id or run_id in {".", ".."}: + raise PlanError(f"invalid run_id: {run_id!r}") + if shard_size < 1: + raise PlanError(f"shard_size must be >= 1; got {shard_size}") + ordered = [str(instance_id) for instance_id in instance_ids] + if not ordered: + raise PlanError("cannot plan a run with no instance ids") + duplicates = sorted({x for x in ordered if ordered.count(x) > 1}) + if duplicates: + raise PlanError( + "instance ids must be unique; duplicated: " + ", ".join(duplicates[:10]) + ) + + digest = plan_digest(run_id, ordered) + units: list[Unit] = [] + for shard, start in enumerate(range(0, len(ordered), shard_size)): + chunk = tuple(ordered[start : start + shard_size]) + units.append( + Unit( + unit_id=f"{run_id}.s{shard:02d}", + run_id=run_id, + shard=shard, + instance_ids=chunk, + ) + ) + return UnitPlan( + run_id=run_id, shard_size=shard_size, digest=digest, units=tuple(units) + ) + + +def read_plan(path: Path) -> UnitPlan: + try: + raw = msgspec.json.decode(path.read_bytes(), type=dict) + except (OSError, msgspec.DecodeError) as exc: + raise PlanError(f"could not read unit plan at {path}") from exc + try: + units = tuple( + Unit( + unit_id=str(entry["unit_id"]), + run_id=str(entry["run_id"]), + shard=int(entry["shard"]), + instance_ids=tuple(str(x) for x in entry["instance_ids"]), + ) + for entry in raw["units"] + ) + plan = UnitPlan( + run_id=str(raw["run_id"]), + shard_size=int(raw["shard_size"]), + digest=str(raw["digest"]), + units=units, + ) + except (KeyError, TypeError, ValueError) as exc: + raise PlanError(f"malformed unit plan at {path}") from exc + + recomputed = plan_digest(plan.run_id, list(plan.instance_ids)) + if recomputed != plan.digest: + raise PlanError( + f"unit plan at {path} is inconsistent: recorded digest " + f"{plan.digest[:12]} does not match its own instance list " + f"({recomputed[:12]})" + ) + return plan diff --git a/tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py b/tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py new file mode 100644 index 000000000..6343b7536 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit plan and work-queue semantics.""" + +from __future__ import annotations + +import threading + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + ClaimError, + UnitOutcome, + UnitResult, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import ( + PlanError, + plan_units, + read_plan, +) + +pytestmark = pytest.mark.unit + + +def make_ids(n: int) -> list[str]: + return [f"repo__proj-{i:03d}" for i in range(n)] + + +@pytest.fixture +def queue(tmp_path): + plan = plan_units("run-a", make_ids(25), shard_size=10) + return WorkQueue(tmp_path / "wq", plan) + + +def result_for(queue: WorkQueue, unit_id: str, **overrides) -> UnitResult: + unit = queue.plan.unit(unit_id) + payload = { + "unit_id": unit_id, + "run_id": unit.run_id, + "plan_digest": queue.plan.digest, + "outcome": UnitOutcome.SUCCEEDED, + "accounted_instance_ids": unit.instance_ids, + "resolved_instance_ids": unit.instance_ids[:1], + } + payload.update(overrides) + return UnitResult(**payload) + + +class TestPlan: + def test_shards_in_order_with_a_short_tail(self): + plan = plan_units("run-a", make_ids(25), shard_size=10) + assert [len(unit.instance_ids) for unit in plan.units] == [10, 10, 5] + assert plan.unit_ids == ("run-a.s00", "run-a.s01", "run-a.s02") + # The short tail is never padded: a padded shard would claim ids it + # never ran and the merge gate compares ids, not counts. + assert plan.instance_ids == tuple(make_ids(25)) + + def test_digest_depends_on_order(self): + ids = make_ids(20) + assert ( + plan_units("r", ids).digest != plan_units("r", list(reversed(ids))).digest + ) + + def test_digest_depends_on_run_id(self): + ids = make_ids(20) + assert plan_units("r1", ids).digest != plan_units("r2", ids).digest + + def test_duplicate_instance_ids_are_refused(self): + with pytest.raises(PlanError, match="unique"): + plan_units("r", ["a", "b", "a"]) + + def test_empty_plan_is_refused(self): + with pytest.raises(PlanError): + plan_units("r", []) + + def test_plan_round_trips_and_self_verifies(self, tmp_path): + plan = plan_units("run-a", make_ids(12), shard_size=5) + path = plan.write(tmp_path) + assert read_plan(path).digest == plan.digest + + def test_rewriting_a_different_plan_is_refused(self, tmp_path): + plan_units("run-a", make_ids(10)).write(tmp_path) + with pytest.raises(PlanError, match="refusing to overwrite"): + plan_units("run-a", make_ids(11)).write(tmp_path) + + def test_tampered_plan_file_is_detected(self, tmp_path): + plan = plan_units("run-a", make_ids(10)) + path = plan.write(tmp_path) + raw = path.read_text().replace(plan.digest, "0" * 64) + path.write_text(raw) + with pytest.raises(PlanError, match="inconsistent"): + read_plan(path) + + +class TestClaims: + def test_a_second_claim_loses(self, queue): + assert queue.claim("run-a.s00") is not None + assert queue.claim("run-a.s00") is None + + def test_exactly_one_thread_wins_a_contested_claim(self, queue): + winners: list[object] = [] + barrier = threading.Barrier(8) + + def contend(): + barrier.wait() + if queue.claim("run-a.s01") is not None: + winners.append(object()) + + threads = [threading.Thread(target=contend) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert len(winners) == 1 + + def test_claiming_a_unit_outside_the_plan_is_an_error(self, queue): + with pytest.raises(ClaimError): + queue.claim("other-run.s00") + + def test_release_removes_the_whole_directory(self, queue): + queue.claim("run-a.s00") + assert queue.release("run-a.s00") + # Removing only `owner` would leave an ownerless directory that still + # hides the unit -- a relabelled problem, not a fix. + assert not (queue.claims_dir / "run-a.s00").exists() + assert "run-a.s00" in queue.available_unit_ids() + + def test_owner_record_carries_identity(self, queue): + record = queue.claim("run-a.s00") + stored = queue.owner("run-a.s00") + assert stored is not None + assert stored.pid == record.pid + assert stored.boot_id == record.boot_id + assert stored.plan_digest == queue.plan.digest + + +class TestAvailability: + def test_claims_and_results_both_hide_a_unit(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s01")) + assert queue.available_unit_ids() == ["run-a.s02"] + + def test_publish_releases_the_claim(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s00")) + # An abandoned or published unit that keeps its claim makes claims/ and + # results/ disagree for the rest of the run. + assert queue.claimed_unit_ids() == set() + + def test_abandon_publishes_and_releases(self, queue): + queue.claim("run-a.s00") + queue.abandon(result_for(queue, "run-a.s00", outcome=UnitOutcome.FAILED)) + stored = queue.result("run-a.s00") + assert stored is not None and stored.abandoned + assert queue.claimed_unit_ids() == set() + + def test_result_from_another_plan_is_refused(self, queue): + bad = result_for(queue, "run-a.s00", plan_digest="0" * 64) + with pytest.raises(ClaimError, match="plan digest"): + queue.publish(bad) + + +class TestRequeue: + def test_deleting_only_the_result_does_not_requeue(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s00")) + # Re-claim so a tombstone exists, mimicking an interrupted retry. + queue.claim("run-a.s00") + (queue.results_dir / "run-a.s00.json").unlink() + assert "run-a.s00" not in queue.available_unit_ids() + + def test_deleting_only_the_claim_does_not_requeue(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s00")) + queue.release("run-a.s00") + assert "run-a.s00" not in queue.available_unit_ids() + + def test_requeue_removes_result_claim_and_attempts(self, queue): + queue.claim("run-a.s00") + queue.record_attempt(result_for(queue, "run-a.s00", outcome=UnitOutcome.FAILED)) + queue.record_attempt(result_for(queue, "run-a.s00", outcome=UnitOutcome.INFRA)) + queue.publish(result_for(queue, "run-a.s00")) + queue.claim("run-a.s00") + + removed = queue.requeue("run-a.s00") + + assert len(removed["results"]) == 1 + assert len(removed["claims"]) == 1 + assert len(removed["attempts"]) == 2 + assert "run-a.s00" in queue.available_unit_ids() + assert queue.attempts("run-a.s00") == 0 + + def test_requeue_outside_the_plan_is_an_error(self, queue): + with pytest.raises(ClaimError): + queue.requeue("other-run.s00") + + +class TestAttemptLedger: + def test_environment_faults_do_not_consume_the_budget(self, queue): + for _ in range(5): + queue.record_attempt( + result_for(queue, "run-a.s00", outcome=UnitOutcome.ENV_FAULT) + ) + # A broken host is a property of the host, not of the unit. Charging it + # to the unit abandons good units for landing in the wrong place. + assert queue.attempts("run-a.s00") == 0 + + def test_counted_failures_increment(self, queue): + assert ( + queue.record_attempt( + result_for(queue, "run-a.s00", outcome=UnitOutcome.FAILED) + ) + == 1 + ) + assert ( + queue.record_attempt( + result_for(queue, "run-a.s00", outcome=UnitOutcome.INFRA) + ) + == 2 + ) + + def test_evidence_is_snapshotted_before_a_retry_overwrites_it( + self, queue, tmp_path + ): + source = tmp_path / "unit-run" + source.mkdir() + (source / "status.json").write_text('{"attempt": 1}') + (source / "swe_bench_agent.log").write_text("first attempt log") + + target = queue.snapshot_evidence("run-a.s00", source, attempt=1) + + # The retry reuses the run directory, so a unit that fails then succeeds + # would otherwise leave only the success's artifacts behind. + (source / "status.json").write_text('{"attempt": 2}') + assert (target / "status.json").read_text() == '{"attempt": 1}' + assert (target / "swe_bench_agent.log.tail").read_text() == "first attempt log" + + +class TestReopen: + def test_reopen_reads_the_plan_from_disk(self, queue): + queue.publish(result_for(queue, "run-a.s00")) + reopened = WorkQueue.open(queue.root) + assert reopened.plan.digest == queue.plan.digest + assert reopened.completed_unit_ids() == {"run-a.s00"} From d4d9ad058fe48d22e87b7acdc5228a806d11d3e4 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:39:43 -0700 Subject: [PATCH 05/25] feat(swe-bench): all-or-nothing merge gate scoped to exactly one run id merge_run(wq, run_id) refuses to emit an accuracy number unless every planned unit has a terminal result, none is abandoned, every unit accounts for exactly its planned instance IDS (a set comparison, never a count), the union equals the plan with no cross-shard duplicates, every plan_digest matches, and no unit carries an infra error. Refusal is a structured MergeRefusal naming the offending units and ids; there is no force flag and no partial-credit path. There is deliberately no --all: merge_run takes a required run id and treats a foreign run id or digest as a hard error, not a skip. verify_inventory() cross-checks claims, results and the id-union as independent producers, so a blind spot shared by one instrument cannot certify itself. --- .../swe_bench_distributed/__init__.py | 5 + .../evaluation/swe_bench_distributed/merge.py | 230 ++++++++++++++++++ .../swe_bench_distributed/test_merge.py | 189 ++++++++++++++ 3 files changed, 424 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/merge.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_merge.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 2042c3d55..3507289bb 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,6 +11,7 @@ exactly once. """ +from .merge import MergeRefusal, MergeResult, merge_run, verify_inventory from .queue import ( ClaimError, UnitOutcome, @@ -21,10 +22,14 @@ __all__ = [ "ClaimError", + "MergeRefusal", + "MergeResult", "Unit", "UnitOutcome", "UnitPlan", "UnitResult", "WorkQueue", + "merge_run", "plan_units", + "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py b/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py new file mode 100644 index 000000000..a8dbee7cf --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py @@ -0,0 +1,230 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The merge gate: refuse to emit an accuracy unless every id is accounted for. + +The single most important property of a sharded accuracy run is that it never +divides the results of 190 instances by 200. The gate is all-or-nothing by +design: there is no force flag and no partial-credit path, because a partial +number is indistinguishable from a real one once it leaves this module. + +The gate is also scoped to exactly one run. There is no ``merge_all``. Merging +"every run that looks finished" once re-merged hundreds of banked results +belonging to unrelated configurations into one number. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from .queue import UnitOutcome, UnitResult, WorkQueue +from .units import UnitPlan + + +class MergeRefusal(RuntimeError): + """The gate refused to produce an accuracy number. + + ``reasons`` lists every independent failure, so one merge attempt reports + everything wrong rather than the first thing wrong. + """ + + def __init__(self, run_id: str, reasons: list[str]) -> None: + self.run_id = run_id + self.reasons = reasons + super().__init__( + f"refusing to score run {run_id!r}: " + + "; ".join(reasons[:10]) + + (f" (+{len(reasons) - 10} more)" if len(reasons) > 10 else "") + ) + + +@dataclass(slots=True) +class MergeResult: + run_id: str + plan_digest: str + total_instances: int + resolved_instances: int + unit_count: int + + @property + def resolved_rate(self) -> float: + return self.resolved_instances / self.total_instances + + def to_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "plan_digest": self.plan_digest, + "total_instances": self.total_instances, + "resolved_instances": self.resolved_instances, + "resolved_rate": self.resolved_rate, + "unit_count": self.unit_count, + } + + +@dataclass(slots=True) +class InventoryReport: + """Cross-check of three independently produced views of the same run. + + The units the plan asked for, the units the queue recorded results for, and + the instance ids those results claim to have covered are produced by + different code paths. Checking one against itself is how a verification pass + can agree with a broken system: the instrument shares the blind spot. These + must agree with each other. + """ + + missing_units: list[str] = field(default_factory=list) + foreign_units: list[str] = field(default_factory=list) + unreadable_units: list[str] = field(default_factory=list) + ownerless_claims: list[str] = field(default_factory=list) + claims_without_results: list[str] = field(default_factory=list) + + @property + def consistent(self) -> bool: + return not ( + self.missing_units + or self.foreign_units + or self.unreadable_units + or self.ownerless_claims + ) + + +def verify_inventory(queue: WorkQueue) -> InventoryReport: + """Compare the plan, the claim directory and the result directory.""" + report = InventoryReport() + plan_units = set(queue.plan.unit_ids) + + result_files = {path.stem for path in queue.results_dir.glob("*.json")} + report.foreign_units = sorted(result_files - plan_units) + report.missing_units = sorted(plan_units - result_files) + for unit_id in sorted(result_files & plan_units): + if queue.result(unit_id) is None: + report.unreadable_units.append(unit_id) + + for unit_id in sorted(queue.claimed_unit_ids()): + if queue.owner(unit_id) is None: + report.ownerless_claims.append(unit_id) + if unit_id not in result_files: + report.claims_without_results.append(unit_id) + return report + + +def merge_run(queue: WorkQueue, run_id: str) -> MergeResult: + """Score one run, or refuse. + + ``run_id`` is required and must match the queue's plan. Passing another + run's id is an error, not a filter. + """ + plan: UnitPlan = queue.plan + if run_id != plan.run_id: + raise MergeRefusal( + run_id, + [ + f"queue at {queue.root} holds run {plan.run_id!r}, not {run_id!r}; " + "a merge is always scoped to exactly one run" + ], + ) + + reasons: list[str] = [] + inventory = verify_inventory(queue) + if inventory.foreign_units: + reasons.append( + "results present for units outside the plan: " + + ", ".join(inventory.foreign_units[:5]) + ) + if inventory.unreadable_units: + reasons.append( + "unreadable result records: " + ", ".join(inventory.unreadable_units[:5]) + ) + if inventory.ownerless_claims: + reasons.append( + "claims with no readable owner: " + + ", ".join(inventory.ownerless_claims[:5]) + ) + if inventory.missing_units: + reasons.append( + f"{len(inventory.missing_units)} of {len(plan.units)} units have no " + "result: " + ", ".join(inventory.missing_units[:5]) + ) + + results: dict[str, UnitResult] = queue.results() + seen_ids: dict[str, str] = {} + resolved: set[str] = set() + + for unit in plan.units: + result = results.get(unit.unit_id) + if result is None: + continue + if result.plan_digest != plan.digest: + reasons.append( + f"{unit.unit_id}: result belongs to plan {result.plan_digest[:12]}, " + f"not {plan.digest[:12]}" + ) + continue + if result.abandoned: + reasons.append(f"{unit.unit_id}: abandoned after {result.attempt} attempts") + continue + if result.outcome is not UnitOutcome.SUCCEEDED: + reasons.append(f"{unit.unit_id}: outcome {result.outcome.value}") + continue + if result.infra_error_count > 0: + reasons.append( + f"{unit.unit_id}: {result.infra_error_count} instance(s) lost to " + "infrastructure" + ) + continue + + expected = set(unit.instance_ids) + accounted = set(result.accounted_instance_ids) + if len(result.accounted_instance_ids) != len(accounted): + reasons.append(f"{unit.unit_id}: duplicate instance ids in its own result") + continue + # Compare ids, never counts. A shard with one duplicate and one missing + # id has the right count and the wrong content. + if accounted != expected: + missing = sorted(expected - accounted) + extra = sorted(accounted - expected) + detail = [] + if missing: + detail.append(f"missing {', '.join(missing[:5])}") + if extra: + detail.append(f"unplanned {', '.join(extra[:5])}") + reasons.append(f"{unit.unit_id}: " + "; ".join(detail)) + continue + + for instance_id in result.accounted_instance_ids: + previous = seen_ids.get(instance_id) + if previous is not None: + reasons.append( + f"instance {instance_id} accounted for by both {previous} and " + f"{unit.unit_id}" + ) + continue + seen_ids[instance_id] = unit.unit_id + unplanned_resolved = set(result.resolved_instance_ids) - expected + if unplanned_resolved: + reasons.append( + f"{unit.unit_id}: resolved ids outside its shard: " + + ", ".join(sorted(unplanned_resolved)[:5]) + ) + continue + resolved.update(result.resolved_instance_ids) + + planned_ids = set(plan.instance_ids) + if not reasons and set(seen_ids) != planned_ids: + unaccounted = sorted(planned_ids - set(seen_ids)) + reasons.append( + f"{len(unaccounted)} planned instance(s) unaccounted for: " + + ", ".join(unaccounted[:5]) + ) + + if reasons: + raise MergeRefusal(run_id, reasons) + + return MergeResult( + run_id=run_id, + plan_digest=plan.digest, + total_instances=len(planned_ids), + resolved_instances=len(resolved), + unit_count=len(plan.units), + ) diff --git a/tests/unit/evaluation/swe_bench_distributed/test_merge.py b/tests/unit/evaluation/swe_bench_distributed/test_merge.py new file mode 100644 index 000000000..528ab9335 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_merge.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The merge gate: all-or-nothing, id-based, scoped to one run.""" + +from __future__ import annotations + +import inspect + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.merge import ( + MergeRefusal, + merge_run, + verify_inventory, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + UnitOutcome, + UnitResult, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import plan_units + +pytestmark = pytest.mark.unit + +IDS = [f"repo__proj-{i:02d}" for i in range(20)] + + +@pytest.fixture +def queue(tmp_path): + return WorkQueue(tmp_path / "wq", plan_units("run-a", IDS, shard_size=10)) + + +def publish(queue: WorkQueue, unit_id: str, **overrides) -> None: + unit = queue.plan.unit(unit_id) + payload = { + "unit_id": unit_id, + "run_id": unit.run_id, + "plan_digest": queue.plan.digest, + "outcome": UnitOutcome.SUCCEEDED, + "accounted_instance_ids": unit.instance_ids, + "resolved_instance_ids": unit.instance_ids[:3], + } + payload.update(overrides) + queue.publish(UnitResult(**payload)) + + +def publish_all(queue: WorkQueue) -> None: + for unit_id in queue.plan.unit_ids: + publish(queue, unit_id) + + +class TestHappyPath: + def test_full_accounting_scores(self, queue): + publish_all(queue) + result = merge_run(queue, "run-a") + assert result.total_instances == 20 + assert result.resolved_instances == 6 + assert result.resolved_rate == pytest.approx(0.3) + assert result.unit_count == 2 + + +class TestRefusals: + def test_a_missing_unit_refuses(self, queue): + publish(queue, "run-a.s00") + # 10 results must never be divided by 20. + with pytest.raises(MergeRefusal, match="have no result"): + merge_run(queue, "run-a") + + def test_an_abandoned_unit_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", abandoned=True, attempt=3) + with pytest.raises(MergeRefusal, match="abandoned"): + merge_run(queue, "run-a") + + def test_a_non_success_outcome_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", outcome=UnitOutcome.FAILED) + with pytest.raises(MergeRefusal, match="outcome failed"): + merge_run(queue, "run-a") + + def test_infrastructure_damage_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", infra_error_count=2) + with pytest.raises(MergeRefusal, match="lost to infrastructure"): + merge_run(queue, "run-a") + + def test_a_missing_id_refuses_even_though_the_count_is_wrong_by_one(self, queue): + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + publish(queue, "run-a.s01", accounted_instance_ids=unit.instance_ids[:-1]) + with pytest.raises(MergeRefusal, match="missing"): + merge_run(queue, "run-a") + + def test_a_swapped_id_refuses_although_the_count_matches(self, queue): + # The whole point of comparing ids rather than counts: this shard has + # exactly ten entries and the wrong content. + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + swapped = unit.instance_ids[:-1] + ("some__other-99",) + publish(queue, "run-a.s01", accounted_instance_ids=swapped) + with pytest.raises(MergeRefusal, match="unplanned"): + merge_run(queue, "run-a") + + def test_a_duplicated_id_within_one_unit_refuses(self, queue): + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + duped = unit.instance_ids[:-1] + (unit.instance_ids[0],) + publish(queue, "run-a.s01", accounted_instance_ids=duped) + with pytest.raises(MergeRefusal, match="duplicate"): + merge_run(queue, "run-a") + + def test_resolved_ids_outside_the_shard_refuse(self, queue): + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + publish( + queue, + "run-a.s01", + resolved_instance_ids=(*unit.instance_ids[:2], IDS[0]), + ) + with pytest.raises(MergeRefusal, match="outside its shard"): + merge_run(queue, "run-a") + + def test_a_foreign_plan_digest_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01") + path = queue.results_dir / "run-a.s01.json" + path.write_text(path.read_text().replace(queue.plan.digest, "f" * 64)) + with pytest.raises(MergeRefusal, match="belongs to plan"): + merge_run(queue, "run-a") + + def test_a_result_outside_the_plan_refuses(self, queue): + publish_all(queue) + (queue.results_dir / "other-run.s00.json").write_text("{}") + with pytest.raises(MergeRefusal, match="outside the plan"): + merge_run(queue, "run-a") + + def test_an_unreadable_result_refuses(self, queue): + publish_all(queue) + (queue.results_dir / "run-a.s00.json").write_text("not json") + with pytest.raises(MergeRefusal, match="unreadable"): + merge_run(queue, "run-a") + + def test_every_reason_is_reported_at_once(self, queue): + publish(queue, "run-a.s00", infra_error_count=1) + with pytest.raises(MergeRefusal) as excinfo: + merge_run(queue, "run-a") + assert len(excinfo.value.reasons) >= 2 + + +class TestScoping: + def test_a_merge_is_always_scoped_to_one_run(self, queue): + publish_all(queue) + with pytest.raises(MergeRefusal, match="scoped to exactly one run"): + merge_run(queue, "some-other-run") + + def test_there_is_no_merge_all(self): + # "Merge everything that looks finished" once combined hundreds of + # banked results from unrelated configurations into one number. + signature = inspect.signature(merge_run) + assert "run_id" in signature.parameters + assert signature.parameters["run_id"].default is inspect.Parameter.empty + assert not hasattr( + __import__( + "inference_endpoint.evaluation.swe_bench_distributed.merge", + fromlist=["merge"], + ), + "merge_all", + ) + + +class TestInventory: + def test_a_complete_run_is_consistent(self, queue): + publish_all(queue) + assert verify_inventory(queue).consistent + + def test_an_ownerless_claim_is_an_inventory_error(self, queue): + publish_all(queue) + claim_dir = queue.claims_dir / "run-a.s00" + claim_dir.mkdir(parents=True) + # Checking `owner` files with one tool and claim directories with + # another is how a verification pass agrees with a broken system. + report = verify_inventory(queue) + assert report.ownerless_claims == ["run-a.s00"] + assert not report.consistent + + def test_claims_without_results_are_reported(self, queue): + queue.claim("run-a.s00") + assert verify_inventory(queue).claims_without_results == ["run-a.s00"] From 2551a2721e6e1b11fe9439eb1723bad8f79e3030 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 13:17:21 -0700 Subject: [PATCH 06/25] feat(swe-bench): withhold the headline accuracy, publish the honest ones The merge gate refuses a bad run, and that refusal is the most important property here. But a refusal that carries no numbers is not the end of the story: somebody still has to report *something*, and with the gate silent they compute it by hand from the artifacts -- which is exactly how a run that lost 106 of 200 instances to infrastructure came to be reported as 47.0% and compared against a complete-run reference of 70.67%. It was read as a model regression. It was attrition. `assess_run()` performs the whole of the gate's arithmetic without deciding anything, and returns a `CompletenessReport`. `merge_run()` becomes the strict all-or-nothing wrapper over it and attaches the report to both `MergeResult` and `MergeRefusal`, so a caller never has to choose between "a number" and "no information". `resolved_rate` is published only when the run is *structurally complete* -- every planned instance id accounted for exactly once -- **and** zero instances were lost to infrastructure. These are two different questions and conflating them gets both wrong. An instance the model attempted and failed is a legitimate score; one our own harness dropped never had the chance. A run short of instances has the wrong denominator; a complete run that leaned on the infrastructure has the wrong provenance. Two numbers are published either way: * `conditional_resolved_rate` -- resolved over the instances that actually completed. Honest about what it measures and not comparable to a complete-run reference. * `resolved_rate_lower_bound` -- resolved over everything planned. Infrastructure losses can only ever *add* resolutions, so this bounds the truth from below even on a badly degraded run. alongside `incomplete_instance_ids`, `infra_lost_instances`, `infra_lost_unit_ids` and a `resolved_rate_withheld_reason` that says which of the two conditions failed and by how much. Ported from the banked campaign's `wq_merge.sh:7-9`: "shard_merge.py refuses to print an accuracy unless all 20 shards account for exactly their own 10 ids, and that refusal is the single most important property in this campaign." What is added here is that the refusal now shows its working. Tests: `TestCompletenessGate` covers both decision boundaries -- complete vs incomplete, and infra-lost vs genuinely-empty (a model that resolved nothing is a score, not a casualty) -- plus an abandoned unit counting as infrastructure loss, the numbers surviving a refusal, and `assess_run` not raising on the run it is describing. Against the parent commit a refusal carries no `report` and no conditional or lower-bound figure at all. --- .../swe_bench_distributed/__init__.py | 11 +- .../evaluation/swe_bench_distributed/merge.py | 201 ++++++++++++++++-- .../swe_bench_distributed/test_merge.py | 135 ++++++++++++ 3 files changed, 334 insertions(+), 13 deletions(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 3507289bb..c97b98f55 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,7 +11,14 @@ exactly once. """ -from .merge import MergeRefusal, MergeResult, merge_run, verify_inventory +from .merge import ( + CompletenessReport, + MergeRefusal, + MergeResult, + assess_run, + merge_run, + verify_inventory, +) from .queue import ( ClaimError, UnitOutcome, @@ -22,6 +29,7 @@ __all__ = [ "ClaimError", + "CompletenessReport", "MergeRefusal", "MergeResult", "Unit", @@ -29,6 +37,7 @@ "UnitPlan", "UnitResult", "WorkQueue", + "assess_run", "merge_run", "plan_units", "verify_inventory", diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py b/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py index a8dbee7cf..5ef6379fc 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py @@ -27,11 +27,23 @@ class MergeRefusal(RuntimeError): ``reasons`` lists every independent failure, so one merge attempt reports everything wrong rather than the first thing wrong. + + ``report`` carries the :class:`CompletenessReport` for the same run. A + refusal is not an absence of information: the conditional rate, the lower + bound and the ids that went missing are exactly what the operator needs in + order to act, and withholding them alongside the headline is what makes + people go and compute the wrong number by hand instead. """ - def __init__(self, run_id: str, reasons: list[str]) -> None: + def __init__( + self, + run_id: str, + reasons: list[str], + report: CompletenessReport | None = None, + ) -> None: self.run_id = run_id self.reasons = reasons + self.report = report super().__init__( f"refusing to score run {run_id!r}: " + "; ".join(reasons[:10]) @@ -39,6 +51,134 @@ def __init__(self, run_id: str, reasons: list[str]) -> None: ) +@dataclass(slots=True) +class CompletenessReport: + """What a run accounted for, and whether that permits an accuracy number. + + ``resolved_rate`` is published only when the run is *structurally complete* + -- every planned instance id reached a terminal state exactly once -- **and** + zero instances were lost to infrastructure. Those are two different + questions and conflating them gets both wrong: + + * An instance the model attempted and failed is a legitimate score. An + instance our own harness dropped is not: it never had the chance. + * A run that is short of instances has the wrong denominator; a run that is + complete but leaned on the infrastructure has the wrong provenance. + + Withholding the headline is the point. A run that lost 106 of 200 instances + to infrastructure and printed ``resolved / planned`` reported 47.0% as + though it were accuracy, and that number was then compared against a + complete-run reference of 70.67% and read as a model regression. It was + attrition. + + Two numbers are therefore *always* published, refusal or not: + + ``conditional_resolved_rate`` + Resolved over the instances that actually completed. Honest about what + it measures, and not comparable to a complete-run reference. + ``resolved_rate_lower_bound`` + Resolved over everything planned. Infrastructure losses can only ever + *add* resolutions, so this bounds the true rate from below even on a + badly degraded run. + """ + + run_id: str + plan_digest: str + total_instances: int + accounted_instance_ids: tuple[str, ...] = () + resolved_instance_ids: tuple[str, ...] = () + incomplete_instance_ids: tuple[str, ...] = () + infra_lost_instances: int = 0 + infra_lost_unit_ids: tuple[str, ...] = () + unit_count: int = 0 + reasons: list[str] = field(default_factory=list) + + @property + def accounted_instances(self) -> int: + return len(self.accounted_instance_ids) + + @property + def resolved_instances(self) -> int: + return len(self.resolved_instance_ids) + + @property + def complete(self) -> bool: + """Every planned instance id reached a terminal state exactly once.""" + return not self.incomplete_instance_ids + + @property + def publishable(self) -> bool: + return self.complete and self.infra_lost_instances == 0 and not self.reasons + + @property + def conditional_resolved_rate(self) -> float | None: + if not self.accounted_instances: + return None + return self.resolved_instances / self.accounted_instances + + @property + def resolved_rate_lower_bound(self) -> float | None: + if not self.total_instances: + return None + return self.resolved_instances / self.total_instances + + @property + def resolved_rate(self) -> float | None: + """The headline number, or ``None`` when it must be withheld.""" + if not self.publishable: + return None + return self.resolved_rate_lower_bound + + @property + def withheld_reason(self) -> str | None: + if self.publishable: + return None + why: list[str] = [] + if self.incomplete_instance_ids: + why.append( + f"{len(self.incomplete_instance_ids)} of {self.total_instances} " + "instances never reached a terminal state" + ) + if self.infra_lost_instances: + why.append( + f"{self.infra_lost_instances} instance(s) were lost to " + "infrastructure, not to the model" + ) + if self.reasons and not why: + why.append("; ".join(self.reasons[:5])) + conditional = self.conditional_resolved_rate + lower = self.resolved_rate_lower_bound + return ( + "NO PUBLISHABLE ACCURACY: " + + "; ".join(why) + + ". Lower bound " + + ("n/a" if lower is None else f"{lower:.4f}") + + " (infrastructure losses can only add resolutions); conditional " + + ("n/a" if conditional is None else f"{conditional:.4f}") + + f" over the {self.accounted_instances} instance(s) that completed. " + "Neither is comparable to a complete-run reference." + ) + + def to_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "plan_digest": self.plan_digest, + "complete": self.complete, + "total_instances": self.total_instances, + "accounted_instances": self.accounted_instances, + "resolved_instances": self.resolved_instances, + "unit_count": self.unit_count, + "incomplete_instance_ids": list(self.incomplete_instance_ids), + "infra_lost_instances": self.infra_lost_instances, + "infra_lost_unit_ids": list(self.infra_lost_unit_ids), + "resolved_rate": self.resolved_rate, + "conditional_resolved_rate": self.conditional_resolved_rate, + "resolved_rate_lower_bound": self.resolved_rate_lower_bound, + "resolved_rate_withheld_reason": self.withheld_reason, + "reasons": list(self.reasons), + } + + @dataclass(slots=True) class MergeResult: run_id: str @@ -46,6 +186,9 @@ class MergeResult: total_instances: int resolved_instances: int unit_count: int + #: The accounting this result was published from. Present on the success + #: path too, so a caller never has to decide which of two shapes it holds. + report: CompletenessReport | None = None @property def resolved_rate(self) -> float: @@ -59,6 +202,7 @@ def to_dict(self) -> dict[str, Any]: "resolved_instances": self.resolved_instances, "resolved_rate": self.resolved_rate, "unit_count": self.unit_count, + **(self.report.to_dict() if self.report is not None else {}), } @@ -109,11 +253,13 @@ def verify_inventory(queue: WorkQueue) -> InventoryReport: return report -def merge_run(queue: WorkQueue, run_id: str) -> MergeResult: - """Score one run, or refuse. +def assess_run(queue: WorkQueue, run_id: str) -> CompletenessReport: + """Account for every planned instance id. Never raises on a bad run. - ``run_id`` is required and must match the queue's plan. Passing another - run's id is an error, not a filter. + This is the whole of the gate's arithmetic, separated from the decision to + refuse, so that a refused run still yields the conditional rate, the lower + bound and the ids that went missing. :func:`merge_run` is the strict + all-or-nothing wrapper over it. """ plan: UnitPlan = queue.plan if run_id != plan.run_id: @@ -150,6 +296,8 @@ def merge_run(queue: WorkQueue, run_id: str) -> MergeResult: results: dict[str, UnitResult] = queue.results() seen_ids: dict[str, str] = {} resolved: set[str] = set() + infra_lost = 0 + infra_lost_units: set[str] = set() for unit in plan.units: result = results.get(unit.unit_id) @@ -163,15 +311,21 @@ def merge_run(queue: WorkQueue, run_id: str) -> MergeResult: continue if result.abandoned: reasons.append(f"{unit.unit_id}: abandoned after {result.attempt} attempts") + infra_lost += len(unit.instance_ids) + infra_lost_units.add(unit.unit_id) continue if result.outcome is not UnitOutcome.SUCCEEDED: reasons.append(f"{unit.unit_id}: outcome {result.outcome.value}") continue if result.infra_error_count > 0: + # Recorded, not merely refused: how many instances the harness lost + # is the difference between an accuracy and an attrition figure. reasons.append( f"{unit.unit_id}: {result.infra_error_count} instance(s) lost to " "infrastructure" ) + infra_lost += result.infra_error_count + infra_lost_units.add(unit.unit_id) continue expected = set(unit.instance_ids) @@ -211,20 +365,43 @@ def merge_run(queue: WorkQueue, run_id: str) -> MergeResult: resolved.update(result.resolved_instance_ids) planned_ids = set(plan.instance_ids) - if not reasons and set(seen_ids) != planned_ids: - unaccounted = sorted(planned_ids - set(seen_ids)) + unaccounted = sorted(planned_ids - set(seen_ids)) + if unaccounted: reasons.append( f"{len(unaccounted)} planned instance(s) unaccounted for: " + ", ".join(unaccounted[:5]) ) - if reasons: - raise MergeRefusal(run_id, reasons) - - return MergeResult( + return CompletenessReport( run_id=run_id, plan_digest=plan.digest, total_instances=len(planned_ids), - resolved_instances=len(resolved), + accounted_instance_ids=tuple(sorted(seen_ids)), + resolved_instance_ids=tuple(sorted(resolved & planned_ids)), + incomplete_instance_ids=tuple(sorted(planned_ids - set(seen_ids))), + infra_lost_instances=infra_lost, + infra_lost_unit_ids=tuple(sorted(infra_lost_units)), unit_count=len(plan.units), + reasons=reasons, + ) + + +def merge_run(queue: WorkQueue, run_id: str) -> MergeResult: + """Score one run, or refuse. + + ``run_id`` is required and must match the queue's plan. Passing another + run's id is an error, not a filter. + """ + report = assess_run(queue, run_id) + if not report.publishable: + raise MergeRefusal( + run_id, report.reasons or [report.withheld_reason or ""], report + ) + return MergeResult( + run_id=run_id, + plan_digest=report.plan_digest, + total_instances=report.total_instances, + resolved_instances=report.resolved_instances, + unit_count=report.unit_count, + report=report, ) diff --git a/tests/unit/evaluation/swe_bench_distributed/test_merge.py b/tests/unit/evaluation/swe_bench_distributed/test_merge.py index 528ab9335..900833797 100644 --- a/tests/unit/evaluation/swe_bench_distributed/test_merge.py +++ b/tests/unit/evaluation/swe_bench_distributed/test_merge.py @@ -11,6 +11,7 @@ from inference_endpoint.evaluation.swe_bench_distributed.merge import ( MergeRefusal, + assess_run, merge_run, verify_inventory, ) @@ -187,3 +188,137 @@ def test_an_ownerless_claim_is_an_inventory_error(self, queue): def test_claims_without_results_are_reported(self, queue): queue.claim("run-a.s00") assert verify_inventory(queue).claims_without_results == ["run-a.s00"] + + +class TestCompletenessGate: + """`resolved_rate` is published only for a complete, uncontaminated run. + + A run that lost 106 of its 200 instances to infrastructure once printed + 47.0% as though it were accuracy, and that was then compared against a + complete-run reference of 70.67% and read as a model regression. It was + attrition. The gate refuses the headline; the conditional rate and the + lower bound are published either way so nobody has to recompute them by + hand from the artifacts. + """ + + def test_a_complete_run_publishes_the_headline(self, queue): + publish_all(queue) + + report = assess_run(queue, "run-a") + + assert report.complete + assert report.publishable + assert report.resolved_rate == pytest.approx(0.3) + assert report.withheld_reason is None + + def test_an_incomplete_run_withholds_the_headline(self, queue): + publish(queue, "run-a.s00") + + report = assess_run(queue, "run-a") + + assert not report.complete + assert report.resolved_rate is None + assert "NO PUBLISHABLE ACCURACY" in report.withheld_reason + + def test_an_incomplete_run_names_the_instances_it_lost(self, queue): + publish(queue, "run-a.s00") + + report = assess_run(queue, "run-a") + + assert list(report.incomplete_instance_ids) == sorted(IDS[10:]) + assert report.accounted_instances == 10 + + def test_the_conditional_rate_is_published_even_when_refused(self, queue): + publish(queue, "run-a.s00") + + report = assess_run(queue, "run-a") + + # 3 resolved of the 10 that actually ran. + assert report.conditional_resolved_rate == pytest.approx(0.3) + # 3 resolved of the 20 that were planned: infrastructure losses can + # only ever add resolutions, so this bounds the truth from below. + assert report.resolved_rate_lower_bound == pytest.approx(0.15) + + def test_infra_loss_withholds_the_headline_on_a_structurally_complete_run( + self, queue + ): + """Complete is not enough. The losses have to be the model's.""" + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", infra_error_count=2) + + report = assess_run(queue, "run-a") + + assert report.infra_lost_instances == 2 + assert list(report.infra_lost_unit_ids) == ["run-a.s01"] + assert report.resolved_rate is None + assert "lost to" in report.withheld_reason + + def test_a_genuinely_empty_result_is_not_infra_loss(self, queue): + """A model that resolved nothing is a score, not a casualty.""" + publish(queue, "run-a.s00", resolved_instance_ids=()) + publish(queue, "run-a.s01", resolved_instance_ids=()) + + report = assess_run(queue, "run-a") + + assert report.complete + assert report.infra_lost_instances == 0 + assert report.publishable + assert report.resolved_rate == pytest.approx(0.0) + + def test_an_abandoned_unit_is_counted_as_infrastructure_loss(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", abandoned=True, attempt=3) + + report = assess_run(queue, "run-a") + + assert report.infra_lost_instances == 10 + assert report.resolved_rate is None + + def test_a_refusal_still_carries_the_report(self, queue): + publish(queue, "run-a.s00") + + with pytest.raises(MergeRefusal) as excinfo: + merge_run(queue, "run-a") + + report = excinfo.value.report + assert report is not None + assert report.resolved_rate is None + assert report.conditional_resolved_rate == pytest.approx(0.3) + assert report.incomplete_instance_ids + + def test_a_published_result_carries_the_report_too(self, queue): + publish_all(queue) + + result = merge_run(queue, "run-a") + + assert result.report is not None + assert result.report.publishable + assert result.to_dict()["resolved_rate_withheld_reason"] is None + + def test_the_serialized_report_names_every_published_field(self, queue): + publish(queue, "run-a.s00") + + payload = assess_run(queue, "run-a").to_dict() + + assert payload["resolved_rate"] is None + assert payload["conditional_resolved_rate"] == pytest.approx(0.3) + assert payload["resolved_rate_lower_bound"] == pytest.approx(0.15) + assert payload["incomplete_instance_ids"] + assert payload["resolved_rate_withheld_reason"] + assert payload["complete"] is False + + def test_a_run_with_nothing_published_has_no_rate_at_all(self, queue): + report = assess_run(queue, "run-a") + + assert report.conditional_resolved_rate is None + assert report.resolved_rate is None + assert report.resolved_rate_lower_bound == pytest.approx(0.0) + + def test_assess_run_does_not_raise_on_a_broken_run(self, queue): + """The arithmetic must survive the run it is describing.""" + publish(queue, "run-a.s00", accounted_instance_ids=(IDS[0], IDS[0])) + + report = assess_run(queue, "run-a") + + assert not report.publishable + assert report.reasons From 7f97f3ed2c63d1b92a9fe3c7b1a91dc555fe1d15 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 19 Aug 2026 00:49:53 -0700 Subject: [PATCH 07/25] fix(benchmark): an accuracy run that produces no number must fail An accuracy run could complete every unit of work, exit 0, and report `N/A`. `finalize_benchmark()` scored, wrote `accuracy_results.json` with `score: null`, printed the summary, and returned -- so `main.py` exited 0 and every wrapper downstream read the run as a pass. This is not hypothetical. A distributed SWE-bench run drove all 20 of its units to terminal records; 17 were abandoned to infrastructure failures, the merge gate correctly refused, `score()` returned None -- and the sbatch wrapper wrote `disposition=run completed (driver rc=0, results=20/20)` over a run with no accuracy number at all. rc=0, all work "done", no number is the failure shape that costs whole GPU allocations, so make it loud where it originates instead of asking each caller to notice. `_require_accuracy_numbers()` runs last, after every artifact is on disk, so the failure never costs the evidence needed to diagnose it. A real number flagged `complete=False` (a partial headline) still passes: the number exists and the entry already says it is partial. A PERF-mode run owes nothing for an externally-scored dataset it never dispatched. Also count what was evaluated, not what was loaded, in the accuracy-only summary line: SWE-bench Verified loads all 500 rows and scores `num_instances` of them, so the run above printed "500 samples evaluated" directly beneath its own "unit=200" headline. --- .../commands/benchmark/execute.py | 66 ++++++- tests/unit/commands/test_benchmark.py | 162 ++++++++++++++++++ 2 files changed, 223 insertions(+), 5 deletions(-) diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index a6ad406a5..c1894d2b2 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -1128,11 +1128,20 @@ def _summarize_and_log_metrics( logger.info(f"Completed in {perf_elapsed:.1f}s") if ctx.accuracy_only: - acc_total = sum( - ds.dataset.num_samples() * ds.num_repeats - for ds in ctx.eval_configs - if ds.dataset_type == DatasetType.ACCURACY - ) + # Count what was actually evaluated, not what was loaded. An external + # scorer (SKIP_ENDPOINT_PHASE) evaluates its own clamped subset -- + # SWE-bench Verified loads all 500 rows but scores `num_instances` of + # them -- and reporting the loaded size instead contradicts the + # per-dataset `unit=` line in the same summary. + acc_total = 0 + for ec in ctx.eval_configs: + if ec.dataset_type != DatasetType.ACCURACY: + continue + external = effective_external_sample_count(ec) + if external is not None: + acc_total += external + else: + acc_total += ec.dataset.num_samples() * ec.num_repeats logger.info(f"Accuracy-only: {acc_total} samples evaluated") else: logger.info( @@ -1208,6 +1217,53 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: # after the report artifacts so a write failure here can't discard them. write_accuracy_results(ctx.report_dir, accuracy_scores) + # Every artifact is on disk; only now may the run be declared a failure. + _require_accuracy_numbers(ctx, accuracy_scores) + + +def _require_accuracy_numbers( + ctx: BenchmarkContext, accuracy_scores: list[dict[str, Any]] +) -> None: + """Fail a run that was asked for accuracy and produced no number. + + A scorer can complete every unit of work, exit cleanly, and still hand back + ``score=None`` -- an externally-scored run (``SKIP_ENDPOINT_PHASE``) whose + merge gate refused, a scorer whose responses never arrived. Without this + check the process exits 0, ``report.txt`` prints ``N/A``, and every wrapper + downstream (sbatch disposition, CI gate, dashboard) reads the run as a pass. + That failure shape -- rc=0, all work "done", no number -- is the one that + costs whole GPU allocations, so it is made loud here rather than left to + each caller to notice. + + Scored-but-partial (a real number with ``complete=False``) is not failed: + the number exists and the entry already says it is partial. + """ + # A PERF-mode run skips externally-scored datasets entirely (they are never + # dispatched), so they are not owed a number here. + expected = { + ec.dataset_name + for ec in ctx.eval_configs + if ec.dataset_type == DatasetType.ACCURACY + and not (ctx.test_mode == TestMode.PERF and ec.scorer.SKIP_ENDPOINT_PHASE) + } + if not expected: + return + + scored = { + entry["dataset_name"] + for entry in accuracy_scores + if entry.get("dataset_type") == DatasetType.ACCURACY.value + and entry.get("score") is not None + } + missing = sorted(expected - scored) + if missing: + raise ExecutionError( + "accuracy scoring produced no score for: " + + ", ".join(missing) + + f". Artifacts were preserved in {ctx.report_dir}; see " + "accuracy/accuracy_results.json for the per-dataset detail." + ) + def run_benchmark( config: BenchmarkConfig, diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index a5ad0e6a4..82f78052a 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -155,6 +155,45 @@ def score_single_sample(self, value, ground_truth): return 0.0 +class _ScorelessExternalScorer(Scorer, scorer_id="_test_scoreless_external"): + """Does all its work, exits cleanly, and returns no number. + + Models the real shape of the SWE-bench fleet scorer whose merge gate + refuses: every unit reached a terminal record, nothing raised, and the + headline is ``None``. + """ + + SKIP_ENDPOINT_PHASE = True + + @classmethod + def external_sample_count(cls, extras): + return 2 + + def score_single_sample(self, value, ground_truth): + return 0.0 + + def score(self): + self.complete = False + return None, 1 + + +class _PartialButScoredScorer(Scorer, scorer_id="_test_partial_but_scored"): + """A real number flagged partial — reported, not failed.""" + + SKIP_ENDPOINT_PHASE = True + + @classmethod + def external_sample_count(cls, extras): + return 2 + + def score_single_sample(self, value, ground_truth): + return 0.0 + + def score(self): + self.complete = False + return 0.25, 1 + + class _OrdinaryAccuracyScorer(Scorer, scorer_id="_test_ordinary_accuracy"): def score_single_sample(self, value, ground_truth): return 0.0 @@ -2563,6 +2602,129 @@ def _make_report(state: str) -> Report: ) +class TestAccuracyRunWithoutANumberFails: + """rc=0, all work done, no number is a FAILURE. + + This is the regression that cost a whole 200-instance GPU run: the + distributed SWE-bench scorer drove all 20 units to terminal records, the + merge gate refused (17 abandoned), ``score()`` returned ``None``, + ``report.txt`` printed ``N/A`` — and the process exited 0, so the sbatch + wrapper wrote ``disposition=run completed``. Any scorer path that finishes + its units but produces no accuracy must fail loudly instead. + """ + + def _eval_config(self, scorer, dataset, report_dir: Path, name: str): + return AccuracyConfiguration( + scorer=scorer, + extractor=None, + dataset_name=name, + dataset=dataset, + report_dir=report_dir, + ground_truth_column=None, + num_repeats=1, + dataset_type=DatasetType.ACCURACY, + ) + + def _ctx(self, tmp_path, scorer, name="external_accuracy"): + dataset = _make_loaded_dataset() + return _make_benchmark_context( + config=OfflineConfig(**_OFFLINE_KWARGS), + report_dir=tmp_path, + test_mode=TestMode.ACC, + dataloader=dataset, + eval_configs=[self._eval_config(scorer, dataset, tmp_path, name)], + ) + + @pytest.mark.unit + def test_scoreless_accuracy_run_raises(self, tmp_path): + ctx = self._ctx(tmp_path, _ScorelessExternalScorer) + + with pytest.raises(ExecutionError, match="produced no score"): + finalize_benchmark(ctx, _make_benchmark_result(tmp_path)) + + @pytest.mark.unit + def test_artifacts_survive_the_failure(self, tmp_path): + """The failure must not cost the evidence needed to diagnose it.""" + ctx = self._ctx(tmp_path, _ScorelessExternalScorer) + + with pytest.raises(ExecutionError): + finalize_benchmark(ctx, _make_benchmark_result(tmp_path)) + + results = json.loads( + (tmp_path / "accuracy" / "accuracy_results.json").read_text() + ) + entry = results["accuracy_scores"][0] + assert entry["dataset_name"] == "external_accuracy" + assert entry["score"] is None + assert entry["complete"] is False + + @pytest.mark.unit + def test_one_scored_dataset_does_not_excuse_a_scoreless_peer(self, tmp_path): + """Averaging over datasets must not hide a dataset that scored nothing.""" + dataset = _make_loaded_dataset() + ctx = _make_benchmark_context( + config=OfflineConfig(**_OFFLINE_KWARGS), + report_dir=tmp_path, + test_mode=TestMode.ACC, + dataloader=dataset, + eval_configs=[ + self._eval_config(_SelfContainedScorer, dataset, tmp_path, "good"), + self._eval_config(_ScorelessExternalScorer, dataset, tmp_path, "bad"), + ], + ) + + with pytest.raises(ExecutionError, match="bad"): + finalize_benchmark(ctx, _make_benchmark_result(tmp_path)) + + @pytest.mark.unit + def test_partial_but_numeric_score_still_passes(self, tmp_path): + """A real number flagged incomplete is reportable, not a failure.""" + ctx = self._ctx(tmp_path, _PartialButScoredScorer) + + finalize_benchmark(ctx, _make_benchmark_result(tmp_path)) + + results = json.loads( + (tmp_path / "accuracy" / "accuracy_results.json").read_text() + ) + assert results["accuracy_scores"][0]["score"] == 0.25 + assert results["accuracy_scores"][0]["complete"] is False + + @pytest.mark.unit + def test_perf_only_run_owes_no_accuracy_number(self, tmp_path): + """PERF mode never dispatches an external scorer, so it owes nothing.""" + dataset = _make_loaded_dataset() + ctx = _make_benchmark_context( + config=OfflineConfig(**_OFFLINE_KWARGS), + report_dir=tmp_path, + test_mode=TestMode.PERF, + dataloader=dataset, + eval_configs=[ + self._eval_config( + _ScorelessExternalScorer, dataset, tmp_path, "external_accuracy" + ) + ], + ) + + finalize_benchmark(ctx, _make_benchmark_result(tmp_path)) + + @pytest.mark.unit + def test_external_scorer_sample_count_is_the_evaluated_count( + self, tmp_path, caplog + ): + """'N samples evaluated' must match the dataset's own unit= line. + + SWE-bench Verified loads 500 rows and scores ``num_instances`` of them; + reporting 500 against a ``unit=200`` headline is how a wrong scope goes + unnoticed. + """ + ctx = self._ctx(tmp_path, _PartialButScoredScorer) + + with caplog.at_level(logging.INFO, logger=execute_mod.logger.name): + finalize_benchmark(ctx, _make_benchmark_result(tmp_path)) + + assert "Accuracy-only: 2 samples evaluated" in caplog.text + + class TestScorerMethodSync: """Ensure ScorerMethod enum stays in sync with the scorer registry.""" From 42689ad310fa3efc0a182cf2ff9829b0243582b1 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 19 Aug 2026 00:52:16 -0700 Subject: [PATCH 08/25] fix(swebench-service): give Pyxis container creation its own deadline Under Pyxis, creating the container is its own piece of infrastructure work: `--container-image` makes enroot import a multi-GB SWE-bench image and slurmstepd launch a step for it. That was charged against `environment.timeout` -- the per-*command* budget, 300s in both templates, sized for `pytest`-scale work inside an already-running container -- because `PyxisSweBenchRunner._configure_environment` dropped the template's `pull_timeout: 3600` as a docker-only key and `PyxisEnvironment.__init__` had nothing else to use. A create budget must be separate from a per-command budget because the two scale with completely different things. A command's cost depends on the task; a create's cost depends on how much other work is contending for the node. Measured on an idle node, one create is ~35s and eight concurrent creates finish in 55s wall -- so 300s looks generous right up until it isn't. In the run that exposed this, four SWE-bench services drove 40 concurrent agents across 5,148 srun steps in 78 minutes, every step requesting all 144 CPUs, on a node also running four vLLM engines. Creation slowed by an order of magnitude, `subprocess.run(timeout=timeout_s + 30)` SIGKILLed the step, and 96 steps died at a uniform 5m47s-5m56s -- 330s plus step-accounting skew, against 3-11s for every ordinary command step. 17 of 20 units were lost and the run produced no accuracy number at all. The registry was never the bottleneck; step contention was. Carry `pull_timeout` through to a distinct `create_timeout_s` (default 3600) and use it for the create step only. Command steps keep `timeout`. Also stop discarding srun's own output. Both infrastructure-failure paths in `run_srun_step()` raised a fixed string and threw away the captured stream, so an import failure, an out-of-space enroot, and a step that never got resources were one indistinguishable message -- the 17 lost units above could not be told apart from their artifacts. The failure now carries srun's last 2000 characters, names the deadline it blew, and reports srun's exit code. Finally, make creation measurable while it happens rather than only afterwards in `sacct`: with `SWEBENCH_PYXIS_CREATE_TIMING_PATH` set, each create appends one JSONL record with its duration and outcome. Off by default, and a sink that cannot be written degrades to nothing -- a create that succeeded and could not be logged is still a create that succeeded. --- .../swebench_service/pyxis_environment.py | 89 ++++++++- .../swebench_service/runner.py | 8 +- .../swebench_service/test_runner.py | 172 +++++++++++++++++- 3 files changed, 262 insertions(+), 7 deletions(-) diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py index ddef3a7db..6795c82e8 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import logging import os import platform @@ -10,6 +11,7 @@ import subprocess import tempfile import threading +import time import uuid from pathlib import Path from typing import Any @@ -160,16 +162,27 @@ def run_srun_step( timeout=timeout_s + 30, env=safe_srun_env(), ) + except subprocess.TimeoutExpired as exc: + if failure_path is not None: + failure_path.touch() + raise RunnerError( + f"Pyxis step exceeded its {timeout_s + 30}s deadline and was killed" + + _srun_evidence(exc.output) + ) from exc except (OSError, subprocess.SubprocessError) as exc: if failure_path is not None: failure_path.touch() raise RunnerError( - "Pyxis infrastructure failure before the command completed" + "Pyxis infrastructure failure before the command completed: " + f"{type(exc).__name__}: {exc}" ) from exc if status_path.read_text().strip() != f"finished:{result.returncode}": if failure_path is not None: failure_path.touch() - raise RunnerError("Pyxis infrastructure failure before the command completed") + raise RunnerError( + "Pyxis infrastructure failure before the command completed " + f"(srun exited {result.returncode})" + _srun_evidence(result.stdout) + ) return result @@ -184,6 +197,56 @@ def enroot_container_name(job_id: str, container_name: str) -> str: return f"pyxis_{job_id}_{container_name}" +#: Opt-in JSONL sink for container-create durations. Off unless set, so this +#: adds nothing to a normal run. Creation is the step whose cost was invisible +#: -- it was only ever observable as a uniform block of SIGKILLs in `sacct`, +#: after the run was already lost -- so measuring it has to be possible without +#: re-deriving it from step accounting. +_CREATE_TIMING_ENV = "SWEBENCH_PYXIS_CREATE_TIMING_PATH" + + +def _record_create_timing(image: str | Path, seconds: float, *, ok: bool) -> None: + path = os.environ.get(_CREATE_TIMING_ENV) + if not path: + return + record = { + "ts": time.time(), + "image": str(image), + "secs": round(seconds, 2), + "ok": ok, + "pid": os.getpid(), + } + try: + # One short line per create, O_APPEND from many concurrent workers. + with open(path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(record) + "\n") + except OSError: + # Observability must never be able to fail a run. A create that + # succeeded and could not be logged is still a create that succeeded. + logger.debug("could not record Pyxis create timing", exc_info=True) + + +def _srun_evidence(output: str | bytes | None, limit: int = 2000) -> str: + """Attach srun's own words to a Pyxis failure. + + srun/pyxis/enroot report the actual cause -- image import failure, no space + left, a step that never got resources -- on the stream this function + captures. Dropping it turns every distinct infrastructure failure into one + indistinguishable message, which is exactly what made a 200-instance run's + 17 lost units undiagnosable from its artifacts. + """ + if not output: + return "" + if isinstance(output, bytes): + output = output.decode("utf-8", errors="replace") + text = output.strip() + if not text: + return "" + if len(text) > limit: + text = "..." + text[-limit:] + return f"\n--- srun output ---\n{text}" + + def resolve_image(image_registry: str, instance_id: str) -> str: if Path(instance_id).name != instance_id or instance_id in {".", ".."}: raise RunnerError(f"invalid SWE-bench instance ID: {instance_id}") @@ -206,6 +269,19 @@ class PyxisEnvironmentConfig(BaseModel): validation_alias=AliasChoices("timeout_s", "timeout"), serialization_alias="timeout", ) + #: Deadline for *creating* the container, which under Pyxis includes the + #: enroot import of a multi-GB SWE-bench image from a remote registry. + #: Deliberately separate from ``timeout_s``: that is a per-*command* + #: budget, sized for `pytest`-scale work inside an already-running + #: container. Charging an image import against it made every agent whose + #: image was not already in the enroot cache fail once the registry was + #: shared by enough concurrent workers to push a single import past ~5 + #: minutes. Defaults to, and accepts, mini-swe-agent's ``pull_timeout``. + create_timeout_s: int = Field( + default=3600, + validation_alias=AliasChoices("create_timeout_s", "pull_timeout"), + serialization_alias="pull_timeout", + ) interpreter: list[str] = Field(default_factory=lambda: ["bash", "-c"]) infrastructure_failure_path: Path | None = None @@ -220,6 +296,7 @@ def __init__(self, **kwargs: Any): self._tmp_dir.chmod(0o1777) self._lock = threading.Lock() self._cleaned = False + started = time.monotonic() try: # A no-op initializes and validates the named persistent container. run_srun_step( @@ -229,14 +306,18 @@ def __init__(self, **kwargs: Any): workdir=self.config.cwd, argv=["true"], status_path=self._tmp_dir / Path(_STEP_STATUS).name, - timeout_s=self.config.timeout_s, + timeout_s=self.config.create_timeout_s, failure_path=self.config.infrastructure_failure_path, ) except RunnerError as exc: + _record_create_timing( + self.config.image, time.monotonic() - started, ok=False + ) self.cleanup() raise RunnerError( - f"failed to start Pyxis container for {self.config.image}" + f"failed to start Pyxis container for {self.config.image}: {exc}" ) from exc + _record_create_timing(self.config.image, time.monotonic() - started, ok=True) def execute( self, action: dict[str, Any], cwd: str = "", *, timeout: int | None = None diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 62f414a82..8682b40a7 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -647,7 +647,13 @@ def __init__( def _configure_environment( self, environment_cfg: dict[str, Any], run_id: str ) -> None: - for key in ("run_args", "pull_timeout", "container_timeout"): + # ``pull_timeout`` is the template's image-acquisition budget and is + # exactly what the Pyxis container-create step needs, so it is carried + # over rather than dropped. Without it the create step fell back to the + # per-command ``timeout`` (300s in both templates) and every image + # import slower than that was killed as an "infrastructure failure". + # ``run_args``/``container_timeout`` stay dropped: both are docker-only. + for key in ("run_args", "container_timeout"): environment_cfg.pop(key, None) environment_cfg["environment_class"] = ( "swebench_service.pyxis_environment.PyxisEnvironment" diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 3955aacfe..44b8cc9b6 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import json import logging import stat import subprocess @@ -714,8 +715,9 @@ def test_pyxis_patch_config_selects_pyxis_environment(tmp_path): assert environment["cwd"] == "/testbed" assert environment["run_id"] == "run-1" assert "run_args" not in environment - assert "pull_timeout" not in environment assert "container_timeout" not in environment + # Carried over, not dropped: the Pyxis create step *is* the image pull. + assert environment["pull_timeout"] == 3600 def test_pyxis_resolves_registry_image_from_instance_id(): @@ -1052,12 +1054,178 @@ def test_pyxis_environment_raises_when_srun_never_starts_command(monkeypatch, tm ), ) - with pytest.raises(RunnerError, match="before the command completed"): + with pytest.raises(RunnerError, match=r"exceeded its 60s deadline"): environment.execute({"command": "pytest -q"}) assert failure_path.exists() +def test_pyxis_container_create_uses_the_pull_budget_not_the_command_budget( + monkeypatch, tmp_path +): + """Creating the container is an image import, not a shell command. + + Under Pyxis, `--container-image` triggers an enroot import of a multi-GB + SWE-bench image from a remote registry. Charging that against the + per-command `timeout` (300s in both templates) killed the create step as + soon as enough concurrent workers shared the registry -- 96 srun steps of + one 200-instance run were SIGKILLed at a uniform ~5m50s (= 300 + 30 grace), + which the service reported as an undiagnosable "failed to start Pyxis + container" and cost 17 of 20 units. + """ + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + timeouts: list[float] = [] + + def fake_run(command, **kwargs): + timeouts.append(kwargs["timeout"]) + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment( + image=tmp_path / "task.sqsh", + run_id="run-1", + timeout=300, + pull_timeout=3600, + ) + environment.execute({"command": "pytest -q"}) + + create_timeout, command_timeout = timeouts[0], timeouts[1] + assert create_timeout == 3600 + 30 + assert command_timeout == 300 + 30 + assert create_timeout > command_timeout, ( + "container creation must not be bounded by the per-command timeout" + ) + environment.cleanup() + + +def test_pyxis_container_create_budget_defaults_without_a_template_value( + monkeypatch, tmp_path +): + """A template that never mentions pull_timeout still gets a pull budget.""" + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + timeouts: list[float] = [] + + def fake_run(command, **kwargs): + timeouts.append(kwargs["timeout"]) + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment( + image=tmp_path / "task.sqsh", run_id="run-1", timeout=300 + ) + + assert timeouts[0] == 3600 + 30 + environment.cleanup() + + +def test_pyxis_records_create_timing_when_enabled(monkeypatch, tmp_path): + """Container-create cost must be measurable without re-deriving it. + + Creation was only ever observable after the fact, as a uniform block of + SIGKILLed steps in `sacct` -- by which point the run was already lost. + """ + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + timing = tmp_path / "creates.jsonl" + monkeypatch.setenv("SWEBENCH_PYXIS_CREATE_TIMING_PATH", str(timing)) + + def fake_run(command, **kwargs): + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="run-1") + environment.cleanup() + + records = [json.loads(line) for line in timing.read_text().splitlines()] + assert len(records) == 1 + assert records[0]["ok"] is True + assert records[0]["secs"] >= 0 + assert records[0]["image"].endswith("task.sqsh") + + +def test_pyxis_records_create_timing_for_a_failed_create(monkeypatch, tmp_path): + """A create that failed is the one whose duration matters most.""" + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + timing = tmp_path / "creates.jsonl" + monkeypatch.setenv("SWEBENCH_PYXIS_CREATE_TIMING_PATH", str(timing)) + + monkeypatch.setattr( + subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess(command, 1, stdout=""), + ) + + with pytest.raises(RunnerError): + PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="run-1") + + records = [json.loads(line) for line in timing.read_text().splitlines()] + assert [r["ok"] for r in records] == [False] + + +def test_pyxis_create_timing_is_off_by_default(monkeypatch, tmp_path): + """No env var, no writes, no behaviour change on a normal run.""" + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + monkeypatch.delenv("SWEBENCH_PYXIS_CREATE_TIMING_PATH", raising=False) + + def fake_run(command, **kwargs): + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="run-1") + environment.cleanup() + + assert list(tmp_path.glob("*.jsonl")) == [] + + +def test_pyxis_create_timing_never_fails_the_run(monkeypatch, tmp_path): + """An unwritable sink degrades to nothing; it does not lose the container.""" + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + monkeypatch.setenv( + "SWEBENCH_PYXIS_CREATE_TIMING_PATH", str(tmp_path / "nope" / "creates.jsonl") + ) + + def fake_run(command, **kwargs): + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="run-1") + environment.cleanup() + + +def test_pyxis_failure_carries_srun_output(monkeypatch, tmp_path): + """srun's own words must survive into the error. + + Without them every distinct infrastructure failure -- import failure, no + space left, a step that never got resources -- collapses into one + indistinguishable message and cannot be diagnosed from the artifacts. + """ + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + + def fake_run(command, **kwargs): + # Step never wrote its status file: srun died before the command ran. + return subprocess.CompletedProcess( + command, 1, stdout="slurmstepd: error: pyxis: no space left\n", stderr="" + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(RunnerError, match="no space left") as exc_info: + PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="run-1") + + assert "failed to start Pyxis container" in str(exc_info.value) + + def test_pyxis_environment_preserves_command_failure(monkeypatch, tmp_path): monkeypatch.setenv("SLURM_JOB_ID", "1738605") monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") From 378b99568b957eabfd64af57124dd3b4a3be7433 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 13:11:07 -0700 Subject: [PATCH 09/25] fix(swebench-service): make a non-launching Pyxis step machine-readable Builds on "give Pyxis container creation its own deadline", which attached srun's own output to these failures and made them *readable*. They are still not *actionable*: nothing in the text distinguishes "the step never launched" from "the command ran and its report was lost", and only the first can be retried without risking double execution. Measured signature, from an isolated probe with no model and no GPU (20 nodes, 200 workers, 6273 ordinary shell steps): 63 steps failed, and in all 63 the status file still read `pending` -- not `started` -- with srun's output empty. The step script never ran its first line. The command provably did not execute, so re-running it cannot double-apply an edit, a removal or a test run. That is the entire safety argument, and it is only available if the status bytes are captured rather than compared and thrown away. Two changes: * `StepNotLaunched(RunnerError)` records `srun_rc`, the observed `status`, and `provable_non_execution` (`status == "pending"` and no sentinel). It subclasses `RunnerError`, so every existing `except RunnerError` is unaffected, and the status bytes now appear in the message too. * An in-band sentinel, `__MLPERF_STEP_RC__ `, becomes the primary result channel. It travels on srun's stdout, so a step reports its outcome without depending on a readable shared filesystem -- a real failure mode of its own on a distributed one -- and it is what makes "no sentinel" half of the provability test. The nonce makes it unforgeable by the command's own output; it is stripped before the output is returned. This deliberately stops at reporting. Nothing here retries. Tests: `test_step_failure_reports_whether_non_execution_is_provable` covers the three decision points (pending / started / finished), `test_step_reports_its_return_code_in_band`, `test_step_sentinel_cannot_be_forged_by_command_output`, `test_read_step_sentinel_ignores_unrelated_output` and `test_step_not_launched_is_a_runner_error`. Against the parent commit the raised error has no `provable_non_execution`, no `srun_rc`, no `status`, and does not quote the status bytes. --- .../evaluation/swebench_service/README.md | 16 +++ .../swebench_service/pyxis_environment.py | 101 +++++++++++++-- .../swebench_service/test_runner.py | 116 ++++++++++++++++++ 3 files changed, 221 insertions(+), 12 deletions(-) diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index f599fefda..9a11f316e 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -75,6 +75,22 @@ overlapping `srun` step in that container, preserving filesystem changes across turns. Tool commands run in private PID namespaces so one trajectory cannot signal processes belonging to another trajectory. +Each step reports its outcome through two channels. The primary one is in band: the +step script prints `__MLPERF_STEP_RC__ ` on `srun`'s stdout, which needs +no readable shared filesystem and is stripped from the command output before it is +returned. The fallback is the status file written into the container's `/tmp` mount. + +When a step reports through neither, `StepNotLaunched` (a `RunnerError`) is raised. +Besides `srun`'s own output it carries `srun_rc`, the observed `status` bytes, and +`provable_non_execution` -- true only when the status file is still `pending` and no +sentinel arrived, meaning the step script did not run even its first line and the +command definitely did not execute. Anything else leaves open that it did. Callers +deciding whether re-running is safe must use that flag rather than the message text. + +Cluster note: on a busy controller these failures cluster around slurmctld RPC rate +limiting (`Job credential expired`). Pacing step creation below the controller's +`rl_refill_rate` is a deployment concern rather than a property of this package. + After generation, the Pyxis worker evaluates each prediction in a fresh `srun` container step because the Docker-based SWE-bench evaluator cannot run on the compute node. It mounts the patch, SWE-bench evaluation script, and output file into diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py index 6795c82e8..12a6dc433 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py @@ -57,18 +57,75 @@ "ENROOT_CONFIG_PATH", ) _STEP_STATUS = "/tmp/.mlperf_srun_status" +#: In-band marker the step script prints alongside its own return code. It is +#: the primary result channel: it travels back on srun's stdout and so needs no +#: readable shared filesystem. The status file remains the fallback. +_STEP_SENTINEL = "__MLPERF_STEP_RC__" +#: The status file contents before the step script runs its very first line. +_STEP_STATUS_PENDING = "pending" _STEP_SCRIPT = r"""set +e status_path=$1 timeout_s=$2 -shift 2 -printf 'started\n' > "$status_path" +nonce=$3 +shift 3 +printf 'started\n' > "$status_path" 2>/dev/null unshare --pid --fork --mount-proc timeout "$timeout_s" "$@" returncode=$? -printf 'finished:%s\n' "$returncode" > "$status_path" +printf 'finished:%s\n' "$returncode" > "$status_path" 2>/dev/null +printf '\n__MLPERF_STEP_RC__ %s %s\n' "$nonce" "$returncode" exit "$returncode" """ +class StepNotLaunched(RunnerError): + """An `srun` step that reported through neither result channel. + + Subclasses :class:`RunnerError` so every existing ``except RunnerError`` + keeps working, and records the facts a caller needs to reason about the + failure rather than only read about it: + + ``srun_rc`` + `srun`'s own exit status. + ``status`` + The bytes actually observed in the step status file. + ``provable_non_execution`` + True only when the status file was still ``pending`` and no in-band + sentinel arrived -- the step script did not run even its first line, so + the command definitely did not execute. Anything else leaves open that + it did, which is the distinction anyone deciding whether a re-run is + safe has to make. + """ + + def __init__( + self, + message: str, + *, + provable_non_execution: bool, + srun_rc: int | None, + status: str, + ) -> None: + super().__init__(message) + self.provable_non_execution = provable_non_execution + self.srun_rc = srun_rc + self.status = status + + +def read_step_sentinel(text: str, nonce: str) -> tuple[int | None, str]: + """Return ``(returncode, output_without_the_sentinel)`` if the step reported. + + ``(None, text)`` when the step did not report in band. The nonce makes the + marker unforgeable by the command's own output. + """ + tag = f"{_STEP_SENTINEL} {nonce} " + for line in reversed((text or "").splitlines()): + if not line.startswith(tag): + continue + value = line[len(tag) :].strip() + if value.lstrip("-").isdigit(): + return int(value), text[: text.rindex(line)].rstrip("\n") + return None, text + + def safe_srun_env() -> dict[str, str]: return {name: os.environ[name] for name in _SAFE_SRUN_ENV if name in os.environ} @@ -134,7 +191,8 @@ def run_srun_step( workdir: str | None = None, stderr: int = subprocess.STDOUT, ) -> subprocess.CompletedProcess[str]: - status_path.write_text("pending\n") + nonce = uuid.uuid4().hex + status_path.write_text(f"{_STEP_STATUS_PENDING}\n") status_path.chmod(0o666) command = build_srun_command( image=image, @@ -148,6 +206,7 @@ def run_srun_step( "pyxis-step", _STEP_STATUS, str(timeout_s), + nonce, *argv, ], ) @@ -176,14 +235,32 @@ def run_srun_step( "Pyxis infrastructure failure before the command completed: " f"{type(exc).__name__}: {exc}" ) from exc - if status_path.read_text().strip() != f"finished:{result.returncode}": - if failure_path is not None: - failure_path.touch() - raise RunnerError( - "Pyxis infrastructure failure before the command completed " - f"(srun exited {result.returncode})" + _srun_evidence(result.stdout) - ) - return result + + # Primary channel: the step reported its own return code in band. + reported, cleaned = read_step_sentinel(result.stdout, nonce) + if reported is not None: + result.stdout = cleaned + result.returncode = reported + return result + + # Fallback channel: the status file the step script wrote into the mount. + try: + status = status_path.read_text().strip() + except OSError as exc: + status = f"" + if status == f"finished:{result.returncode}": + return result + + if failure_path is not None: + failure_path.touch() + raise StepNotLaunched( + "Pyxis infrastructure failure before the command completed " + f"(srun exited {result.returncode}, status={status!r})" + + _srun_evidence(result.stdout), + provable_non_execution=status == _STEP_STATUS_PENDING, + srun_rc=result.returncode, + status=status, + ) def enroot_container_name(job_id: str, container_name: str) -> str: diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 44b8cc9b6..c04549cce 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -22,8 +22,10 @@ ) from inference_endpoint.evaluation.swebench_service.swebench_service.pyxis_environment import ( PyxisEnvironment, + StepNotLaunched, build_srun_command, enroot_container_name, + read_step_sentinel, resolve_image, safe_srun_env, ) @@ -1226,6 +1228,120 @@ def fake_run(command, **kwargs): assert "failed to start Pyxis container" in str(exc_info.value) +def _bare_environment(tmp_path, failure_path=None): + environment = object.__new__(PyxisEnvironment) + environment.config = types.SimpleNamespace( + cwd="/testbed", + env={}, + timeout_s=30, + interpreter=["bash", "-c"], + infrastructure_failure_path=failure_path, + ) + environment.name = "mswe_run-1_abcd1234" + environment._tmp_dir = tmp_path + return environment + + +@pytest.mark.parametrize( + ("status", "provable"), + [ + # The step script never ran its first line: the command provably did + # not execute, so re-running it cannot double-apply anything. + ("pending\n", True), + # The step script started; the command may well have executed. + ("started\n", False), + # A report for some other return code: the command ran. + ("finished:0\n", False), + ], +) +def test_step_failure_reports_whether_non_execution_is_provable( + monkeypatch, tmp_path, status, provable +): + """srun's text says *what* broke; this says whether a re-run is safe. + + Attaching srun's output made these failures readable. It does not make them + machine-actionable: nothing in the text distinguishes "the step never + launched" from "the command ran and its report was lost", and only the first + can be retried without risking double execution. + """ + environment = _bare_environment(tmp_path) + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + + def fake_run(command, **kwargs): + (tmp_path / Path("/tmp/.mlperf_srun_status").name).write_text(status) + return subprocess.CompletedProcess(command, 7, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(StepNotLaunched) as exc_info: + environment.execute({"command": "pytest -q"}) + + failure = exc_info.value + assert failure.provable_non_execution is provable + assert failure.status == status.strip() + assert failure.srun_rc == 7 + assert repr(status.strip()) in str(failure) + + +def test_step_not_launched_is_a_runner_error(): + """Existing ``except RunnerError`` handlers must keep working unchanged.""" + assert issubclass(StepNotLaunched, RunnerError) + + +def test_step_reports_its_return_code_in_band(monkeypatch, tmp_path): + """The sentinel is authoritative and is stripped from the output. + + It removes the shared-filesystem dependency from the success path: a step + can report its result even where the status file is unreadable, which on a + distributed filesystem is a real failure mode of its own. + """ + environment = _bare_environment(tmp_path) + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + + def fake_run(command, **kwargs): + nonce = command[command.index("pyxis-step") + 3] + return subprocess.CompletedProcess( + command, + 0, + stdout=f"real output\n\n__MLPERF_STEP_RC__ {nonce} 3\n", + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + output = environment.execute({"command": "false"}) + + assert output["returncode"] == 3 + assert output["output"] == "real output" + + +def test_step_sentinel_cannot_be_forged_by_command_output(monkeypatch, tmp_path): + environment = _bare_environment(tmp_path) + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + + def fake_run(command, **kwargs): + return subprocess.CompletedProcess( + command, 1, stdout="__MLPERF_STEP_RC__ deadbeef 0\n", stderr="" + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(StepNotLaunched): + environment.execute({"command": "echo spoof"}) + + +def test_read_step_sentinel_ignores_unrelated_output(): + assert read_step_sentinel("no marker here\n", "abc") == (None, "no marker here\n") + assert read_step_sentinel("out\n__MLPERF_STEP_RC__ abc x\n", "abc") == ( + None, + "out\n__MLPERF_STEP_RC__ abc x\n", + ) + assert read_step_sentinel("out\n__MLPERF_STEP_RC__ abc -1\n", "abc") == (-1, "out") + + def test_pyxis_environment_preserves_command_failure(monkeypatch, tmp_path): monkeypatch.setenv("SLURM_JOB_ID", "1738605") monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") From 57df1dfcea099b1371c746542615cfe453fb8f7c Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:40:25 -0700 Subject: [PATCH 10/25] feat(swe-bench): conservative claim reaper + PID-only memory guard reaper.py releases a stale claim only when it has no result, its heartbeat is past stale_after, AND its owner is provably gone. Liveness is a pluggable protocol: LocalProcessLiveness pairs pid with boot id so a recycled pid on a rebooted host is not read as a live owner, and SlurmStepLiveness treats a step missing from scontrol inside a live job as dead, because the job-level rule alone deadlocks the queue forever. An indeterminate probe releases NOTHING - a false reap creates two owners, duplicate results and a wrong denominator. guards.py kills a runaway graded test only under a full conjunction (RSS over threshold AND cwd inside the testbed AND a container-supervisor ancestor). Kills are by PID and refuse self and any ancestor of self; there is no pattern-kill path in the module at all, and a test greps the source to keep it that way. Each term reports its evidence count, and HealthVerdict.combine returns INDETERMINATE rather than UNHEALTHY when a term has zero evidence, so a conjunctive guard cannot collapse into its weakest clause. --- .../swe_bench_distributed/__init__.py | 11 + .../swe_bench_distributed/guards.py | 314 ++++++++++++++++++ .../swe_bench_distributed/reaper.py | 242 ++++++++++++++ .../test_guards_and_reaper.py | 295 ++++++++++++++++ .../swe_bench_distributed/test_liveness.py | 163 +++++++++ 5 files changed, 1025 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/guards.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_liveness.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index c97b98f55..a7b6f488c 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,6 +11,7 @@ exactly once. """ +from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .merge import ( CompletenessReport, MergeRefusal, @@ -25,20 +26,30 @@ UnitResult, WorkQueue, ) +from .reaper import LocalProcessLiveness, OwnerLiveness, SlurmStepLiveness, reap from .units import Unit, UnitPlan, plan_units __all__ = [ "ClaimError", "CompletenessReport", + "HealthTerm", + "HealthVerdict", + "LocalProcessLiveness", + "MemoryGuard", "MergeRefusal", "MergeResult", + "OwnerLiveness", + "SlurmStepLiveness", "Unit", "UnitOutcome", "UnitPlan", "UnitResult", "WorkQueue", "assess_run", + "combine_terms", + "kill_by_pid", "merge_run", "plan_units", + "reap", "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/guards.py b/src/inference_endpoint/evaluation/swe_bench_distributed/guards.py new file mode 100644 index 000000000..92801fe28 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/guards.py @@ -0,0 +1,314 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resource guards for graded SWE-bench evaluation. + +A graded test runs inside an evaluation container with no memory limit. A model +patch that makes a test allocate without bound will take the host down, and when +a whole client fleet shares one scheduler step, one host's OOM destroys every +peer's work along with it -- ``--kill-on-bad-exit=0`` does **not** prevent that, +because the scheduler escalates OOM separately from task exit codes. + +Killing such a process is correct, not a distortion. A patch that makes a graded +test allocate without bound is a failing patch, exactly as a patch that makes it +loop forever is; the alternative to killing was never "the test passes", it was +"the host dies and the instance still never completes". The kill is recorded as +a marker file and the classifier books it as a genuine failure. + +TWO RULES THAT ARE ENFORCED BY CONSTRUCTION HERE: + +1. **Kill by pid, never by pattern.** A pattern such as ``runtests.py`` can + appear in the guard's own command line, and a long-lived daemon can carry a + dead process's argv for days. This module contains no ``pkill``/``pgrep`` + path at all, and :func:`kill_by_pid` refuses self and its own ancestors. +2. **A conjunctive guard must not degenerate.** When one honest term of an + AND-guard permanently loses its data source, the conjunction collapses into + its remaining, weaker clauses and starts firing on healthy targets -- that is + how an idle watchdog killed a live bring-up. :func:`combine_terms` therefore + returns ``INDETERMINATE``, never ``UNHEALTHY``, if any term has no evidence. +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import time +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path + +logger = logging.getLogger(__name__) + +DEFAULT_KILL_BYTES = 150 * 1024**3 +DEFAULT_WARN_BYTES = 100 * 1024**3 +#: Ancestors that prove a process is inside a container supervisor. +CONTAINER_SUPERVISORS = ("conmon", "containerd-shim", "runc", "enroot", "crun") +_ANCESTOR_DEPTH = 6 + + +class HealthVerdict(StrEnum): + HEALTHY = "healthy" + UNHEALTHY = "unhealthy" + INDETERMINATE = "indeterminate" + + +@dataclass(slots=True) +class HealthTerm: + """One clause of a conjunctive guard, with its evidence count. + + ``evidence`` is the number of observations the term actually made. A term + that made none cannot vote, and must not be silently read as ``HEALTHY`` + (which would let the conjunction fire on the strength of the other clauses + alone) nor as ``UNHEALTHY``. + """ + + name: str + verdict: HealthVerdict + evidence: int + detail: str = "" + + +def combine_terms(terms: list[HealthTerm]) -> tuple[HealthVerdict, str]: + """AND the terms, refusing to act on an unevidenced conjunction.""" + if not terms: + return HealthVerdict.INDETERMINATE, "no terms" + blind = [term.name for term in terms if term.evidence <= 0] + if blind: + return ( + HealthVerdict.INDETERMINATE, + "no evidence for term(s): " + + ", ".join(blind) + + " -- a conjunction with a blind term cannot be trusted to be true", + ) + indeterminate = [ + term.name for term in terms if term.verdict is HealthVerdict.INDETERMINATE + ] + if indeterminate: + return ( + HealthVerdict.INDETERMINATE, + "indeterminate term(s): " + ", ".join(indeterminate), + ) + healthy = [term.name for term in terms if term.verdict is HealthVerdict.HEALTHY] + if healthy: + return HealthVerdict.HEALTHY, "healthy term(s): " + ", ".join(healthy) + return HealthVerdict.UNHEALTHY, "; ".join( + f"{term.name}: {term.detail}" for term in terms + ) + + +class SelfKillRefused(RuntimeError): + """Refused to signal this process or one of its ancestors.""" + + +def ancestors( + pid: int, *, depth: int = _ANCESTOR_DEPTH, proc: Path | None = None +) -> list[int]: + """Parent pids of ``pid``, nearest first.""" + root = proc if proc is not None else Path("/proc") + found: list[int] = [] + current = pid + for _ in range(depth): + try: + stat = (root / str(current) / "status").read_text() + except OSError: + break + parent = None + for line in stat.splitlines(): + if line.startswith("PPid:"): + try: + parent = int(line.split()[1]) + except (IndexError, ValueError): + parent = None + break + if parent is None or parent <= 0 or parent in found: + break + found.append(parent) + current = parent + return found + + +def kill_by_pid( + pid: int, *, sig: int = signal.SIGKILL, proc: Path | None = None +) -> bool: + """Signal exactly one pid. + + Refuses this process and any of its ancestors. There is deliberately no + pattern-matching variant of this function: matching by command line is how a + guard kills itself, or kills whatever inherited a stale argv. + """ + if pid <= 0: + raise SelfKillRefused(f"refusing to signal pid {pid}") + if pid == os.getpid(): + raise SelfKillRefused("refusing to signal self") + if pid in ancestors(os.getpid(), proc=proc): + raise SelfKillRefused(f"refusing to signal ancestor pid {pid}") + try: + os.kill(pid, sig) + except ProcessLookupError: + return False + except OSError: + logger.warning("could not signal pid %d", pid, exc_info=True) + return False + return True + + +@dataclass(slots=True) +class ProcessSample: + pid: int + rss_bytes: int + #: Name of the container this process belongs to, if it could be resolved. + #: This is what determines the phase, so an unresolvable name is not "not a + #: test" -- see :meth:`MemoryGuard.phase_for`. + container_name: str | None = None + ancestor_names: tuple[str, ...] = () + #: Advisory only. Deliberately NOT a predicate: see MemoryGuard's docstring. + cwd: str = "" + + +@dataclass(slots=True) +class GuardAction: + pid: int + rss_bytes: int + verdict: HealthVerdict + reason: str + killed: bool = False + terms: list[HealthTerm] = field(default_factory=list) + + +class MemoryGuard: + """Kill a runaway graded test, and only a runaway graded test. + + A process is a candidate only when **both** terms hold: + + * resident memory at or above ``kill_bytes`` (default 150 GiB; a healthy + graded test uses single-digit GiB, so the headroom is roughly thirty-fold) + * it has a container-supervisor ancestor -- it is inside a container + + THERE IS DELIBERATELY NO WORKING-DIRECTORY TERM. An earlier version required + the process's cwd to be inside the testbed, on the reasoning that a graded + test runs there. It does not always: a runaway that had grown to 667 GiB was + skipped for 105 minutes because its cwd was ``/tmp``. Every additional + conjunct is another way for the guard to miss what it exists to catch, so + the predicate set is the smallest one that cannot match a benchmark client, + an engine, a login shell or the guard itself -- all of which fail the + container term. ``cwd`` is still sampled, as advisory detail only. + """ + + def __init__( + self, + *, + kill_bytes: int = DEFAULT_KILL_BYTES, + warn_bytes: int = DEFAULT_WARN_BYTES, + killed_dir: Path | None = None, + supervisors: tuple[str, ...] = CONTAINER_SUPERVISORS, + eval_container_prefixes: tuple[str, ...] = ("sweb.eval",), + agent_container_prefixes: tuple[str, ...] = ("minisweagent",), + ) -> None: + self.kill_bytes = kill_bytes + self.warn_bytes = warn_bytes + self.killed_dir = killed_dir + self.supervisors = supervisors + self.eval_container_prefixes = eval_container_prefixes + self.agent_container_prefixes = agent_container_prefixes + + def phase_for(self, sample: ProcessSample) -> str: + """Which phase a runaway belongs to, from its container name. + + Fails closed to ``"unknown"``. An unresolvable container name must not + stop the kill -- the process is still a confirmed runaway inside a + container -- but it must also not be booked as an eval kill, because + only an eval kill turns an instance's error into a genuine failure. + """ + name = sample.container_name or "" + if any(name.startswith(prefix) for prefix in self.eval_container_prefixes): + return "eval" + if any(name.startswith(prefix) for prefix in self.agent_container_prefixes): + return "agent" + return "unknown" + + def evaluate(self, sample: ProcessSample) -> GuardAction: + terms = [ + HealthTerm( + name="rss", + verdict=( + HealthVerdict.UNHEALTHY + if sample.rss_bytes >= self.kill_bytes + else HealthVerdict.HEALTHY + ), + evidence=1 if sample.rss_bytes >= 0 else 0, + detail=f"{sample.rss_bytes / 1024**3:.1f} GiB", + ), + HealthTerm( + name="in_container", + verdict=( + HealthVerdict.UNHEALTHY + if any(name in self.supervisors for name in sample.ancestor_names) + else HealthVerdict.HEALTHY + ), + evidence=len(sample.ancestor_names), + detail=f"ancestors={list(sample.ancestor_names)}", + ), + ] + verdict, reason = combine_terms(terms) + return GuardAction( + pid=sample.pid, + rss_bytes=sample.rss_bytes, + verdict=verdict, + reason=reason, + terms=terms, + ) + + def act( + self, + sample: ProcessSample, + *, + instance_id: str | None = None, + phase: str | None = None, + apply: bool = False, + ) -> GuardAction: + """Evaluate and, when ``apply``, kill by pid and record a marker. + + The marker is written *before* the kill: a SIGKILLed test leaves an + ambiguous log, so the record of having killed it is the only reliable + evidence, and it has to exist even if the process dies first. + """ + action = self.evaluate(sample) + if action.verdict is not HealthVerdict.UNHEALTHY or not apply: + return action + resolved_phase = phase if phase is not None else self.phase_for(sample) + if self.killed_dir is not None and instance_id: + self.record_kill(instance_id, sample, phase=resolved_phase) + action.killed = kill_by_pid(sample.pid) + return action + + def record_kill( + self, instance_id: str, sample: ProcessSample, *, phase: str = "eval" + ) -> Path: + """Write the ``....json`` marker. + + Phase is load-bearing: only ``eval`` markers make an instance's error a + genuine failure. An ``agent`` kill merely makes one tool call return an + error observation and the agent carries on, so it must never influence + classification. + """ + import socket + + assert self.killed_dir is not None + self.killed_dir.mkdir(parents=True, exist_ok=True) + host = socket.gethostname() + path = self.killed_dir / f"{phase}.{instance_id}.{host}.{sample.pid}.json" + path.write_text( + json.dumps( + { + "phase": phase, + "instance_id": instance_id, + "host": host, + "pid": sample.pid, + "rss_bytes": sample.rss_bytes, + "killed_at": time.time(), + } + ) + ) + return path diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py b/src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py new file mode 100644 index 000000000..0905cee42 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Return orphaned claims to the queue. Nothing else. + +A false reap is the worst thing this system can do. Releasing a claim whose +owner is still running puts the unit back in the queue while it is executing, a +second worker takes it, both write results, and the run has duplicate work, a +wrong denominator, and no error anywhere -- the exact silent corruption the +atomic claim exists to prevent, reintroduced by the janitor. + +Therefore the reaper is conservative in one specific direction: **uncertainty +never escalates.** If liveness cannot be determined, nothing is released. +""" + +from __future__ import annotations + +import logging +import os +import subprocess +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Protocol + +from .queue import OwnerRecord, WorkQueue + +logger = logging.getLogger(__name__) + +_PROBE_TIMEOUT_S = 60 + + +class Liveness(StrEnum): + ALIVE = "alive" + DEAD = "dead" + #: Could not tell. Treated as ALIVE for the purpose of reaping. + INDETERMINATE = "indeterminate" + + +@dataclass(frozen=True, slots=True) +class LivenessVerdict: + state: Liveness + #: Which layer decided. ``"step"`` gets a shorter staleness threshold: a + #: step that died inside a live job took its tasks with it immediately, so + #: there is no reason to wait an hour to believe it. + scope: str = "process" + detail: str = "" + + +class OwnerLiveness(Protocol): + """Decides whether the process that claimed a unit still exists.""" + + def probe(self, owner: OwnerRecord) -> LivenessVerdict: ... + + +class LocalProcessLiveness: + """Liveness by pid, scoped to one host and one boot. + + A pid on its own is not evidence: after a reboot the same number can belong + to something unrelated, so an owner from a different boot of this host is + dead, and an owner from a different host is indeterminate (we cannot see it). + """ + + def __init__(self, *, host: str | None = None, boot: str | None = None) -> None: + import socket + + from .queue import boot_id + + self.host = host if host is not None else socket.gethostname() + self.boot = boot if boot is not None else boot_id() + + def probe(self, owner: OwnerRecord) -> LivenessVerdict: + if owner.host != self.host: + return LivenessVerdict( + Liveness.INDETERMINATE, "process", f"owner is on {owner.host}" + ) + if owner.boot_id and owner.boot_id != self.boot: + return LivenessVerdict( + Liveness.DEAD, "process", "host rebooted since claim" + ) + if owner.pid <= 0: + return LivenessVerdict(Liveness.INDETERMINATE, "process", "no pid recorded") + try: + os.kill(owner.pid, 0) + except ProcessLookupError: + return LivenessVerdict(Liveness.DEAD, "process", "pid gone") + except PermissionError: + # Exists, owned by someone else. + return LivenessVerdict(Liveness.ALIVE, "process", "pid exists") + except OSError: + return LivenessVerdict(Liveness.INDETERMINATE, "process", "kill(0) failed") + return LivenessVerdict(Liveness.ALIVE, "process", "pid exists") + + +class SlurmStepLiveness: + """Liveness by SLURM job *and step*. + + An owner is dead when its job is absent from ``squeue``, **or** when the job + is alive but its step is gone. The second clause is not optional: a step can + die inside a live job (a killed srun, an OOM-terminated step) and SLURM + kills that step's tasks, but the job never leaves ``squeue``, so a + job-level-only rule blocks those units for the entire life of the + allocation. + + Step liveness comes from ``scontrol show step``, never ``squeue -s``: on the + clusters this was built for ``squeue -s`` reports only ``.extern`` and never + the worker step, so using it would mark every live step dead and falsely + reap every claim. + + Every failure to read SLURM yields ``INDETERMINATE``. An unavailable + ``squeue`` must never be read as "no jobs are running". + """ + + def __init__(self, *, timeout_s: int = _PROBE_TIMEOUT_S) -> None: + self.timeout_s = timeout_s + + def _run(self, argv: list[str]) -> str | None: + try: + completed = subprocess.run( + argv, capture_output=True, text=True, timeout=self.timeout_s + ) + except (OSError, subprocess.SubprocessError): + logger.warning("reaper: %s unavailable; releasing nothing", argv[0]) + return None + if completed.returncode != 0: + return None + return completed.stdout + + def live_job_ids(self) -> set[str] | None: + out = self._run(["squeue", "-h", "-o", "%i"]) + if out is None: + return None + ids: set[str] = set() + for token in out.split(): + token = token.strip() + if not token: + continue + ids.add(token) + ids.add(token.split("_")[0].split(".")[0]) + if not ids and os.environ.get("SLURM_JOB_ID"): + # An empty queue is legitimate in general, but not while we are + # ourselves inside a job. That is what a broken squeue looks like. + logger.warning( + "reaper: squeue returned empty while inside a job; releasing nothing" + ) + return None + return ids + + def live_step_ids(self, job_id: str) -> set[str] | None: + out = self._run(["scontrol", "show", "step", str(job_id)]) + if out is None: + return None + steps = { + token.split("=", 1)[1].split(".", 1)[1] + for token in out.split() + if token.startswith("StepId=") and "." in token.split("=", 1)[1] + } + # A successful scontrol listing no step at all is implausible while the + # job exists (there is always .extern): indeterminate, not empty. + return steps or None + + def probe(self, owner: OwnerRecord) -> LivenessVerdict: + if not owner.slurm_job_id: + return LivenessVerdict(Liveness.INDETERMINATE, "job", "no job id recorded") + jobs = self.live_job_ids() + if jobs is None: + return LivenessVerdict(Liveness.INDETERMINATE, "job", "squeue unreadable") + if owner.slurm_job_id not in jobs: + return LivenessVerdict(Liveness.DEAD, "job", "job absent from squeue") + if not owner.slurm_step_id: + return LivenessVerdict( + Liveness.ALIVE, "job", "job present, no step recorded" + ) + steps = self.live_step_ids(owner.slurm_job_id) + if steps is None: + # Indeterminate step liveness must not become MORE aggressive than + # the job-level answer, which is "alive". + return LivenessVerdict(Liveness.ALIVE, "job", "step list unreadable") + if owner.slurm_step_id in steps: + return LivenessVerdict(Liveness.ALIVE, "step", "step present") + return LivenessVerdict(Liveness.DEAD, "step", "step gone inside a live job") + + +@dataclass(slots=True) +class ReapReport: + released: list[str] = field(default_factory=list) + kept: dict[str, str] = field(default_factory=dict) + dry_run: bool = True + + def __bool__(self) -> bool: # pragma: no cover - convenience + return bool(self.released) + + +def reap( + queue: WorkQueue, + liveness: OwnerLiveness, + *, + stale_after_s: float = 3600.0, + step_stale_after_s: float = 900.0, + apply: bool = False, + now: float | None = None, +) -> ReapReport: + """Release claims whose owner is provably gone and which produced no result. + + All three conditions must hold: no result, a stale-enough heartbeat, and a + ``DEAD`` liveness verdict. A verdict scoped to ``"step"`` uses the shorter + ``step_stale_after_s``: when a step dies inside a job that stays in the + queue, the job-level rule alone never fires and those units stay blocked for + the entire life of the allocation. + """ + report = ReapReport(dry_run=not apply) + completed = queue.completed_unit_ids() + for unit_id in sorted(queue.claimed_unit_ids()): + if unit_id in completed: + # Claims for completed units are harmless bookkeeping. + report.kept[unit_id] = "has result" + continue + age = queue.heartbeat_age(unit_id, now=now) + if age is None: + report.kept[unit_id] = "no heartbeat to age" + continue + owner = queue.owner(unit_id) + if owner is None: + # We cannot prove anything about an unreadable owner, so age alone + # decides. A claim holding anything but pure bookkeeping is not ours + # to reason about at all. + if not queue.is_pure_bookkeeping(unit_id): + report.kept[unit_id] = "claim holds non-bookkeeping contents" + continue + verdict = LivenessVerdict(Liveness.DEAD, "process", "owner unreadable") + else: + verdict = liveness.probe(owner) + threshold = step_stale_after_s if verdict.scope == "step" else stale_after_s + if age < threshold: + report.kept[unit_id] = f"heartbeat {age:.0f}s < {threshold:.0f}s" + continue + if verdict.state is not Liveness.DEAD: + report.kept[unit_id] = f"owner {verdict.state.value}: {verdict.detail}" + continue + report.released.append(unit_id) + if apply: + queue.release(unit_id) + return report diff --git a/tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py b/tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py new file mode 100644 index 000000000..aebdec290 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resource guards and the claim reaper.""" + +from __future__ import annotations + +import os +import time + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed import guards as guards_mod +from inference_endpoint.evaluation.swe_bench_distributed.guards import ( + DEFAULT_KILL_BYTES, + HealthTerm, + HealthVerdict, + MemoryGuard, + ProcessSample, + SelfKillRefused, + combine_terms, + kill_by_pid, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + UnitOutcome, + UnitResult, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.reaper import ( + Liveness, + LivenessVerdict, + reap, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import plan_units + +pytestmark = pytest.mark.unit + +GIB = 1024**3 + + +def executable_source(module) -> str: + """Module source with comments and string literals removed.""" + import tokenize + + kept = [] + with open(module.__file__, "rb") as handle: + for token in tokenize.tokenize(handle.readline): + if token.type in {tokenize.COMMENT, tokenize.STRING}: + continue + kept.append(token.string) + return " ".join(kept) + + +class FakeLiveness: + def __init__(self, verdict: LivenessVerdict) -> None: + self.verdict = verdict + + def probe(self, owner): + return self.verdict + + +@pytest.fixture +def queue(tmp_path): + plan = plan_units("run-a", [f"i-{i}" for i in range(20)], shard_size=10) + return WorkQueue(tmp_path / "wq", plan) + + +class TestConjunction: + def test_all_unhealthy_terms_fire(self): + terms = [ + HealthTerm("a", HealthVerdict.UNHEALTHY, evidence=1), + HealthTerm("b", HealthVerdict.UNHEALTHY, evidence=1), + ] + assert combine_terms(terms)[0] is HealthVerdict.UNHEALTHY + + def test_a_blind_term_makes_the_conjunction_indeterminate(self): + # When an honest term permanently loses its data source, an AND-guard + # collapses into its remaining, weaker clauses and starts firing on + # healthy targets. That is how an idle watchdog killed a live bring-up. + terms = [ + HealthTerm("loud", HealthVerdict.UNHEALTHY, evidence=1), + HealthTerm("blind", HealthVerdict.UNHEALTHY, evidence=0), + ] + verdict, reason = combine_terms(terms) + assert verdict is HealthVerdict.INDETERMINATE + assert "blind" in reason + + def test_a_blind_term_never_yields_unhealthy(self): + terms = [HealthTerm("blind", HealthVerdict.UNHEALTHY, evidence=0)] + assert combine_terms(terms)[0] is not HealthVerdict.UNHEALTHY + + def test_one_healthy_term_spares_the_target(self): + terms = [ + HealthTerm("a", HealthVerdict.UNHEALTHY, evidence=1), + HealthTerm("b", HealthVerdict.HEALTHY, evidence=1), + ] + assert combine_terms(terms)[0] is HealthVerdict.HEALTHY + + def test_no_terms_is_indeterminate(self): + assert combine_terms([])[0] is HealthVerdict.INDETERMINATE + + +class TestKillDiscipline: + def test_there_is_no_pattern_kill_path_in_the_module(self): + # A pattern such as "runtests.py" can appear in the guard's own command + # line, and a long-lived daemon can carry a dead process's argv for days. + # Executable code is inspected with comments and strings removed, so the + # docstring explaining the rule cannot satisfy the test for it. + code = executable_source(guards_mod) + assert "pkill" not in code + assert "pgrep" not in code + + def test_the_guard_never_shells_out(self): + # There is no command line to match against in the first place. + code = executable_source(guards_mod) + assert "subprocess" not in code + assert "os.system" not in code + + def test_killing_self_is_refused(self): + with pytest.raises(SelfKillRefused, match="self"): + kill_by_pid(os.getpid()) + + def test_killing_an_ancestor_is_refused(self): + with pytest.raises(SelfKillRefused, match="ancestor"): + kill_by_pid(os.getppid()) + + def test_a_nonsense_pid_is_refused(self): + with pytest.raises(SelfKillRefused): + kill_by_pid(0) + + +class TestMemoryGuard: + def runaway(self, **overrides): + payload = { + "pid": 4242, + "rss_bytes": 200 * GIB, + "container_name": "sweb.eval.arm64.repo__proj-1", + "ancestor_names": ("bash", "conmon"), + } + payload.update(overrides) + return ProcessSample(**payload) + + def test_a_runaway_graded_test_is_unhealthy(self): + action = MemoryGuard().evaluate(self.runaway()) + assert action.verdict is HealthVerdict.UNHEALTHY + + def test_a_runaway_outside_the_testbed_is_still_caught(self): + # An earlier version required cwd inside /testbed. A runaway that had + # grown to 667 GiB was skipped for 105 minutes because its cwd was /tmp, + # so cwd is advisory detail and never a predicate. + action = MemoryGuard().evaluate(self.runaway(cwd="/tmp")) + assert action.verdict is HealthVerdict.UNHEALTHY + assert {term.name for term in action.terms} == {"rss", "in_container"} + + def test_a_large_process_outside_a_container_is_spared(self): + action = MemoryGuard().evaluate(self.runaway(ancestor_names=("bash", "sshd"))) + assert action.verdict is HealthVerdict.HEALTHY + + def test_a_normal_test_is_spared(self): + action = MemoryGuard().evaluate(self.runaway(rss_bytes=3 * GIB)) + assert action.verdict is HealthVerdict.HEALTHY + + def test_unreadable_ancestry_is_indeterminate_not_a_kill(self): + action = MemoryGuard().evaluate(self.runaway(ancestor_names=())) + assert action.verdict is HealthVerdict.INDETERMINATE + + def test_the_default_threshold_leaves_wide_headroom(self): + assert DEFAULT_KILL_BYTES >= 100 * GIB + + @pytest.mark.parametrize( + ("container_name", "phase"), + [ + ("sweb.eval.arm64.repo__proj-1", "eval"), + ("minisweagent-abc123", "agent"), + ("something-else", "unknown"), + (None, "unknown"), + ], + ) + def test_phase_comes_from_the_container_name_and_fails_closed( + self, container_name, phase + ): + # Only an eval kill turns an instance's error into a genuine failure, so + # an unresolvable name must not be booked as one. + guard = MemoryGuard() + assert guard.phase_for(self.runaway(container_name=container_name)) == phase + + def test_the_marker_is_written_before_the_kill(self, tmp_path, monkeypatch): + killed_dir = tmp_path / "killed" + order: list[str] = [] + monkeypatch.setattr( + guards_mod, + "kill_by_pid", + lambda pid, **kwargs: order.append("kill") or True, + ) + guard = MemoryGuard(killed_dir=killed_dir) + original = guard.record_kill + + def traced(*args, **kwargs): + order.append("marker") + return original(*args, **kwargs) + + monkeypatch.setattr(guard, "record_kill", traced) + guard.act(self.runaway(), instance_id="repo__proj-1", apply=True) + + # A SIGKILLed test leaves an ambiguous log, so the record of having + # killed it must survive even if the process dies first. + assert order == ["marker", "kill"] + assert list(killed_dir.glob("eval.repo__proj-1.*.json")) + + def test_dry_evaluation_does_not_kill(self, tmp_path, monkeypatch): + monkeypatch.setattr( + guards_mod, "kill_by_pid", lambda *a, **k: pytest.fail("killed") + ) + action = MemoryGuard(killed_dir=tmp_path).act( + self.runaway(), instance_id="x", apply=False + ) + assert not action.killed + + +class TestReaper: + def stale_claim(self, queue, unit_id="run-a.s00", age=7200.0): + queue.claim(unit_id) + heartbeat = queue.claims_dir / unit_id / "hb" + past = time.time() - age + os.utime(heartbeat, (past, past)) + + def test_a_dead_owner_with_no_result_is_released(self, queue): + self.stale_claim(queue) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == ["run-a.s00"] + assert "run-a.s00" in queue.available_unit_ids() + + def test_a_live_owner_is_never_released(self, queue): + self.stale_claim(queue) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.ALIVE)), apply=True) + # A false reap gives one unit two owners, duplicate results and a wrong + # denominator, with no error anywhere. + assert report.released == [] + + def test_an_indeterminate_probe_releases_nothing(self, queue): + self.stale_claim(queue) + report = reap( + queue, + FakeLiveness(LivenessVerdict(Liveness.INDETERMINATE)), + apply=True, + ) + assert report.released == [] + assert "indeterminate" in report.kept["run-a.s00"] + + def test_a_fresh_heartbeat_is_never_released(self, queue): + queue.claim("run-a.s00") + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == [] + + def test_a_claim_with_a_result_is_never_released(self, queue): + self.stale_claim(queue) + unit = queue.plan.unit("run-a.s00") + queue.results_dir.joinpath("run-a.s00.json").write_text( + UnitResult( + unit_id="run-a.s00", + run_id="run-a", + plan_digest=queue.plan.digest, + outcome=UnitOutcome.SUCCEEDED, + accounted_instance_ids=unit.instance_ids, + ).to_dict() + and '{"unit_id": "run-a.s00"}' + ) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == [] + + def test_a_dead_step_uses_the_shorter_threshold(self, queue): + # A step that dies inside a live job takes its tasks with it at once, so + # waiting an hour would block those units for the whole allocation. + self.stale_claim(queue, age=1200.0) + report = reap( + queue, + FakeLiveness(LivenessVerdict(Liveness.DEAD, scope="step")), + stale_after_s=3600.0, + step_stale_after_s=900.0, + apply=True, + ) + assert report.released == ["run-a.s00"] + + def test_dry_run_reports_without_releasing(self, queue): + self.stale_claim(queue) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD))) + assert report.released == ["run-a.s00"] + assert queue.claimed_unit_ids() == {"run-a.s00"} + + def test_a_claim_with_unexpected_contents_is_left_alone(self, queue): + self.stale_claim(queue) + (queue.claims_dir / "run-a.s00" / "surprise").write_text("x") + (queue.claims_dir / "run-a.s00" / "owner").unlink() + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == [] diff --git a/tests/unit/evaluation/swe_bench_distributed/test_liveness.py b/tests/unit/evaluation/swe_bench_distributed/test_liveness.py new file mode 100644 index 000000000..e65b32b9b --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_liveness.py @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Owner-liveness probes. Uncertainty must never escalate to DEAD.""" + +from __future__ import annotations + +import os +import socket +import subprocess + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.queue import OwnerRecord +from inference_endpoint.evaluation.swe_bench_distributed.reaper import ( + Liveness, + LocalProcessLiveness, + SlurmStepLiveness, +) + +pytestmark = pytest.mark.unit + + +def owner(**overrides) -> OwnerRecord: + payload = { + "unit_id": "run-a.s00", + "host": socket.gethostname(), + "pid": os.getpid(), + "boot_id": "boot-1", + "plan_digest": "d" * 64, + "claimed_at": 0.0, + } + payload.update(overrides) + return OwnerRecord(**payload) + + +class TestLocalProcessLiveness: + def test_a_live_pid_on_this_boot_is_alive(self): + probe = LocalProcessLiveness(boot="boot-1") + assert probe.probe(owner()).state is Liveness.ALIVE + + def test_a_missing_pid_is_dead(self): + probe = LocalProcessLiveness(boot="boot-1") + # 2**22 is above the default pid_max on Linux, so it cannot exist. + assert probe.probe(owner(pid=2**22)).state is Liveness.DEAD + + def test_a_different_boot_is_dead(self): + # After a reboot the same pid number can belong to something unrelated, + # so a live-looking pid is not evidence that the owner survived. + probe = LocalProcessLiveness(boot="boot-2") + assert probe.probe(owner()).state is Liveness.DEAD + + def test_another_host_is_indeterminate_not_dead(self): + probe = LocalProcessLiveness(boot="boot-1") + assert probe.probe(owner(host="elsewhere")).state is Liveness.INDETERMINATE + + def test_a_missing_pid_record_is_indeterminate(self): + probe = LocalProcessLiveness(boot="boot-1") + assert probe.probe(owner(pid=0)).state is Liveness.INDETERMINATE + + +class FakeSlurm(SlurmStepLiveness): + def __init__(self, responses): + super().__init__() + self.responses = responses + self.calls: list[list[str]] = [] + + def _run(self, argv): + self.calls.append(argv) + response = self.responses.get(argv[0]) + if isinstance(response, Exception): + raise response + return response + + +class TestSlurmStepLiveness: + def slurm_owner(self, **overrides): + return owner(slurm_job_id="1000", slurm_step_id="3", **overrides) + + def test_a_job_absent_from_squeue_is_dead(self): + probe = FakeSlurm({"squeue": "2000\n"}) + verdict = probe.probe(self.slurm_owner()) + assert verdict.state is Liveness.DEAD + assert verdict.scope == "job" + + def test_a_live_job_and_step_is_alive(self): + probe = FakeSlurm( + { + "squeue": "1000\n", + "scontrol": "StepId=1000.3 State=RUNNING StepId=1000.extern", + } + ) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_a_dead_step_inside_a_live_job_is_dead(self): + # A step can die inside a live job -- a killed srun, an OOM-terminated + # step -- and the job never leaves the queue, so a job-level-only rule + # blocks those units for the entire allocation. + probe = FakeSlurm( + {"squeue": "1000\n", "scontrol": "StepId=1000.extern State=RUNNING"} + ) + verdict = probe.probe(self.slurm_owner()) + assert verdict.state is Liveness.DEAD + assert verdict.scope == "step" + + def test_step_liveness_uses_scontrol_not_squeue_s(self): + probe = FakeSlurm( + {"squeue": "1000\n", "scontrol": "StepId=1000.3 State=RUNNING"} + ) + probe.probe(self.slurm_owner()) + # `squeue -s` reports only `.extern` on the clusters this targets, so it + # would mark every live step dead and falsely reap every claim. + assert ["scontrol", "show", "step", "1000"] in probe.calls + assert not any("-s" in argv for argv in probe.calls if argv[0] == "squeue") + + def test_an_unreadable_squeue_is_indeterminate(self): + probe = FakeSlurm({"squeue": None}) + assert probe.probe(self.slurm_owner()).state is Liveness.INDETERMINATE + + def test_an_unreadable_step_list_falls_back_to_the_job_answer(self): + # Indeterminate step liveness must never be more aggressive than the + # job-level answer, which is "alive". + probe = FakeSlurm({"squeue": "1000\n", "scontrol": None}) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_an_empty_scontrol_listing_is_treated_as_unreadable(self): + probe = FakeSlurm({"squeue": "1000\n", "scontrol": "no steps here"}) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_an_owner_without_a_job_id_is_indeterminate(self): + probe = FakeSlurm({"squeue": "1000\n"}) + assert probe.probe(owner()).state is Liveness.INDETERMINATE + + def test_an_empty_queue_inside_a_job_is_implausible(self, monkeypatch): + # An empty successful squeue is what a broken squeue looks like. It must + # never be read as "no jobs are running" while we are inside a job. + monkeypatch.setenv("SLURM_JOB_ID", "1000") + probe = FakeSlurm({"squeue": ""}) + assert probe.live_job_ids() is None + + def test_an_empty_queue_outside_a_job_is_trusted(self, monkeypatch): + monkeypatch.delenv("SLURM_JOB_ID", raising=False) + probe = FakeSlurm({"squeue": ""}) + assert probe.live_job_ids() == set() + + def test_array_job_ids_are_matched_by_base_id(self): + probe = FakeSlurm( + {"squeue": "1000_4\n", "scontrol": "StepId=1000.3 State=RUNNING"} + ) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_a_failing_command_is_indeterminate_not_dead(self): + probe = SlurmStepLiveness(timeout_s=1) + + def boom(argv, **kwargs): + raise subprocess.SubprocessError("no slurm here") + + original = subprocess.run + subprocess.run = boom + try: + assert probe.live_job_ids() is None + finally: + subprocess.run = original From 9d7ea7ffbceadb7ec6c3cf89df9aa94159f1ad17 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 13:23:02 -0700 Subject: [PATCH 11/25] feat(swe-bench): retry infrastructure faults only where non-execution is proved The reaper returns a claim whose owner is provably gone. This is the same argument one level down: an operation that provably never ran can be run again, and one that may have run cannot. Measured signature, from an isolated probe with no model and no GPU (20 nodes, 200 workers, 6273 ordinary shell steps): 63 steps failed, and in all 63 the step's status file still read `pending` -- not `started` -- with no in-band sentinel. The step script never executed its first line. Re-running those commands cannot double-apply an edit, a removal or a test run. That is the entire safety argument, and it is why the gate is `provable_non_execution` rather than "an error happened". A failure that does not make that claim is re-raised immediately and does not consume the budget, which also makes every exception type this module has never heard of safe by default. `infra_retry` provides three things: * `retry_on_provable_non_execution(...)` -- bounded attempts, and the gate. The evidence is read as an attribute rather than an isinstance check, so the producer of the evidence and this consumer stay decoupled. * `InfraRetryLedger` -- every attempt and its outcome, appended as JSONL so a run that dies still leaves its retry history behind, and held in memory so the counters survive a ledger that cannot be written. Accounting must never be able to take a run down. * `summary()` -- `infra_retries_total`, `instances_saved_by_retry`, `infra_retries_exhausted`, the succeeded-on-attempt distribution, and `run_quality: CLEAN | OK_WITH_RETRIES | DEGRADED`. The counting is the point. The banked campaign retried environment faults without limit and without counting them (`wq_worker.sh:41` WQ_MAX_ATTEMPTS=5, `:256` "ENVIRONMENT FAULTS DO NOT CONSUME THE UNIT'S ATTEMPT BUDGET"), which is exactly why nobody knew how many there had been. A retry loop that quietly absorbs the defect it compensates for turns a broken cluster into an invisible one. Measured effect of adding this loop: RunnerError 59 -> 7 and resolve 47.0% -> 70.0% against a banked 70.67% on the identical 200 instances -- a rescue on that scale is not a clean run, and `run_quality` says so even at 200/200. This is the fleet-side half: the decision rule, the accounting and the quality verdict. The next commit applies the same rule inside the SWE-bench service, where the Pyxis step that produces `provable_non_execution` runs. Tests: `TestTheSafetyGate` covers both sides of the decision boundary (`pending` retried, `started` never retried, unfamiliar exception never retried) plus the bound; `TestAccounting` covers recovery, exhaustion, not-retryable, ledger durability and a ledger that cannot be written; `TestRunQuality` covers all three verdicts including DEGRADED on volume alone, where every operation eventually succeeded. --- .../swe_bench_distributed/__init__.py | 14 + .../swe_bench_distributed/infra_retry.py | 261 ++++++++++++++++++ .../swe_bench_distributed/test_infra_retry.py | 243 ++++++++++++++++ 3 files changed, 518 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index a7b6f488c..7269ab520 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -12,6 +12,14 @@ """ from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid +from .infra_retry import ( + InfraRetryLedger, + RetryOutcome, + RetryRecord, + RunQuality, + is_provable_non_execution, + retry_on_provable_non_execution, +) from .merge import ( CompletenessReport, MergeRefusal, @@ -34,11 +42,15 @@ "CompletenessReport", "HealthTerm", "HealthVerdict", + "InfraRetryLedger", "LocalProcessLiveness", "MemoryGuard", "MergeRefusal", "MergeResult", "OwnerLiveness", + "RetryOutcome", + "RetryRecord", + "RunQuality", "SlurmStepLiveness", "Unit", "UnitOutcome", @@ -47,9 +59,11 @@ "WorkQueue", "assess_run", "combine_terms", + "is_provable_non_execution", "kill_by_pid", "merge_run", "plan_units", "reap", + "retry_on_provable_non_execution", "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py b/src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py new file mode 100644 index 000000000..20d51e19d --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Retry infrastructure faults, but only where non-execution is *provable*. + +A retry is a correctness decision, not a convenience. Re-running a command that +may already have run can apply an edit twice, delete something twice, or double +a test run, and none of those announce themselves. So the gate here is not "an +error happened" -- it is "the work provably did not happen". + +The evidence comes from the failure itself. An exception may expose +``provable_non_execution``: for a Pyxis step that is the status file still +reading ``pending`` **and** no in-band sentinel, meaning the step script did not +run even its first line. Anything that does not make that claim is not retried, +which is the safe default for every exception type this module has never heard +of. + +Retries are bounded and, more importantly, **counted**. The banked campaign this +is ported from retried environment faults without limit and without counting +them (``wq_worker.sh:41`` ``WQ_MAX_ATTEMPTS=5``, with ``:256`` "ENVIRONMENT +FAULTS DO NOT CONSUME THE UNIT'S ATTEMPT BUDGET"), which is precisely why nobody +knew how many there had been. A retry loop that quietly absorbs the defect it +compensates for turns a broken cluster into an invisible one: the measured +effect of adding this loop was ``RunnerError`` 59 -> 7 and resolve 47.0% -> +70.0% against a banked 70.67% on the identical 200 instances, and a run that +needs that much rescuing is not a clean run even when it finishes. +""" + +from __future__ import annotations + +import json +import logging +import threading +import time +from collections import Counter +from collections.abc import Callable +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +DEFAULT_MAX_ATTEMPTS = 3 +#: Above this share of operations needing a retry, a run is DEGRADED even if +#: every unit eventually succeeded. Rescuing one operation in fifty is not a +#: healthy fleet, it is a fleet that happened to be caught. +DEGRADED_RETRY_FRACTION = 0.02 + + +class RetryOutcome(StrEnum): + #: Provably never executed; another attempt follows. + RETRYING = "retrying" + #: A later attempt succeeded. + RECOVERED = "recovered" + #: Provably never executed, but the attempt budget ran out. + EXHAUSTED = "exhausted" + #: The work may have executed. Retrying could double-apply it, so this is a + #: hard failure by construction. + NOT_RETRYABLE = "not_retryable" + + +class RunQuality(StrEnum): + CLEAN = "CLEAN" + OK_WITH_RETRIES = "OK_WITH_RETRIES" + DEGRADED = "DEGRADED" + + +def is_provable_non_execution(error: BaseException) -> bool: + """Whether ``error`` proves its operation never ran. + + Read as an attribute rather than an isinstance check so the producer of the + evidence (the Pyxis step runner) and this consumer stay decoupled. An + exception that does not claim the property is never retried. + """ + return getattr(error, "provable_non_execution", False) is True + + +@dataclass(frozen=True, slots=True) +class RetryRecord: + target: str + attempt: int + outcome: RetryOutcome + detail: str | None = None + at: float = 0.0 + + def to_dict(self) -> dict[str, Any]: + return { + "target": self.target, + "attempt": self.attempt, + "outcome": self.outcome.value, + "detail": self.detail, + "at": self.at, + } + + +class InfraRetryLedger: + """Every provable non-execution and what became of it. + + Appends one JSON line per event when given a path, so a run that dies still + leaves its retry history behind, and holds the same records in memory for + :meth:`summary`. Accounting must never be able to take a run down, so a + write failure is logged and swallowed -- but the in-memory counters are + updated first, so the summary is correct even then. + """ + + def __init__(self, path: Path | None = None) -> None: + self.path = Path(path) if path is not None else None + self._lock = threading.Lock() + self._records: list[RetryRecord] = [] + self._operations = 0 + + @property + def records(self) -> list[RetryRecord]: + with self._lock: + return list(self._records) + + @property + def operations(self) -> int: + """Operations submitted to the retry wrapper: the denominator.""" + with self._lock: + return self._operations + + def note_operation(self) -> None: + with self._lock: + self._operations += 1 + + def record( + self, + *, + target: str, + attempt: int, + outcome: RetryOutcome, + detail: str | None = None, + ) -> None: + entry = RetryRecord( + target=target, + attempt=attempt, + outcome=outcome, + detail=detail, + at=time.time(), + ) + with self._lock: + self._records.append(entry) + path = self.path + if path is not None: + try: + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry.to_dict()) + "\n") + except OSError: + logger.warning( + "could not append to the infra retry ledger", exc_info=True + ) + + def summary(self) -> dict[str, Any]: + """Counters a report can publish without re-deriving anything.""" + records = self.records + operations = self.operations + recovered = {r.target for r in records if r.outcome is RetryOutcome.RECOVERED} + exhausted = {r.target for r in records if r.outcome is RetryOutcome.EXHAUSTED} + return { + "infra_retries_total": len(records), + "infra_retry_operations": operations, + "infra_retry_outcomes": dict( + Counter(r.outcome.value for r in records).most_common() + ), + "infra_retry_succeeded_on_attempt": dict( + Counter( + str(r.attempt) + for r in records + if r.outcome is RetryOutcome.RECOVERED + ).most_common() + ), + # A target that recovered and later exhausted was not saved. + "instances_saved_by_retry": len(recovered - exhausted), + "infra_retries_exhausted": sum( + 1 for r in records if r.outcome is RetryOutcome.EXHAUSTED + ), + "run_quality": self.run_quality().value, + } + + def run_quality(self) -> RunQuality: + records = self.records + if any( + r.outcome in (RetryOutcome.EXHAUSTED, RetryOutcome.NOT_RETRYABLE) + for r in records + ): + return RunQuality.DEGRADED + if not records: + return RunQuality.CLEAN + operations = max(1, self.operations) + if len(records) > DEGRADED_RETRY_FRACTION * operations: + return RunQuality.DEGRADED + return RunQuality.OK_WITH_RETRIES + + +def retry_on_provable_non_execution[T]( + operation: Callable[[], T], + *, + target: str, + ledger: InfraRetryLedger | None = None, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + backoff_s: float = 2.0, + max_backoff_s: float = 30.0, + sleep: Callable[[float], None] = time.sleep, +) -> T: + """Call ``operation``, retrying only failures that prove it never ran. + + Raises the last failure when the budget is exhausted, and re-raises + immediately -- without consuming the budget -- for anything that does not + prove non-execution. + """ + if max_attempts < 1: + raise ValueError("max_attempts must be at least 1") + if ledger is not None: + ledger.note_operation() + for attempt in range(1, max_attempts + 1): + try: + result = operation() + except Exception as exc: + if not is_provable_non_execution(exc): + # The work may have run. Retrying could double-apply it. + if ledger is not None: + ledger.record( + target=target, + attempt=attempt, + outcome=RetryOutcome.NOT_RETRYABLE, + detail=f"{type(exc).__name__}: {exc}", + ) + raise + if attempt == max_attempts: + if ledger is not None: + ledger.record( + target=target, + attempt=attempt, + outcome=RetryOutcome.EXHAUSTED, + detail=f"{type(exc).__name__}: {exc}", + ) + raise + if ledger is not None: + ledger.record( + target=target, + attempt=attempt, + outcome=RetryOutcome.RETRYING, + detail=f"{type(exc).__name__}: {exc}", + ) + logger.warning( + "%s provably never executed (attempt %d/%d): %s -- retrying", + target, + attempt, + max_attempts, + exc, + ) + sleep(min(max_backoff_s, backoff_s * attempt)) + continue + if attempt > 1 and ledger is not None: + ledger.record( + target=target, attempt=attempt, outcome=RetryOutcome.RECOVERED + ) + return result + raise AssertionError("unreachable") # pragma: no cover diff --git a/tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py b/tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py new file mode 100644 index 000000000..9b803fe35 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Retry only where non-execution is provable, and always count the retries.""" + +from __future__ import annotations + +import json + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.infra_retry import ( + InfraRetryLedger, + RetryOutcome, + RunQuality, + is_provable_non_execution, + retry_on_provable_non_execution, +) + +pytestmark = pytest.mark.unit + + +class NeverLaunched(RuntimeError): + """Stands in for a step whose status file still read ``pending``.""" + + provable_non_execution = True + + +class MayHaveRun(RuntimeError): + """Stands in for a step that reached ``started`` before it failed.""" + + provable_non_execution = False + + +def _never_sleep(_seconds: float) -> None: + return None + + +def _run(operation, **kwargs): + kwargs.setdefault("target", "run-a.s00") + kwargs.setdefault("sleep", _never_sleep) + return retry_on_provable_non_execution(operation, **kwargs) + + +class TestTheSafetyGate: + def test_a_provable_non_execution_is_retried(self): + calls = [] + + def operation(): + calls.append(1) + if len(calls) < 3: + raise NeverLaunched("status=pending") + return "ok" + + assert _run(operation, max_attempts=3) == "ok" + assert len(calls) == 3 + + def test_a_failure_that_may_have_run_is_never_retried(self): + """The whole safety argument. Re-running could double-apply the work.""" + calls = [] + + def operation(): + calls.append(1) + raise MayHaveRun("status=started") + + with pytest.raises(MayHaveRun): + _run(operation, max_attempts=5) + + assert len(calls) == 1 + + def test_an_unfamiliar_exception_is_not_retried(self): + """Absence of the claim is not evidence for it.""" + calls = [] + + def operation(): + calls.append(1) + raise ValueError("something else entirely") + + with pytest.raises(ValueError): + _run(operation, max_attempts=5) + + assert len(calls) == 1 + + def test_the_attempt_budget_is_bounded(self): + calls = [] + + def operation(): + calls.append(1) + raise NeverLaunched("status=pending") + + with pytest.raises(NeverLaunched): + _run(operation, max_attempts=4) + + assert len(calls) == 4 + + def test_a_successful_first_attempt_costs_nothing(self): + ledger = InfraRetryLedger() + + assert _run(lambda: "ok", ledger=ledger) == "ok" + + assert ledger.records == [] + assert ledger.run_quality() is RunQuality.CLEAN + + @pytest.mark.parametrize( + ("error", "provable"), + [(NeverLaunched(""), True), (MayHaveRun(""), False), (ValueError(""), False)], + ) + def test_provability_is_read_from_the_failure(self, error, provable): + assert is_provable_non_execution(error) is provable + + +class TestAccounting: + def test_a_recovered_operation_is_counted_and_attributed(self): + ledger = InfraRetryLedger() + calls = [] + + def operation(): + calls.append(1) + if len(calls) < 2: + raise NeverLaunched("status=pending") + return "ok" + + _run(operation, ledger=ledger, max_attempts=3) + summary = ledger.summary() + + assert summary["infra_retries_total"] == 2 + assert summary["instances_saved_by_retry"] == 1 + assert summary["infra_retries_exhausted"] == 0 + assert summary["infra_retry_succeeded_on_attempt"] == {"2": 1} + assert summary["infra_retry_outcomes"] == {"retrying": 1, "recovered": 1} + + def test_an_exhausted_operation_is_counted_as_exhausted(self): + ledger = InfraRetryLedger() + + with pytest.raises(NeverLaunched): + _run( + lambda: (_ for _ in ()).throw(NeverLaunched("status=pending")), + ledger=ledger, + max_attempts=2, + ) + summary = ledger.summary() + + assert summary["infra_retries_exhausted"] == 1 + assert summary["instances_saved_by_retry"] == 0 + assert summary["run_quality"] == RunQuality.DEGRADED.value + + def test_a_not_retryable_failure_is_recorded(self): + ledger = InfraRetryLedger() + + with pytest.raises(MayHaveRun): + _run( + lambda: (_ for _ in ()).throw(MayHaveRun("status=started")), + ledger=ledger, + ) + + assert ledger.records[0].outcome is RetryOutcome.NOT_RETRYABLE + + def test_the_ledger_is_durable(self, tmp_path): + """A run that dies must still leave its retry history behind.""" + path = tmp_path / "infra_retries.jsonl" + ledger = InfraRetryLedger(path) + calls = [] + + def operation(): + calls.append(1) + if len(calls) < 2: + raise NeverLaunched("status=pending") + return "ok" + + _run(operation, ledger=ledger, max_attempts=3) + + rows = [json.loads(line) for line in path.read_text().splitlines()] + assert [row["outcome"] for row in rows] == ["retrying", "recovered"] + assert all(row["target"] == "run-a.s00" for row in rows) + + def test_accounting_never_takes_the_run_down(self, tmp_path): + """A ledger that cannot be written must not fail the operation.""" + ledger = InfraRetryLedger(tmp_path / "no-such-dir" / "retries.jsonl") + calls = [] + + def operation(): + calls.append(1) + if len(calls) < 2: + raise NeverLaunched("status=pending") + return "ok" + + assert _run(operation, ledger=ledger, max_attempts=3) == "ok" + # In-memory counters are still correct. + assert ledger.summary()["instances_saved_by_retry"] == 1 + + +class TestRunQuality: + def test_no_retries_is_clean(self): + ledger = InfraRetryLedger() + for _ in range(100): + _run(lambda: "ok", ledger=ledger) + + assert ledger.summary()["run_quality"] == RunQuality.CLEAN.value + + def test_a_few_retries_is_ok_with_retries(self): + ledger = InfraRetryLedger() + for _ in range(200): + _run(lambda: "ok", ledger=ledger) + calls = [] + + def operation(): + calls.append(1) + if len(calls) < 2: + raise NeverLaunched("status=pending") + return "ok" + + _run(operation, ledger=ledger, max_attempts=3) + + assert ledger.summary()["run_quality"] == RunQuality.OK_WITH_RETRIES.value + + def test_many_retries_is_degraded_even_when_everything_succeeded(self): + """A run that leaned on the retry loop is not a clean run.""" + ledger = InfraRetryLedger() + for index in range(10): + calls = [] + + def operation(calls=calls): + calls.append(1) + if len(calls) < 2: + raise NeverLaunched("status=pending") + return "ok" + + _run(operation, ledger=ledger, target=f"unit-{index}", max_attempts=3) + + assert ledger.summary()["run_quality"] == RunQuality.DEGRADED.value + + def test_an_exhaustion_is_degraded_regardless_of_volume(self): + ledger = InfraRetryLedger() + for _ in range(1000): + _run(lambda: "ok", ledger=ledger) + with pytest.raises(NeverLaunched): + _run( + lambda: (_ for _ in ()).throw(NeverLaunched("status=pending")), + ledger=ledger, + max_attempts=2, + ) + + assert ledger.run_quality() is RunQuality.DEGRADED From 2889689d4fe0c25b965232be4f4fe1baa28c7732 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 13:55:36 -0700 Subject: [PATCH 12/25] feat(swebench-service): re-attempt a Pyxis step that provably never launched Wires the retry decision into the place the failure actually happens. `run_srun_step()` now re-attempts a step only when `StepNotLaunched` reports `provable_non_execution` -- the status file still `pending` and no in-band sentinel, so the step script did not run even its first line and the command definitely did not execute. A `StepNotLaunched` that reached `started`, and every other failure, is raised immediately: re-running work that may already have run can apply an edit twice, delete twice, or double a test run, and none of those announce themselves. Measured signature, from an isolated probe with no model and no GPU (20 nodes, 200 workers, 6273 ordinary shell steps): 63 steps failed and in all 63 the status file still read `pending`. Measured effect of retrying exactly those: `RunnerError` 59 -> 7 and resolve 47.0% -> 70.0% against a banked 70.67% on the identical 200 instances. Bounded by `SWEBENCH_PYXIS_STEP_RETRIES` (default 3, set 1 to disable), and every attempt and outcome is appended to `SWEBENCH_PYXIS_INFRA_RETRY_LOG` when set. The record shape is deliberately identical to `swe_bench_distributed.infra_retry.RetryRecord`, which gains `InfraRetryLedger.from_jsonl()` to read it back and publish `infra_retries_total`, `instances_saved_by_retry`, `infra_retries_exhausted` and `run_quality`. The service is an isolated subproject and must not import the benchmark client, so the two halves share a file format rather than a module -- and a test on each side pins that agreement, because if it breaks the retries stop reaching the run-level quality flag and a rescued run looks clean. A retry loop that quietly absorbs the defect it compensates for turns a broken cluster into an invisible one. That is why the accounting is not optional and why `run_quality` reports DEGRADED on volume alone. Tests: `TestStepRetry` covers both sides of the boundary (`pending` retried, `started` never retried), the bound, the recorded outcomes for recovery and exhaustion, and a log path that cannot be written. `TestReadingBackAWritten Ledger` covers the cross-process format, including a truncated final line from a run that died. An autouse fixture pins the existing single-shot tests to one attempt so they keep asserting single-shot behaviour. --- .../swe_bench_distributed/infra_retry.py | 41 ++++++ .../swebench_service/pyxis_environment.py | 107 +++++++++++++- .../swe_bench_distributed/test_infra_retry.py | 49 +++++++ .../swebench_service/test_runner.py | 137 ++++++++++++++++++ 4 files changed, 333 insertions(+), 1 deletion(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py b/src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py index 20d51e19d..013a6f7bb 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/infra_retry.py @@ -152,6 +152,47 @@ def record( "could not append to the infra retry ledger", exc_info=True ) + @classmethod + def from_jsonl(cls, path: Path) -> InfraRetryLedger: + """Load a ledger written by another process. + + The SWE-bench service is an isolated subproject that must not import the + benchmark client, so its Pyxis step runner writes this same record shape + directly. Sharing a file format rather than a module is the only way the + two halves can agree, and reading it here is what turns per-step retries + into a run-level `run_quality`. + + Unparseable lines are skipped: a truncated final line from a run that + died is expected, and losing the whole history to it would be worse. + """ + ledger = cls(path) + try: + text = Path(path).read_text(encoding="utf-8") + except OSError: + return ledger + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + raw = json.loads(line) + record = RetryRecord( + target=str(raw["target"]), + attempt=int(raw["attempt"]), + outcome=RetryOutcome(raw["outcome"]), + detail=raw.get("detail"), + at=float(raw.get("at") or 0.0), + ) + except (ValueError, KeyError, TypeError): + logger.warning("skipping unreadable infra retry record") + continue + ledger._records.append(record) + # Every first attempt that needed a retry represents one operation; a + # writer that only logs failures cannot report the clean denominator, so + # it is reported as unknown rather than guessed. + ledger._operations = sum(1 for r in ledger._records if r.attempt == 1) + return ledger + def summary(self) -> dict[str, Any]: """Counters a report can publish without re-deriving anything.""" records = self.records diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py index 12a6dc433..18c05bfc4 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py @@ -179,7 +179,7 @@ def build_srun_command( return command -def run_srun_step( +def _run_srun_step_once( *, argv: list[str], status_path: Path, @@ -324,6 +324,111 @@ def _srun_evidence(output: str | bytes | None, limit: int = 2000) -> str: return f"\n--- srun output ---\n{text}" +#: Bounded re-attempts for a step that provably never launched. Set to 1 to +#: disable. A retry here is only ever reached when the step script did not run +#: its first line, so it cannot double-apply work -- see run_srun_step. +_STEP_RETRIES_ENV = "SWEBENCH_PYXIS_STEP_RETRIES" +_DEFAULT_STEP_RETRIES = 3 +#: Optional JSONL sink recording every retry and its outcome. The schema matches +#: `swe_bench_distributed.infra_retry.RetryRecord`, which reads it back to +#: publish infra_retries_total / instances_saved_by_retry / run_quality. The two +#: sides cannot share code: this is an isolated subproject that must not import +#: the benchmark client, so they share a file format instead. +_STEP_RETRY_LOG_ENV = "SWEBENCH_PYXIS_INFRA_RETRY_LOG" +_RETRY_LOG_LOCK = threading.Lock() + + +def _step_retry_attempts() -> int: + raw = os.environ.get(_STEP_RETRIES_ENV, "").strip() + if not raw: + return _DEFAULT_STEP_RETRIES + try: + return max(1, int(raw)) + except ValueError: + logger.warning("ignoring non-numeric %s=%r", _STEP_RETRIES_ENV, raw) + return _DEFAULT_STEP_RETRIES + + +def _record_step_retry( + *, target: str, attempt: int, outcome: str, detail: str | None = None +) -> None: + path = os.environ.get(_STEP_RETRY_LOG_ENV) + if not path: + return + record = { + "target": target, + "attempt": attempt, + "outcome": outcome, + "detail": detail, + "at": time.time(), + } + try: + # Accounting must never be able to take a run down. + with _RETRY_LOG_LOCK, open(path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(record) + "\n") + except OSError: + logger.debug("could not append to the infra retry log", exc_info=True) + + +def run_srun_step(**kwargs: Any) -> subprocess.CompletedProcess[str]: + """Run one `srun` step, re-attempting only a *provable* non-launch. + + Retrying is a correctness decision, not a convenience: re-running a command + that may already have run can apply an edit twice, delete twice, or double a + test run, and none of those announce themselves. So the only failure retried + here is :class:`StepNotLaunched` with ``provable_non_execution`` -- the status + file still ``pending`` and no in-band sentinel, meaning the step script did + not execute even its first line. Every other failure, including a + ``StepNotLaunched`` that reached ``started``, is raised immediately. + + Measured signature, from an isolated probe with no model and no GPU (20 + nodes, 200 workers, 6273 ordinary shell steps): 63 steps failed and in all 63 + the status file still read ``pending``. + + Every attempt and outcome is appended to ``SWEBENCH_PYXIS_INFRA_RETRY_LOG`` + when set. A retry loop that quietly absorbs the defect it compensates for + turns a broken cluster into an invisible one. + """ + attempts = _step_retry_attempts() + target = str(kwargs.get("name") or kwargs.get("image") or "pyxis-step") + for attempt in range(1, attempts + 1): + try: + result = _run_srun_step_once(**kwargs) + except StepNotLaunched as exc: + if not exc.provable_non_execution: + # The command may have run. Another attempt could double it. + _record_step_retry( + target=target, + attempt=attempt, + outcome="not_retryable", + detail=f"srun_rc={exc.srun_rc} status={exc.status!r}", + ) + raise + outcome = "exhausted" if attempt == attempts else "retrying" + _record_step_retry( + target=target, + attempt=attempt, + outcome=outcome, + detail=f"srun_rc={exc.srun_rc} status={exc.status!r}", + ) + if attempt == attempts: + raise + logger.warning( + "Pyxis step provably never launched (attempt %d/%d, srun rc=%s, " + "status=%r); retrying", + attempt, + attempts, + exc.srun_rc, + exc.status, + ) + time.sleep(min(30.0, 2.0 * attempt)) + continue + if attempt > 1: + _record_step_retry(target=target, attempt=attempt, outcome="recovered") + return result + raise AssertionError("unreachable") # pragma: no cover + + def resolve_image(image_registry: str, instance_id: str) -> str: if Path(instance_id).name != instance_id or instance_id in {".", ".."}: raise RunnerError(f"invalid SWE-bench instance ID: {instance_id}") diff --git a/tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py b/tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py index 9b803fe35..a34471ca0 100644 --- a/tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py +++ b/tests/unit/evaluation/swe_bench_distributed/test_infra_retry.py @@ -241,3 +241,52 @@ def test_an_exhaustion_is_degraded_regardless_of_volume(self): ) assert ledger.run_quality() is RunQuality.DEGRADED + + +class TestReadingBackAWrittenLedger: + """The SWE-bench service writes this shape from another process. + + It is an isolated subproject and cannot import this package, so the two + halves share a file format. If that agreement breaks, per-step retries stop + reaching the run-level `run_quality` and the run looks clean. + """ + + def test_a_written_ledger_round_trips(self, tmp_path): + path = tmp_path / "infra_retries.jsonl" + source = InfraRetryLedger(path) + source.record(target="unit-1", attempt=1, outcome=RetryOutcome.RETRYING) + source.record(target="unit-1", attempt=2, outcome=RetryOutcome.RECOVERED) + + loaded = InfraRetryLedger.from_jsonl(path) + + assert [r.outcome for r in loaded.records] == [ + RetryOutcome.RETRYING, + RetryOutcome.RECOVERED, + ] + assert loaded.summary()["instances_saved_by_retry"] == 1 + + def test_a_truncated_final_line_does_not_lose_the_history(self, tmp_path): + path = tmp_path / "infra_retries.jsonl" + path.write_text( + '{"target": "u", "attempt": 1, "outcome": "retrying", "at": 1.0}\n' + '{"target": "u", "attempt": 2, "outcome": "recov' + ) + + loaded = InfraRetryLedger.from_jsonl(path) + + assert len(loaded.records) == 1 + + def test_a_missing_ledger_is_an_empty_clean_one(self, tmp_path): + loaded = InfraRetryLedger.from_jsonl(tmp_path / "never-written.jsonl") + + assert loaded.records == [] + assert loaded.run_quality() is RunQuality.CLEAN + + def test_an_exhaustion_written_elsewhere_still_degrades_the_run(self, tmp_path): + path = tmp_path / "infra_retries.jsonl" + path.write_text( + '{"target": "u", "attempt": 1, "outcome": "retrying", "at": 1.0}\n' + '{"target": "u", "attempt": 2, "outcome": "exhausted", "at": 2.0}\n' + ) + + assert InfraRetryLedger.from_jsonl(path).run_quality() is RunQuality.DEGRADED diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index c04549cce..b82cb0d7f 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -7,6 +7,7 @@ import subprocess import sys import threading +import time import types from pathlib import Path from typing import Literal, get_type_hints @@ -14,6 +15,7 @@ import msgspec.json import pytest import yaml + from inference_endpoint.evaluation.swebench_service.swebench_service import ( pyxis_worker as worker_mod, ) @@ -44,6 +46,17 @@ pytestmark = pytest.mark.unit +@pytest.fixture(autouse=True) +def _single_step_attempt(monkeypatch): + """Most tests assert single-shot step behaviour. + + The step runner re-attempts a *provable* non-launch, so without this every + such test would run its fake three times and sleep between them. Retry + behaviour has its own tests, which opt back in explicitly. + """ + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "1") + + def test_pyxis_implementation_is_confined_to_environment_and_worker_modules(): package_dir = Path(runner_mod.__file__).parent @@ -1284,6 +1297,130 @@ def fake_run(command, **kwargs): assert repr(status.strip()) in str(failure) +class TestStepRetry: + """Re-attempt only a provable non-launch, and count every attempt. + + Re-running a command that may already have run can apply an edit twice, + delete twice, or double a test run. So the gate is not "an error happened". + """ + + @pytest.fixture(autouse=True) + def _fast(self, monkeypatch): + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + + def _environment(self, tmp_path): + return _bare_environment(tmp_path) + + def test_a_provable_non_launch_is_retried(self, monkeypatch, tmp_path): + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "3") + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if len(calls) < 3: + # Status file untouched: still "pending". + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + output = self._environment(tmp_path).execute({"command": "pytest -q"}) + + assert output["returncode"] == 0 + assert len(calls) == 3 + + def test_a_step_that_started_is_never_retried(self, monkeypatch, tmp_path): + """It may have executed. Another attempt could double-apply it.""" + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "5") + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + (tmp_path / ".mlperf_srun_status").write_text("started\n") + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(StepNotLaunched): + self._environment(tmp_path).execute({"command": "rm -rf build"}) + + assert len(calls) == 1 + + def test_the_attempt_budget_is_bounded(self, monkeypatch, tmp_path): + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "4") + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(StepNotLaunched): + self._environment(tmp_path).execute({"command": "pytest -q"}) + + assert len(calls) == 4 + + def test_every_attempt_is_recorded(self, monkeypatch, tmp_path): + log = tmp_path / "infra_retries.jsonl" + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "3") + monkeypatch.setenv("SWEBENCH_PYXIS_INFRA_RETRY_LOG", str(log)) + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if len(calls) < 2: + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + self._environment(tmp_path).execute({"command": "pytest -q"}) + + rows = [json.loads(line) for line in log.read_text().splitlines()] + assert [row["outcome"] for row in rows] == ["retrying", "recovered"] + + def test_an_exhausted_step_is_recorded_as_exhausted(self, monkeypatch, tmp_path): + log = tmp_path / "infra_retries.jsonl" + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "2") + monkeypatch.setenv("SWEBENCH_PYXIS_INFRA_RETRY_LOG", str(log)) + monkeypatch.setattr( + subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess( + command, 1, stdout="", stderr="" + ), + ) + + with pytest.raises(StepNotLaunched): + self._environment(tmp_path).execute({"command": "pytest -q"}) + + rows = [json.loads(line) for line in log.read_text().splitlines()] + assert [row["outcome"] for row in rows] == ["retrying", "exhausted"] + + def test_accounting_never_takes_the_step_down(self, monkeypatch, tmp_path): + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "3") + monkeypatch.setenv( + "SWEBENCH_PYXIS_INFRA_RETRY_LOG", str(tmp_path / "nope" / "retries.jsonl") + ) + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if len(calls) < 2: + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert self._environment(tmp_path).execute({"command": "x"})["returncode"] == 0 + + def test_step_not_launched_is_a_runner_error(): """Existing ``except RunnerError`` handlers must keep working unchanged.""" assert issubclass(StepNotLaunched, RunnerError) From fd3ea461588562b67bfd177638a878030c0fb8fa Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 12:53:46 -0700 Subject: [PATCH 13/25] fix(swebench-service): score the predictions a failed agent phase left behind One worker's infrastructure failure discarded the entire eval phase. The agent phase runs `--workers` trajectories concurrently. `_run_agent` re-raises whatever the phase raised, so a single worker that could not start its container -- or whose `srun` step never launched -- took the exception all the way out of `_run()`. That happens *before* `preds.json` is ever looked at, so the predictions every other worker had already written were never scored. Observed: a 200-instance run with 137 predictions on disk reported as a total loss, exit non-zero, no accuracy number, and had to be re-scored by hand from the retained artifacts. The GPU allocation that produced those 137 predictions was gone by then. Eval is now robust to individual worker failure: whatever predictions exist are always scored. The failure is not hidden -- * it is written to the new `agent_phase_error.txt` run artifact, with secrets redacted, and served through the existing artifact route; * it is logged at ERROR; * it is chained as `__cause__` onto the `preds.json` failure when the phase genuinely produced nothing, so an empty run still fails loudly. `RunCancelled` is explicitly not tolerated: a cancelled run is not a degraded run and must not proceed to eval. Tests: `test_run_scores_predictions_left_behind_by_a_failed_agent_phase` (the eval phase runs and the artifact is written), `test_run_still_fails_when_the_agent_phase_produced_nothing` (no false pass, cause chained), `test_run_redacts_secrets_from_the_agent_phase_error`, `test_run_does_not_tolerate_cancellation` and `test_agent_phase_error_is_a_retrievable_artifact`. The existing cleanup-after-failure test asserted that the agent error propagated verbatim and is updated to assert the chained failure instead. --- .../evaluation/swe_bench_scorer.py | 1 + .../evaluation/swebench_service/README.md | 16 ++ .../swebench_service/artifacts.py | 1 + .../swebench_service/runner.py | 59 +++++++- .../swebench_service/test_runner.py | 142 +++++++++++++++++- 5 files changed, 211 insertions(+), 8 deletions(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_scorer.py b/src/inference_endpoint/evaluation/swe_bench_scorer.py index a91d63734..98a76d2a1 100644 --- a/src/inference_endpoint/evaluation/swe_bench_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_scorer.py @@ -67,6 +67,7 @@ class SWEBenchScorer(Scorer, scorer_id="swe_bench_scorer"): "artifacts.download", } SAFE_ARTIFACT_NAMES: ClassVar[set[str]] = { + "agent_phase_error.txt", "preds.json", "swe_bench_agent.log", "swe_bench_eval.log", diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index 9a11f316e..c815e64cc 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -109,6 +109,22 @@ allocation. `scancel` does not reclaim them either -- ending the job does not remove Enroot containers -- which is why the removal has to be both correctly named and audible. +### Agent-phase failures + +The agent phase runs `--workers` trajectories concurrently. If it fails after +some workers have already written their patches, the service still evaluates the +predictions that reached `preds.json` rather than discarding them: the failure is +recorded in the `agent_phase_error.txt` artifact and logged at ERROR, and the run +continues into the eval phase. Instances with no prediction are simply absent from +the results, which the harness reports as unresolved. + +A run whose agent phase produced no predictions at all still fails, with the agent +error chained as the cause. Cancellation is never tolerated this way: it +propagates immediately and no eval phase runs. + +Consumers that need to know a run was degraded should fetch `agent_phase_error.txt`; +its presence means some instances may be missing from `swe_bench_results.json`. + The benchmark client submits a run to this service only in `ACC` or `BOTH` mode; the default `PERF` mode skips external evaluation. diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py index fd207dc7c..d24d1ee8d 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py @@ -42,6 +42,7 @@ re.compile(r"(://[^:/\s]+:)[^@\s/]+(@)"), ) SAFE_ARTIFACT_NAMES = { + "agent_phase_error.txt", "preds.json", "swe_bench_agent.log", "swe_bench_eval.log", diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 8682b40a7..e54167b2a 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -84,6 +84,8 @@ def detach(self, process: subprocess.Popen[str]) -> None: _LOG_TAIL_MAX_BYTES = 64 * 1024 _LOG_TAIL_MAX_LINES = 50 _RUN_LABEL = "com.mlcommons.endpoints.swebench-run" +#: Written when the agent phase failed but predictions were still scored. +AGENT_PHASE_ERROR_FILE = "agent_phase_error.txt" _PROCESS_TERMINATE_TIMEOUT_S = 10 _SWEBENCH_DATASETS = { "verified": "princeton-nlp/SWE-bench_Verified", @@ -298,7 +300,7 @@ def _run( request, run_id=run_dir.name, ) - self._run_agent( + agent_error = self._run_agent_tolerantly( request, patched_config, output_dir, @@ -309,7 +311,10 @@ def _run( preds_path = output_dir / "preds.json" if not preds_path.exists(): - raise RunnerError("mini-extra did not produce preds.json") + # A genuinely empty agent phase still fails loudly: there is + # nothing to score. The agent error, if any, is the cause. + error = RunnerError("mini-extra did not produce preds.json") + raise error from agent_error self._validate_prediction_ids(request, preds_path) shutil.copy2(preds_path, run_dir / "preds.json") @@ -319,6 +324,56 @@ def _run( shutil.copy2(result_path, run_dir / "swe_bench_results.json") return msgspec.json.decode(result_path.read_bytes(), type=dict) + def _run_agent_tolerantly( + self, + request: RunRequest, + patched_config: Path, + output_dir: Path, + run_dir: Path, + secret_values: set[str], + cancel_token: CancellationToken | None, + ) -> BaseException | None: + """Run the agent phase; record a failure instead of discarding the run. + + The agent phase fans out across many concurrent workers, and a single + worker's infrastructure failure (a container that would not start, an + `srun` step that never launched) propagates out of the whole phase. The + predictions every *other* worker already wrote are on disk and are + perfectly scoreable, but the exception reached ``run()`` before + ``preds.json`` was ever looked at, so the entire eval phase was skipped + and a run with 137 of 200 predictions reported as a total loss. + + The failure is not hidden. It is written to ``agent_phase_error.txt``, + served as a run artifact, logged at ERROR, and chained onto the + ``preds.json`` failure when the phase really did produce nothing. + Cancellation is not a failure to tolerate: it propagates unchanged. + """ + try: + self._run_agent( + request, + patched_config, + output_dir, + run_dir, + secret_values, + cancel_token, + ) + except RunCancelled: + raise + except Exception as exc: + atomic_write_bytes( + run_dir / AGENT_PHASE_ERROR_FILE, + redact_text(f"{type(exc).__name__}: {exc}\n", secret_values).encode(), + ) + logger.error( + "SWE-bench agent phase failed for run %s; continuing to eval so " + "the predictions already on disk are still scored: %s", + run_dir.name, + exc, + exc_info=True, + ) + return exc + return None + def _load_template(self, request: RunRequest) -> dict[str, Any]: template_path = self._template_dir / TEMPLATE_FILES[request.template] with template_path.open() as f: diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index b82cb0d7f..45ea87743 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -16,6 +16,9 @@ import pytest import yaml +from inference_endpoint.evaluation.swebench_service.swebench_service import ( + artifacts as artifacts_mod, +) from inference_endpoint.evaluation.swebench_service.swebench_service import ( pyxis_worker as worker_mod, ) @@ -435,15 +438,18 @@ def test_run_cleans_labeled_containers_after_success(monkeypatch, tmp_path): @pytest.mark.parametrize( - ("error", "match"), + ("error", "raised", "match"), [ - (RuntimeError("agent failed"), "agent failed"), - (RunnerError("subprocess timed out"), "timed out"), - (RunCancelled("subprocess cancelled"), "cancelled"), + # A non-cancellation agent failure that leaves no prediction behind + # surfaces as the empty-predictions failure, with the agent error + # chained as its cause. + (RuntimeError("agent failed"), RunnerError, "did not produce preds.json"), + (RunnerError("subprocess timed out"), RunnerError, "did not produce preds"), + (RunCancelled("subprocess cancelled"), RunCancelled, "cancelled"), ], ) def test_run_cleans_labeled_containers_after_failure( - monkeypatch, tmp_path, error, match + monkeypatch, tmp_path, error, raised, match ): runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) cleaned: list[str] = [] @@ -454,12 +460,136 @@ def fail_agent(*args, **kwargs): monkeypatch.setattr(runner, "_run_agent", fail_agent) monkeypatch.setattr(runner, "_cleanup_containers", cleaned.append) - with pytest.raises(type(error), match=match): + with pytest.raises(raised, match=match) as exc_info: runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-2") + if raised is not RunCancelled: + assert exc_info.value.__cause__ is error assert cleaned == ["run-2"] +def test_run_scores_predictions_left_behind_by_a_failed_agent_phase( + monkeypatch, tmp_path +): + """One worker's infrastructure failure must not discard the eval phase. + + The agent phase fans out across many workers. When one of them dies the + exception propagates out of the whole phase, but every prediction the other + workers wrote is already on disk. Before this fix a run with predictions for + most of its instances was reported as a total loss and never scored at all. + """ + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + run_dir = tmp_path / "run-partial" + scored: list[Path] = [] + + def partial_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + raise RunnerError("Pyxis infrastructure failure before the command completed") + + def fake_run_eval( + request, preds_path, output_dir, run_dir, secret_values, cancel_token=None + ): + scored.append(preds_path) + result_path = output_dir / "result.json" + result_path.write_text('{"resolved_instances":1,"submitted_instances":1}') + return result_path + + monkeypatch.setattr(runner, "_run_agent", partial_agent) + monkeypatch.setattr(runner, "_run_eval", fake_run_eval) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + result = runner.run(_request(["http://endpoint:30000"]), run_dir) + + assert result == {"resolved_instances": 1, "submitted_instances": 1} + assert scored, "eval phase never ran" + error_text = (run_dir / "agent_phase_error.txt").read_text() + assert "RunnerError" in error_text + assert "Pyxis infrastructure failure" in error_text + + +def test_agent_phase_error_is_a_retrievable_artifact(): + assert "agent_phase_error.txt" in artifacts_mod.SAFE_ARTIFACT_NAMES + + +def test_run_redacts_secrets_from_the_agent_phase_error(monkeypatch, tmp_path): + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + run_dir = tmp_path / "run-secret" + request = _request(["http://endpoint:30000"]) + request.endpoint_api_key = "real-secret" + + def leaky_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + raise RunnerError("connection to http://endpoint:30000 with real-secret failed") + + def fake_run_eval( + request, preds_path, output_dir, run_dir, secret_values, cancel_token=None + ): + result_path = output_dir / "result.json" + result_path.write_text("{}") + return result_path + + monkeypatch.setattr(runner, "_run_agent", leaky_agent) + monkeypatch.setattr(runner, "_run_eval", fake_run_eval) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + runner.run(request, run_dir) + + error_text = (run_dir / "agent_phase_error.txt").read_text() + assert "real-secret" not in error_text + assert "" in error_text + + +def test_run_still_fails_when_the_agent_phase_produced_nothing(monkeypatch, tmp_path): + """Tolerating the failure must not turn an empty run into a pass.""" + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + cause = RunnerError("every worker died") + + def dead_agent(*args, **kwargs): + raise cause + + monkeypatch.setattr(runner, "_run_agent", dead_agent) + monkeypatch.setattr( + runner, + "_run_eval", + lambda *a, **k: pytest.fail("eval must not run without predictions"), + ) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + with pytest.raises(RunnerError, match="did not produce preds.json") as exc_info: + runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-empty") + + assert exc_info.value.__cause__ is cause + + +def test_run_does_not_tolerate_cancellation(monkeypatch, tmp_path): + """Cancellation is not a worker failure and must propagate unchanged.""" + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + run_dir = tmp_path / "run-cancelled" + + def cancelled_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + raise RunCancelled("subprocess cancelled") + + monkeypatch.setattr(runner, "_run_agent", cancelled_agent) + monkeypatch.setattr( + runner, + "_run_eval", + lambda *a, **k: pytest.fail("eval must not run after cancellation"), + ) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + with pytest.raises(RunCancelled): + runner.run(_request(["http://endpoint:30000"]), run_dir) + + assert not (run_dir / "agent_phase_error.txt").exists() + + def test_run_cleans_harness_containers_after_eval_cancellation(monkeypatch, tmp_path): runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) cleaned: list[tuple[str, dict]] = [] From c319426aa14c111f873835d066049c373cb11810 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:40:57 -0700 Subject: [PATCH 14/25] feat(swe-bench): eval-phase infra-vs-genuine error classifier The Pyxis sentinel only covers the agent phase; eval-phase error_ids were counted as real outcomes and never retried, which is what produced 24 of 25 permanently-bad runs on the source cluster. classify.py reads the SWE-bench report's error_ids and each instance's run_instance.log and classifies them through an ORDERED rule list, first match wins. The order is load-bearing: BuildImageError is checked before everything because its message embeds the other rules' needles, CONMON_EAGAIN and TEST_TIMEOUT precede WEDGE_EVAL, and PATCH_APPLY_FAILED is last. Anything unclassifiable is UNKNOWN and UNKNOWN is GENUINE, asserted by a membership test: a false bad-run costs one redo, a false retry biases the measurement toward optimism. Memory-kill markers are consumed by phase - an eval-phase kill is a genuine failure (an unbounded allocation is a failing patch), an agent-phase kill is recorded for audit only, since the agent merely gets an error observation and the instance still reaches a real outcome. --- .../swe_bench_distributed/__init__.py | 14 ++ .../swe_bench_distributed/classify.py | 234 ++++++++++++++++++ .../swe_bench_distributed/test_classify.py | 171 +++++++++++++ 3 files changed, 419 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/classify.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_classify.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 7269ab520..8e2ae103e 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,6 +11,14 @@ exactly once. """ +from .classify import ( + GENUINE_KINDS, + INFRA_KINDS, + ErrorKind, + UnitClassification, + classify_eval_log, + classify_unit, +) from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .infra_retry import ( InfraRetryLedger, @@ -38,8 +46,11 @@ from .units import Unit, UnitPlan, plan_units __all__ = [ + "GENUINE_KINDS", + "INFRA_KINDS", "ClaimError", "CompletenessReport", + "ErrorKind", "HealthTerm", "HealthVerdict", "InfraRetryLedger", @@ -53,11 +64,14 @@ "RunQuality", "SlurmStepLiveness", "Unit", + "UnitClassification", "UnitOutcome", "UnitPlan", "UnitResult", "WorkQueue", "assess_run", + "classify_eval_log", + "classify_unit", "combine_terms", "is_provable_non_execution", "kill_by_pid", diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py new file mode 100644 index 000000000..b7c06ac25 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Split a unit's error instances into infrastructure damage and genuine failures. + +A SWE-bench run reports three per-instance outcomes: resolved, unresolved, and +*error*. The agent phase has an infrastructure sentinel (the Pyxis +``infrastructure_failure_path``), but the eval phase has none: an instance whose +evaluation container wedged is booked as ``error``, which counts as "accounted +for", so the unit is published successful and is never retried. Those instances +silently poison a run that can then never reach a full result. + +Classification exists to catch exactly that. It reads each error instance's +``run_instance.log`` and assigns one kind. + +BIAS RULE -- this is the whole design and it is deliberately asymmetric. If an +error cannot be classified confidently it is treated as GENUINE, never as +infrastructure. A false bad-run costs one redo; a false retry silently biases +the measurement toward optimism, and an optimistic accuracy number is worse than +no number. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path + +logger = logging.getLogger(__name__) + + +class ErrorKind(StrEnum): + """One classification of an error instance.""" + + # Infrastructure: a defect in the runtime we provided, safe to retry. + CONTAINER_EXEC_REFUSED = "container_exec_refused" + CONTAINER_FORK_EAGAIN = "container_fork_eagain" + RUNTIME_READ_TIMEOUT = "runtime_read_timeout" + IMAGE_BUILD_TIMEOUT = "image_build_timeout" + IMAGE_BUILD_ERROR = "image_build_error" + STEP_INFRASTRUCTURE_FAILURE = "step_infrastructure_failure" + ENDPOINT_CHANGED = "endpoint_changed" + + # Genuine: a real outcome of the model's patch, or unreadable. Never retried. + TEST_TIMEOUT = "test_timeout" + TEST_MEMORY_EXCEEDED = "test_memory_exceeded" + PATCH_APPLY_FAILED = "patch_apply_failed" + UNKNOWN = "unknown" + + +#: Retryable. Every member is a defect in infrastructure we control. +INFRA_KINDS: frozenset[ErrorKind] = frozenset( + { + ErrorKind.CONTAINER_EXEC_REFUSED, + ErrorKind.CONTAINER_FORK_EAGAIN, + ErrorKind.RUNTIME_READ_TIMEOUT, + ErrorKind.IMAGE_BUILD_TIMEOUT, + ErrorKind.IMAGE_BUILD_ERROR, + ErrorKind.STEP_INFRASTRUCTURE_FAILURE, + ErrorKind.ENDPOINT_CHANGED, + } +) + +#: Never retried. +#: +#: ``TEST_TIMEOUT`` is a plausible model outcome: a patch that makes the suite +#: loop is a failing patch. ``TEST_MEMORY_EXCEEDED`` is its exact parallel -- a +#: patch that makes a graded test allocate without bound is a failing patch, and +#: the alternative to killing it was never "the test passes", it was "the host +#: OOMs and the instance still never completes". ``PATCH_APPLY_FAILED`` is the +#: model emitting a diff that does not apply; SWE-bench books it as ``error`` +#: rather than ``unresolved``, but it is model behaviour. ``UNKNOWN`` is the bias +#: rule. +GENUINE_KINDS: frozenset[ErrorKind] = frozenset( + { + ErrorKind.TEST_TIMEOUT, + ErrorKind.TEST_MEMORY_EXCEEDED, + ErrorKind.PATCH_APPLY_FAILED, + ErrorKind.UNKNOWN, + } +) + +# ORDERED. First match wins, and the order is load-bearing. +# +# CONTAINER_FORK_EAGAIN and TEST_TIMEOUT come BEFORE CONTAINER_EXEC_REFUSED: a +# timed-out or fork-failed evaluation frequently *also* emits "container state +# improper" while the harness tears the container down, and reading that as a +# wedge would retry a genuine model outcome. +# +# PATCH_APPLY_FAILED is checked LAST: if a container also wedged, the wedge +# wins, because a wedged container's verdict is unreliable either way. Only a +# log with no infrastructure signature at all reaches this rule. +_RULES: tuple[tuple[ErrorKind, tuple[str, ...]], ...] = ( + ( + ErrorKind.CONTAINER_FORK_EAGAIN, + ("fork/exec /usr/bin/conmon: resource temporarily unavailable",), + ), + (ErrorKind.TEST_TIMEOUT, ("Test timed out after",)), + ( + ErrorKind.CONTAINER_EXEC_REFUSED, + ( + "can only create exec sessions on running containers", + "container state improper", + ), + ), + (ErrorKind.RUNTIME_READ_TIMEOUT, ("Read timed out. (read timeout=",)), + ( + ErrorKind.PATCH_APPLY_FAILED, + ( + "Reversed (or previously applied) patch detected", + ">>>>> Patch Apply Failed", + "hunk FAILED", + "hunk failed", + ), + ), +) + + +def classify_eval_log(text: str) -> ErrorKind: + """Classify one instance's evaluation log. + + ``BuildImageError`` is checked before the ordered rules because its message + embeds the same "Read timed out" / "500" strings the other rules look for, + so any other order misattributes a build failure. + """ + if "BuildImageError" in text: + if "Read timed out" in text: + return ErrorKind.IMAGE_BUILD_TIMEOUT + return ErrorKind.IMAGE_BUILD_ERROR + for kind, needles in _RULES: + if any(needle in text for needle in needles): + return kind + return ErrorKind.UNKNOWN + + +@dataclass(slots=True) +class UnitClassification: + """Per-kind counts for one unit's error instances.""" + + kinds: dict[ErrorKind, int] = field(default_factory=dict) + error_instance_ids: tuple[str, ...] = () + #: False when the run's report could not be read at all. "Not measured" and + #: "measured zero" are different, and conflating them once let a damaged + #: unit into a clean set. + measured: bool = False + + @property + def infra_count(self) -> int: + return sum(count for kind, count in self.kinds.items() if kind in INFRA_KINDS) + + @property + def genuine_count(self) -> int: + return sum(count for kind, count in self.kinds.items() if kind in GENUINE_KINDS) + + @property + def should_retry(self) -> bool: + return self.infra_count > 0 + + def as_counts(self) -> dict[str, int]: + return {kind.value: count for kind, count in sorted(self.kinds.items())} + + +def _find_instance_log(output_dir: Path, instance_id: str) -> Path | None: + patterns = ( + f"logs/run_evaluation/*/*/{instance_id}/run_instance.log", + f"logs/run_evaluation/*/*/*/{instance_id}/run_instance.log", + ) + for pattern in patterns: + for match in sorted(output_dir.glob(pattern)): + return match + return None + + +def memory_kill_markers(killed_dir: Path, instance_id: str) -> bool: + """True only for an *eval*-phase memory kill. + + Phase is load-bearing and the two cases must never be collapsed. An eval + kill destroyed a graded result, so the instance's error is a genuine + failure. An agent kill merely makes one tool call return an error + observation and the agent carries on, so the instance still reaches a real + outcome; that marker exists for audit and must not influence classification. + + A marker beats any log heuristic: a SIGKILLed test leaves an ambiguous log, + but the kill itself is a fact recorded before acting. + """ + return any(killed_dir.glob(f"eval.{instance_id}.*.json")) + + +def classify_unit( + output_dir: Path, + error_instance_ids: list[str] | tuple[str, ...] | None, + *, + killed_dir: Path | None = None, + infrastructure_failure: bool = False, + endpoint_changed: bool = False, +) -> UnitClassification: + """Classify every error instance of one unit. + + ``infrastructure_failure`` carries the Pyxis agent-phase sentinel, and + ``endpoint_changed`` carries a mismatch between the inference endpoint + fingerprint recorded at claim time and at publish time -- an engine + restarted under a live client produces a plausible-looking run that must not + be scored. + """ + kinds: dict[ErrorKind, int] = {} + + if infrastructure_failure: + kinds[ErrorKind.STEP_INFRASTRUCTURE_FAILURE] = ( + kinds.get(ErrorKind.STEP_INFRASTRUCTURE_FAILURE, 0) + 1 + ) + if endpoint_changed: + kinds[ErrorKind.ENDPOINT_CHANGED] = kinds.get(ErrorKind.ENDPOINT_CHANGED, 0) + 1 + + if error_instance_ids is None: + return UnitClassification(kinds=kinds, error_instance_ids=(), measured=False) + + ids = tuple(str(x) for x in error_instance_ids) + for instance_id in ids: + if killed_dir is not None and memory_kill_markers(killed_dir, instance_id): + kinds[ErrorKind.TEST_MEMORY_EXCEEDED] = ( + kinds.get(ErrorKind.TEST_MEMORY_EXCEEDED, 0) + 1 + ) + continue + log_path = _find_instance_log(output_dir, instance_id) + kind = ErrorKind.UNKNOWN + if log_path is not None: + try: + kind = classify_eval_log(log_path.read_text(errors="replace")) + except OSError: + logger.debug("could not read %s", log_path, exc_info=True) + kinds[kind] = kinds.get(kind, 0) + 1 + + return UnitClassification(kinds=kinds, error_instance_ids=ids, measured=True) diff --git a/tests/unit/evaluation/swe_bench_distributed/test_classify.py b/tests/unit/evaluation/swe_bench_distributed/test_classify.py new file mode 100644 index 000000000..2b079b315 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_classify.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Infrastructure-versus-genuine classification of error instances.""" + +from __future__ import annotations + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.classify import ( + GENUINE_KINDS, + INFRA_KINDS, + ErrorKind, + classify_eval_log, + classify_unit, +) + +pytestmark = pytest.mark.unit + + +def write_log(output_dir, instance_id: str, text: str) -> None: + log_dir = output_dir / "logs" / "run_evaluation" / "run-1" / "model" / instance_id + log_dir.mkdir(parents=True, exist_ok=True) + (log_dir / "run_instance.log").write_text(text) + + +class TestBiasRule: + def test_unknown_is_genuine_never_infra(self): + # A false bad-run costs one redo; a false retry biases the measurement + # toward optimism. Unclassifiable therefore means "keep the result". + assert ErrorKind.UNKNOWN in GENUINE_KINDS + assert ErrorKind.UNKNOWN not in INFRA_KINDS + + def test_kinds_are_partitioned(self): + assert not (INFRA_KINDS & GENUINE_KINDS) + assert INFRA_KINDS | GENUINE_KINDS == set(ErrorKind) + + def test_model_outcomes_are_genuine(self): + for kind in ( + ErrorKind.TEST_TIMEOUT, + ErrorKind.TEST_MEMORY_EXCEEDED, + ErrorKind.PATCH_APPLY_FAILED, + ): + assert kind in GENUINE_KINDS + + +class TestLogRules: + @pytest.mark.parametrize( + ("text", "expected"), + [ + ( + "fork/exec /usr/bin/conmon: resource temporarily unavailable", + ErrorKind.CONTAINER_FORK_EAGAIN, + ), + ("Test timed out after 1800s", ErrorKind.TEST_TIMEOUT), + ( + "can only create exec sessions on running containers", + ErrorKind.CONTAINER_EXEC_REFUSED, + ), + ("container state improper", ErrorKind.CONTAINER_EXEC_REFUSED), + ("Read timed out. (read timeout=60)", ErrorKind.RUNTIME_READ_TIMEOUT), + ( + "Reversed (or previously applied) patch detected", + ErrorKind.PATCH_APPLY_FAILED, + ), + ("1 out of 3 hunk FAILED", ErrorKind.PATCH_APPLY_FAILED), + ("nothing recognisable here", ErrorKind.UNKNOWN), + ], + ) + def test_each_rule(self, text, expected): + assert classify_eval_log(text) is expected + + def test_timeout_wins_over_teardown_noise(self): + # A timed-out evaluation also emits "container state improper" while the + # harness tears the container down. Reading that as a wedge would retry + # a genuine model outcome, so rule order is load-bearing. + text = "Test timed out after 1800s\ncontainer state improper\n" + assert classify_eval_log(text) is ErrorKind.TEST_TIMEOUT + + def test_fork_failure_wins_over_teardown_noise(self): + text = ( + "fork/exec /usr/bin/conmon: resource temporarily unavailable\n" + "can only create exec sessions on running containers\n" + ) + assert classify_eval_log(text) is ErrorKind.CONTAINER_FORK_EAGAIN + + def test_a_wedge_wins_over_patch_apply(self): + # A wedged container's verdict is unreliable either way, so the wedge + # decides and the unit is retried. + text = "container state improper\nhunk FAILED\n" + assert classify_eval_log(text) is ErrorKind.CONTAINER_EXEC_REFUSED + + def test_build_error_is_checked_before_every_other_rule(self): + # BuildImageError's message embeds the same needles the other rules look + # for, so any other ordering misattributes a build failure. + assert ( + classify_eval_log("BuildImageError: Read timed out. (read timeout=60)") + is ErrorKind.IMAGE_BUILD_TIMEOUT + ) + assert ( + classify_eval_log("BuildImageError: 500 Server Error") + is ErrorKind.IMAGE_BUILD_ERROR + ) + + +class TestClassifyUnit: + def test_infra_and_genuine_are_counted_separately(self, tmp_path): + write_log(tmp_path, "a-1", "container state improper") + write_log(tmp_path, "a-2", "Test timed out after 1800s") + + classification = classify_unit(tmp_path, ["a-1", "a-2"]) + + assert classification.infra_count == 1 + assert classification.genuine_count == 1 + assert classification.should_retry + + def test_only_genuine_errors_do_not_trigger_a_retry(self, tmp_path): + write_log(tmp_path, "a-1", "Test timed out after 1800s") + assert not classify_unit(tmp_path, ["a-1"]).should_retry + + def test_a_missing_log_is_unknown_and_therefore_genuine(self, tmp_path): + classification = classify_unit(tmp_path, ["absent"]) + assert classification.kinds == {ErrorKind.UNKNOWN: 1} + assert not classification.should_retry + + def test_none_error_ids_means_not_measured(self, tmp_path): + # "We did not measure" and "we measured zero" are different; conflating + # them once let a damaged unit into a clean set. + classification = classify_unit(tmp_path, None) + assert classification.measured is False + + def test_empty_error_ids_means_measured_zero(self, tmp_path): + classification = classify_unit(tmp_path, []) + assert classification.measured is True + assert classification.infra_count == 0 + + def test_eval_memory_kill_marker_is_genuine(self, tmp_path): + killed = tmp_path / "killed" + killed.mkdir() + (killed / "eval.a-1.host.999.json").write_text("{}") + write_log(tmp_path, "a-1", "container state improper") + + classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed) + + # The marker beats the log: a SIGKILLed test leaves an ambiguous log, + # but the kill is a fact recorded before acting. + assert classification.kinds == {ErrorKind.TEST_MEMORY_EXCEEDED: 1} + assert not classification.should_retry + + def test_agent_phase_kill_marker_does_not_classify(self, tmp_path): + killed = tmp_path / "killed" + killed.mkdir() + (killed / "agent.a-1.host.999.json").write_text("{}") + write_log(tmp_path, "a-1", "Test timed out after 1800s") + + # An agent kill only makes one tool call return an error observation; + # the instance still reaches a real outcome, so the marker is audit-only. + classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed) + assert classification.kinds == {ErrorKind.TEST_TIMEOUT: 1} + + def test_step_infrastructure_failure_forces_a_retry(self, tmp_path): + classification = classify_unit(tmp_path, [], infrastructure_failure=True) + assert classification.should_retry + assert classification.kinds == {ErrorKind.STEP_INFRASTRUCTURE_FAILURE: 1} + + def test_a_changed_endpoint_forces_a_retry(self, tmp_path): + # An engine restarted under a live client produces a plausible run that + # scores near zero and exits successfully. + classification = classify_unit(tmp_path, [], endpoint_changed=True) + assert classification.should_retry + assert classification.kinds == {ErrorKind.ENDPOINT_CHANGED: 1} From 1f42e6fcd19bc096f7122b14d420a00107db96a5 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 14:15:27 -0700 Subject: [PATCH 15/25] fix(swebench-service): a failed eval container must not discard the whole report The same defect as the agent phase, one phase later. `pyxis_worker` grades each prediction in its own container concurrently, collected the per-instance failures, and then raised `RunnerError` *before* `make_run_report()`. So one wedged evaluation container threw away every other instance's grade -- the work was done, the reports were on disk, and nothing was ever written. The eval phase is now robust to individual instance failure. Whatever was graded is reported: an instance with no `report.json` is counted as an error by `make_run_report`, which is the correct and visible outcome, and is exactly what an operator needs to see. A run in which *no* instance could be evaluated still fails -- but only after the report has been written, so the run can be diagnosed from its own artifacts rather than from nothing. The losses are not hidden and, more importantly, not left as prose in a log: `eval_infra_failures.txt` lists `instance_iderror` per lost instance, is copied beside `swe_bench_results.json` and is served through the existing artifact route. That distinction is load bearing. An instance the harness dropped is not an instance the model failed, and a consumer that cannot separate them reads attrition as an accuracy regression -- which is precisely the misreport the completeness gate exists to prevent. Leaving the evidence only in a human-readable log would leave the gate unable to see it. Found while auditing which cluster-side runtime patches the package had made redundant: this one had not been, and it would have compromised the very run intended to validate the package. Tests: `test_pyxis_worker_reports_the_instances_one_bad_container_did_not_kill` (the report is produced, only the failed instance is listed), `test_pyxis_worker_still_fails_when_no_instance_could_be_evaluated` (no false pass, and the report is written first), `test_pyxis_worker_records_no_failure_file_for_a_clean_eval`, `test_eval_infra_failures_are_published_beside_the_results` and `test_eval_infra_failures_is_a_retrievable_artifact`. Four fail against the parent commit. The existing propagation test still holds: its single instance is also every instance. --- .../evaluation/swe_bench_scorer.py | 1 + .../evaluation/swebench_service/README.md | 27 ++- .../swebench_service/artifacts.py | 1 + .../swebench_service/pyxis_worker.py | 47 ++++- .../swebench_service/runner.py | 9 + .../swebench_service/test_runner.py | 173 ++++++++++++++++++ 6 files changed, 244 insertions(+), 14 deletions(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_scorer.py b/src/inference_endpoint/evaluation/swe_bench_scorer.py index 98a76d2a1..6cb1399ef 100644 --- a/src/inference_endpoint/evaluation/swe_bench_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_scorer.py @@ -68,6 +68,7 @@ class SWEBenchScorer(Scorer, scorer_id="swe_bench_scorer"): } SAFE_ARTIFACT_NAMES: ClassVar[set[str]] = { "agent_phase_error.txt", + "eval_infra_failures.txt", "preds.json", "swe_bench_agent.log", "swe_bench_eval.log", diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index c815e64cc..212d53aac 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -97,8 +97,9 @@ compute node. It mounts the patch, SWE-bench evaluation script, and output file the task image. It preserves SWE-bench 4.1.0's patch-application order, test timeout, captured output, and `get_eval_report` grading. A patch failure or test timeout is an unresolved task; an `srun`, Enroot, or container-start failure is an infrastructure -error that fails the run. The service then aggregates the per-instance reports and -removes its named Pyxis containers. +loss, recorded per instance and reported rather than allowed to fail the whole run +(see *Eval-phase failures*). The service then aggregates the per-instance reports +and removes its named Pyxis containers. Pyxis namespaces a named container by its allocation: `--container-name=X` inside job `N` is the Enroot container `pyxis_N_X`. `PyxisEnvironment.cleanup()` removes @@ -122,8 +123,26 @@ A run whose agent phase produced no predictions at all still fails, with the age error chained as the cause. Cancellation is never tolerated this way: it propagates immediately and no eval phase runs. -Consumers that need to know a run was degraded should fetch `agent_phase_error.txt`; -its presence means some instances may be missing from `swe_bench_results.json`. +### Eval-phase failures + +The same rule applies one phase later. The eval phase grades each prediction in +its own container, concurrently. If some of those containers fail, the rest are +still graded and the run report is still produced: an instance with no +`report.json` is counted as an error by the harness, which is the correct and +visible outcome. The instances that were lost are listed in the +`eval_infra_failures.txt` artifact, one `instance_iderror` per line. + +A run in which *no* instance could be evaluated still fails — but only after the +report has been written, so the run can be diagnosed from its own artifacts. + +### Telling a degraded run apart from a bad one + +Both artifacts are machine-readable on purpose. `agent_phase_error.txt` means +some instances may be missing from `swe_bench_results.json`; `eval_infra_failures.txt` +names the instances the harness lost during grading. Instances lost this way are +*infrastructure* losses, not model failures, and a consumer that cannot separate +the two will read attrition as an accuracy regression. Neither file exists on a +clean run. The benchmark client submits a run to this service only in `ACC` or `BOTH` mode; the default `PERF` mode skips external evaluation. diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py index d24d1ee8d..aa369c032 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py @@ -43,6 +43,7 @@ ) SAFE_ARTIFACT_NAMES = { "agent_phase_error.txt", + "eval_infra_failures.txt", "preds.json", "swe_bench_agent.log", "swe_bench_eval.log", diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py index efd6a6bf3..10472e3b4 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py @@ -15,7 +15,7 @@ from .artifacts import atomic_write_bytes from .pyxis_environment import resolve_image, run_srun_step -from .runner import RunnerError +from .runner import EVAL_INFRA_FAILURES_FILE, RunnerError _PRINT_LOCK = threading.Lock() _INFRASTRUCTURE_FAILURE = ".pyxis_infrastructure_failure" @@ -96,6 +96,18 @@ def get_pyxis_environment(config: dict, instance: dict): swebench.get_sb_environment = original_get_sb_environment +def _record_eval_failures(output_dir: Path, failures: dict[str, str]) -> None: + lines = "".join( + f"{instance_id}\t{detail}\n" for instance_id, detail in sorted(failures.items()) + ) + try: + atomic_write_bytes(output_dir / EVAL_INFRA_FAILURES_FILE, lines.encode()) + except OSError: + # Accounting must never be able to take the report down. + with _PRINT_LOCK: + print("could not record eval infrastructure failures", flush=True) + + def _evaluate_instance( *, test_spec: Any, @@ -212,19 +224,26 @@ def _run_eval(args: argparse.Namespace) -> None: ].instance_id for payload in payloads } - failures = [] + failures: dict[str, str] = {} for future in concurrent.futures.as_completed(futures): + instance_id = futures[future] try: future.result() - except Exception as exc: + except Exception as exc: # noqa: BLE001 -- one instance, not the run with _PRINT_LOCK: - print(f"Pyxis evaluation failed: {exc}", flush=True) - failures.append(futures[future]) - if failures: - raise RunnerError( - "Pyxis infrastructure failure evaluating: " - + ", ".join(sorted(failures)) - ) + print( + f"Pyxis evaluation failed for {instance_id} " + f"(non-fatal): {exc}", + flush=True, + ) + failures[instance_id] = f"{type(exc).__name__}: {exc}" + + # The report is produced even when some instances could not be evaluated. + # An instance with no report.json is counted as an error by make_run_report, + # which is the correct and visible outcome; raising here instead discarded + # every other instance's grade along with it. + if failures: + _record_eval_failures(args.output_dir, failures) output_dir = args.output_dir.resolve() with contextlib.chdir(output_dir): @@ -235,6 +254,14 @@ def _run_eval(args: argparse.Namespace) -> None: client=None, ) + if payloads and len(failures) == len(payloads): + # Nothing was evaluated at all. The report is on disk for diagnosis, + # but a report in which no instance ran is not a result. + raise RunnerError( + "Pyxis infrastructure failure evaluating every instance: " + + ", ".join(sorted(failures)[:10]) + ) + def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser() diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index e54167b2a..8dadb9a06 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -86,6 +86,8 @@ def detach(self, process: subprocess.Popen[str]) -> None: _RUN_LABEL = "com.mlcommons.endpoints.swebench-run" #: Written when the agent phase failed but predictions were still scored. AGENT_PHASE_ERROR_FILE = "agent_phase_error.txt" +#: Written when some instances could not be evaluated but the rest were scored. +EVAL_INFRA_FAILURES_FILE = "eval_infra_failures.txt" _PROCESS_TERMINATE_TIMEOUT_S = 10 _SWEBENCH_DATASETS = { "verified": "princeton-nlp/SWE-bench_Verified", @@ -321,6 +323,13 @@ def _run( result_path = self._run_eval( request, preds_path, output_dir, run_dir, secret_values, cancel_token ) + eval_failures = output_dir / EVAL_INFRA_FAILURES_FILE + if eval_failures.exists(): + # Instances the harness lost during eval. Published beside the + # results so a consumer can tell them apart from instances the model + # genuinely failed; without it a degraded run is indistinguishable + # from a bad one. + shutil.copy2(eval_failures, run_dir / EVAL_INFRA_FAILURES_FILE) shutil.copy2(result_path, run_dir / "swe_bench_results.json") return msgspec.json.decode(result_path.read_bytes(), type=dict) diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 45ea87743..176f644a5 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -2177,3 +2177,176 @@ def test_pyxis_worker_propagates_evaluation_infrastructure_failure( "repo__repo-1", ] ) + + +def _stub_swebench_eval(monkeypatch, predictions, *, make_run_report): + """Stand in for the SWE-bench harness modules the eval worker imports.""" + swebench = types.ModuleType("swebench") + harness = types.ModuleType("swebench.harness") + reporting = types.ModuleType("swebench.harness.reporting") + reporting.make_run_report = make_run_report + test_spec = types.ModuleType("swebench.harness.test_spec") + test_spec_module = types.ModuleType("swebench.harness.test_spec.test_spec") + test_spec_module.make_test_spec = lambda row, arch: types.SimpleNamespace( + instance_id=row["instance_id"], eval_script="pytest -q" + ) + utils = types.ModuleType("swebench.harness.utils") + utils.get_predictions_from_file = lambda *args: predictions.values() + utils.load_swebench_dataset = lambda *args: [ + {"instance_id": instance_id} for instance_id in predictions + ] + for name, module in ( + ("swebench", swebench), + ("swebench.harness", harness), + ("swebench.harness.reporting", reporting), + ("swebench.harness.test_spec", test_spec), + ("swebench.harness.test_spec.test_spec", test_spec_module), + ("swebench.harness.utils", utils), + ): + monkeypatch.setitem(sys.modules, name, module) + + +def _eval_argv(output_dir: Path, instance_ids: list[str]) -> list[str]: + return [ + "eval", + "--dataset-name", + "princeton-nlp/SWE-bench_Verified", + "--split", + "test", + "--predictions-path", + str(output_dir / "preds.json"), + "--max-workers", + "2", + "--run-id", + "endpoints_test", + "--image-registry", + _PYXIS_IMAGE_REGISTRY, + "--output-dir", + str(output_dir), + "--instance-ids", + *instance_ids, + ] + + +def _predictions(*instance_ids: str) -> dict[str, dict[str, str]]: + return { + instance_id: { + "model_name_or_path": "test-model", + "instance_id": instance_id, + "model_patch": "diff --git a/a b/a", + } + for instance_id in instance_ids + } + + +def test_pyxis_worker_reports_the_instances_one_bad_container_did_not_kill( + monkeypatch, tmp_path +): + """One failed eval container must not discard every other instance's grade. + + The per-instance failures were collected and then raised *before* + `make_run_report`, so a single wedged evaluation container threw away the + whole report -- the same defect as the agent phase, one phase later. + """ + output_dir = tmp_path / "output" + output_dir.mkdir() + predictions = _predictions("repo__repo-1", "repo__repo-2", "repo__repo-3") + reported: list[tuple] = [] + + def fake_make_run_report(predictions, dataset, run_id, client): + reported.append((predictions, dataset, run_id, client)) + + _stub_swebench_eval(monkeypatch, predictions, make_run_report=fake_make_run_report) + + def flaky(**kwargs): + if kwargs["test_spec"].instance_id == "repo__repo-2": + raise RunnerError("Pyxis infrastructure failure") + + monkeypatch.setattr(worker_mod, "_evaluate_instance", flaky) + + worker_mod.main(_eval_argv(output_dir, list(predictions))) + + assert len(reported) == 1, "the report was never produced" + failures = (output_dir / "eval_infra_failures.txt").read_text() + assert "repo__repo-2" in failures + assert "RunnerError" in failures + assert "repo__repo-1" not in failures + + +def test_pyxis_worker_still_fails_when_no_instance_could_be_evaluated( + monkeypatch, tmp_path +): + """Tolerating losses must not turn a total loss into a pass.""" + output_dir = tmp_path / "output" + output_dir.mkdir() + predictions = _predictions("repo__repo-1", "repo__repo-2") + reported: list[tuple] = [] + + _stub_swebench_eval( + monkeypatch, + predictions, + make_run_report=lambda *args, **kwargs: reported.append(args), + ) + monkeypatch.setattr( + worker_mod, + "_evaluate_instance", + lambda **kwargs: (_ for _ in ()).throw(RunnerError("no space left")), + ) + + with pytest.raises(RunnerError, match="every instance"): + worker_mod.main(_eval_argv(output_dir, list(predictions))) + + # The report is still written first, so the run can be diagnosed. + assert len(reported) == 1 + + +def test_pyxis_worker_records_no_failure_file_for_a_clean_eval(monkeypatch, tmp_path): + output_dir = tmp_path / "output" + output_dir.mkdir() + predictions = _predictions("repo__repo-1") + + _stub_swebench_eval( + monkeypatch, predictions, make_run_report=lambda *args, **kwargs: None + ) + monkeypatch.setattr(worker_mod, "_evaluate_instance", lambda **kwargs: None) + + worker_mod.main(_eval_argv(output_dir, list(predictions))) + + assert not (output_dir / "eval_infra_failures.txt").exists() + + +def test_eval_infra_failures_are_published_beside_the_results(monkeypatch, tmp_path): + """The losses have to reach the caller, not just the service host's disk.""" + runner = PyxisSweBenchRunner( + project_root=tmp_path, + subprocess_timeout_s=30, + image_registry=_PYXIS_IMAGE_REGISTRY, + ) + run_dir = tmp_path / "run-1" + + def fake_run_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + + def fake_run_eval( + request, preds_path, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "eval_infra_failures.txt").write_text( + "repo__repo-1\tRunnerError: no space left\n" + ) + result_path = output_dir / "result.json" + result_path.write_text("{}") + return result_path + + monkeypatch.setattr(runner, "_run_agent", fake_run_agent) + monkeypatch.setattr(runner, "_run_eval", fake_run_eval) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + runner.run(_request(["http://endpoint:30000"]), run_dir) + + assert "no space left" in (run_dir / "eval_infra_failures.txt").read_text() + + +def test_eval_infra_failures_is_a_retrievable_artifact(): + assert "eval_infra_failures.txt" in artifacts_mod.SAFE_ARTIFACT_NAMES From ce06df392dc73b5e88c77b9c7a193aa21732e1a8 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 12:49:27 -0700 Subject: [PATCH 16/25] fix(swebench-service): always give the agent a credential placeholder A run against a remote engine with no endpoint credential makes zero progress and never ends. `SweBenchRunner._base_env()` auto-filled `OPENAI_API_KEY="EMPTY"` only when the endpoint hostname was `localhost`, `127.0.0.1` or `::1`. For any other host with `endpoint_api_key` unset it did the opposite: it *removed* the variable. litellm then refuses to build the request locally -- litellm.AuthenticationError: Missing credentials -- mini-swe-agent classifies that as transient and retries it every 60s, forever. Not one request reaches the engine, nothing is logged at ERROR, the agent processes stay alive, and the run neither progresses nor terminates. Observed on a 20-node run against a remote GB300 engine: 200 workers, 0 requests served, no failure surfaced. The hostname gate is the defect. An unauthenticated OpenAI-compatible server ignores the credential value whether it is reached over loopback or over the network, so the placeholder is correct in both cases and the distinction only ever suppressed it where it was needed most. Replace the pop with the placeholder. The security property that motivated the pop is kept and made explicit: an ambient `OPENAI_API_KEY` inherited from the service host is still never forwarded to the endpoint -- it is overwritten rather than deleted. Tests: `test_base_env_always_supplies_a_credential_placeholder` covers loopback and remote hosts; the existing `test_base_env_supplies_api_key_only_to_agent_subprocess` encoded the old behaviour and is corrected to assert the placeholder while still proving a configured key wins and an ambient key never leaks. --- .../evaluation/swebench_service/README.md | 14 +++++++++ .../swebench_service/runner.py | 22 ++++++++----- .../swebench_service/test_runner.py | 31 ++++++++++++++++++- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index 212d53aac..ef747df0f 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -18,6 +18,20 @@ The endpoint URL in the benchmark config must be reachable from the service host Service mode supports exactly one endpoint URL and follows the LiveCodeBench-style external-service convention for heavyweight evaluation work. +### Endpoint credentials + +`accuracy_config.extras.swebench_service_auth_token` authenticates the *client to +this service*. The credential the agent presents to the *model endpoint* is +separate and comes from the run's endpoint configuration. + +When no endpoint credential is configured, the agent subprocess is given +`OPENAI_API_KEY=EMPTY`, which is what an unauthenticated OpenAI-compatible server +expects. An `OPENAI_API_KEY` inherited from the service host's environment is never +forwarded to the endpoint; it is replaced by the placeholder. The variable is +always set, regardless of whether the endpoint is on loopback or on another host, +because the client library refuses to issue a request with no credential at all and +retries that refusal indefinitely. + ## Runtime workflow ### Common workflow diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 8dadb9a06..7d5991a3e 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -558,17 +558,23 @@ def _base_env(self, request: RunRequest) -> dict[str, str]: no_proxy_value = ",".join(sorted(no_proxy)) env["NO_PROXY"] = no_proxy_value env["no_proxy"] = no_proxy_value - endpoint_host = ( - urlparse(str(request.endpoint_urls[0])).hostname - if request.endpoint_urls - else None - ) if request.endpoint_api_key: env["OPENAI_API_KEY"] = request.endpoint_api_key - elif endpoint_host in {"localhost", "127.0.0.1", "::1"}: - env["OPENAI_API_KEY"] = "EMPTY" else: - env.pop("OPENAI_API_KEY", None) + # No key was configured for this run. An ambient OPENAI_API_KEY + # inherited from the service host must never reach the endpoint, so + # it is replaced -- but it must be replaced, not removed. litellm + # refuses to build a request without a credential and raises + # `Missing credentials` locally; mini-swe-agent treats that as a + # transient error and retries every 60s indefinitely. The result is + # a run in which zero requests ever reach the engine, nothing is + # logged as an error, and the run never terminates. + # + # The placeholder is not host-dependent. An unauthenticated engine + # ignores the value whether it is on loopback or on another node, + # and gating the placeholder on the hostname is what left every + # remote-engine run hanging. + env["OPENAI_API_KEY"] = "EMPTY" return env def _cleanup_containers( diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 176f644a5..d1ce15f49 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -274,7 +274,36 @@ def test_base_env_supplies_api_key_only_to_agent_subprocess(monkeypatch, tmp_pat monkeypatch.setenv("OPENAI_API_KEY", "ambient-secret") unauthenticated = _request(["http://endpoint:30000"]) - assert "OPENAI_API_KEY" not in runner._base_env(unauthenticated) + assert runner._base_env(unauthenticated)["OPENAI_API_KEY"] == "EMPTY" + assert runner._base_env(authenticated)["OPENAI_API_KEY"] == "real-secret" + + +@pytest.mark.parametrize( + "endpoint", + [ + "http://localhost:30000", + "http://127.0.0.1:30000", + "http://[::1]:30000", + "http://swebench-host:30000", + "https://engine.example.com:8443", + ], +) +def test_base_env_always_supplies_a_credential_placeholder( + monkeypatch, tmp_path, endpoint +): + """A keyless run must never leave OPENAI_API_KEY unset. + + litellm raises ``Missing credentials`` before issuing anything and the agent + retries it forever, so a remote endpoint with no key made zero progress + while the run looked healthy. The placeholder must not depend on whether the + endpoint happens to be loopback. + """ + monkeypatch.setenv("OPENAI_API_KEY", "ambient-secret") + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + + env = runner._base_env(_request([endpoint])) + + assert env["OPENAI_API_KEY"] == "EMPTY" def test_run_agent_filters_exact_instance_ids(monkeypatch, tmp_path): From 1fa5ded523bcfe18795e7b92812a4528dbc8f646 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:41:31 -0700 Subject: [PATCH 17/25] feat(swe-bench): pre-dispatch gates that must prove their own scale run_gates() calls assert_scale() before check() and treats GateScaleError as a gate FAILURE, never a skip. This is the code-level form of the most expensive lesson available: a tool-call gate that exercised the right operation at a 278-token prompt passed, while prompts over 2k tokens silently returned empty, and the run scored 0/80. - CheckpointIdentityGate probes /get_model_info then /v1/models and compares the served model path with == , never startswith or in: the bf16 path is a strict prefix of the fp8 path, so any substring test passes an FP8 engine as bf16. Unidentifiable or ambiguous endpoints fail closed. - ToolCallGate requires a well-formed bash tool call at a prompt of at least min_prompt_tokens measured with the server's own /tokenize, not estimated from characters. No tokenizer means the gate cannot prove its scale, so it fails. - EndpointFingerprintGate records a per-endpoint identity the dispatcher re-checks at publish time, so an engine restarted under a live client cannot yield a 0%-accuracy run that still exits rc=0. --- .../swe_bench_distributed/__init__.py | 22 +- .../evaluation/swe_bench_distributed/gates.py | 423 ++++++++++++++++++ .../swe_bench_distributed/test_gates.py | 224 ++++++++++ 3 files changed, 667 insertions(+), 2 deletions(-) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/gates.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_gates.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 8e2ae103e..58310bf12 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -19,6 +19,16 @@ classify_eval_log, classify_unit, ) +from .gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + Gate, + GateFailure, + GateReport, + GateScaleError, + ToolCallGate, + run_gates, +) from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .infra_retry import ( InfraRetryLedger, @@ -46,13 +56,19 @@ from .units import Unit, UnitPlan, plan_units __all__ = [ - "GENUINE_KINDS", - "INFRA_KINDS", + "CheckpointIdentityGate", "ClaimError", "CompletenessReport", + "EndpointFingerprintGate", "ErrorKind", + "GENUINE_KINDS", + "Gate", + "GateFailure", + "GateReport", + "GateScaleError", "HealthTerm", "HealthVerdict", + "INFRA_KINDS", "InfraRetryLedger", "LocalProcessLiveness", "MemoryGuard", @@ -63,6 +79,7 @@ "RetryRecord", "RunQuality", "SlurmStepLiveness", + "ToolCallGate", "Unit", "UnitClassification", "UnitOutcome", @@ -79,5 +96,6 @@ "plan_units", "reap", "retry_on_provable_non_execution", + "run_gates", "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py new file mode 100644 index 000000000..9330fe012 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -0,0 +1,423 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates on the inference endpoints. + +A gate proves, before a single instance is dispatched, that the endpoints under +test can actually do the thing the benchmark requires. Gates fail closed: an +endpoint that cannot be identified or reached is a failure, never a pass. + +THE SCALE RULE. Every gate must first prove it is testing at the scale it +claims, via :meth:`Gate.assert_scale`, and a scale failure is a *gate failure*, +not a skip. This is not defensive programming; it is the most expensive lesson +in this codebase's history. A tool-call gate that exercised exactly the right +operation with a 278-token prompt passed cleanly while every prompt above 2000 +tokens silently returned an empty completion -- and SWE-bench prompts are all +far larger than 2000 tokens. The gate was green and the run scored zero. A gate +that cannot prove its scale is not a gate. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Protocol +from urllib import error as urllib_error +from urllib import request as urllib_request + +logger = logging.getLogger(__name__) + +_DEFAULT_TIMEOUT_S = 60.0 +#: SWE-bench prompts are far larger than this; the threshold is a floor, not a +#: target. +DEFAULT_MIN_PROMPT_TOKENS = 2000 + + +class GateFailure(RuntimeError): + """A gate refused to let the run start.""" + + +class GateScaleError(GateFailure): + """A gate could not prove it was testing at the scale it claims.""" + + +@dataclass(slots=True) +class GateReport: + name: str + passed: bool + checked: int = 0 + failures: list[tuple[str, str]] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + def summary(self) -> str: + head = ( + f"{self.name}: {'pass' if self.passed else 'FAIL'} ({self.checked} checked)" + ) + detail = "".join( + f"\n {target} -> {reason}" for target, reason in self.failures[:8] + ) + notes = "".join(f"\n note: {note}" for note in self.notes) + return head + detail + notes + + +class Gate(Protocol): + name: str + + def assert_scale(self, targets: list[str]) -> None: + """Prove this gate tests what it claims. Raise :class:`GateScaleError`.""" + ... + + def check(self, targets: list[str]) -> GateReport: ... + + +def _http_json( + url: str, + payload: dict[str, Any] | None = None, + *, + timeout_s: float = _DEFAULT_TIMEOUT_S, + api_key: str | None = None, +) -> dict[str, Any]: + data = None + headers = {"Accept": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + if payload is not None: + data = json.dumps(payload).encode() + headers["Content-Type"] = "application/json" + request = urllib_request.Request(url, data=data, headers=headers) + with urllib_request.urlopen(request, timeout=timeout_s) as response: + return json.loads(response.read()) + + +def run_gates(gates: list[Gate], targets: list[str]) -> list[GateReport]: + """Run every gate; raise :class:`GateFailure` if any refused. + + Every gate runs even after one fails, so one preflight reports every problem + rather than sending the operator round the loop once per endpoint. + """ + reports: list[GateReport] = [] + for gate in gates: + try: + gate.assert_scale(targets) + except GateScaleError as exc: + reports.append( + GateReport( + name=gate.name, + passed=False, + failures=[("", str(exc))], + notes=[ + "a gate that cannot prove its scale is a failing gate, " + "not a skipped one" + ], + ) + ) + continue + reports.append(gate.check(targets)) + + failed = [report for report in reports if not report.passed] + if failed: + raise GateFailure( + "pre-dispatch gate(s) refused:\n" + + "\n".join(report.summary() for report in failed) + ) + return reports + + +class CheckpointIdentityGate: + """Every endpoint must serve exactly the expected checkpoint. + + Two traps, both of which produced silently contaminated results: + + 1. ``/v1/models`` echoes ``--served-model-name``, which operators routinely + set identically for two different checkpoints (e.g. an FP8 and a BF16 + build of the same model). ``/get_model_info`` reports the real model + path, so it is tried first. + 2. Checkpoint names nest: ``Org/Model`` is a strict prefix of + ``Org/Model-FP8``. Any ``startswith``/``in`` test therefore accepts an + FP8 endpoint as BF16. Comparison is ``==`` and nothing else. + """ + + name = "checkpoint_identity" + + def __init__( + self, + expected_model: str, + *, + timeout_s: float = 10.0, + api_key: str | None = None, + ) -> None: + if not expected_model: + raise ValueError("expected_model is required") + self.expected_model = expected_model + self.timeout_s = timeout_s + self.api_key = api_key + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to identify") + + def probe(self, url: str) -> tuple[str | None, str]: + base = url.rstrip("/") + try: + info = _http_json( + f"{base}/get_model_info", + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + model_path = info.get("model_path") + if model_path: + return str(model_path), "get_model_info" + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + pass # not an SGLang endpoint; fall through to the OpenAI route + try: + listing = _http_json( + f"{base}/v1/models", timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + return None, f"unreachable: {type(exc).__name__}" + ids = [ + entry.get("id") for entry in listing.get("data") or [] if entry.get("id") + ] + if len(ids) == 1: + return str(ids[0]), "v1/models" + if len(ids) > 1: + return None, f"ambiguous /v1/models: {ids!r}" + return None, "no model id from either endpoint" + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + sources: set[str] = set() + for url in targets: + identity, source = self.probe(url) + if identity is None: + report.failures.append((url, source)) + elif identity != self.expected_model: # EXACT; never startswith/in + report.failures.append( + (url, f"serves {identity!r}, expected {self.expected_model!r}") + ) + else: + sources.add(source) + if "v1/models" in sources: + report.notes.append( + "identity came from /v1/models, which echoes --served-model-name; " + "that string can be identical across checkpoints, so it cannot " + "separate two builds that share a served name" + ) + report.passed = not report.failures + report.data["expected_model"] = self.expected_model + return report + + +class ToolCallGate: + """Every endpoint must return a well-formed tool call at SWE-bench scale. + + The prompt is measured with the *server's own* tokenizer (``/tokenize``), + never estimated from character count, and a prompt that measures below + ``min_prompt_tokens`` fails the scale assertion rather than passing the + gate. + """ + + name = "tool_call" + + def __init__( + self, + model: str, + *, + min_prompt_tokens: int = DEFAULT_MIN_PROMPT_TOKENS, + prompt: str | None = None, + timeout_s: float = 180.0, + api_key: str | None = None, + tool_name: str = "bash", + ) -> None: + self.model = model + self.min_prompt_tokens = min_prompt_tokens + self.prompt = prompt if prompt is not None else build_scale_prompt() + self.timeout_s = timeout_s + self.api_key = api_key + self.tool_name = tool_name + self._measured: dict[str, int] = {} + + @property + def tools(self) -> list[dict[str, Any]]: + return [ + { + "type": "function", + "function": { + "name": self.tool_name, + "description": "Run a shell command", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + }, + }, + } + ] + + def count_tokens(self, url: str) -> int | None: + try: + response = _http_json( + f"{url.rstrip('/')}/tokenize", + {"model": self.model, "prompt": self.prompt}, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + return None + count = response.get("count") + if count is None: + tokens = response.get("tokens") + count = len(tokens) if isinstance(tokens, list) else None + try: + return int(count) if count is not None else None + except (TypeError, ValueError): + return None + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to gate") + measured = False + for url in targets: + count = self.count_tokens(url) + if count is None: + continue + self._measured[url] = count + measured = True + if count < self.min_prompt_tokens: + raise GateScaleError( + f"{url}: gate prompt measures {count} tokens, below the " + f"{self.min_prompt_tokens}-token floor this gate claims to " + "test. A tool-call gate that passes at a small prompt says " + "nothing about SWE-bench-sized prompts." + ) + if not measured and self.min_prompt_tokens > 0: + raise GateScaleError( + "no endpoint exposed /tokenize, so the gate cannot prove the " + f"prompt reaches {self.min_prompt_tokens} tokens. Serve a " + "tokenizer endpoint or set min_prompt_tokens=0 to accept an " + "unverified prompt size." + ) + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + tokens = self._measured.get(url) + try: + response = _http_json( + f"{url.rstrip('/')}/v1/chat/completions", + { + "model": self.model, + "messages": [{"role": "user", "content": self.prompt}], + "tools": self.tools, + "tool_choice": "auto", + "max_tokens": 256, + "temperature": 0.0, + }, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + report.failures.append((url, f"{type(exc).__name__}: {exc}")) + continue + failure = self._validate(response) + if failure is not None: + report.failures.append((url, f"tokens={tokens}: {failure}")) + report.passed = not report.failures + report.data["measured_tokens"] = dict(self._measured) + return report + + def _validate(self, response: dict[str, Any]) -> str | None: + try: + message = response["choices"][0]["message"] + except (KeyError, IndexError, TypeError): + return "malformed chat completion response" + tool_calls = message.get("tool_calls") + if not tool_calls: + content = (message.get("content") or "")[:120] + return f"no tool_calls; content={content!r}" + function = tool_calls[0].get("function") or {} + if function.get("name") != self.tool_name: + return f"wrong tool {function.get('name')!r}" + try: + arguments = json.loads(function.get("arguments") or "") + except (TypeError, ValueError): + return f"arguments are not valid JSON: {function.get('arguments')!r}" + command = arguments.get("command") + if not isinstance(command, str) or not command.strip(): + return f"malformed arguments {function.get('arguments')!r}" + return None + + +def build_scale_prompt(repetitions: int = 120) -> str: + """A prompt long enough to exercise the large-context path.""" + filler = "\n".join( + f"def helper_{index}(path, flags=None):\n" + f" # legacy shim retained for compatibility with the v{index} api\n" + " result = compute_checksum(path, flags or DEFAULT_FLAGS)\n" + " return normalise(result), path, flags\n" + for index in range(repetitions) + ) + return ( + "You are working in a Python repository checked out at /testbed.\n" + "Below is the current content of /testbed/legacy/helpers.py.\n\n" + "\n" + filler + "\n\n" + "Before proposing any change you must inspect the repository.\n" + "List the files in /testbed using the shell tool. Call the tool; do not " + "answer in prose." + ) + + +class EndpointFingerprintGate: + """Record a per-endpoint fingerprint for later comparison. + + An engine restarted under a live client yields a run that scores near zero + and still exits successfully -- nothing in the result distinguishes it from + a genuinely bad model. The dispatcher therefore records each endpoint's + fingerprint when a unit is claimed and re-reads it when the unit is + published; a change means the unit was scored against something other than + what it was dispatched to, and the unit is requeued rather than counted. + """ + + name = "endpoint_fingerprint" + + def __init__(self, *, timeout_s: float = 10.0, api_key: str | None = None) -> None: + self.timeout_s = timeout_s + self.api_key = api_key + self.fingerprints: dict[str, str] = {} + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to fingerprint") + + def fingerprint(self, url: str) -> str | None: + base = url.rstrip("/") + parts: list[str] = [] + for path in ("/get_model_info", "/v1/models"): + try: + payload = _http_json( + base + path, timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + continue + parts.append(json.dumps(payload, sort_keys=True, default=str)) + if not parts: + return None + import hashlib + + return hashlib.sha256("|".join(parts).encode()).hexdigest()[:16] + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + value = self.fingerprint(url) + if value is None: + report.failures.append( + (url, "could not read an identity to fingerprint") + ) + continue + self.fingerprints[url] = value + report.passed = not report.failures + report.data["fingerprints"] = dict(self.fingerprints) + return report diff --git a/tests/unit/evaluation/swe_bench_distributed/test_gates.py b/tests/unit/evaluation/swe_bench_distributed/test_gates.py new file mode 100644 index 000000000..607b04d3f --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_gates.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates, including the scale rule.""" + +from __future__ import annotations + +import json + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed import gates as gates_mod +from inference_endpoint.evaluation.swe_bench_distributed.gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + GateFailure, + GateScaleError, + ToolCallGate, + build_scale_prompt, + run_gates, +) + +pytestmark = pytest.mark.unit + +ENDPOINT = "http://engine-1:8000" + + +def install_http(monkeypatch, routes): + """Route ``_http_json`` by URL suffix; a missing route raises like a network error.""" + + def fake(url, payload=None, *, timeout_s=60.0, api_key=None): + for suffix, response in routes.items(): + if url.endswith(suffix): + if isinstance(response, Exception): + raise response + if callable(response): + return response(payload) + return response + raise OSError(f"no route for {url}") + + monkeypatch.setattr(gates_mod, "_http_json", fake) + + +def tool_call_response(command="ls /testbed", name="bash", arguments=None): + return { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": name, + "arguments": ( + arguments + if arguments is not None + else json.dumps({"command": command}) + ), + } + } + ] + } + } + ] + } + + +class TestCheckpointIdentity: + def test_exact_match_passes(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]) + assert report.passed + + def test_a_prefix_is_not_a_match(self, monkeypatch): + # "Org/Model" is a strict prefix of "Org/Model-FP8", so any + # startswith/in test would accept an FP8 engine as the BF16 build. + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert not report.passed + assert "Org/Model-FP8" in report.failures[0][1] + + def test_get_model_info_is_preferred_over_v1_models(self, monkeypatch): + # /v1/models echoes --served-model-name, which operators routinely set + # identically for two different checkpoints. + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Org/Model-FP8"}, + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + assert CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]).passed + + def test_v1_models_fallback_warns_about_its_own_ambiguity(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert report.passed + assert any("served-model-name" in note for note in report.notes) + + def test_an_unreachable_endpoint_fails_closed(self, monkeypatch): + install_http( + monkeypatch, + {"/get_model_info": OSError("down"), "/v1/models": OSError("down")}, + ) + assert not CheckpointIdentityGate("Org/Model").check([ENDPOINT]).passed + + def test_ambiguous_model_listing_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "a"}, {"id": "b"}]}, + }, + ) + report = CheckpointIdentityGate("a").check([ENDPOINT]) + assert not report.passed + assert "ambiguous" in report.failures[0][1] + + +class TestToolCallScale: + def test_a_small_prompt_fails_the_scale_assertion(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 278}}) + gate = ToolCallGate("Org/Model", prompt="tiny", min_prompt_tokens=2000) + # A gate exercising the right operation at a 278-token prompt passed + # while every prompt above 2000 tokens silently returned nothing. + with pytest.raises(GateScaleError, match="278 tokens"): + gate.assert_scale([ENDPOINT]) + + def test_a_scale_failure_is_a_gate_failure_not_a_skip(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 100}}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateFailure, match="failing gate"): + run_gates([gate], [ENDPOINT]) + + def test_no_tokenizer_means_the_gate_cannot_prove_its_scale(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateScaleError, match="cannot prove"): + gate.assert_scale([ENDPOINT]) + + def test_scale_can_be_waived_explicitly(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + ToolCallGate("Org/Model", min_prompt_tokens=0).assert_scale([ENDPOINT]) + + def test_the_default_prompt_is_large(self): + assert len(build_scale_prompt()) > 20_000 + + +class TestToolCallCheck: + def _gate(self, monkeypatch, chat_response): + install_http( + monkeypatch, + {"/tokenize": {"count": 4096}, "/v1/chat/completions": chat_response}, + ) + gate = ToolCallGate("Org/Model") + gate.assert_scale([ENDPOINT]) + return gate + + def test_a_well_formed_call_passes(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response()) + report = gate.check([ENDPOINT]) + assert report.passed + assert report.data["measured_tokens"][ENDPOINT] == 4096 + + def test_an_empty_completion_fails(self, monkeypatch): + gate = self._gate(monkeypatch, {"choices": [{"message": {"content": ""}}]}) + report = gate.check([ENDPOINT]) + assert not report.passed + assert "no tool_calls" in report.failures[0][1] + + def test_the_wrong_tool_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(name="python")) + assert not gate.check([ENDPOINT]).passed + + def test_unparseable_arguments_fail(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(arguments="{not json")) + report = gate.check([ENDPOINT]) + assert "not valid JSON" in report.failures[0][1] + + def test_an_empty_command_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(command=" ")) + assert not gate.check([ENDPOINT]).passed + + +class TestFingerprint: + def test_the_fingerprint_changes_with_the_served_model(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "A"}}) + first = EndpointFingerprintGate().fingerprint(ENDPOINT) + install_http(monkeypatch, {"/get_model_info": {"model_path": "B"}}) + second = EndpointFingerprintGate().fingerprint(ENDPOINT) + assert first is not None and first != second + + def test_an_unidentifiable_endpoint_fails(self, monkeypatch): + install_http(monkeypatch, {}) + assert not EndpointFingerprintGate().check([ENDPOINT]).passed + + +class TestRunGates: + def test_every_gate_runs_even_after_one_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Wrong/Model"}, + "/tokenize": {"count": 4096}, + "/v1/chat/completions": {"choices": [{"message": {}}]}, + }, + ) + with pytest.raises(GateFailure) as excinfo: + run_gates( + [CheckpointIdentityGate("Org/Model"), ToolCallGate("Org/Model")], + [ENDPOINT], + ) + message = str(excinfo.value) + assert "checkpoint_identity" in message + assert "tool_call" in message + + def test_no_targets_is_a_failure_not_a_pass(self, monkeypatch): + with pytest.raises(GateFailure): + run_gates([CheckpointIdentityGate("Org/Model")], []) From 8fb1c98fdb91add8fed8fb2249898d763d18447b Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:04 -0700 Subject: [PATCH 18/25] fix(swe-bench): fingerprint endpoint identity, not the time of asking EndpointFingerprintGate hashed the whole /v1/models payload. vLLM stamps that response with a request-time `created` field and mints a fresh `permission[].id` on every call, so two reads of one healthy, untouched engine produce two different fingerprints -- four calls, four values. The dispatcher records a fingerprint when a unit is claimed and re-reads it when the unit is published, and treats any difference as `endpoint_changed`: an infrastructure fault, which requeues the unit. With an unstable fingerprint that comparison is always true, so every unit is retried until it exhausts max_attempts, is published as abandoned, and the merge gate refuses the run. The failure costs the full agent and evaluation time of every attempt first, and reports itself as infrastructure damage rather than as a bug here. Hash only the identity-bearing fields by dropping the per-request ones. The gate still fails closed on an endpoint whose identity cannot be read at all, which is the property it exists to provide. --- .../evaluation/swe_bench_distributed/gates.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py index 9330fe012..302ce3d3e 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -369,6 +369,30 @@ def build_scale_prompt(repetitions: int = 120) -> str: ) +#: Response fields that change on every request and carry no checkpoint +#: identity. vLLM's ``/v1/models`` stamps ``created`` with the request time and +#: mints a fresh ``permission[].id`` per call, so hashing the raw payload makes +#: the fingerprint differ between any two reads of a perfectly healthy engine. +#: The dispatcher compares the claim-time and publish-time fingerprints and +#: treats a difference as ``endpoint_changed`` -- an infrastructure fault -- so +#: an unstable fingerprint retries and then abandons every unit, and the merge +#: gate can never produce a number. +_VOLATILE_IDENTITY_KEYS = frozenset({"created", "created_at", "permission"}) + + +def _strip_volatile(value: Any) -> Any: + """Drop per-request fields so a fingerprint reflects identity, not time.""" + if isinstance(value, dict): + return { + key: _strip_volatile(item) + for key, item in value.items() + if key not in _VOLATILE_IDENTITY_KEYS + } + if isinstance(value, list): + return [_strip_volatile(item) for item in value] + return value + + class EndpointFingerprintGate: """Record a per-endpoint fingerprint for later comparison. @@ -401,7 +425,9 @@ def fingerprint(self, url: str) -> str | None: ) except (urllib_error.URLError, OSError, ValueError, TimeoutError): continue - parts.append(json.dumps(payload, sort_keys=True, default=str)) + parts.append( + json.dumps(_strip_volatile(payload), sort_keys=True, default=str) + ) if not parts: return None import hashlib From be864fa2c113092829dc4b620b08aba3df7a378d Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 16:08:26 -0700 Subject: [PATCH 19/25] fix(benchmark): don't reject an accuracy-only run for listing many endpoints An accuracy-only run that lists more than one endpoint fails during setup: Failed to connect to endpoint: 1 validation error for HTTPClientConfig Value error, num_workers (1) must be a multiple of the number of endpoint URLs (4) ... Got remainder 1. and exits 3 before any dataset is planned or scored. Two forced choices collide. setup_benchmark pins num_workers=1 and max_connections=1 for every TestMode.ACC run, deliberately, so the compliance gate's single_stream assertion holds. HTTPClientConfig separately requires num_workers to divide the endpoint count so each endpoint gets equal workers. One worker cannot divide four endpoints, so the run is refused. What makes this a defect rather than a tight constraint is that the rejected client does no work. Both SWE-bench scorers set SKIP_ENDPOINT_PHASE, so no sample is ever issued through it -- the same run logs "Expected samples: 0" moments earlier. A validator is rejecting a configuration on behalf of a component that never runs, and it takes the whole run down with it. Give the idle client a single endpoint when the run will issue nothing, so the divisibility invariant still means what it says for runs that do issue. Scorers that fan work out across endpoints themselves read the endpoint list from the run's config.yaml rather than from this client, so this does not narrow the run. The proper fix is to skip building an issuer at all when nothing will be issued. That requires a null issuer type, because BenchmarkSession takes a non-None issuer, and is a larger change than this defect warrants on its own; endpoints[:1] is the narrow form of it, not the intended end state. Only reachable with more than one endpoint in accuracy-only mode: a single-endpoint run divides exactly and never surfaces it. Tests: `TestAccuracyOnlyIdleIssuer` covers the failing case (four endpoints, one worker, zero samples -- fails against the previous code) plus the two boundaries it must not disturb: a single-endpoint accuracy run, and an accuracy run that will actually issue, which still receives every endpoint. docs/evaluation/DESIGN.md records why the idle issuer exists and how narrowly this applies. --- docs/evaluation/DESIGN.md | 15 ++++ .../commands/benchmark/execute.py | 19 +++++ tests/unit/commands/test_benchmark.py | 82 +++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/docs/evaluation/DESIGN.md b/docs/evaluation/DESIGN.md index 1074c343d..245ea592b 100644 --- a/docs/evaluation/DESIGN.md +++ b/docs/evaluation/DESIGN.md @@ -93,6 +93,21 @@ Code execution cannot be done safely in-process. The evaluation server runs in a with resource limits. This is a deliberate architecture choice — not a shortcut — and is documented prominently in the dataset README. +**An externally-scored accuracy-only run builds an idle issuer** + +A scorer with `SKIP_ENDPOINT_PHASE` evaluates through its own service, so the run +issues no samples and `total_samples` is zero. `BenchmarkSession` still requires a +non-None issuer, so one is built and never used. Accuracy-only runs are pinned to +`num_workers=1` for deterministic single-stream ordering, and `HTTPClientConfig` +requires `num_workers` to divide the endpoint count, so that idle client is handed a +single endpoint. This is scoped strictly to the zero-sample case: a run that will +issue keeps the full endpoint list and the divisibility invariant applies to it +unchanged. + +Scorers that themselves fan work across several endpoints read the endpoint list from +the run's `config.yaml`, not from this client, so the narrowing does not narrow the +run. + ## Integration Points | Component | Role | diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index c1894d2b2..a45e7c804 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -705,6 +705,25 @@ async def _create_issuer( """Create the HTTP endpoint client + sample issuer, or raise SetupError.""" config = ctx.config endpoints = config.endpoint_config.endpoints + if ctx.accuracy_only and ctx.total_samples == 0: + # This client will not issue a single sample: every accuracy dataset is + # scored externally (Scorer.SKIP_ENDPOINT_PHASE), which is what makes + # total_samples zero. It is built only so the session has an issuer. + # + # It still has to satisfy HTTPClientConfig, which requires num_workers + # to divide the endpoint count -- and accuracy-only runs are forced to + # num_workers=1 for deterministic ordering, so any run listing more than + # one endpoint is rejected before it starts. Hand the idle client a + # single endpoint so the invariant holds; it drives none of them. + # + # Scorers that fan out across endpoints themselves (swe_bench_fleet) + # read the real endpoint list from the run's config.yaml, not from this + # client, so narrowing it here does not narrow the run. + # + # The proper fix is to not build an issuer at all when nothing will be + # issued; that needs a null issuer, because BenchmarkSession requires a + # non-None one. This is the narrow version of that change. + endpoints = endpoints[:1] logger.info(f"Connecting: {endpoints}") try: api_type: APIType = config.endpoint_config.api_type diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index 82f78052a..43c3672ce 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -3721,3 +3721,85 @@ def _run_audit(cfg, base_report_dir): benchmark_spy.assert_not_called() assert audit_calls == [tmp_path / "audit"] + + +class TestAccuracyOnlyIdleIssuer: + """An accuracy-only run whose datasets are all scored externally. + + `setup_benchmark` pins num_workers=1 and max_connections=1 for every + TestMode.ACC run so the compliance gate's single_stream assertion holds, and + `HTTPClientConfig` separately requires num_workers to divide the endpoint + count. One worker cannot divide four endpoints, so listing several endpoints + took the whole run down during setup -- for a client that issues nothing. + """ + + def _ctx(self, tmp_path, endpoints: list[str], *, total_samples: int = 0): + config = OfflineConfig( + endpoint_config={"endpoints": endpoints}, + model_params={"name": "test-model"}, + datasets=[{"path": "test.jsonl"}], + settings=OfflineSettings(client=HTTPClientConfig(num_workers=1)), + ) + ctx = _make_benchmark_context(config, tmp_path, test_mode=TestMode.ACC) + return dataclasses.replace(ctx, total_samples=total_samples) + + async def _capture_endpoints(self, ctx) -> list[str]: + captured: list[str] = [] + + async def _create(http_config, loop): + captured.extend(http_config.endpoint_urls) + return MagicMock() + + with ( + patch.object(execute_mod.HTTPEndpointClient, "create", new=_create), + patch.object(execute_mod, "HttpClientSampleIssuer", MagicMock()), + ): + await execute_mod._create_issuer(ctx, asyncio.get_event_loop()) + return captured + + @pytest.mark.unit + @pytest.mark.asyncio + async def test_many_endpoints_do_not_reject_an_idle_accuracy_run(self, tmp_path): + ctx = self._ctx( + tmp_path, + [f"http://engine-{i}:8000" for i in range(4)], + total_samples=0, + ) + + captured = await self._capture_endpoints(ctx) + + assert len(captured) == 1 + assert captured[0].startswith("http://engine-0:8000") + + @pytest.mark.unit + @pytest.mark.asyncio + async def test_a_single_endpoint_run_is_unchanged(self, tmp_path): + ctx = self._ctx(tmp_path, ["http://engine-0:8000"], total_samples=0) + + captured = await self._capture_endpoints(ctx) + + assert len(captured) == 1 + + @pytest.mark.unit + @pytest.mark.asyncio + async def test_an_issuing_accuracy_run_still_gets_every_endpoint(self, tmp_path): + """The narrowing is scoped to a client that issues nothing. + + A run that will issue samples keeps the full endpoint list, so the + divisibility invariant still means what it says where it matters. + """ + endpoints = [f"http://engine-{i}:8000" for i in range(2)] + config = OfflineConfig( + endpoint_config={"endpoints": endpoints}, + model_params={"name": "test-model"}, + datasets=[{"path": "test.jsonl"}], + settings=OfflineSettings(client=HTTPClientConfig(num_workers=2)), + ) + ctx = dataclasses.replace( + _make_benchmark_context(config, tmp_path, test_mode=TestMode.ACC), + total_samples=8, + ) + + captured = await self._capture_endpoints(ctx) + + assert len(captured) == 2 From 3f4a4fe95f864591b3e9c6c7c2b3690d1481e1ab Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:42:09 -0700 Subject: [PATCH 20/25] feat(swe-bench): SWEBenchFleetScorer - fan out units across a service fleet Wires the pieces into a scorer registered as eval_method: swe_bench_fleet. It is a scheduler in front of the existing SWE-bench service protocol, not a new runtime: a unit is one RunRequest over a shard, so exact instance binding, per-instance containers, artifact allow-listing and cancellation are reused rather than reimplemented. preflight() runs the gates against the inference endpoints and /health against every service, raising SetupError before a single instance is dispatched. score() plans, dispatches with one in-flight run per service, classifies every unit, requeues any unit with infra_error_count > 0 even when the service reported succeeded, and takes the accuracy number only past the merge gate - self.complete comes from the gate, never a count heuristic. Stall quarantine verifies effect rather than status: a service that is /health-OK but has completed no unit within stall_timeout_s is quarantined and its in-flight unit requeued. Also adds scripts/swe_bench_wq.py {status,merge,requeue,reap} for operators. reap is dry-run by default and requeue prints exactly which result, claim and attempt records it removed, because the failure mode in the field was an operator believing a delete had requeued something. --- AGENTS.md | 3 + docs/evaluation/SWE_BENCH_DISTRIBUTED.md | 233 ++++++++++ scripts/swe_bench_wq.py | 174 +++++++ src/inference_endpoint/config/schema.py | 7 +- src/inference_endpoint/evaluation/scoring.py | 5 +- .../evaluation/swe_bench_distributed/fleet.py | 434 ++++++++++++++++++ .../evaluation/swe_bench_fleet_scorer.py | 392 ++++++++++++++++ .../swe_bench_distributed/test_fleet.py | 293 ++++++++++++ .../test_fleet_scorer.py | 78 ++++ 9 files changed, 1617 insertions(+), 2 deletions(-) create mode 100644 docs/evaluation/SWE_BENCH_DISTRIBUTED.md create mode 100644 scripts/swe_bench_wq.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_fleet.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py diff --git a/AGENTS.md b/AGENTS.md index 46e4e42b0..cf314378d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,6 +100,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | | **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | | **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | +| **SWE-bench (distributed)** | `src/inference_endpoint/evaluation/swe_bench_distributed/`, `src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py`, `scripts/swe_bench_wq.py` | `SWEBenchFleetScorer` (`eval_method: swe_bench_fleet`) shards the instance list into units and runs them across several SWE-bench services concurrently, reusing the same service HTTP protocol. Adds what the single-service path lacks: a durable `mkdir`-atomic work queue (resume after a client crash), an eval-phase infra-vs-genuine classifier driving in-unit retry, pre-dispatch gates on the inference endpoints (checkpoint identity, tool call at >=2k-token scale, endpoint fingerprint), a memory guard, and an all-or-nothing merge gate that compares instance **ids** (never counts) and is scoped to exactly one run id. Operator CLI: `scripts/swe_bench_wq.py {status,merge,requeue,reap}` — `requeue` is the only way to re-run a unit; deleting a result leaves the claim tombstone in place. See `docs/evaluation/SWE_BENCH_DISTRIBUTED.md`. | | **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | ### Hot-Path Architecture @@ -266,6 +267,8 @@ src/inference_endpoint/ │ └── adapter.py # VideoGenAdapter (HttpRequestAdapter) + VideoGenAccumulator (no-op) ├── evaluation/ # Accuracy evaluation (extractor, scoring, livecodebench) │ └── swebench_service/ # Isolated uv service for Docker-backed SWE-bench runs +│ ├── swe_bench_distributed/ # Fleet dispatch: unit plan, work queue, reaper, classifier, gates, guards, merge gate +│ └── swe_bench_fleet_scorer.py # SWEBenchFleetScorer (scorer_id swe_bench_fleet) ├── compliance/ # Submission compliance checks (config-lock, accuracy gate, run validity) │ ├── __init__.py │ └── checker.py # check_submission() + Check/ComplianceReport (Edge-Agentic ruleset) diff --git a/docs/evaluation/SWE_BENCH_DISTRIBUTED.md b/docs/evaluation/SWE_BENCH_DISTRIBUTED.md new file mode 100644 index 000000000..898d4e15b --- /dev/null +++ b/docs/evaluation/SWE_BENCH_DISTRIBUTED.md @@ -0,0 +1,233 @@ +# Distributed SWE-bench (`swe_bench_fleet`) + +> Shards a SWE-bench accuracy run across several SWE-bench services, classifies +> infrastructure damage separately from genuine model failures, and refuses to +> emit an accuracy number unless every planned instance is accounted for exactly +> once. + +`swe_bench_scorer` runs the whole instance list as one service run against one +endpoint. That is the right shape for a hundred instances on one Docker host. It +does not survive a 200-instance run spread over many hours and many hosts: a +client crash loses everything, a host that dies takes its instances with it, and +an evaluation container that wedges is booked as an ordinary `error` — accounted +for, never retried, and silently subtracted from the score. + +`swe_bench_fleet` addresses those six gaps and nothing else. It reuses the +service HTTP protocol, the Docker/Pyxis runtimes, the exact instance-id binding, +the artifact allow-list and the secret redaction unchanged. + +## Configuration + +```yaml +datasets: + - name: swe_bench + accuracy_config: + eval_method: swe_bench_fleet + extras: + swebench_service_urls: + - http://swe-host-1:18080 + - http://swe-host-2:18080 + swebench_service_auth_token: ${SWEBENCH_TOKEN} + num_instances: 200 + shard_size: 10 # instances per unit; 200 / 10 = 20 units + max_attempts: 3 + expected_model: Org/Model-FP8 # optional; gates checkpoint identity + min_prompt_tokens: 2000 # tool-call gate scale floor + stall_timeout_s: 10800 +``` + +| Extra | Default | Meaning | +| --- | --- | --- | +| `swebench_service_urls` | required | One URL per service host. Duplicates are refused: two entries for one host is not extra capacity, it is two runs contending for the same container runtime. | +| `shard_size` | 10 | Instances per unit. | +| `max_attempts` | 3 | Counted attempts before a unit is abandoned. Environment faults are not counted. | +| `expected_model` | none | When set, every endpoint must serve exactly this checkpoint. | +| `min_prompt_tokens` | 2000 | Floor the tool-call gate must prove it reaches. `0` waives the proof. | +| `stall_timeout_s` | 10800 | A service completing no unit in this long is quarantined even if healthy. | + +## What runs, in order + +1. **Preflight gates** (`preflight()`, before the benchmark starts) — every + service's `/health`, plus the endpoint gates below. Any failure raises + `SetupError` before a single instance is dispatched, and every gate runs so + one preflight reports every problem. +2. **Plan** — the instance list is split into units, and the plan is written + once to `units.json` with a digest over the run id and the ordered ids. +3. **Dispatch** — one in-flight service run per service; each service loop + claims a unit, submits it, polls, downloads artifacts, classifies, and + publishes or retries. +4. **Merge gate** — `merge_run(queue, run_id)`. All-or-nothing. + +## The gates + +| Gate | Refuses | +| --- | --- | +| `CheckpointIdentityGate` | Any endpoint not serving *exactly* `expected_model`. `/get_model_info` is preferred over `/v1/models`, because the latter echoes `--served-model-name`, which is routinely identical across checkpoints. Comparison is `==`: `Org/Model` is a strict prefix of `Org/Model-FP8`, so any `startswith`/`in` test accepts FP8 as BF16. Unidentifiable means fail. | +| `ToolCallGate` | Any endpoint that does not return a well-formed tool call — right name, `arguments` parse as JSON, non-empty `command`. | +| `EndpointFingerprintGate` | Any endpoint whose identity cannot be read at all. Records a fingerprint compared again at publish time. | + +**The scale rule.** Every gate implements `assert_scale()`, it runs before the +gate's own check, and a scale failure is a *gate failure*, never a skip. +`ToolCallGate` measures its prompt with the server's own `/tokenize` and fails if +the prompt is below `min_prompt_tokens`. This exists because a tool-call gate +that exercised exactly the right operation with a 278-token prompt passed +cleanly while every prompt above 2000 tokens silently returned an empty +completion. SWE-bench prompts are all far above 2000 tokens; the gate was green +and the run scored zero. **A gate that cannot prove its scale is not a gate.** + +## The work queue + +Under `/swe_bench_wq/`: + +``` +units.json immutable plan + digest +claims//owner host, pid, boot id, plan digest, SLURM job/step +claims//hb heartbeat (mtime only) +results/.json terminal record (succeeded OR abandoned) +failed/..json one per counted attempt +failed/env/.*.json environment faults (not counted) +failed/artifacts/.attemptN/ evidence snapshot taken before a retry +``` + +A unit is available when it is in the plan and has **neither** a claim **nor** a +result. Claiming is `os.mkdir` and nothing else — `makedirs(exist_ok=True)` would +hand the unit to every caller. + +### Re-running a unit + +```bash +python scripts/swe_bench_wq.py requeue REPORT_DIR run-a.s07 +``` + +`requeue` is the **only** supported way. Deleting the result file does not +requeue anything: the claim tombstone still hides the unit. `requeue` removes the +result, the claim and the counted attempt records together, and prints exactly +what it removed. + +### Reaping abandoned claims + +```bash +python scripts/swe_bench_wq.py reap REPORT_DIR # dry run +python scripts/swe_bench_wq.py reap REPORT_DIR --apply --slurm +``` + +A claim is released only when it has no result, its heartbeat is stale, **and** +its owner is provably gone. Uncertainty never escalates: if the liveness probe +fails, times out, or returns an implausible answer, nothing is released. A false +reap gives one unit two owners, duplicate results, and a wrong denominator, with +no error anywhere. + +`SlurmStepLiveness` treats an owner as dead when its job is absent from `squeue` +**or** its step is absent from `scontrol show step` while the job lives — a step +can die inside a live job, and the job-level rule alone then blocks those units +for the whole allocation. Step liveness never uses `squeue -s`, which reports +only `.extern` on the clusters this targets and would mark every live step dead. + +## Classification and retry + +Every unit is classified after the service reports success. The rule list is +ordered and first-match-wins; the order is load-bearing. + +| Kind | Class | Why | +| --- | --- | --- | +| `container_fork_eagain`, `container_exec_refused`, `runtime_read_timeout`, `image_build_timeout`, `image_build_error`, `step_infrastructure_failure`, `endpoint_changed` | infra → **retry** | Defects in infrastructure we provided. | +| `test_timeout` | genuine | A patch that makes the suite loop is a failing patch. | +| `test_memory_exceeded` | genuine | A patch that makes a graded test allocate without bound is a failing patch. The alternative to killing it was never "the test passes", it was "the host OOMs and the instance still never completes". | +| `patch_apply_failed` | genuine | The model emitted a diff that does not apply. SWE-bench books it as `error`, but it is model behaviour. | +| `unknown` | genuine | **The bias rule.** | + +**The bias rule is deliberately asymmetric.** An error that cannot be classified +confidently is genuine, never infrastructure. A false bad-run costs one redo; a +false retry biases the measurement toward optimism, and an optimistic accuracy +number is worse than no number. + +`endpoint_changed` deserves its own note: an engine restarted under a live client +yields a run that scores near zero and exits successfully, and nothing in the +result distinguishes it from a genuinely bad model. The endpoint fingerprint +recorded at claim time is re-read at publish time; a change requeues the unit. + +### Attempt accounting + +* **Environment fault** (service unreachable, submit failed) — recorded under + `failed/env/`, does **not** consume the attempt budget, and counts toward + quarantining that service. A broken host is a property of the host, not of the + unit. +* **Infra / failed** — counted. After `max_attempts` the unit is published as + `abandoned` and its claim released, so it stops burning capacity and shows up + loudly in the merge gate instead of spinning forever. + +Before every retry, the small files that explain the failure are snapshotted to +`failed/artifacts/.attemptN/`, because the unit's run directory is reused +and a unit that fails then succeeds would otherwise leave only the success's +artifacts behind. + +## The merge gate + +`merge_run(queue, run_id)` produces a number only when **all** hold: + +1. every planned unit has a terminal result; +2. no result is abandoned; +3. every unit's accounted instance **ids** equal its planned ids exactly — a set + comparison, never a count, because a shard with one duplicate and one missing + id has the right count and the wrong content; +4. the union across units equals the plan, with no id claimed twice; +5. every result carries the plan's digest; +6. no unit lost instances to infrastructure. + +Otherwise it raises `MergeRefusal` listing every reason. There is no force flag, +no partial-credit path, and **no `merge_all`**: `run_id` is required, and merging +"everything that looks finished" once combined hundreds of banked results from +unrelated configurations into one number. + +`verify_inventory()` cross-checks three independently produced views — the plan, +the claim directory and the result directory — and treats disagreement as an +error. Checking one view against itself is how a verification pass agrees with a +broken system. + +## Resource guards + +`MemoryGuard` kills a graded test only when its resident memory is at or above +`kill_bytes` (default 150 GiB) **and** it has a container-supervisor ancestor. +There is deliberately no working-directory term: an earlier version required a +cwd inside the testbed and skipped a runaway that had grown to 667 GiB because +its cwd was `/tmp`. Every extra conjunct is another way for the guard to miss +what it exists to catch. + +Two rules are enforced by construction and by test: + +* **Kill by pid, never by pattern.** There is no `pkill`/`pgrep` path in the + module and it never shells out; `kill_by_pid` refuses this process and its + ancestors. A pattern can match the guard's own command line, and a long-lived + daemon can carry a dead process's argv for days. +* **A conjunctive guard must not degenerate.** `combine_terms()` returns + `INDETERMINATE`, never `UNHEALTHY`, when any term has zero evidence. An + AND-guard whose honest term loses its data source collapses into its weaker + clauses and starts firing on healthy targets. + +The kill marker is written *before* the signal, and its phase is load-bearing: +only `eval.*` markers make an instance's error a genuine failure. An `agent` +kill merely makes one tool call return an error observation, so it is recorded +for audit and must not influence classification. An unresolvable container name +fails closed to `unknown`. + +## Operational notes + +* `--kill-on-bad-exit=0` does **not** prevent a scheduler force-terminating a + whole step when one node OOMs; OOM escalation is separate from task exit codes. + The defence is small, independently retryable units plus `MemoryGuard` acting + before the host dies — not the flag. +* A service that answers `/health` while completing nothing is the silent + failure. Verify the effect, never the status: the dispatcher quarantines a + service that has completed no unit within `stall_timeout_s` and requeues its + work. +* In shell tooling around this queue, remember `grep -c` exits 1 on zero + matches; a pipeline under `set -e` will abort on an empty, correct answer. + +## What is intentionally not here + +Cluster-lifecycle machinery — allocation rotation, holder chaining, image-store +construction and distribution, node suspect lists, multi-configuration campaign +bookkeeping — is out of scope for a benchmark client. The Pyxis runtime pulls +per-instance images from a registry and never builds a store, so that whole +class of image-corruption failure is architecturally absent rather than worked +around. diff --git a/scripts/swe_bench_wq.py b/scripts/swe_bench_wq.py new file mode 100644 index 000000000..b82648df5 --- /dev/null +++ b/scripts/swe_bench_wq.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Operator tool for a distributed SWE-bench work queue. + + swe_bench_wq.py status REPORT_DIR + swe_bench_wq.py merge REPORT_DIR --run-id RUN + swe_bench_wq.py requeue REPORT_DIR UNIT_ID [UNIT_ID ...] + swe_bench_wq.py reap REPORT_DIR [--apply] + +Two deliberate omissions: + +* There is no ``merge --all``. A merge is always scoped to one run id; merging + "everything that looks finished" once combined hundreds of banked results + from unrelated configurations into a single number. +* There is no way to re-run a unit other than ``requeue``. Deleting a result + file does not requeue anything, because the claim tombstone still hides the + unit; ``requeue`` removes the result, the claim and the attempt records + together and prints exactly what it removed. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from inference_endpoint.evaluation.swe_bench_distributed.fleet import ( # noqa: E402 + QUEUE_DIRNAME, +) +from inference_endpoint.evaluation.swe_bench_distributed.merge import ( # noqa: E402 + MergeRefusal, + merge_run, + verify_inventory, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( # noqa: E402 + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.reaper import ( # noqa: E402 + LocalProcessLiveness, + SlurmStepLiveness, + reap, +) + + +def _open(report_dir: Path) -> WorkQueue: + root = report_dir / QUEUE_DIRNAME + if not root.exists(): + root = report_dir + return WorkQueue.open(root) + + +def cmd_status(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + results = queue.results() + claimed = queue.claimed_unit_ids() + inventory = verify_inventory(queue) + print(f"run: {queue.plan.run_id}") + print(f"plan digest: {queue.plan.digest[:16]}") + print(f"units: {len(queue.plan.units)}") + print(f" with result: {len(results)}") + print(f" claimed: {len(claimed)}") + print(f" available: {len(queue.available_unit_ids())}") + abandoned = [uid for uid, result in results.items() if result.abandoned] + if abandoned: + print(f" ABANDONED: {len(abandoned)} -> {', '.join(sorted(abandoned)[:8])}") + infra = [uid for uid, result in results.items() if result.infra_error_count] + if infra: + print(f" infra-damaged:{len(infra)} -> {', '.join(sorted(infra)[:8])}") + if not inventory.consistent: + print("\nINVENTORY DISAGREEMENT (claims, results and ids do not agree):") + for label, values in ( + ("missing results", inventory.missing_units), + ("results outside the plan", inventory.foreign_units), + ("unreadable results", inventory.unreadable_units), + ("ownerless claims", inventory.ownerless_claims), + ): + if values: + print(f" {label}: {len(values)} -> {', '.join(values[:8])}") + return 0 + + +def cmd_merge(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + try: + result = merge_run(queue, args.run_id) + except MergeRefusal as exc: + print(f"REFUSED to score run {exc.run_id}:") + for reason in exc.reasons: + print(f" - {reason}") + return 1 + print(json.dumps(result.to_dict(), indent=2)) + return 0 + + +def cmd_requeue(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + for unit_id in args.unit_ids: + removed = queue.requeue(unit_id) + total = sum(len(paths) for paths in removed.values()) + print(f"{unit_id}: removed {total} record(s)") + for kind, paths in removed.items(): + for path in paths: + print(f" {kind}: {path}") + if total == 0: + print(" (nothing to remove; the unit was already runnable)") + return 0 + + +def cmd_reap(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + liveness = SlurmStepLiveness() if args.slurm else LocalProcessLiveness() + report = reap( + queue, + liveness, + stale_after_s=args.stale, + step_stale_after_s=args.step_stale, + apply=args.apply, + ) + verb = "released" if args.apply else "would release" + print(f"{verb} {len(report.released)} claim(s)") + for unit_id in report.released: + print(f" {unit_id}") + if args.verbose: + for unit_id, reason in sorted(report.kept.items()): + print(f" kept {unit_id}: {reason}") + if not args.apply and report.released: + print("\nthis was a dry run; pass --apply to release") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + status = sub.add_parser("status", help="summarise the queue") + status.add_argument("report_dir", type=Path) + status.set_defaults(func=cmd_status) + + merge = sub.add_parser("merge", help="score exactly one run") + merge.add_argument("report_dir", type=Path) + merge.add_argument( + "--run-id", + required=True, + help="required; a merge is always scoped to one run", + ) + merge.set_defaults(func=cmd_merge) + + requeue = sub.add_parser( + "requeue", help="make units runnable again (result + claim + attempts)" + ) + requeue.add_argument("report_dir", type=Path) + requeue.add_argument("unit_ids", nargs="+") + requeue.set_defaults(func=cmd_requeue) + + reap_parser = sub.add_parser("reap", help="release claims whose owner is gone") + reap_parser.add_argument("report_dir", type=Path) + reap_parser.add_argument("--apply", action="store_true", help="actually release") + reap_parser.add_argument("--slurm", action="store_true", help="use SLURM liveness") + reap_parser.add_argument("--stale", type=float, default=3600.0) + reap_parser.add_argument("--step-stale", type=float, default=900.0) + reap_parser.add_argument("--verbose", action="store_true") + reap_parser.set_defaults(func=cmd_reap) + + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 4b72244bd..c871f3124 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -143,6 +143,7 @@ class ScorerMethod(str, Enum): BFCL_V4 = "bfcl_v4" LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" SWE_BENCH = "swe_bench_scorer" + SWE_BENCH_FLEET = "swe_bench_fleet" # Audit configuration; runnable test registry lives in compliance/. @@ -1126,6 +1127,9 @@ def _resolve_and_validate(self) -> Self: if not self.model_params.name: raise ValueError("Required: --model-params.name [--model]") + # Only the single-service scorer is limited to one endpoint. The fleet + # scorer runs many service runs, each against one endpoint, so it has + # no such restriction. uses_swe_bench = any( dataset.accuracy_config is not None and dataset.accuracy_config.eval_method == ScorerMethod.SWE_BENCH @@ -1271,7 +1275,8 @@ def _resolve_and_validate(self) -> Self: acc = ds.accuracy_config if ( acc is not None - and acc.eval_method == ScorerMethod.SWE_BENCH + and acc.eval_method + in (ScorerMethod.SWE_BENCH, ScorerMethod.SWE_BENCH_FLEET) and (acc.extras is None or acc.extras.get("workers") is None) ): new_extras = {**(acc.extras or {}), "workers": concurrency} diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index 790a17bcb..d626bf9d5 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -2143,5 +2143,8 @@ def score_breakdown(self) -> dict[str, Any] | None: return self._breakdown -# Late import registers the extracted scorer without introducing a cycle. +# Late imports register the extracted scorers without introducing a cycle. +from .swe_bench_fleet_scorer import ( # noqa: E402 + SWEBenchFleetScorer as SWEBenchFleetScorer, +) from .swe_bench_scorer import SWEBenchScorer as SWEBenchScorer # noqa: E402 diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py b/src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py new file mode 100644 index 000000000..44723dabc --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py @@ -0,0 +1,434 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fan a SWE-bench accuracy run out across a fleet of SWE-bench services.""" + +from __future__ import annotations + +import logging +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import urljoin + +import msgspec +import yaml + +from ...exceptions import SetupError +from .classify import classify_unit +from .gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + Gate, + GateFailure, + ToolCallGate, + run_gates, +) +from .merge import MergeRefusal, merge_run +from .queue import UnitOutcome, UnitResult, WorkQueue +from .reaper import LocalProcessLiveness, reap +from .units import Unit, plan_units + +logger = logging.getLogger(__name__) + +QUEUE_DIRNAME = "swe_bench_wq" +UNITS_DIRNAME = "units" + +#: Buckets a SWE-bench run report uses for instances that reached an outcome. +#: ``incomplete_ids`` is deliberately absent: an incomplete instance is exactly +#: what "not accounted for" means, and it must fail the merge gate. +ACCOUNTED_ID_KEYS = ( + "resolved_ids", + "unresolved_ids", + "empty_patch_ids", + "error_ids", +) + + +class ServiceQuarantined(RuntimeError): + """A service was withdrawn from the fleet.""" + + +@dataclass(slots=True) +class ServiceState: + """Per-service bookkeeping for the dispatcher.""" + + url: str + completed_units: int = 0 + consecutive_env_faults: int = 0 + last_progress_at: float = field(default_factory=time.monotonic) + quarantined_reason: str | None = None + + @property + def available(self) -> bool: + return self.quarantined_reason is None + + +@dataclass(slots=True) +class DispatchOutcome: + """What one attempt at one unit produced.""" + + result: UnitResult + terminal: bool + + +class FleetDispatcher: + """Claim units, run them on services, classify, retry, and merge. + + Concurrency is one in-flight service run per service. The service itself + parallelises within a run (``workers`` / ``max_eval_workers``), so a second + concurrent run per service would only contend for the same host. + """ + + def __init__( + self, + *, + queue: WorkQueue, + service_urls: list[str], + submit: Any, + poll: Any, + collect: Any, + fingerprint: Any = None, + max_attempts: int = 3, + stall_timeout_s: float = 3 * 60 * 60, + max_consecutive_env_faults: int = 3, + idle_poll_s: float = 1.0, + unit_root: Path | None = None, + killed_dir: Path | None = None, + ) -> None: + if not service_urls: + raise SetupError("the SWE-bench fleet needs at least one service URL") + self.queue = queue + self.services = {url: ServiceState(url=url) for url in service_urls} + self.submit = submit + self.poll = poll + self.collect = collect + self.fingerprint = fingerprint + self.max_attempts = max_attempts + self.stall_timeout_s = stall_timeout_s + self.max_consecutive_env_faults = max_consecutive_env_faults + self.idle_poll_s = idle_poll_s + self.unit_root = unit_root + self.killed_dir = killed_dir + self._lock = threading.Lock() + self._in_flight = 0 + + # ------------------------------------------------------------ dispatch -- + + def run(self) -> None: + """Drive every planned unit to a terminal result.""" + with ThreadPoolExecutor(max_workers=len(self.services)) as pool: + futures = [ + pool.submit(self._service_loop, url) for url in list(self.services) + ] + for future in futures: + future.result() + + def _service_loop(self, url: str) -> None: + state = self.services[url] + while state.available: + fingerprint = self._fingerprint(url) + unit = self._take_unit(fingerprint) + if unit is None: + # An empty queue does not mean the run is finished. A unit in + # flight on another service can be released back at any moment + # -- a failed attempt, a quarantined peer -- and a worker that + # exits on the first empty poll leaves that unit for nobody. + # Only "nothing available AND nothing in flight" ends the run. + if self._idle_is_terminal(): + return + time.sleep(self.idle_poll_s) + continue + try: + outcome = self._attempt(unit, state, fingerprint) + self._settle(unit, state, outcome) + finally: + with self._lock: + self._in_flight -= 1 + self._check_stall(state) + + def _take_unit(self, fingerprint: str | None) -> Unit | None: + """Claim the next available unit and mark it in flight, atomically. + + Claiming and counting must happen under one lock. With the increment + after the claim there is a window in which a peer sees "nothing + available" (this unit is claimed) and "nothing in flight" (not yet + counted), concludes the run is over, and exits -- leaving the unit with + nobody to retry it if this attempt fails. + """ + with self._lock: + for unit_id in self.queue.available_unit_ids(): + if self.queue.claim(unit_id, endpoint_fingerprint=fingerprint) is None: + continue # another process won the filesystem claim + self._in_flight += 1 + return self.queue.plan.unit(unit_id) + return None + + def _idle_is_terminal(self) -> bool: + with self._lock: + return self._in_flight == 0 and not self.queue.available_unit_ids() + + def _fingerprint(self, url: str) -> str | None: + if self.fingerprint is None: + return None + try: + return self.fingerprint() + except Exception: # noqa: BLE001 - a fingerprint is advisory at claim time + logger.debug("could not fingerprint endpoints for %s", url, exc_info=True) + return None + + def _attempt( + self, unit: Unit, state: ServiceState, claim_fingerprint: str | None + ) -> DispatchOutcome: + started = time.monotonic() + base = UnitResult( + unit_id=unit.unit_id, + run_id=unit.run_id, + plan_digest=self.queue.plan.digest, + outcome=UnitOutcome.FAILED, + service_url=state.url, + endpoint_fingerprint=claim_fingerprint, + ) + try: + service_run_id = self.submit(state.url, unit) + except Exception as exc: # noqa: BLE001 - any submit failure is the host's + base.outcome = UnitOutcome.ENV_FAULT + base.detail = f"submit failed: {type(exc).__name__}: {exc}" + base.duration_s = time.monotonic() - started + return DispatchOutcome(result=base, terminal=False) + + base.service_run_id = service_run_id + try: + status = self.poll(state.url, service_run_id) + except Exception as exc: # noqa: BLE001 + base.outcome = UnitOutcome.ENV_FAULT + base.detail = f"poll failed: {type(exc).__name__}: {exc}" + base.duration_s = time.monotonic() - started + return DispatchOutcome(result=base, terminal=False) + + base.duration_s = time.monotonic() - started + if status.get("status") != "succeeded": + base.outcome = UnitOutcome.FAILED + base.detail = ( + f"service run ended {status.get('status')}: {status.get('error')}" + ) + return DispatchOutcome(result=base, terminal=False) + + report, output_dir = self.collect(state.url, service_run_id, unit, status) + # An engine restarted mid-unit yields a plausible run that scores near + # zero and exits successfully. Comparing the fingerprint is the only + # thing that distinguishes it from a genuinely bad model. + publish_fingerprint = self._fingerprint(state.url) + endpoint_changed = ( + claim_fingerprint is not None + and publish_fingerprint is not None + and claim_fingerprint != publish_fingerprint + ) + + accounted, resolved = accounted_and_resolved(report) + classification = classify_unit( + output_dir, + report.get("error_ids"), + killed_dir=self.killed_dir, + infrastructure_failure=bool(report.get("infrastructure_failure")), + endpoint_changed=endpoint_changed, + ) + base.accounted_instance_ids = accounted + base.resolved_instance_ids = resolved + base.infra_error_count = classification.infra_count + base.genuine_error_count = classification.genuine_count + base.error_kinds = classification.as_counts() + + if classification.should_retry: + # The agent phase succeeded and the service said so, but instances + # were lost to infrastructure. Publishing this as a success is how a + # run silently becomes unable to ever reach a full result. + base.outcome = UnitOutcome.INFRA + base.detail = ( + f"{classification.infra_count} instance(s) lost to infrastructure: " + + ", ".join(f"{k}={v}" for k, v in base.error_kinds.items()) + ) + return DispatchOutcome(result=base, terminal=False) + + missing = set(unit.instance_ids) - set(accounted) + if missing: + base.outcome = UnitOutcome.FAILED + base.detail = f"{len(missing)} instance(s) unaccounted for" + return DispatchOutcome(result=base, terminal=False) + + base.outcome = UnitOutcome.SUCCEEDED + return DispatchOutcome(result=base, terminal=True) + + def _settle( + self, unit: Unit, state: ServiceState, outcome: DispatchOutcome + ) -> None: + result = outcome.result + if outcome.terminal: + self.queue.publish(result) + state.completed_units += 1 + state.consecutive_env_faults = 0 + state.last_progress_at = time.monotonic() + return + + if self.unit_root is not None: + attempt = self.queue.attempts(unit.unit_id) + 1 + self.queue.snapshot_evidence( + unit.unit_id, self.unit_root / unit.unit_id, attempt + ) + + attempts = self.queue.record_attempt(result) + + if result.outcome is UnitOutcome.ENV_FAULT: + # The unit is fine; the host is not. Do not charge the unit, and + # withdraw the service if it keeps doing this. + state.consecutive_env_faults += 1 + if state.consecutive_env_faults >= self.max_consecutive_env_faults: + state.quarantined_reason = ( + f"{state.consecutive_env_faults} consecutive environment faults" + ) + logger.error( + "quarantining SWE-bench service %s: %s", + state.url, + state.quarantined_reason, + ) + self.queue.release(unit.unit_id) + return + + state.consecutive_env_faults = 0 + if attempts >= self.max_attempts: + # Stop burning slots. An abandoned unit is a loud, terminal record + # that the merge gate refuses, not a unit that spins forever. + self.queue.abandon(result) + state.last_progress_at = time.monotonic() + return + self.queue.release(unit.unit_id) + + def _check_stall(self, state: ServiceState) -> None: + """Withdraw a service that is healthy but not producing. + + Health is not progress. A service that answers ``/health`` while + completing nothing is the silent failure mode: verify the effect, never + the status. + """ + if not state.available: + return + idle = time.monotonic() - state.last_progress_at + if idle > self.stall_timeout_s: + state.quarantined_reason = ( + f"no unit completed in {idle:.0f}s despite a healthy service" + ) + logger.error( + "quarantining SWE-bench service %s: %s", + state.url, + state.quarantined_reason, + ) + + @property + def quarantined(self) -> dict[str, str]: + return { + url: state.quarantined_reason + for url, state in self.services.items() + if state.quarantined_reason is not None + } + + +def accounted_and_resolved( + report: dict[str, Any], +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Extract the accounted and resolved instance ids from a SWE-bench report. + + Ids, not counts. A shard with one duplicate and one missing id has the right + count and the wrong content, and only an id comparison catches it. + """ + accounted: list[str] = [] + seen: set[str] = set() + for key in ACCOUNTED_ID_KEYS: + for instance_id in report.get(key) or (): + text = str(instance_id) + if text in seen: + # Preserve the duplicate so the merge gate can refuse it rather + # than silently deduplicating a real accounting bug. + accounted.append(text) + continue + seen.add(text) + accounted.append(text) + resolved = tuple(str(x) for x in report.get("resolved_ids") or ()) + return tuple(accounted), resolved + + +def build_gates( + *, + expected_model: str | None, + tool_call_model: str | None, + min_prompt_tokens: int, + api_key: str | None = None, +) -> tuple[list[Gate], EndpointFingerprintGate]: + """Assemble the pre-dispatch gates. + + ``expected_model`` is optional only because not every deployment pins a + checkpoint path; when it is set the identity gate is mandatory. + """ + fingerprint_gate = EndpointFingerprintGate(api_key=api_key) + gates: list[Gate] = [] + if expected_model: + gates.append(CheckpointIdentityGate(expected_model, api_key=api_key)) + if tool_call_model: + gates.append( + ToolCallGate( + tool_call_model, + min_prompt_tokens=min_prompt_tokens, + api_key=api_key, + ) + ) + gates.append(fingerprint_gate) + return gates, fingerprint_gate + + +def load_benchmark_config(report_dir: Path) -> dict[str, Any]: + config_path = report_dir / "config.yaml" + if not config_path.exists(): + raise FileNotFoundError( + f"config.yaml not found at {config_path}. The fleet scorer must run " + "inside a benchmark that has already written its config." + ) + with config_path.open() as handle: + config = yaml.safe_load(handle) + if not isinstance(config, dict): + raise ValueError(f"benchmark config at {config_path} must be a YAML mapping") + return config + + +def write_merge_artifacts( + report_dir: Path, payload: dict[str, Any], name: str = "swe_bench_merge.json" +) -> Path: + path = report_dir / name + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_bytes(msgspec.json.encode(payload)) + tmp.replace(path) + return path + + +__all__ = [ + "ACCOUNTED_ID_KEYS", + "QUEUE_DIRNAME", + "UNITS_DIRNAME", + "DispatchOutcome", + "FleetDispatcher", + "GateFailure", + "LocalProcessLiveness", + "MergeRefusal", + "ServiceQuarantined", + "ServiceState", + "accounted_and_resolved", + "build_gates", + "load_benchmark_config", + "merge_run", + "plan_units", + "reap", + "run_gates", + "urljoin", + "write_merge_artifacts", +] diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py new file mode 100644 index 000000000..2848d9213 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -0,0 +1,392 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SWE-bench accuracy scorer that fans out across a fleet of SWE-bench services. + +:class:`~inference_endpoint.evaluation.swe_bench_scorer.SWEBenchScorer` runs the +whole instance list as one service run. This scorer shards it, runs the shards +concurrently on several services, refuses to score a run whose instances are not +all accounted for, and retries the shards that lost instances to infrastructure +rather than the model. + +Configured entirely through ``accuracy_config.extras``:: + + accuracy_config: + eval_method: swe_bench_fleet + extras: + swebench_service_urls: + - http://swe-host-1:18080 + - http://swe-host-2:18080 + shard_size: 10 + max_attempts: 3 + expected_model: Org/Model-FP8 # optional; gates the checkpoint + min_prompt_tokens: 2000 +""" + +from __future__ import annotations + +import logging +import time +from pathlib import Path +from typing import Any, ClassVar +from urllib.parse import urljoin + +import msgspec + +from ..dataset_manager.dataset import Dataset +from ..exceptions import SetupError +from .extractor import Extractor +from .scoring import Scorer +from .swe_bench_distributed.fleet import ( + QUEUE_DIRNAME, + UNITS_DIRNAME, + FleetDispatcher, + accounted_and_resolved, + build_gates, + load_benchmark_config, + write_merge_artifacts, +) +from .swe_bench_distributed.gates import GateFailure, run_gates +from .swe_bench_distributed.merge import MergeRefusal, merge_run +from .swe_bench_distributed.queue import WorkQueue +from .swe_bench_distributed.units import Unit, plan_units +from .swe_bench_scorer import SWEBenchScorer + +logger = logging.getLogger(__name__) + + +class SWEBenchFleetScorer(Scorer, scorer_id="swe_bench_fleet"): + """Distributed SWE-bench scoring across N services.""" + + REQUIRES_EXTRACTOR: ClassVar[bool] = False + SKIP_ENDPOINT_PHASE: ClassVar[bool] = True + DEFAULT_SHARD_SIZE: ClassVar[int] = 10 + DEFAULT_MAX_ATTEMPTS: ClassVar[int] = 3 + DEFAULT_MIN_PROMPT_TOKENS: ClassVar[int] = 2000 + DEFAULT_STALL_TIMEOUT_S: ClassVar[int] = 3 * 60 * 60 + DEFAULT_SERVICE_TIMEOUT_S: ClassVar[int] = 24 * 60 * 60 + DEFAULT_POLL_INTERVAL_S: ClassVar[float] = 5.0 + + def __init__( + self, + dataset_name: str, + dataset: Dataset, + report_dir: Any, + extractor: type[Extractor] | None = None, + ground_truth_column: str | None = "instance_id", + **extras: Any, + ) -> None: + super().__init__( + dataset_name=dataset_name, + dataset=dataset, + report_dir=report_dir, + extractor=extractor, + ground_truth_column=ground_truth_column or "instance_id", + ) + self.report_dir = self.report_dir.resolve() + self.options = self._resolve_options(extras) + + # --------------------------------------------------------------- config -- + + @classmethod + def _service_urls(cls, extras: dict[str, Any]) -> list[str]: + raw = extras.get("swebench_service_urls") + if raw is None: + single = extras.get("swebench_service_url") + raw = [single] if single else [] + if isinstance(raw, str): + raw = [part.strip() for part in raw.split(",") if part.strip()] + urls = [SWEBenchScorer._normalize_service_url(url) for url in raw or []] + if not urls: + raise SetupError( + "accuracy_config.extras.swebench_service_urls is required for " + "swe_bench_fleet; list one URL per SWE-bench service host." + ) + duplicates = sorted({url for url in urls if urls.count(url) > 1}) + if duplicates: + # Two entries for one host is not extra capacity; it is two + # concurrent runs contending for the same Docker/Pyxis runtime. + raise SetupError( + "duplicate SWE-bench service URLs: " + ", ".join(duplicates) + ) + return urls + + @classmethod + def _resolve_options(cls, extras: dict[str, Any]) -> dict[str, Any]: + options = dict(SWEBenchScorer._resolve_dataset_options(extras)) + options["service_urls"] = cls._service_urls(extras) + options["auth_token"] = extras.get("swebench_service_auth_token") or None + options["num_instances"] = SWEBenchScorer._get_extra_int( + extras, + "num_instances", + default=SWEBenchScorer.DEFAULT_NUM_INSTANCES, + min_value=1, + ) + options["shard_size"] = SWEBenchScorer._get_extra_int( + extras, "shard_size", default=cls.DEFAULT_SHARD_SIZE, min_value=1 + ) + options["workers"] = SWEBenchScorer._get_extra_int( + extras, "workers", default=SWEBenchScorer.DEFAULT_WORKERS, min_value=1 + ) + options["max_eval_workers"] = SWEBenchScorer._get_extra_int( + extras, + "max_eval_workers", + default=SWEBenchScorer.DEFAULT_MAX_EVAL_WORKERS, + min_value=1, + ) + options["max_attempts"] = SWEBenchScorer._get_extra_int( + extras, "max_attempts", default=cls.DEFAULT_MAX_ATTEMPTS, min_value=1 + ) + options["min_prompt_tokens"] = SWEBenchScorer._get_extra_int( + extras, + "min_prompt_tokens", + default=cls.DEFAULT_MIN_PROMPT_TOKENS, + min_value=0, + ) + options["stall_timeout_s"] = SWEBenchScorer._get_extra_int( + extras, + "stall_timeout_s", + default=cls.DEFAULT_STALL_TIMEOUT_S, + min_value=1, + ) + options["service_timeout_s"] = SWEBenchScorer._get_extra_int( + extras, + "service_timeout_s", + default=cls.DEFAULT_SERVICE_TIMEOUT_S, + min_value=1, + ) + options["poll_interval_s"] = SWEBenchScorer._get_extra_float( + extras, + "poll_interval_s", + default=cls.DEFAULT_POLL_INTERVAL_S, + min_value=0, + ) + options["swebench_template"] = SWEBenchScorer._resolve_service_template(extras) + options["expected_model"] = extras.get("expected_model") or None + options["run_id"] = str(extras.get("run_id") or "swe_bench") + return options + + @classmethod + def dataset_loader_kwargs(cls, extras: dict[str, Any]) -> dict[str, Any]: + return SWEBenchScorer._resolve_dataset_options(extras) + + @classmethod + def external_sample_count(cls, extras: dict[str, Any]) -> int | None: + return SWEBenchScorer.external_sample_count(extras) + + # ------------------------------------------------------------ preflight -- + + @classmethod + def preflight( + cls, extras: dict[str, Any], *, loaded_sample_count: int | None = None + ) -> None: + """Health-check every service and run the pre-dispatch gates. + + Every problem is reported from one preflight. A run that starts against + a mis-served checkpoint, or against an endpoint that cannot emit a tool + call at SWE-bench prompt scale, produces a plausible-looking low score + hours later and costs the whole run. + """ + options = cls._resolve_options(extras) + for url in options["service_urls"]: + SWEBenchScorer._check_health(url, options["auth_token"]) + + endpoints = extras.get("endpoint_urls") or [] + if not endpoints: + logger.info( + "swe_bench_fleet: no endpoint URLs available at preflight; " + "checkpoint and tool-call gates run at dispatch instead" + ) + return + gates, _ = build_gates( + expected_model=options["expected_model"], + tool_call_model=extras.get("model_name"), + min_prompt_tokens=options["min_prompt_tokens"], + api_key=extras.get("endpoint_api_key"), + ) + try: + run_gates(gates, list(endpoints)) + except GateFailure as exc: + raise SetupError(str(exc)) from exc + + def score_single_sample(self, value: str, ground_truth: str) -> float: + raise RuntimeError( + "SWEBenchFleetScorer scores whole units through services; call score()." + ) + + # ---------------------------------------------------------------- score -- + + def score(self) -> tuple[float | None, int]: + self.complete = True + config = load_benchmark_config(self.report_dir) + model_params = config.get("model_params") or {} + model_name = model_params.get("name") + if not model_name: + raise ValueError("model_params.name is required in the benchmark config") + endpoint_config = config.get("endpoint_config") or {} + endpoint_urls = list(endpoint_config.get("endpoints") or []) + if not endpoint_urls: + raise SetupError("the benchmark config lists no endpoint URLs") + + instance_ids = self._instance_ids() + if not instance_ids: + logger.warning("swe_bench_fleet: no instances selected") + self.complete = False + return None, 1 + + gates, fingerprint_gate = build_gates( + expected_model=self.options["expected_model"], + tool_call_model=model_name, + min_prompt_tokens=self.options["min_prompt_tokens"], + api_key=endpoint_config.get("api_key"), + ) + try: + run_gates(gates, endpoint_urls) + except GateFailure as exc: + raise SetupError(str(exc)) from exc + + plan = plan_units( + self.options["run_id"], instance_ids, shard_size=self.options["shard_size"] + ) + queue = WorkQueue(self.report_dir / QUEUE_DIRNAME, plan) + unit_root = self.report_dir / UNITS_DIRNAME + unit_root.mkdir(parents=True, exist_ok=True) + + self._model_name = model_name + self._endpoint_urls = endpoint_urls + self._endpoint_api_key = endpoint_config.get("api_key") + self._generation_params = SWEBenchScorer._generation_params(model_params) + self._unit_root = unit_root + + def fingerprint() -> str | None: + values = [fingerprint_gate.fingerprint(url) for url in endpoint_urls] + if any(value is None for value in values): + return None + return "|".join(v for v in values if v is not None) + + dispatcher = FleetDispatcher( + queue=queue, + service_urls=self.options["service_urls"], + submit=self._submit_unit, + poll=self._poll_unit, + collect=self._collect_unit, + fingerprint=fingerprint, + max_attempts=self.options["max_attempts"], + stall_timeout_s=self.options["stall_timeout_s"], + unit_root=unit_root, + ) + dispatcher.run() + + payload: dict[str, Any] = { + "run_id": plan.run_id, + "plan_digest": plan.digest, + "services": self.options["service_urls"], + "quarantined": dispatcher.quarantined, + } + try: + merged = merge_run(queue, plan.run_id) + except MergeRefusal as exc: + payload["refused"] = exc.reasons + write_merge_artifacts(self.report_dir, payload) + logger.error("swe_bench_fleet: %s", exc) + self.complete = False + return None, 1 + + payload["merge"] = merged.to_dict() + write_merge_artifacts(self.report_dir, payload) + logger.info( + "swe_bench_fleet: resolved %d / %d (%.1f%%) across %d units", + merged.resolved_instances, + merged.total_instances, + merged.resolved_rate * 100, + merged.unit_count, + ) + return merged.resolved_rate, 1 + + # --------------------------------------------------------- service glue -- + + def _instance_ids(self) -> list[str]: + if self.dataset.dataframe is None: + raise RuntimeError( + "SWEBench dataset must be loaded before scoring; call dataset.load()." + ) + frame = self.dataset.dataframe + total = min(self.options["num_instances"], len(frame)) + return [ + str(instance_id) + for instance_id in frame.iloc[:total][self.ground_truth_column].tolist() + ] + + def _submit_unit(self, service_url: str, unit: Unit) -> str: + payload = { + "model_name": self._model_name, + # The service accepts exactly one endpoint URL per run; the fleet's + # parallelism comes from running many units, not many endpoints. + "endpoint_urls": self._endpoint_urls[:1], + "endpoint_api_key": self._endpoint_api_key, + "generation_params": self._generation_params, + "subset": self.options["subset"], + "split": self.options["split"], + "num_instances": len(unit.instance_ids), + "workers": self.options["workers"], + "max_eval_workers": self.options["max_eval_workers"], + "evaluated_instance_ids": list(unit.instance_ids), + "template": self.options["swebench_template"], + } + submitted = SWEBenchScorer._http_json( + urljoin(service_url, "v1/runs"), + method="POST", + payload=payload, + timeout_s=30.0, + auth_token=self.options["auth_token"], + ) + run_id = str(submitted.get("run_id") or "") + if not run_id: + raise SetupError(f"{service_url} did not return a run_id") + return run_id + + def _poll_unit(self, service_url: str, service_run_id: str) -> dict[str, Any]: + deadline = time.monotonic() + self.options["service_timeout_s"] + status: dict[str, Any] = {"status": "queued"} + while status.get("status") not in {"succeeded", "failed", "cancelled"}: + if time.monotonic() >= deadline: + SWEBenchScorer._cancel_service_run( + service_url, service_run_id, self.options["auth_token"] + ) + raise SetupError( + f"timed out waiting for {service_url} run {service_run_id}" + ) + time.sleep(self.options["poll_interval_s"]) + status = SWEBenchScorer._http_json( + urljoin(service_url, f"v1/runs/{service_run_id}"), + timeout_s=30.0, + auth_token=self.options["auth_token"], + ) + return status + + def _collect_unit( + self, + service_url: str, + service_run_id: str, + unit: Unit, + status: dict[str, Any], + ) -> tuple[dict[str, Any], Path]: + target = self._unit_root / unit.unit_id + target.mkdir(parents=True, exist_ok=True) + SWEBenchScorer._download_artifacts( + service_url, status, target, self.options["auth_token"] + ) + report = status.get("result") + if not isinstance(report, dict): + results_path = target / "swe_bench_results.json" + if results_path.exists(): + try: + report = msgspec.json.decode(results_path.read_bytes(), type=dict) + except msgspec.DecodeError: + report = {} + else: + report = {} + return report, target + + +__all__ = ["SWEBenchFleetScorer", "accounted_and_resolved"] diff --git a/tests/unit/evaluation/swe_bench_distributed/test_fleet.py b/tests/unit/evaluation/swe_bench_distributed/test_fleet.py new file mode 100644 index 000000000..02ee81d3b --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_fleet.py @@ -0,0 +1,293 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fleet dispatch: fan-out, classification-driven retry, quarantine, merge.""" + +from __future__ import annotations + +import itertools + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.fleet import ( + FleetDispatcher, + accounted_and_resolved, +) +from inference_endpoint.evaluation.swe_bench_distributed.merge import ( + MergeRefusal, + merge_run, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + UnitOutcome, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import plan_units + +pytestmark = pytest.mark.unit + +IDS = [f"repo__proj-{i:02d}" for i in range(30)] +SERVICES = ["http://svc-a:18080", "http://svc-b:18080"] + + +@pytest.fixture +def queue(tmp_path): + return WorkQueue(tmp_path / "wq", plan_units("run-a", IDS, shard_size=10)) + + +class FakeFleet: + """A scripted stand-in for the SWE-bench service HTTP protocol.""" + + def __init__(self, queue: WorkQueue, tmp_path, *, resolved_per_unit: int = 4): + self.queue = queue + self.tmp_path = tmp_path + self.resolved_per_unit = resolved_per_unit + self.counter = itertools.count() + self.submitted: list[tuple[str, str]] = [] + self.submit_errors: dict[str, Exception] = {} + self.status_for_unit: dict[str, str] = {} + self.error_ids_for_unit: dict[str, list[str]] = {} + self.fingerprints: list[str] = ["fp-1"] + + def submit(self, service_url, unit): + error = self.submit_errors.get(service_url) + if error is not None: + raise error + self.submitted.append((service_url, unit.unit_id)) + return f"svc-run-{next(self.counter)}" + + def poll(self, service_url, service_run_id): + unit_id = self.submitted[-1][1] + return {"status": self.status_for_unit.get(unit_id, "succeeded")} + + def collect(self, service_url, service_run_id, unit, status): + error_ids = self.error_ids_for_unit.get(unit.unit_id, []) + remaining = [i for i in unit.instance_ids if i not in error_ids] + resolved = remaining[: self.resolved_per_unit] + unresolved = remaining[self.resolved_per_unit :] + output_dir = self.tmp_path / "units" / unit.unit_id + output_dir.mkdir(parents=True, exist_ok=True) + report = { + "resolved_ids": resolved, + "unresolved_ids": unresolved, + "error_ids": error_ids, + "empty_patch_ids": [], + } + return report, output_dir + + def fingerprint(self): + return self.fingerprints[0] + + def write_eval_log(self, unit_id: str, instance_id: str, text: str) -> None: + log_dir = ( + self.tmp_path + / "units" + / unit_id + / "logs" + / "run_evaluation" + / "r" + / "m" + / instance_id + ) + log_dir.mkdir(parents=True, exist_ok=True) + (log_dir / "run_instance.log").write_text(text) + + +def dispatcher_for(queue, fleet, **overrides): + kwargs = { + "queue": queue, + "service_urls": SERVICES, + "submit": fleet.submit, + "poll": fleet.poll, + "collect": fleet.collect, + "fingerprint": fleet.fingerprint, + "max_attempts": 3, + } + kwargs.update(overrides) + return FleetDispatcher(**kwargs) + + +class TestAccounting: + def test_every_outcome_bucket_counts_as_accounted(self): + report = { + "resolved_ids": ["a"], + "unresolved_ids": ["b"], + "empty_patch_ids": ["c"], + "error_ids": ["d"], + } + accounted, resolved = accounted_and_resolved(report) + assert set(accounted) == {"a", "b", "c", "d"} + assert resolved == ("a",) + + def test_incomplete_instances_are_not_accounted(self): + # "Incomplete" is exactly what unaccounted means; counting it would let + # a partial shard through the merge gate. + accounted, _ = accounted_and_resolved( + {"resolved_ids": ["a"], "incomplete_ids": ["b"]} + ) + assert accounted == ("a",) + + def test_duplicates_are_preserved_for_the_gate_to_refuse(self): + accounted, _ = accounted_and_resolved( + {"resolved_ids": ["a"], "unresolved_ids": ["a"]} + ) + assert accounted == ("a", "a") + + +class TestHappyPath: + def test_all_units_complete_and_merge(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet).run() + + assert len(queue.completed_unit_ids()) == 3 + result = merge_run(queue, "run-a") + assert result.total_instances == 30 + assert result.resolved_instances == 12 + + def test_work_is_spread_across_services(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet).run() + assert len({service for service, _ in fleet.submitted}) >= 1 + assert len(fleet.submitted) == 3 + + def test_every_unit_runs_exactly_once(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet).run() + dispatched = [unit_id for _, unit_id in fleet.submitted] + assert sorted(dispatched) == sorted(queue.plan.unit_ids) + + +class TestInfraRetry: + def test_an_eval_infra_error_requeues_a_succeeded_run(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.error_ids_for_unit["run-a.s00"] = [IDS[0]] + fleet.write_eval_log("run-a.s00", IDS[0], "container state improper") + + dispatcher_for(queue, fleet, max_attempts=1).run() + + # The service said "succeeded" and every instance was accounted for, so + # nothing but classification distinguishes this from a real result. + stored = queue.result("run-a.s00") + assert stored is not None + assert stored.abandoned + assert stored.outcome is UnitOutcome.INFRA + assert stored.infra_error_count == 1 + with pytest.raises(MergeRefusal): + merge_run(queue, "run-a") + + def test_a_genuine_error_is_scored_not_retried(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.error_ids_for_unit["run-a.s00"] = [IDS[0]] + fleet.write_eval_log("run-a.s00", IDS[0], "Test timed out after 1800s") + + dispatcher_for(queue, fleet).run() + + stored = queue.result("run-a.s00") + assert stored is not None + assert stored.outcome is UnitOutcome.SUCCEEDED + assert stored.genuine_error_count == 1 + assert merge_run(queue, "run-a").total_instances == 30 + + def test_a_transient_infra_error_succeeds_on_retry(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.error_ids_for_unit["run-a.s00"] = [IDS[0]] + fleet.write_eval_log("run-a.s00", IDS[0], "container state improper") + + original_collect = fleet.collect + + def collect_once(service_url, service_run_id, unit, status): + result = original_collect(service_url, service_run_id, unit, status) + fleet.error_ids_for_unit.pop(unit.unit_id, None) + return result + + dispatcher_for(queue, fleet, collect=collect_once).run() + + stored = queue.result("run-a.s00") + assert stored is not None and stored.outcome is UnitOutcome.SUCCEEDED + assert queue.attempts("run-a.s00") == 1 + assert merge_run(queue, "run-a").total_instances == 30 + + def test_a_unit_is_abandoned_after_max_attempts(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.status_for_unit["run-a.s00"] = "failed" + + dispatcher_for(queue, fleet, max_attempts=2).run() + + stored = queue.result("run-a.s00") + assert stored is not None and stored.abandoned + assert queue.attempts("run-a.s00") == 2 + # An abandoned unit must be loud, not silent: it stops burning slots and + # the gate refuses the run. + assert queue.claimed_unit_ids() == set() + with pytest.raises(MergeRefusal, match="abandoned"): + merge_run(queue, "run-a") + + +class TestEndpointFingerprint: + def test_a_restarted_engine_requeues_the_unit(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + original_collect = fleet.collect + + def collect_then_restart(service_url, service_run_id, unit, status): + result = original_collect(service_url, service_run_id, unit, status) + if unit.unit_id == "run-a.s00" and fleet.fingerprints[0] == "fp-1": + fleet.fingerprints[0] = "fp-2" + return result + + dispatcher_for(queue, fleet, collect=collect_then_restart, max_attempts=1).run() + + stored = queue.result("run-a.s00") + assert stored is not None + # The run "succeeded" and every instance was accounted for; only the + # fingerprint says it was scored against a different engine. + assert stored.outcome is UnitOutcome.INFRA + assert "endpoint_changed" in stored.error_kinds + + +class TestEnvironmentFaults: + def test_a_submit_failure_does_not_charge_the_unit(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.submit_errors[SERVICES[0]] = OSError("service host is broken") + + dispatcher_for(queue, fleet).run() + + # A broken host is a property of the host. The units still complete, on + # the other service, with a clean attempt ledger. + assert len(queue.completed_unit_ids()) == 3 + assert all(queue.attempts(unit_id) == 0 for unit_id in queue.plan.unit_ids) + assert merge_run(queue, "run-a").total_instances == 30 + + def test_a_persistently_broken_service_is_quarantined(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.submit_errors[SERVICES[0]] = OSError("service host is broken") + + dispatcher = dispatcher_for(queue, fleet, max_consecutive_env_faults=2) + dispatcher.run() + + assert SERVICES[0] in dispatcher.quarantined + assert SERVICES[1] not in dispatcher.quarantined + + +class TestStallQuarantine: + def test_a_healthy_but_unproductive_service_is_withdrawn(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher = dispatcher_for(queue, fleet, stall_timeout_s=-1) + dispatcher.run() + # Health is not progress. A service answering /health while completing + # nothing is the silent failure: verify the effect, never the status. + assert dispatcher.quarantined + assert all( + "no unit completed" in reason for reason in dispatcher.quarantined.values() + ) + + +class TestResume: + def test_a_restarted_client_does_not_redo_completed_units(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet, service_urls=SERVICES[:1]).run() + first_pass = len(fleet.submitted) + + reopened = WorkQueue.open(queue.root) + dispatcher_for(reopened, fleet, service_urls=SERVICES[:1]).run() + + assert len(fleet.submitted) == first_pass + assert merge_run(reopened, "run-a").total_instances == 30 diff --git a/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py b/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py new file mode 100644 index 000000000..b96026227 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Configuration and registration of the fleet scorer.""" + +from __future__ import annotations + +import pytest + +from inference_endpoint.config.schema import ScorerMethod +from inference_endpoint.evaluation.scoring import Scorer +from inference_endpoint.evaluation.swe_bench_fleet_scorer import SWEBenchFleetScorer +from inference_endpoint.exceptions import SetupError + +pytestmark = pytest.mark.unit + +URLS = ["http://svc-a:18080", "http://svc-b:18080"] + + +class TestRegistration: + def test_the_scorer_is_registered(self): + assert Scorer.get("swe_bench_fleet") is SWEBenchFleetScorer + + def test_the_scorer_method_enum_is_in_sync(self): + assert ScorerMethod.SWE_BENCH_FLEET.value in Scorer.available_scorers() + + def test_it_skips_the_endpoint_phase(self): + # Like the single-service scorer, this one drives the run itself rather + # than consuming responses collected by the load generator. + assert SWEBenchFleetScorer.SKIP_ENDPOINT_PHASE + assert not SWEBenchFleetScorer.REQUIRES_EXTRACTOR + + +class TestOptions: + def test_service_urls_are_normalised(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": ["http://svc-a:18080/"]} + ) + assert options["service_urls"] == ["http://svc-a:18080/"] + + def test_a_comma_separated_string_is_accepted(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": "http://svc-a:18080, http://svc-b:18080"} + ) + assert len(options["service_urls"]) == 2 + + def test_the_single_service_key_still_works(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_url": "http://svc-a:18080"} + ) + assert options["service_urls"] == ["http://svc-a:18080/"] + + def test_no_service_urls_is_a_setup_error(self): + with pytest.raises(SetupError, match="swebench_service_urls is required"): + SWEBenchFleetScorer._resolve_options({}) + + def test_duplicate_service_urls_are_refused(self): + # Two entries for one host is not extra capacity; it is two concurrent + # runs contending for the same container runtime. + with pytest.raises(SetupError, match="duplicate"): + SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": ["http://svc-a:18080", "http://svc-a:18080/"]} + ) + + def test_defaults_are_sane(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": URLS} + ) + assert options["shard_size"] == 10 + assert options["max_attempts"] == 3 + # The tool-call gate's floor must stay at SWE-bench prompt scale. + assert options["min_prompt_tokens"] == 2000 + + def test_a_bad_shard_size_is_rejected(self): + with pytest.raises(SetupError, match="shard_size"): + SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": URLS, "shard_size": 0} + ) From 170b0ea57da9879ae47958b7c388322aedae1726 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:26 -0700 Subject: [PATCH 21/25] fix(swe-bench): validate model_params before reading generation settings score() reads the run's settings back from the report directory's config.yaml, which yaml.safe_load() returns as plain dictionaries. It then handed that mapping to SWEBenchScorer._generation_params(), which calls .model_dump() on it, so the fleet scorer raised AttributeError: 'dict' object has no attribute 'model_dump' on every run, after the plan and the work queue had been written but before a single unit was dispatched. Re-validate the mapping into ModelParams instead of re-implementing the field selection here, so the fleet path and the single-service path stay in agreement about which generation settings are forwarded to the service. --- .../evaluation/swe_bench_fleet_scorer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py index 2848d9213..d3a932224 100644 --- a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -255,7 +255,15 @@ def score(self) -> tuple[float | None, int]: self._model_name = model_name self._endpoint_urls = endpoint_urls self._endpoint_api_key = endpoint_config.get("api_key") - self._generation_params = SWEBenchScorer._generation_params(model_params) + # load_benchmark_config() yaml.safe_load()s config.yaml, so model_params + # is a plain mapping here, while _generation_params() expects the + # pydantic ModelParams. Re-validate rather than re-implement the field + # selection, so the fleet path and the single-service path agree. + from ..config.schema import ModelParams + + self._generation_params = SWEBenchScorer._generation_params( + ModelParams.model_validate(model_params) + ) self._unit_root = unit_root def fingerprint() -> str | None: From cb4a20261d4f6ab1f91890480f16009d33367da1 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:45 -0700 Subject: [PATCH 22/25] feat(swe-bench): bind each unit to an endpoint by shard index Every unit was submitted with endpoint_urls[:1], so a fleet configured with N engines sent all of its work to the first one and left the other N-1 idle. The comment justified this by noting that the service accepts exactly one endpoint per run and that the fleet's parallelism comes from running many units -- true, but it does not follow that every unit must pick the same one. Two consequences. The obvious one is a throughput ceiling: concurrency is bounded by one engine no matter how much hardware the run was given. The serious one is a measurement hazard -- a single engine's behaviour decides the whole run's accuracy, so one degraded engine is indistinguishable from a degraded model, which is precisely the confusion the endpoint fingerprint exists to prevent. Bind unit -> endpoint by shard index instead. The mapping is deterministic, so a retried unit lands on the endpoint it was originally measured against and stays comparable to its first attempt, and a run with one endpoint behaves exactly as before. --- .../evaluation/swe_bench_fleet_scorer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py index d3a932224..a946edfc2 100644 --- a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -326,11 +326,17 @@ def _instance_ids(self) -> list[str]: ] def _submit_unit(self, service_url: str, unit: Unit) -> str: + # The service accepts exactly one endpoint URL per run, so a unit is + # bound to exactly one endpoint. Sending every unit to endpoint 0 would + # funnel the whole fleet through a single engine while the rest idle, + # which is both a throughput ceiling and a measurement hazard: one + # engine's behaviour would decide the entire run's accuracy. Binding by + # shard index spreads units deterministically -- the same unit always + # gets the same endpoint, so a retry is comparable to its first attempt. + endpoint = self._endpoint_urls[unit.shard % len(self._endpoint_urls)] payload = { "model_name": self._model_name, - # The service accepts exactly one endpoint URL per run; the fleet's - # parallelism comes from running many units, not many endpoints. - "endpoint_urls": self._endpoint_urls[:1], + "endpoint_urls": [endpoint], "endpoint_api_key": self._endpoint_api_key, "generation_params": self._generation_params, "subset": self.options["subset"], From 8419d6734e9b0471df656d81687b47b8dc66000e Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 13:24:24 -0700 Subject: [PATCH 23/25] docs(swe-bench): document the completeness gate and the infrastructure retry Also surface the completeness report on the refusal paths that already existed: `SWEBenchFleetScorer` writes it into the run's merge artifacts and `swe_bench_wq.py merge` prints it. A refusal is not an absence of information, and leaving the conditional rate and the lower bound unpublished is what makes somebody recompute a headline by hand from the artifacts -- which is how a run that lost 106 of 200 instances came to be reported as 47.0% accuracy. --- docs/evaluation/SWE_BENCH_DISTRIBUTED.md | 80 +++++++++++++++++++ scripts/swe_bench_wq.py | 3 + .../evaluation/swe_bench_fleet_scorer.py | 6 ++ 3 files changed, 89 insertions(+) diff --git a/docs/evaluation/SWE_BENCH_DISTRIBUTED.md b/docs/evaluation/SWE_BENCH_DISTRIBUTED.md index 898d4e15b..b005c1194 100644 --- a/docs/evaluation/SWE_BENCH_DISTRIBUTED.md +++ b/docs/evaluation/SWE_BENCH_DISTRIBUTED.md @@ -179,11 +179,91 @@ no partial-credit path, and **no `merge_all`**: `run_id` is required, and mergin "everything that looks finished" once combined hundreds of banked results from unrelated configurations into one number. +### What a refusal still publishes + +A refusal that carries no numbers is not the end of the story -- somebody still +has to report *something*, and with the gate silent they compute it by hand. That +is how a run which lost 106 of its 200 instances to infrastructure came to be +reported as **47.0%** and compared against a complete-run reference of 70.67%. It +was read as a model regression. It was attrition. + +`assess_run(queue, run_id)` performs the same arithmetic without deciding +anything and returns a `CompletenessReport`, which is attached to both +`MergeResult` and `MergeRefusal` and written to the run's merge artifacts. + +| Field | Always present | Meaning | +| --- | --- | --- | +| `resolved_rate` | yes, may be `null` | The headline. `null` unless the run is structurally complete **and** lost nothing to infrastructure. | +| `conditional_resolved_rate` | yes | Resolved over the instances that actually completed. Honest about what it measures; **not** comparable to a complete-run reference. | +| `resolved_rate_lower_bound` | yes | Resolved over everything planned. Infrastructure losses can only ever *add* resolutions, so this bounds the truth from below. | +| `incomplete_instance_ids` | yes | Exactly which instances never reached a terminal state. | +| `infra_lost_instances`, `infra_lost_unit_ids` | yes | What the harness lost, as distinct from what the model failed. | +| `resolved_rate_withheld_reason` | yes, may be `null` | Which of the two conditions failed, and by how much. | + +The two conditions are deliberately separate and conflating them gets both +wrong. An instance the model attempted and failed is a legitimate score; one the +harness dropped never had the chance. A run short of instances has the wrong +denominator; a complete run that leaned on the infrastructure has the wrong +provenance. A model that resolved nothing at all is a score, not a casualty, and +publishes `0.0` rather than withholding. + +Ported from `wq_merge.sh:7-9` in the banked campaign: "shard_merge.py refuses to +print an accuracy unless all 20 shards account for exactly their own 10 ids, and +that refusal is the single most important property in this campaign." What is +added here is that the refusal shows its working. + `verify_inventory()` cross-checks three independently produced views — the plan, the claim directory and the result directory — and treats disagreement as an error. Checking one view against itself is how a verification pass agrees with a broken system. +## Infrastructure retry + +`retry_on_provable_non_execution()` retries an operation **only** when the +failure proves the work never happened, and never merely because an error +occurred. Re-running something that may already have run can apply an edit +twice, delete twice, or double a test run, and none of those announce +themselves. + +The evidence is the exception's `provable_non_execution` attribute, read as an +attribute rather than an isinstance check so producer and consumer stay +decoupled. For a Pyxis step it means the status file still read `pending` **and** +no in-band sentinel arrived -- the step script did not run even its first line. +Anything that does not make that claim is re-raised immediately and does not +consume the attempt budget, which makes every exception type this module has +never heard of safe by default. + +Measured signature, from an isolated probe with no model and no GPU (20 nodes, +200 workers, 6273 ordinary shell steps): 63 steps failed and all 63 were still +`pending`. + +Retries are bounded and, more importantly, counted. `InfraRetryLedger` appends +one JSON line per event so a run that dies still leaves its retry history, and +`summary()` publishes: + +| Field | Meaning | +| --- | --- | +| `infra_retries_total` | Every retry event. | +| `instances_saved_by_retry` | Targets that recovered and did not later exhaust. | +| `infra_retries_exhausted` | Targets the budget could not rescue. | +| `infra_retry_succeeded_on_attempt` | Which attempt each recovery landed on. | +| `run_quality` | `CLEAN` / `OK_WITH_RETRIES` / `DEGRADED`. | + +`run_quality` is `DEGRADED` on any exhaustion or not-retryable failure, and also +when more than 2% of operations needed a retry **even if every one of them +eventually succeeded**. The banked campaign retried environment faults without +limit and without counting them (`wq_worker.sh:41` `WQ_MAX_ATTEMPTS=5`, `:256` +"ENVIRONMENT FAULTS DO NOT CONSUME THE UNIT'S ATTEMPT BUDGET"), which is exactly +why nobody knew how many there had been. Measured effect of adding this loop: +`RunnerError` 59 -> 7 and resolve 47.0% -> 70.0% against a banked 70.67% on the +identical 200 instances. A rescue on that scale is not a clean run, and +`run_quality` says so at 200/200. + +Cluster guidance, deliberately not in the package: on a busy controller these +non-launches cluster around slurmctld RPC rate limiting (`Job credential +expired`). Pacing step creation below the controller's `rl_refill_rate` prevents +them; the retry loop only recovers from them. + ## Resource guards `MemoryGuard` kills a graded test only when its resident memory is at or above diff --git a/scripts/swe_bench_wq.py b/scripts/swe_bench_wq.py index b82648df5..6215def80 100644 --- a/scripts/swe_bench_wq.py +++ b/scripts/swe_bench_wq.py @@ -92,6 +92,9 @@ def cmd_merge(args: argparse.Namespace) -> int: print(f"REFUSED to score run {exc.run_id}:") for reason in exc.reasons: print(f" - {reason}") + if exc.report is not None: + # The honest numbers, so nobody has to recompute a headline by hand. + print(json.dumps(exc.report.to_dict(), indent=2)) return 1 print(json.dumps(result.to_dict(), indent=2)) return 0 diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py index a946edfc2..092fe54df 100644 --- a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -295,6 +295,12 @@ def fingerprint() -> str | None: merged = merge_run(queue, plan.run_id) except MergeRefusal as exc: payload["refused"] = exc.reasons + # A refusal is not an absence of information. Publishing the + # conditional rate, the lower bound and the ids that went missing + # is what stops somebody recomputing a headline by hand from the + # artifacts and reporting attrition as accuracy. + if exc.report is not None: + payload["completeness"] = exc.report.to_dict() write_merge_artifacts(self.report_dir, payload) logger.error("swe_bench_fleet: %s", exc) self.complete = False From 1d7d04c3515da7707b7f85947cfc9f82450d32fc Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Fri, 4 Sep 2026 11:44:09 -0700 Subject: [PATCH 24/25] Add persistent Pyxis SWE-bench runtime --- .../swebench_pyxis/README.md | 33 + .../swebench_pyxis/swebench_pyxis.sbatch | 59 + .../evaluation/swebench_service/README.md | 97 +- .../swebench_service/__main__.py | 23 +- .../swebench_service/pyxis_environment.py | 1217 ++++++++++++++++- .../swebench_service/pyxis_worker.py | 46 +- .../swebench_service/runner.py | 157 ++- .../swebench_service/test_runner.py | 1007 +++++++++++++- 8 files changed, 2503 insertions(+), 136 deletions(-) create mode 100644 examples/10_Agentic_Inference/swebench_pyxis/README.md create mode 100755 examples/10_Agentic_Inference/swebench_pyxis/swebench_pyxis.sbatch diff --git a/examples/10_Agentic_Inference/swebench_pyxis/README.md b/examples/10_Agentic_Inference/swebench_pyxis/README.md new file mode 100644 index 000000000..192159ab5 --- /dev/null +++ b/examples/10_Agentic_Inference/swebench_pyxis/README.md @@ -0,0 +1,33 @@ +# Distributed SWE-bench with Slurm and Pyxis + +This example starts one authenticated SWE-bench service inside an existing +multi-node Slurm allocation. Instance images must already be staged on the +nodes named by the shared node map. + +The repository does not choose cluster policy. Supply the account, partition, +node count, time limit, storage paths, and scheduler-control values when the +job is submitted. A typical submission is: + +```bash +sbatch \ + --account="${SLURM_ACCOUNT}" \ + --partition="${SLURM_PARTITION}" \ + --nodes="${SWEBENCH_NODES}" \ + --time="${SLURM_TIME}" \ + --export=ALL,REPO_ROOT,RUN_ROOT,SWEBENCH_IMAGE_DIR,SWEBENCH_NODE_MAP,SWEBENCH_SERVICE_AUTH_TOKEN_FILE,SWEBENCH_PYXIS_STEP_RATE_PER_S,SWEBENCH_PYXIS_STEP_RATE_STATE_PATH,SWEBENCH_PYXIS_STEP_CONCURRENCY,SWEBENCH_PYXIS_CREATE_CONCURRENCY,SWEBENCH_PYXIS_STEP_LAUNCH_GRACE_S,SWEBENCH_PYXIS_STEP_RETRIES \ + examples/10_Agentic_Inference/swebench_pyxis/swebench_pyxis.sbatch +``` + +`SWEBENCH_IMAGE_DIR` must resolve to a node-local directory with one +`.sqsh` file for each assignment. `SWEBENCH_NODE_MAP` is a shared +file containing one `instance_id node` pair per line. All nodes must resolve +the image directory to their own staged copy. + +Set `SWEBENCH_PYXIS_PERSISTENT_EXEC=1` to keep one command-server step alive +inside each writable environment. The pacing and concurrency settings are +deployment controls; measure values that remain below the target Slurm +controller's step-creation capacity rather than copying another site's values. + +The service prints its URL after readiness. Start additional services on +different allocation nodes when using `swe_bench_fleet`, and provide their +unique URLs through `swebench_service_urls`. diff --git a/examples/10_Agentic_Inference/swebench_pyxis/swebench_pyxis.sbatch b/examples/10_Agentic_Inference/swebench_pyxis/swebench_pyxis.sbatch new file mode 100755 index 000000000..4180774ae --- /dev/null +++ b/examples/10_Agentic_Inference/swebench_pyxis/swebench_pyxis.sbatch @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +#SBATCH --job-name=swebench-pyxis +#SBATCH --ntasks-per-node=1 +#SBATCH --exclusive +#SBATCH --stepmgr + +set -euo pipefail + +: "${REPO_ROOT:?set REPO_ROOT to this repository checkout}" +: "${RUN_ROOT:?set RUN_ROOT to durable shared storage}" +: "${SWEBENCH_IMAGE_DIR:?set SWEBENCH_IMAGE_DIR to staged node-local images}" +: "${SWEBENCH_NODE_MAP:?set SWEBENCH_NODE_MAP to the shared instance/node map}" +: "${SWEBENCH_SERVICE_AUTH_TOKEN_FILE:?set a protected token-file path}" +: "${SWEBENCH_PYXIS_STEP_RATE_PER_S:?set the measured scheduler admission rate}" +: "${SWEBENCH_PYXIS_STEP_RATE_STATE_PATH:?set a shared pacing-state path}" +: "${SWEBENCH_PYXIS_STEP_CONCURRENCY:?set the scheduler-step concurrency}" +: "${SWEBENCH_PYXIS_CREATE_CONCURRENCY:?set the create-step concurrency}" +: "${SWEBENCH_PYXIS_STEP_LAUNCH_GRACE_S:?set the scheduler launch grace}" +: "${SWEBENCH_PYXIS_STEP_RETRIES:?set the safe non-execution attempt count}" + +token_mode=$(stat -c '%a' "$SWEBENCH_SERVICE_AUTH_TOKEN_FILE") +if [[ "$token_mode" != 600 ]]; then + echo "token file must have mode 600, found ${token_mode}" >&2 + exit 2 +fi + +mkdir -p "$RUN_ROOT" +export TMPDIR="$RUN_ROOT/tmp" +mkdir -p "$TMPDIR" +export SLURMD_NODENAME=$(hostname -s) + +# Nested agent steps bind to the allocation, not this service step. +unset SLURM_STEP_ID SLURM_STEPID SLURM_STEP_NODELIST SLURM_STEP_NUM_NODES \ + SLURM_STEP_NUM_TASKS SLURM_STEP_TASKS_PER_NODE SLURM_STEP_LAUNCHER_PORT \ + SLURM_STEP_RESV_PORTS SLURM_SRUN_COMM_HOST SLURM_SRUN_COMM_PORT \ + SLURM_TASK_PID SLURM_PROCID SLURM_LOCALID SLURM_NODEID SLURM_GTIDS \ + SLURM_NTASKS SLURM_NPROCS SLURM_NNODES SLURM_JOB_NUM_NODES \ + SLURM_NTASKS_PER_NODE SLURM_TASKS_PER_NODE SLURM_DISTRIBUTION \ + SLURM_LAUNCH_NODE_IPADDR SLURM_JOBID SLURM_CPUS_PER_TASK \ + SLURM_JOB_CPUS_PER_NODE + +export SWEBENCH_PYXIS_PERSISTENT_EXEC=${SWEBENCH_PYXIS_PERSISTENT_EXEC:-1} +export SWEBENCH_PYXIS_INFRA_RETRY_LOG=${SWEBENCH_PYXIS_INFRA_RETRY_LOG:-$RUN_ROOT/infra_retries.jsonl} +export SWEBENCH_PYXIS_PERSISTENT_STATS_PATH=${SWEBENCH_PYXIS_PERSISTENT_STATS_PATH:-$RUN_ROOT/persistent_exec.jsonl} +export SWEBENCH_PYXIS_STEP_RATE_STATS_PATH=${SWEBENCH_PYXIS_STEP_RATE_STATS_PATH:-$RUN_ROOT/step_rate.json} + +exec uv run \ + --project "$REPO_ROOT/src/inference_endpoint/evaluation/swebench_service" \ + python -m swebench_service \ + --host 0.0.0.0 \ + --port "${SWEBENCH_SERVICE_PORT:-18080}" \ + --artifact-root "$RUN_ROOT/service_artifacts" \ + --subprocess-timeout-s "${SWEBENCH_SUBPROCESS_TIMEOUT_S:-86400}" \ + --runtime pyxis \ + --image-dir "$SWEBENCH_IMAGE_DIR" \ + --node-map "$SWEBENCH_NODE_MAP" \ + --auth-token-file "$SWEBENCH_SERVICE_AUTH_TOKEN_FILE" diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index ef747df0f..9355431ad 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -20,8 +20,8 @@ external-service convention for heavyweight evaluation work. ### Endpoint credentials -`accuracy_config.extras.swebench_service_auth_token` authenticates the *client to -this service*. The credential the agent presents to the *model endpoint* is +`accuracy_config.extras.swebench_service_auth_token` authenticates the _client to +this service_. The credential the agent presents to the _model endpoint_ is separate and comes from the run's endpoint configuration. When no endpoint credential is configured, the agent subprocess is given @@ -68,31 +68,56 @@ registry requires authentication. Launch the service on the compute node inside active one-node Slurm allocation. The runtime requires `SLURM_JOB_ID` and `SLURMD_NODENAME` and assumes the node is exclusive to the user. +For a staged multi-node allocation, use `--image-dir /node/local/images +--node-map /shared/instance_nodes.txt` instead of `--image-registry`. The map +contains one `instance_id node` pair per line. Each environment and eval step is +routed to that node, and terminal container cleanup fans out over every mapped +node. The image directory name must be valid on every target node and contain +`.sqsh` for each assignment. Run the worker allocation with Slurm +step manager enabled (`#SBATCH --stepmgr`) when the site supports it. + Each `srun` step is given an explicit allow-list of environment variables rather than the service's whole environment, so that inherited `SLURM_JOB_ID` / `SLURM_STEP_ID` cannot corrupt a nested `srun`. Several entries on that list are load bearing on real clusters: -| Variable | Why it must reach the step | -| --- | --- | -| `SLURM_CONF` | Without it the child `srun` falls back to `/etc/slurm/slurm.conf` and aborts on a configless or multi-cluster site. | -| `http_proxy`, `https_proxy`, `no_proxy` (+ uppercase) | Enroot performs the registry pull inside the step and needs the caller's proxy policy. | -| `ENROOT_TEMP_PATH`, `ENROOT_CONFIG_PATH` | Enroot creates the container inside the step. Dropping these discards the operator's override, so the multi-gigabyte create-time temp lands back on the device holding the unpacked rootfs. | +| Variable | Why it must reach the step | +| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SLURM_CONF` | Without it the child `srun` falls back to `/etc/slurm/slurm.conf` and aborts on a configless or multi-cluster site. | +| `http_proxy`, `https_proxy`, `no_proxy` (+ uppercase) | Enroot performs the registry pull inside the step and needs the caller's proxy policy. | +| `ENROOT_TEMP_PATH`, `ENROOT_CONFIG_PATH` | Enroot creates the container inside the step. Dropping these discards the operator's override, so the multi-gigabyte create-time temp lands back on the device holding the unpacked rootfs. | Credentials such as `OPENAI_API_KEY`, `HF_TOKEN` and the service auth token are never forwarded, and no other `SLURM_*` variable is. During generation, the service still uses mini-swe-agent for the agent loop and model requests, but replaces its Docker environment with `PyxisEnvironment`. Every -trajectory receives a named, writable Pyxis container. Each tool call becomes an -overlapping `srun` step in that container, preserving filesystem changes across -turns. Tool commands run in private PID namespaces so one trajectory cannot signal -processes belonging to another trajectory. - -Each step reports its outcome through two channels. The primary one is in band: the -step script prints `__MLPERF_STEP_RC__ ` on `srun`'s stdout, which needs -no readable shared filesystem and is stripped from the command output before it is -returned. The fallback is the status file written into the container's `/tmp` mount. +trajectory receives a named, writable Pyxis container. By default, each tool call +becomes an overlapping `srun` step in that container, preserving filesystem +changes across turns. Tool commands run in private PID namespaces so one +trajectory cannot signal processes belonging to another trajectory. + +An experimental persistent execution mode is available by setting +`SWEBENCH_PYXIS_PERSISTENT_EXEC=1`. It starts one long-lived, paced `srun` command +server after creating each named container. Later `execute()` calls use atomic, +per-nonce request and response files in the environment's private `/tmp` mount and +create no additional Slurm steps. The Bash server preserves separate stdout and +stderr, applies the normal command timeout in a private PID namespace, and +publishes `pending` -> `started` -> `finished` state before an atomic completion +manifest. It does not require Python in the task image. + +The mode never replays an active request after its local `srun` client dies: +even a locally observed `pending` file cannot prove the remote step is gone and +will not claim the request later. Cleanup sends a file-based poison pill, then +performs bounded TERM/KILL/reap before the normal exact Enroot-container +removal. Keep this mode opt-in until it passes a CPU-only A/B with the site's +real staged images and Slurm configuration. + +Each step reports its outcome through two channels. The primary one is in band: +the step script prints `__MLPERF_STEP_RC__ ` on `srun`'s +stdout, which needs no readable shared filesystem and is stripped from the +command output before it is returned. The fallback is the status file written +into the container's `/tmp` mount. When a step reports through neither, `StepNotLaunched` (a `RunnerError`) is raised. Besides `srun`'s own output it carries `srun_rc`, the observed `status` bytes, and @@ -100,10 +125,37 @@ Besides `srun`'s own output it carries `srun_rc`, the observed `status` bytes, a sentinel arrived, meaning the step script did not run even its first line and the command definitely did not execute. Anything else leaves open that it did. Callers deciding whether re-running is safe must use that flag rather than the message text. +An outer `srun` timeout is classified through the same channels after Python has +killed and reaped the scheduler client: `pending` remains safely retryable, +`started` is never replayed, and a sentinel or `finished:` is accepted as a +completed command. Partial scheduler output is retained in every failure. Cluster note: on a busy controller these failures cluster around slurmctld RPC rate limiting (`Job credential expired`). Pacing step creation below the controller's -`rl_refill_rate` is a deployment concern rather than a property of this package. +`rl_refill_rate` is deployment-specific. Set +`SWEBENCH_PYXIS_STEP_RATE_PER_S` to a measured safe global step rate for the +service process. Set `SWEBENCH_PYXIS_STEP_RATE_STATE_PATH` to one shared file +to coordinate that budget across several service processes. The limiter admits +execution, retry, and cleanup steps at that fixed rate and is disabled when the +rate variable is unset. On OCI AGA, whose +controller refills 10 RPC tokens/s and where one step consumes several RPCs, a +3,240-step validation completed without a scheduler failure at 1.5 steps/s. + +The reusable deployment controls are environment variables so they propagate to +the isolated agent and eval subprocesses: + +| Variable | Purpose | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `SWEBENCH_PYXIS_STEP_RATE_PER_S` | Fixed scheduler-step admission rate; unset disables pacing. | +| `SWEBENCH_PYXIS_STEP_RATE_STATE_PATH` | Optional shared lock/state file that makes the configured rate global across service processes. | +| `SWEBENCH_PYXIS_STEP_RATE_STATS_PATH` | Optional atomically replaced JSON snapshot written only by a process that issued steps. | +| `SWEBENCH_PYXIS_STEP_CONCURRENCY` | Optional bound on all in-flight scheduler steps. | +| `SWEBENCH_PYXIS_CREATE_CONCURRENCY` | Optional additional bound on concurrent image/container creation. | +| `SWEBENCH_PYXIS_STEP_LAUNCH_GRACE_S` | Extra outer deadline for scheduler launch; it does not change the command's inner timeout. | +| `SWEBENCH_PYXIS_STEP_RETRIES` | Attempts for provable non-execution only; defaults to 3. | +| `SWEBENCH_PYXIS_INFRA_RETRY_LOG` | Optional JSONL retry accounting sink. | +| `SWEBENCH_PYXIS_PERSISTENT_EXEC` | Opt in to one long-lived command-server step per environment; unset/false preserves one `srun` per command. Experimental pending a site CPU A/B. | +| `SWEBENCH_PYXIS_PERSISTENT_STATS_PATH` | Optional JSONL sink with per-environment command, failure, byte, and scheduler-step counters. | After generation, the Pyxis worker evaluates each prediction in a fresh `srun` container step because the Docker-based SWE-bench evaluator cannot run on the @@ -112,7 +164,7 @@ the task image. It preserves SWE-bench 4.1.0's patch-application order, test tim captured output, and `get_eval_report` grading. A patch failure or test timeout is an unresolved task; an `srun`, Enroot, or container-start failure is an infrastructure loss, recorded per instance and reported rather than allowed to fail the whole run -(see *Eval-phase failures*). The service then aggregates the per-instance reports +(see _Eval-phase failures_). The service then aggregates the per-instance reports and removes its named Pyxis containers. Pyxis namespaces a named container by its allocation: `--container-name=X` inside @@ -124,6 +176,11 @@ allocation. `scancel` does not reclaim them either -- ending the job does not remove Enroot containers -- which is why the removal has to be both correctly named and audible. +The parameterized Slurm/Pyxis service recipe is in +`examples/10_Agentic_Inference/swebench_pyxis/`. It keeps cluster allocation, +storage, image staging, and scheduler-control values in the deployment +environment rather than embedding site policy in the service. + ### Agent-phase failures The agent phase runs `--workers` trajectories concurrently. If it fails after @@ -146,7 +203,7 @@ still graded and the run report is still produced: an instance with no visible outcome. The instances that were lost are listed in the `eval_infra_failures.txt` artifact, one `instance_iderror` per line. -A run in which *no* instance could be evaluated still fails — but only after the +A run in which _no_ instance could be evaluated still fails — but only after the report has been written, so the run can be diagnosed from its own artifacts. ### Telling a degraded run apart from a bad one @@ -154,7 +211,7 @@ report has been written, so the run can be diagnosed from its own artifacts. Both artifacts are machine-readable on purpose. `agent_phase_error.txt` means some instances may be missing from `swe_bench_results.json`; `eval_infra_failures.txt` names the instances the harness lost during grading. Instances lost this way are -*infrastructure* losses, not model failures, and a consumer that cannot separate +_infrastructure_ losses, not model failures, and a consumer that cannot separate the two will read attrition as an accuracy regression. Neither file exists on a clean run. diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/__main__.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/__main__.py index 74f13087b..0f5b9c37e 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/__main__.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/__main__.py @@ -33,20 +33,37 @@ def main() -> None: parser.add_argument("--max-concurrent-runs", type=int, default=1) parser.add_argument("--subprocess-timeout-s", type=int, default=24 * 60 * 60) parser.add_argument("--runtime", choices=("docker", "pyxis"), default="docker") - parser.add_argument("--image-registry") + image_group = parser.add_mutually_exclusive_group() + image_group.add_argument("--image-registry") + image_group.add_argument("--image-dir", type=Path) + parser.add_argument( + "--node-map", + type=Path, + help="optional file with one 'instance_id node' Pyxis assignment per line", + ) auth_group = parser.add_mutually_exclusive_group() auth_group.add_argument("--auth-token") + auth_group.add_argument("--auth-token-file", type=Path) auth_group.add_argument("--allow-unauthenticated", action="store_true") parser.add_argument("--max-stored-runs", type=int, default=100) args = parser.parse_args() + auth_token = args.auth_token + if args.auth_token_file is not None: + try: + auth_token = args.auth_token_file.read_text().strip() + except OSError as exc: + parser.error(f"could not read --auth-token-file: {exc}") + if not auth_token: + parser.error("--auth-token-file is empty") + config = ServiceConfig( host=args.host, port=args.port, artifact_root=Path(args.artifact_root), max_concurrent_runs=args.max_concurrent_runs, subprocess_timeout_s=args.subprocess_timeout_s, - auth_token=args.auth_token, + auth_token=auth_token, allow_unauthenticated=args.allow_unauthenticated, max_stored_runs=args.max_stored_runs, ) @@ -55,6 +72,8 @@ def main() -> None: project_root=Path(__file__).resolve().parents[1], subprocess_timeout_s=config.subprocess_timeout_s, image_registry=args.image_registry, + image_dir=args.image_dir, + node_map=args.node_map, ) web.run_app(create_app(config, runner=runner), host=config.host, port=config.port) diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py index 18c05bfc4..2f7fb86b7 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_environment.py @@ -3,16 +3,24 @@ from __future__ import annotations +import fcntl +import hashlib +import hmac import json import logging +import math import os import platform import re +import secrets +import shutil import subprocess import tempfile import threading import time import uuid +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, ExitStack, contextmanager, nullcontext from pathlib import Path from typing import Any @@ -69,13 +77,155 @@ nonce=$3 shift 3 printf 'started\n' > "$status_path" 2>/dev/null -unshare --pid --fork --mount-proc timeout "$timeout_s" "$@" +timeout_log="${status_path}.timeout.${nonce}" +: > "$timeout_log" +lc_all_was_set=0 +[ "${LC_ALL+x}" = x ] && lc_all_was_set=1 +original_lc_all=${LC_ALL-} +unshare --pid --fork --mount-proc \ + env LC_ALL=C timeout --verbose -k 5 "$timeout_s" \ + bash -c ' + if [ "$1" -eq 1 ]; then export LC_ALL=$2; else unset LC_ALL; fi + shift 2 + exec "$@" 2>&3 + ' pyxis-command "$lc_all_was_set" "$original_lc_all" "$@" \ + 3>&2 2>"$timeout_log" returncode=$? -printf 'finished:%s\n' "$returncode" > "$status_path" 2>/dev/null -printf '\n__MLPERF_STEP_RC__ %s %s\n' "$nonce" "$returncode" +cat "$timeout_log" >&2 +timed_out=0 +grep -q '^timeout: sending signal ' "$timeout_log" && timed_out=1 +rm -f "$timeout_log" +printf 'finished:%s:%s\n' "$returncode" "$timed_out" > "$status_path" 2>/dev/null +printf '\n__MLPERF_STEP_RC__ %s %s %s\n' "$nonce" "$returncode" "$timed_out" exit "$returncode" """ +# Host-level inspection commands such as ``enroot list`` must run in the +# node's existing namespaces. Some sites prohibit a nested PID namespace for +# those commands even though the same isolation is required for commands run +# inside a writable SWE-bench container. Keep the status and sentinel +# protocol identical so opting out of PID isolation does not bypass timeout or +# non-launch classification. +_HOST_STEP_SCRIPT = r"""set +e +status_path=$1 +timeout_s=$2 +nonce=$3 +shift 3 +printf 'started\n' > "$status_path" 2>/dev/null +timeout_log="${status_path}.timeout.${nonce}" +: > "$timeout_log" +lc_all_was_set=0 +[ "${LC_ALL+x}" = x ] && lc_all_was_set=1 +original_lc_all=${LC_ALL-} +env LC_ALL=C timeout --verbose -k 5 "$timeout_s" \ + bash -c ' + if [ "$1" -eq 1 ]; then export LC_ALL=$2; else unset LC_ALL; fi + shift 2 + exec "$@" 2>&3 + ' pyxis-command "$lc_all_was_set" "$original_lc_all" "$@" \ + 3>&2 2>"$timeout_log" +returncode=$? +cat "$timeout_log" >&2 +timed_out=0 +grep -q '^timeout: sending signal ' "$timeout_log" && timed_out=1 +rm -f "$timeout_log" +printf 'finished:%s:%s\n' "$returncode" "$timed_out" > "$status_path" 2>/dev/null +printf '\n__MLPERF_STEP_RC__ %s %s %s\n' "$nonce" "$returncode" "$timed_out" +exit "$returncode" +""" + +_PERSISTENT_EXEC_ENV = "SWEBENCH_PYXIS_PERSISTENT_EXEC" +_PERSISTENT_STATS_ENV = "SWEBENCH_PYXIS_PERSISTENT_STATS_PATH" +_PERSISTENT_ROOT = "/tmp/.mlperf_persistent_exec" +_PERSISTENT_POLL_S = 0.05 +_PERSISTENT_STATS_LOCK = threading.Lock() +_PERSISTENT_SERVER_SCRIPT = r"""set -u +root=$1 +generation=$2 +secret=$3 +shift 3 +interpreter=("$@") + +atomic_write() { + path=$1 + value=$2 + temporary="${path}.tmp.$$" + printf '%s\n' "$value" > "$temporary" || exit 70 + mv -f -- "$temporary" "$path" || exit 70 +} + +mkdir -p "$root/requests" || exit 70 +atomic_write "$root/server_status" started +atomic_write "$root/ready" "$generation" + +while :; do + if [ -f "$root/stop" ]; then + atomic_write "$root/server_status" stopped + exit 0 + fi + handled=0 + for request in "$root"/requests/*; do + [ -d "$request" ] || continue + mkdir "$request/claim" 2>/dev/null || continue + status=$(cat "$request/status" 2>/dev/null || printf unknown) + if [ "$status" != pending ]; then + rmdir "$request/claim" 2>/dev/null || true + continue + fi + handled=1 + atomic_write "$request/status" started + timeout_s=$(cat "$request/timeout" 2>/dev/null || printf invalid) + separate=$(cat "$request/separate" 2>/dev/null || printf 0) + case "$timeout_s" in + ''|*[!0-9]*) returncode=125; timed_out=0 ;; + *) + cwd=$(cat "$request/cwd" 2>/dev/null || printf /testbed) + command=$(cat "$request/command" 2>/dev/null || printf '') + if [ "$separate" = 1 ]; then + ( + cd -- "$cwd" || exit 125 + unshare --pid --fork --mount-proc \ + timeout -k 5 "$timeout_s" "${interpreter[@]}" "$command" + ) > "$request/stdout.tmp" 2> "$request/stderr.tmp" + else + ( + cd -- "$cwd" || exit 125 + unshare --pid --fork --mount-proc \ + timeout -k 5 "$timeout_s" "${interpreter[@]}" "$command" + ) > "$request/stdout.tmp" 2>&1 + : > "$request/stderr.tmp" + fi + returncode=$? + case "$returncode" in 124|137) timed_out=1 ;; *) timed_out=0 ;; esac + ;; + esac + [ -f "$request/stdout.tmp" ] || : > "$request/stdout.tmp" + [ -f "$request/stderr.tmp" ] || : > "$request/stderr.tmp" + mv -f -- "$request/stdout.tmp" "$request/stdout" || exit 70 + mv -f -- "$request/stderr.tmp" "$request/stderr" || exit 70 + stdout_size=$(wc -c < "$request/stdout") || exit 70 + stderr_size=$(wc -c < "$request/stderr") || exit 70 + nonce=${request##*/} + digest=$( + { + printf '%s\0%s\0%s\0%s\0%s\0%s\0' \ + "$secret" "$nonce" "$returncode" "$stdout_size" \ + "$stderr_size" "$timed_out" + cat "$request/stdout" + printf '\0' + cat "$request/stderr" + printf '\0%s' "$secret" + } | sha256sum + ) || exit 70 + digest=${digest%% *} + atomic_write "$request/status" "finished:$returncode" + atomic_write "$request/complete" \ + "$returncode $stdout_size $stderr_size $timed_out $digest" + done + [ "$handled" -eq 1 ] || sleep 0.05 +done +""" + class StepNotLaunched(RunnerError): """An `srun` step that reported through neither result channel. @@ -103,11 +253,15 @@ def __init__( provable_non_execution: bool, srun_rc: int | None, status: str, + stdout: str = "", + stderr: str = "", ) -> None: super().__init__(message) self.provable_non_execution = provable_non_execution self.srun_rc = srun_rc self.status = status + self.stdout = stdout + self.stderr = stderr def read_step_sentinel(text: str, nonce: str) -> tuple[int | None, str]: @@ -116,18 +270,476 @@ def read_step_sentinel(text: str, nonce: str) -> tuple[int | None, str]: ``(None, text)`` when the step did not report in band. The nonce makes the marker unforgeable by the command's own output. """ + reported, _timed_out, cleaned = _read_step_sentinel_details(text, nonce) + return reported, cleaned + + +def _read_step_sentinel_details(text: str, nonce: str) -> tuple[int | None, bool, str]: tag = f"{_STEP_SENTINEL} {nonce} " for line in reversed((text or "").splitlines()): if not line.startswith(tag): continue - value = line[len(tag) :].strip() - if value.lstrip("-").isdigit(): - return int(value), text[: text.rindex(line)].rstrip("\n") - return None, text + fields = line[len(tag) :].split() + if ( + fields + and fields[0].lstrip("-").isdigit() + and (len(fields) == 1 or fields[1] in {"0", "1"}) + ): + return ( + int(fields[0]), + len(fields) > 1 and fields[1] == "1", + text[: text.rindex(line)].rstrip("\n"), + ) + return None, False, text + + +def _output_text(output: str | bytes | None) -> str: + if output is None: + return "" + if isinstance(output, bytes): + return output.decode("utf-8", errors="replace") + return output + + +def _read_step_status(status_path: Path) -> str: + try: + return status_path.read_text().strip() + except OSError as exc: + return f"" + + +def _atomic_write_text(path: Path, text: str) -> None: + temporary = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") + temporary.write_text(text) + os.replace(temporary, path) + + +def _read_sized_file( + path: Path, + size: int, + deadline: float, + *, + read_bytes: Callable[[Path], bytes] | None = None, + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> bytes: + """Read an atomically published response, retrying stale partial views.""" + reader = read_bytes or (lambda target: target.read_bytes()) + while True: + try: + data = reader(path) + except OSError: + data = b"" + if len(data) == size: + return data + if len(data) > size: + raise RunnerError( + f"persistent Pyxis response {path} grew past manifest size " + f"({len(data)} > {size})" + ) + if monotonic() >= deadline: + raise RunnerError( + f"persistent Pyxis response {path} remained partial " + f"({len(data)} of {size} bytes)" + ) + sleep(_PERSISTENT_POLL_S) + + +def _persistent_response_digest( + *, + secret: str, + nonce: str, + returncode: int, + stdout: bytes, + stderr: bytes, + timed_out: bool, +) -> str: + prefix = "\0".join( + ( + secret, + nonce, + str(returncode), + str(len(stdout)), + str(len(stderr)), + "1" if timed_out else "0", + ) + ).encode() + payload = prefix + b"\0" + stdout + b"\0" + stderr + b"\0" + secret.encode() + return hashlib.sha256(payload).hexdigest() + + +def _env_flag(name: str) -> bool: + raw = os.environ.get(name, "").strip().lower() + if not raw: + return False + if raw in {"1", "true", "yes", "on"}: + return True + if raw in {"0", "false", "no", "off"}: + return False + raise RunnerError(f"{name} must be a boolean (1/0, true/false, yes/no, on/off)") + + +class _PersistentExecChannel: + """One long-lived scheduler step serving atomic file-based requests.""" + + def __init__( + self, + *, + protocol_dir: Path, + command_factory: Callable[[str, str], list[str]], + admission_factory: Callable[[], AbstractContextManager[None]] = nullcontext, + failure_path: Path | None = None, + launch_timeout_s: float = 30.0, + driver_grace_s: float = 30.0, + shutdown_grace_s: float = 3.0, + capture_stderr_separately: bool = False, + retry_target: str = "persistent-pyxis-server", + ) -> None: + self.protocol_dir = protocol_dir + self._requests_dir = protocol_dir / "requests" + self._command_factory = command_factory + self._admission_factory = admission_factory + self._failure_path = failure_path + self._launch_timeout_s = launch_timeout_s + self._driver_grace_s = driver_grace_s + self._shutdown_grace_s = shutdown_grace_s + self._capture_stderr_separately = capture_stderr_separately + self._retry_target = retry_target + self._secret = secrets.token_hex(32) + self._process: subprocess.Popen[bytes] | None = None + self._stdout_handle: Any = None + self._stderr_handle: Any = None + self._lock = threading.Lock() + self._closed = False + self.stats: dict[str, int] = { + "server_starts": 0, + "commands": 0, + "command_failures": 0, + "nonzero_commands": 0, + "safe_restarts": 0, + "stdout_bytes": 0, + "stderr_bytes": 0, + } + self._requests_dir.mkdir(parents=True, exist_ok=True) + + def _log_tail(self) -> tuple[str, str]: + def tail(path: Path) -> str: + try: + return path.read_text(errors="replace")[-2000:] + except OSError: + return "" + + return tail(self.protocol_dir / "server.stdout"), tail( + self.protocol_dir / "server.stderr" + ) + + def _close_handles(self) -> None: + for handle_name in ("_stdout_handle", "_stderr_handle"): + handle = getattr(self, handle_name) + if handle is not None: + try: + handle.close() + finally: + setattr(self, handle_name, None) + + def _discard_process(self) -> None: + process = self._process + if process is not None: + try: + process.wait(timeout=0) + except subprocess.TimeoutExpired: + pass + self._process = None + self._close_handles() + + def _launch_once(self) -> None: + generation = uuid.uuid4().hex + (self.protocol_dir / "ready").unlink(missing_ok=True) + (self.protocol_dir / "stop").unlink(missing_ok=True) + _atomic_write_text(self.protocol_dir / "server_status", "pending\n") + stdout_path = self.protocol_dir / "server.stdout" + stderr_path = self.protocol_dir / "server.stderr" + self._stdout_handle = open(stdout_path, "ab", buffering=0) + self._stderr_handle = open(stderr_path, "ab", buffering=0) + try: + with self._admission_factory(): + self._process = subprocess.Popen( + self._command_factory(generation, self._secret), + stdin=subprocess.DEVNULL, + stdout=self._stdout_handle, + stderr=self._stderr_handle, + env=safe_srun_env(), + ) + self.stats["server_starts"] += 1 + deadline = time.monotonic() + self._launch_timeout_s + while True: + try: + ready = (self.protocol_dir / "ready").read_text().strip() + except OSError: + ready = "" + if ready == generation: + return + returncode = self._process.poll() + if returncode is not None: + status = _read_step_status(self.protocol_dir / "server_status") + stdout, stderr = self._log_tail() + raise StepNotLaunched( + "persistent Pyxis server died before readiness " + f"(rc={returncode}, status={status!r})" + + _srun_evidence(stdout, stderr), + provable_non_execution=status == _STEP_STATUS_PENDING, + srun_rc=returncode, + status=status, + stdout=stdout, + stderr=stderr, + ) + if time.monotonic() >= deadline: + status = _read_step_status(self.protocol_dir / "server_status") + stdout, stderr = self._log_tail() + raise StepNotLaunched( + "persistent Pyxis server did not become ready within " + f"{self._launch_timeout_s}s (status={status!r})" + + _srun_evidence(stdout, stderr), + provable_non_execution=status == _STEP_STATUS_PENDING, + srun_rc=None, + status=status, + stdout=stdout, + stderr=stderr, + ) + time.sleep(_PERSISTENT_POLL_S) + except Exception: + self._terminate_process() + raise + + def start(self) -> None: + attempts = _step_retry_attempts() + target = self._retry_target + for attempt in range(1, attempts + 1): + try: + self._launch_once() + except StepNotLaunched as exc: + if not exc.provable_non_execution or attempt == attempts: + if self._failure_path is not None: + self._failure_path.touch() + _record_step_retry( + target=target, + attempt=attempt, + outcome=( + "not_retryable" + if not exc.provable_non_execution + else "exhausted" + ), + detail=f"srun_rc={exc.srun_rc} status={exc.status!r}", + ) + raise + _record_step_retry( + target=target, + attempt=attempt, + outcome="retrying", + detail=f"srun_rc={exc.srun_rc} status={exc.status!r}", + ) + time.sleep(min(30.0, 2.0 * attempt)) + continue + if attempt > 1: + _record_step_retry(target=target, attempt=attempt, outcome="recovered") + return + raise AssertionError("unreachable") # pragma: no cover + + def _terminate_process(self) -> None: + process = self._process + if process is None: + self._close_handles() + return + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=self._shutdown_grace_s) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=self._shutdown_grace_s) + else: + process.wait() + self._discard_process() + + def _read_completion( + self, request_dir: Path, deadline: float + ) -> subprocess.CompletedProcess[str] | None: + complete_path = request_dir / "complete" + try: + manifest = complete_path.read_text().strip().split() + except OSError: + return None + if ( + len(manifest) != 5 + or not all(value.lstrip("-").isdigit() for value in manifest[:4]) + or re.fullmatch(r"[0-9a-f]{64}", manifest[4]) is None + ): + if time.monotonic() >= deadline: + raise RunnerError( + f"persistent Pyxis completion marker is invalid: {manifest!r}" + ) + return None + returncode, stdout_size, stderr_size, timed_out = map(int, manifest[:4]) + digest = manifest[4] + if stdout_size < 0 or stderr_size < 0 or timed_out not in {0, 1}: + raise RunnerError("persistent Pyxis completion marker has invalid fields") + stdout = _read_sized_file(request_dir / "stdout", stdout_size, deadline) + stderr = _read_sized_file(request_dir / "stderr", stderr_size, deadline) + expected_digest = _persistent_response_digest( + secret=self._secret, + nonce=request_dir.name, + returncode=returncode, + stdout=stdout, + stderr=stderr, + timed_out=bool(timed_out), + ) + if not hmac.compare_digest(digest, expected_digest): + raise RunnerError("persistent Pyxis completion digest did not verify") + self.stats["stdout_bytes"] += len(stdout) + self.stats["stderr_bytes"] += len(stderr) + if returncode != 0: + self.stats["nonzero_commands"] += 1 + result = subprocess.CompletedProcess( + ["persistent-pyxis-exec"], + returncode, + stdout=stdout.decode("utf-8", errors="replace"), + stderr=stderr.decode("utf-8", errors="replace"), + ) + result.__dict__["timed_out"] = bool(timed_out) + return result + + def execute( + self, *, command: str, cwd: str, timeout_s: int + ) -> subprocess.CompletedProcess[str]: + with self._lock: + if self._closed: + raise RunnerError("persistent Pyxis channel is closed") + if self._process is None or self._process.poll() is not None: + self.stats["command_failures"] += 1 + if self._failure_path is not None: + self._failure_path.touch() + raise RunnerError("persistent Pyxis server is not running") + nonce = uuid.uuid4().hex + temporary = self._requests_dir / f".{nonce}.{os.getpid()}.tmp" + request_dir = self._requests_dir / nonce + temporary.mkdir() + (temporary / "command").write_text(command) + (temporary / "cwd").write_text(cwd) + (temporary / "timeout").write_text(str(timeout_s)) + (temporary / "separate").write_text( + "1\n" if self._capture_stderr_separately else "0\n" + ) + (temporary / "status").write_text(f"{_STEP_STATUS_PENDING}\n") + os.replace(temporary, request_dir) + self.stats["commands"] += 1 + deadline = time.monotonic() + timeout_s + self._driver_grace_s + while True: + try: + completed = self._read_completion(request_dir, deadline) + except RunnerError: + self.stats["command_failures"] += 1 + if self._failure_path is not None: + self._failure_path.touch() + raise + if completed is not None: + try: + shutil.rmtree(request_dir) + except OSError: + # Finished requests are ignored by the server and + # nonces are never reused. The private temp tree is + # removed with the environment, so response cleanup + # cannot turn a completed command into a failure. + logger.debug( + "could not remove completed persistent request %s", + request_dir, + exc_info=True, + ) + return completed + returncode = self._process.poll() if self._process is not None else None + if returncode is not None: + status = _read_step_status(request_dir / "status") + stdout, stderr = self._log_tail() + self.stats["command_failures"] += 1 + if self._failure_path is not None: + self._failure_path.touch() + raise StepNotLaunched( + "persistent Pyxis server died while a request was active " + f"(rc={returncode}, status={status!r})" + + _srun_evidence(stdout, stderr), + # Local srun-client death cannot prove that its remote + # step also died. Never start a second server while the + # first could still claim this request. + provable_non_execution=False, + srun_rc=returncode, + status=status, + stdout=stdout, + stderr=stderr, + ) + if time.monotonic() >= deadline: + status = _read_step_status(request_dir / "status") + self.stats["command_failures"] += 1 + if self._failure_path is not None: + self._failure_path.touch() + raise StepNotLaunched( + "persistent Pyxis request exceeded its driver deadline " + f"(status={status!r})", + # A live server can cross pending->started immediately + # after this observation, so deadline expiry is never a + # safe replay decision. + provable_non_execution=False, + srun_rc=None, + status=status, + ) + time.sleep(_PERSISTENT_POLL_S) + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + _atomic_write_text(self.protocol_dir / "stop", "stop\n") + process = self._process + if process is not None and process.poll() is None: + try: + process.wait(timeout=self._shutdown_grace_s) + except subprocess.TimeoutExpired: + self._terminate_process() + else: + self._discard_process() + else: + self._discard_process() def safe_srun_env() -> dict[str, str]: - return {name: os.environ[name] for name in _SAFE_SRUN_ENV if name in os.environ} + env = {name: os.environ[name] for name in _SAFE_SRUN_ENV if name in os.environ} + # The caller may deliberately put tempfile.TemporaryDirectory on shared + # storage so a container routed to another node can mount it. Passing + # that same TMPDIR to slurmstepd is a separate concern: the remote node may + # not have the path yet, and its fallback warning is merged into the + # agent's command observation. Resolve the host-side mount first, then + # keep only the child step's temporary files node-local. + env["TMPDIR"] = "/tmp" + return env + + +_STEP_LAUNCH_GRACE_ENV = "SWEBENCH_PYXIS_STEP_LAUNCH_GRACE_S" + + +def _step_launch_grace_s() -> float: + raw = os.environ.get(_STEP_LAUNCH_GRACE_ENV, "").strip() + if not raw: + return 0.0 + try: + grace = float(raw) + except ValueError as exc: + raise RunnerError( + f"{_STEP_LAUNCH_GRACE_ENV} must be a finite number at least zero" + ) from exc + if not math.isfinite(grace) or grace < 0: + raise RunnerError( + f"{_STEP_LAUNCH_GRACE_ENV} must be a finite number at least zero" + ) + return grace def build_srun_command( @@ -137,12 +749,13 @@ def build_srun_command( name: str | None = None, mounts: list[tuple[Path, str]] | None = None, workdir: str | None = None, + node: str | None = None, ) -> list[str]: job_id = os.environ.get("SLURM_JOB_ID", "").strip() if not job_id: raise RunnerError("Pyxis runtime requires SLURM_JOB_ID") - node = os.environ.get("SLURMD_NODENAME", "").strip() - if not node: + target_node = (node or os.environ.get("SLURMD_NODENAME") or "").strip() + if not target_node: raise RunnerError("Pyxis runtime requires SLURMD_NODENAME") command = [ "srun", @@ -150,7 +763,7 @@ def build_srun_command( f"--jobid={job_id}", "-N1", "-n1", - f"--nodelist={node}", + f"--nodelist={target_node}", ] if image is not None: image_ref = str(image.resolve()) if isinstance(image, Path) else image @@ -189,7 +802,9 @@ def _run_srun_step_once( name: str | None = None, mounts: list[tuple[Path, str]] | None = None, workdir: str | None = None, + node: str | None = None, stderr: int = subprocess.STDOUT, + isolate_pid_namespace: bool = True, ) -> subprocess.CompletedProcess[str]: nonce = uuid.uuid4().hex status_path.write_text(f"{_STEP_STATUS_PENDING}\n") @@ -199,17 +814,19 @@ def _run_srun_step_once( name=name, mounts=mounts, workdir=workdir, + node=node, argv=[ "bash", "-c", - _STEP_SCRIPT, + _STEP_SCRIPT if isolate_pid_namespace else _HOST_STEP_SCRIPT, "pyxis-step", - _STEP_STATUS, + (_STEP_STATUS if isolate_pid_namespace else str(status_path.resolve())), str(timeout_s), nonce, *argv, ], ) + outer_timeout_s = timeout_s + 30 + _step_launch_grace_s() try: result = subprocess.run( command, @@ -218,15 +835,43 @@ def _run_srun_step_once( errors="replace", stdout=subprocess.PIPE, stderr=stderr, - timeout=timeout_s + 30, + timeout=outer_timeout_s, env=safe_srun_env(), ) except subprocess.TimeoutExpired as exc: - if failure_path is not None: - failure_path.touch() - raise RunnerError( - f"Pyxis step exceeded its {timeout_s + 30}s deadline and was killed" - + _srun_evidence(exc.output) + # subprocess.run() has already killed and reaped its Popen child before + # re-raising TimeoutExpired. Preserve the final communicate() output, + # then consult the same two completion channels as the normal path. + # A scheduler client can outlive a command that finished, or time out + # before the step script ran at all; those outcomes are not equivalent. + stdout = _output_text(exc.output) + captured_stderr = _output_text(exc.stderr) + reported, timed_out, cleaned = _read_step_sentinel_details(stdout, nonce) + if reported is not None: + completed = subprocess.CompletedProcess( + command, reported, stdout=cleaned, stderr=captured_stderr + ) + completed.__dict__["timed_out"] = timed_out + return completed + status = _read_step_status(status_path) + finished = re.fullmatch(r"finished:(-?\d+):([01])", status) + if finished is not None: + completed = subprocess.CompletedProcess( + command, + int(finished.group(1)), + stdout=stdout, + stderr=captured_stderr, + ) + completed.__dict__["timed_out"] = finished.group(2) == "1" + return completed + raise StepNotLaunched( + f"Pyxis step exceeded its {outer_timeout_s}s outer deadline " + f"(status={status!r})" + _srun_evidence(stdout, captured_stderr), + provable_non_execution=status == _STEP_STATUS_PENDING, + srun_rc=None, + status=status, + stdout=stdout, + stderr=captured_stderr, ) from exc except (OSError, subprocess.SubprocessError) as exc: if failure_path is not None: @@ -237,29 +882,31 @@ def _run_srun_step_once( ) from exc # Primary channel: the step reported its own return code in band. - reported, cleaned = read_step_sentinel(result.stdout, nonce) + reported, timed_out, cleaned = _read_step_sentinel_details(result.stdout, nonce) if reported is not None: result.stdout = cleaned result.returncode = reported + result.__dict__["timed_out"] = timed_out return result # Fallback channel: the status file the step script wrote into the mount. - try: - status = status_path.read_text().strip() - except OSError as exc: - status = f"" - if status == f"finished:{result.returncode}": + status = _read_step_status(status_path) + if status in { + f"finished:{result.returncode}:0", + f"finished:{result.returncode}:1", + }: + result.__dict__["timed_out"] = status.endswith(":1") return result - if failure_path is not None: - failure_path.touch() raise StepNotLaunched( "Pyxis infrastructure failure before the command completed " f"(srun exited {result.returncode}, status={status!r})" - + _srun_evidence(result.stdout), + + _srun_evidence(result.stdout, result.stderr), provable_non_execution=status == _STEP_STATUS_PENDING, srun_rc=result.returncode, status=status, + stdout=_output_text(result.stdout), + stderr=_output_text(result.stderr), ) @@ -303,7 +950,7 @@ def _record_create_timing(image: str | Path, seconds: float, *, ok: bool) -> Non logger.debug("could not record Pyxis create timing", exc_info=True) -def _srun_evidence(output: str | bytes | None, limit: int = 2000) -> str: +def _srun_evidence(*outputs: str | bytes | None, limit: int = 2000) -> str: """Attach srun's own words to a Pyxis failure. srun/pyxis/enroot report the actual cause -- image import failure, no space @@ -312,11 +959,16 @@ def _srun_evidence(output: str | bytes | None, limit: int = 2000) -> str: indistinguishable message, which is exactly what made a 200-instance run's 17 lost units undiagnosable from its artifacts. """ - if not output: - return "" - if isinstance(output, bytes): - output = output.decode("utf-8", errors="replace") - text = output.strip() + parts: list[str] = [] + for output in outputs: + if not output: + continue + if isinstance(output, bytes): + output = output.decode("utf-8", errors="replace") + output = output.strip() + if output: + parts.append(output) + text = "\n".join(parts) if not text: return "" if len(text) > limit: @@ -336,6 +988,215 @@ def _srun_evidence(output: str | bytes | None, limit: int = 2000) -> str: #: the benchmark client, so they share a file format instead. _STEP_RETRY_LOG_ENV = "SWEBENCH_PYXIS_INFRA_RETRY_LOG" _RETRY_LOG_LOCK = threading.Lock() +_RETRY_LOG_ERRORS: dict[str, list[str]] = {} + +#: Optional process-wide admission rate for new Slurm steps. One Pyxis agent +#: command is one ``srun --overlap`` step, and a step costs several controller +#: RPCs. At large worker counts, limiting concurrent steps is insufficient: +#: sustained creation can exhaust slurmctld's RPC bucket and leave srun +#: backing off until its job credential expires. The deployment must set this +#: below its controller-specific safe rate; an unset value preserves the +#: package's existing unpaced behavior. +_STEP_RATE_ENV = "SWEBENCH_PYXIS_STEP_RATE_PER_S" +_STEP_RATE_STATE_ENV = "SWEBENCH_PYXIS_STEP_RATE_STATE_PATH" +_STEP_RATE_STATS_ENV = "SWEBENCH_PYXIS_STEP_RATE_STATS_PATH" +_STEP_PACER_UNSET = object() + + +class _StepPacer: + """Thread-safe fixed-rate admission for step creation. + + Slots are reserved under the lock and slept outside it. This avoids both a + burst after a quiet period and holding the lock while one caller waits. + """ + + def __init__( + self, + rate_per_s: float, + *, + state_path: Path | None = None, + monotonic: Any = time.monotonic, + wall_time: Any = time.time, + sleep: Any = time.sleep, + ) -> None: + self.rate_per_s = rate_per_s + self._interval_s = 1.0 / rate_per_s + self._monotonic = monotonic + self._wall_time = wall_time + self._sleep = sleep + self._state_path = state_path + self._next_slot = 0.0 + self._started_at = monotonic() + self._last_report_at = 0.0 + self._issued = 0 + self._waited_s = 0.0 + self._lock = threading.Lock() + + def wait(self) -> None: + with self._lock: + if self._state_path is None: + now = self._monotonic() + slot = max(now, self._next_slot) + self._next_slot = slot + self._interval_s + else: + self._state_path.parent.mkdir(parents=True, exist_ok=True) + try: + with self._state_path.open("a+", encoding="utf-8") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + handle.seek(0) + raw = handle.read().strip() + now = self._wall_time() + try: + next_slot = float(raw) if raw else 0.0 + except ValueError: + next_slot = 0.0 + slot = max(now, next_slot) + handle.seek(0) + handle.truncate() + handle.write(f"{slot + self._interval_s:.9f}\n") + handle.flush() + os.fsync(handle.fileno()) + except OSError as exc: + raise RunnerError( + f"could not reserve shared Pyxis step-rate slot in " + f"{self._state_path}: {exc}" + ) from exc + self._issued += 1 + delay = slot - now + if delay > 0: + self._sleep(delay) + self._record_stats(delay) + + def _record_stats(self, delay: float) -> None: + path_text = os.environ.get(_STEP_RATE_STATS_ENV, "").strip() + with self._lock: + self._waited_s += max(0.0, delay) + now = self._monotonic() + if not path_text or now - self._last_report_at < 60.0: + return + self._last_report_at = now + elapsed = max(1e-6, now - self._started_at) + payload = { + "pid": os.getpid(), + "budget_steps_per_s": self.rate_per_s, + "steps_issued": self._issued, + "observed_steps_per_s": round(self._issued / elapsed, 3), + "time_spent_throttled_s": round(self._waited_s, 1), + "elapsed_s": round(elapsed, 1), + } + path = Path(path_text.replace("{pid}", str(os.getpid()))) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + temporary.write_text(json.dumps(payload, indent=1) + "\n") + os.replace(temporary, path) + except OSError: + temporary.unlink(missing_ok=True) + logger.debug("could not write Pyxis step-rate stats", exc_info=True) + + +_STEP_PACER_CONFIG_LOCK = threading.Lock() +_STEP_PACER_SETTING: object | tuple[str, str] = _STEP_PACER_UNSET +_STEP_PACER: _StepPacer | None = None + +_STEP_CONCURRENCY_ENV = "SWEBENCH_PYXIS_STEP_CONCURRENCY" +_CREATE_CONCURRENCY_ENV = "SWEBENCH_PYXIS_CREATE_CONCURRENCY" +_LIMITERS_CONFIG_LOCK = threading.Lock() +_LIMITERS_SETTING: tuple[str, str] | None = None +_STEP_LIMITER: threading.BoundedSemaphore | None = None +_CREATE_LIMITER: threading.BoundedSemaphore | None = None + + +def _optional_positive_int(name: str) -> int | None: + raw = os.environ.get(name, "").strip() + if not raw: + return None + try: + value = int(raw) + except ValueError as exc: + raise RunnerError(f"{name} must be an integer greater than zero") from exc + if value <= 0: + raise RunnerError(f"{name} must be an integer greater than zero") + return value + + +def _configured_limiters() -> tuple[ + threading.BoundedSemaphore | None, threading.BoundedSemaphore | None +]: + global _LIMITERS_SETTING, _STEP_LIMITER, _CREATE_LIMITER + + setting = ( + os.environ.get(_STEP_CONCURRENCY_ENV, "").strip(), + os.environ.get(_CREATE_CONCURRENCY_ENV, "").strip(), + ) + with _LIMITERS_CONFIG_LOCK: + if setting != _LIMITERS_SETTING: + step_limit = _optional_positive_int(_STEP_CONCURRENCY_ENV) + create_limit = _optional_positive_int(_CREATE_CONCURRENCY_ENV) + _STEP_LIMITER = ( + threading.BoundedSemaphore(step_limit) if step_limit else None + ) + _CREATE_LIMITER = ( + threading.BoundedSemaphore(create_limit) if create_limit else None + ) + _LIMITERS_SETTING = setting + return _STEP_LIMITER, _CREATE_LIMITER + + +@contextmanager +def srun_step_admission(*, is_create: bool = False) -> Iterator[None]: + """Apply rate and concurrency controls to one scheduler step.""" + step_limiter, create_limiter = _configured_limiters() + with ExitStack() as stack: + # One order everywhere prevents create and general admission deadlocks. + if is_create and create_limiter is not None: + stack.enter_context(create_limiter) + if step_limiter is not None: + stack.enter_context(step_limiter) + # Take the rate slot only once the concurrency permit is held. Pacing + # before the semaphore lets already-paced callers accumulate behind a + # long image create and burst together when permits are released. + _pace_srun_step() + yield + + +def _pace_srun_step() -> None: + """Wait for the process-wide step-admission slot when configured.""" + global _STEP_PACER_SETTING, _STEP_PACER + + setting = ( + os.environ.get(_STEP_RATE_ENV, "").strip(), + os.environ.get(_STEP_RATE_STATE_ENV, "").strip(), + ) + with _STEP_PACER_CONFIG_LOCK: + if setting != _STEP_PACER_SETTING: + rate_setting, state_setting = setting + if not rate_setting: + pacer = None + else: + try: + rate_per_s = float(rate_setting) + except ValueError as exc: + raise RunnerError( + f"{_STEP_RATE_ENV} must be a finite number greater than zero" + ) from exc + if not math.isfinite(rate_per_s) or rate_per_s <= 0: + raise RunnerError( + f"{_STEP_RATE_ENV} must be a finite number greater than zero" + ) + pacer = _StepPacer( + rate_per_s, + state_path=Path(state_setting) if state_setting else None, + ) + logger.info( + "limiting Pyxis step creation to %.3f/s%s", + rate_per_s, + (f" using shared state {state_setting}" if state_setting else ""), + ) + _STEP_PACER = pacer + _STEP_PACER_SETTING = setting + pacer = _STEP_PACER + if pacer is not None: + pacer.wait() def _step_retry_attempts() -> int: @@ -367,9 +1228,41 @@ def _record_step_retry( with _RETRY_LOG_LOCK, open(path, "a", encoding="utf-8") as handle: handle.write(json.dumps(record) + "\n") except OSError: + key = str(Path(path).resolve()) + with _RETRY_LOG_LOCK: + _RETRY_LOG_ERRORS.setdefault(key, []).append( + f"could not append retry record for {target} attempt {attempt}" + ) logger.debug("could not append to the infra retry log", exc_info=True) +def read_step_retry_log(path: Path) -> tuple[list[dict[str, Any]], list[str]]: + """Read retry JSONL without letting corrupt accounting look like success.""" + with _RETRY_LOG_LOCK: + accounting_errors = list(_RETRY_LOG_ERRORS.get(str(path.resolve()), ())) + if not path.exists(): + return [], accounting_errors + try: + lines = path.read_text().splitlines() + except OSError as exc: + return [], [*accounting_errors, f"could not read {path}: {exc}"] + records: list[dict[str, Any]] = [] + errors: list[str] = accounting_errors + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + errors.append(f"line {line_number}: invalid JSON: {exc.msg}") + continue + if not isinstance(record, dict) or not isinstance(record.get("outcome"), str): + errors.append(f"line {line_number}: expected an object with string outcome") + continue + records.append(record) + return records, errors + + def run_srun_step(**kwargs: Any) -> subprocess.CompletedProcess[str]: """Run one `srun` step, re-attempting only a *provable* non-launch. @@ -391,12 +1284,18 @@ def run_srun_step(**kwargs: Any) -> subprocess.CompletedProcess[str]: """ attempts = _step_retry_attempts() target = str(kwargs.get("name") or kwargs.get("image") or "pyxis-step") + is_create = kwargs.get("image") is not None for attempt in range(1, attempts + 1): try: - result = _run_srun_step_once(**kwargs) + # Every actual attempt, including a retry, is a new scheduler step. + with srun_step_admission(is_create=is_create): + result = _run_srun_step_once(**kwargs) except StepNotLaunched as exc: if not exc.provable_non_execution: # The command may have run. Another attempt could double it. + failure_path = kwargs.get("failure_path") + if failure_path is not None: + Path(failure_path).touch() _record_step_retry( target=target, attempt=attempt, @@ -412,6 +1311,9 @@ def run_srun_step(**kwargs: Any) -> subprocess.CompletedProcess[str]: detail=f"srun_rc={exc.srun_rc} status={exc.status!r}", ) if attempt == attempts: + failure_path = kwargs.get("failure_path") + if failure_path is not None: + Path(failure_path).touch() raise logger.warning( "Pyxis step provably never launched (attempt %d/%d, srun rc=%s, " @@ -425,13 +1327,28 @@ def run_srun_step(**kwargs: Any) -> subprocess.CompletedProcess[str]: continue if attempt > 1: _record_step_retry(target=target, attempt=attempt, outcome="recovered") + result.__dict__["srun_attempts"] = attempt return result raise AssertionError("unreachable") # pragma: no cover -def resolve_image(image_registry: str, instance_id: str) -> str: +def _validate_instance_id(instance_id: str) -> None: if Path(instance_id).name != instance_id or instance_id in {".", ".."}: raise RunnerError(f"invalid SWE-bench instance ID: {instance_id}") + + +def resolve_image( + image_registry: str | None, + instance_id: str, + *, + image_dir: Path | None = None, +) -> str | Path: + """Resolve one SWE-bench image from a registry or a staged local store.""" + _validate_instance_id(instance_id) + if image_dir is not None: + return image_dir / f"{instance_id}.sqsh" + if image_registry is None: + raise RunnerError("Pyxis runtime requires an image registry or image directory") image_registry = image_registry.rstrip("/") if "#" not in image_registry: host, separator, repository = image_registry.partition("/") @@ -441,9 +1358,42 @@ def resolve_image(image_registry: str, instance_id: str) -> str: return f"{image_registry}/sweb.eval.arm64.{instance_id.lower()}:v4.1.0-arm64" +def load_node_map(path: Path | None) -> dict[str, str]: + """Load an ``instance_id node`` routing file used by multi-node Pyxis.""" + if path is None: + return {} + try: + lines = path.read_text().splitlines() + except OSError as exc: + raise RunnerError(f"could not read Pyxis node map {path}: {exc}") from exc + assignments: dict[str, str] = {} + for line_number, line in enumerate(lines, start=1): + if not line.strip() or line.lstrip().startswith("#"): + continue + fields = line.split() + if len(fields) != 2: + raise RunnerError( + f"invalid Pyxis node map line {line_number}: expected instance_id node" + ) + instance_id, node = fields + _validate_instance_id(instance_id) + if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", node) is None: + raise RunnerError( + f"invalid Pyxis node name on line {line_number}: {node!r}" + ) + previous = assignments.setdefault(instance_id, node) + if previous != node: + raise RunnerError( + f"conflicting Pyxis node assignments for {instance_id}: " + f"{previous} and {node}" + ) + return assignments + + class PyxisEnvironmentConfig(BaseModel): image: str | Path run_id: str + node: str | None = None cwd: str = "/testbed" env: dict[str, str] = Field(default_factory=dict) timeout_s: int = Field( @@ -466,22 +1416,35 @@ class PyxisEnvironmentConfig(BaseModel): ) interpreter: list[str] = Field(default_factory=lambda: ["bash", "-c"]) infrastructure_failure_path: Path | None = None + capture_stderr_separately: bool = False + persistent_exec: bool = False class PyxisEnvironment: def __init__(self, **kwargs: Any): self.config = PyxisEnvironmentConfig(**kwargs) + self._persistent_enabled = self.config.persistent_exec or _env_flag( + _PERSISTENT_EXEC_ENV + ) + self._persistent_channel: _PersistentExecChannel | None = None + self._scheduler_steps = 0 + self._direct_command_steps = 0 + self._cleanup_steps = 0 safe_run_id = re.sub(r"[^A-Za-z0-9_.-]", "-", self.config.run_id)[:24] self.name = f"mswe_{safe_run_id}_{uuid.uuid4().hex[:8]}" self._tmp = tempfile.TemporaryDirectory(prefix=f"pyxis_{self.name}_") self._tmp_dir = Path(self._tmp.name) - self._tmp_dir.chmod(0o1777) + # Pyxis remaps container root to the submitting host uid, so the + # environment's private mount does not need world access. Keeping the + # TemporaryDirectory's owner-only mode prevents other host users from + # reading tool commands, responses, and the persistent-channel secret. + self._tmp_dir.chmod(0o700) self._lock = threading.Lock() self._cleaned = False started = time.monotonic() try: # A no-op initializes and validates the named persistent container. - run_srun_step( + create_result = run_srun_step( image=self.config.image, name=self.name, mounts=[(self._tmp_dir, "/tmp")], @@ -490,45 +1453,147 @@ def __init__(self, **kwargs: Any): status_path=self._tmp_dir / Path(_STEP_STATUS).name, timeout_s=self.config.create_timeout_s, failure_path=self.config.infrastructure_failure_path, + node=self.config.node, ) - except RunnerError as exc: + self._scheduler_steps += getattr(create_result, "srun_attempts", 1) + if self._persistent_enabled: + protocol_dir = self._tmp_dir / Path(_PERSISTENT_ROOT).name + protocol_dir.mkdir() + self._persistent_channel = _PersistentExecChannel( + protocol_dir=protocol_dir, + command_factory=self._persistent_server_command, + admission_factory=lambda: srun_step_admission(), + failure_path=self.config.infrastructure_failure_path, + launch_timeout_s=30.0 + _step_launch_grace_s(), + capture_stderr_separately=self.config.capture_stderr_separately, + retry_target=f"persistent-pyxis-server:{self.name}", + ) + self._persistent_channel.start() + except Exception as exc: _record_create_timing( self.config.image, time.monotonic() - started, ok=False ) self.cleanup() + if not isinstance(exc, RunnerError): + exc = RunnerError( + "persistent Pyxis initialization failed: " + f"{type(exc).__name__}: {exc}" + ) raise RunnerError( f"failed to start Pyxis container for {self.config.image}: {exc}" ) from exc _record_create_timing(self.config.image, time.monotonic() - started, ok=True) - def execute( - self, action: dict[str, Any], cwd: str = "", *, timeout: int | None = None - ) -> dict[str, Any]: - command = action.get("command", "") - logger.debug("Executing Pyxis command: %s", command) + def _persistent_server_command(self, generation: str, secret: str) -> list[str]: argv = ["env"] argv.extend(f"{key}={value}" for key, value in self.config.env.items()) - argv.extend([*self.config.interpreter, command]) - result = run_srun_step( + argv.extend( + [ + "bash", + "-c", + _PERSISTENT_SERVER_SCRIPT, + "pyxis-persistent-server", + _PERSISTENT_ROOT, + generation, + secret, + *self.config.interpreter, + ] + ) + return build_srun_command( argv=argv, - status_path=self._tmp_dir / Path(_STEP_STATUS).name, - timeout_s=timeout or self.config.timeout_s, - failure_path=self.config.infrastructure_failure_path, name=self.name, mounts=[(self._tmp_dir, "/tmp")], - workdir=cwd or self.config.cwd, + workdir=self.config.cwd, + node=self.config.node, ) + + def _persistent_stats(self) -> dict[str, Any]: + channel_stats = ( + dict(self._persistent_channel.stats) + if self._persistent_channel is not None + else {} + ) + server_steps = channel_stats.get("server_starts", 0) + commands = channel_stats.get("commands", 0) + scheduler_steps = self._scheduler_steps + server_steps + return { + "enabled": self._persistent_enabled, + "scheduler_steps": scheduler_steps, + "container_create_steps": self._scheduler_steps + - self._direct_command_steps + - self._cleanup_steps, + "server_start_steps": server_steps, + "direct_command_steps": self._direct_command_steps, + "cleanup_steps": self._cleanup_steps, + "persistent_commands": commands, + "scheduler_steps_per_persistent_command": ( + round(scheduler_steps / commands, 6) if commands else None + ), + **channel_stats, + } + + def _write_persistent_stats(self) -> None: + path = os.environ.get(_PERSISTENT_STATS_ENV, "").strip() + if not path: + return + payload = { + "at": time.time(), + "pid": os.getpid(), + "environment": self.name, + **self._persistent_stats(), + } + try: + with _PERSISTENT_STATS_LOCK, open(path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(payload) + "\n") + except OSError: + logger.debug("could not append persistent Pyxis stats", exc_info=True) + + def execute( + self, action: dict[str, Any], cwd: str = "", *, timeout: int | None = None + ) -> dict[str, Any]: + command = action.get("command", "") + logger.debug("Executing Pyxis command: %s", command) + capture_stderr_separately = getattr( + self.config, "capture_stderr_separately", False + ) + timeout_s = timeout or self.config.timeout_s + persistent_channel = getattr(self, "_persistent_channel", None) + if persistent_channel is not None: + result = persistent_channel.execute( + command=command, + cwd=cwd or self.config.cwd, + timeout_s=timeout_s, + ) + else: + argv = ["env"] + argv.extend(f"{key}={value}" for key, value in self.config.env.items()) + argv.extend([*self.config.interpreter, command]) + result = run_srun_step( + argv=argv, + status_path=self._tmp_dir / Path(_STEP_STATUS).name, + timeout_s=timeout_s, + failure_path=self.config.infrastructure_failure_path, + name=self.name, + mounts=[(self._tmp_dir, "/tmp")], + workdir=cwd or self.config.cwd, + node=self.config.node, + stderr=( + subprocess.PIPE if capture_stderr_separately else subprocess.STDOUT + ), + ) + attempts = getattr(result, "srun_attempts", 1) + if hasattr(self, "_scheduler_steps"): + self._scheduler_steps += attempts + self._direct_command_steps += attempts output: dict[str, Any] - if result.returncode == 124: + if result.returncode == 124 or getattr(result, "timed_out", False): output = { "output": result.stdout, "returncode": -1, "exception_info": "The command timed out", "extra": { "exception_type": "TimeoutExpired", - "exception": ( - f"command timed out after {timeout or self.config.timeout_s}s" - ), + "exception": (f"command timed out after {timeout_s}s"), }, } else: @@ -537,6 +1602,8 @@ def execute( "returncode": result.returncode, "exception_info": "", } + if capture_stderr_separately or persistent_channel is not None: + output.setdefault("extra", {})["stderr"] = result.stderr or "" lines = output.get("output", "").lstrip().splitlines(keepends=True) if ( lines @@ -571,6 +1638,7 @@ def serialize(self) -> dict[str, Any]: "environment_type": ( f"{self.__class__.__module__}.{self.__class__.__name__}" ), + "persistent_exec_stats": self._persistent_stats(), } } } @@ -581,18 +1649,41 @@ def cleanup(self) -> None: return self._cleaned = True try: + persistent_channel = getattr(self, "_persistent_channel", None) + if persistent_channel is not None: + try: + persistent_channel.close() + except (OSError, subprocess.SubprocessError): + # A failed poison/TERM/KILL must not prevent the exact + # named-container removal below. That cleanup is what + # reclaims the writable rootfs and any surviving step. + logger.warning( + "Could not stop persistent Pyxis command server %s", + self.name, + exc_info=True, + ) job_id = os.environ.get("SLURM_JOB_ID", "").strip() if job_id: container = enroot_container_name(job_id, self.name) try: - completed = subprocess.run( - build_srun_command(argv=["enroot", "remove", "-f", container]), - check=False, - capture_output=True, - text=True, - timeout=30, - env=safe_srun_env(), - ) + # Cleanup creates a Slurm step too; leaving it outside the + # process-wide admission path recreates the same RPC burst + # when many workers finish together. + with srun_step_admission(): + if hasattr(self, "_scheduler_steps"): + self._scheduler_steps += 1 + self._cleanup_steps += 1 + completed = subprocess.run( + build_srun_command( + argv=["enroot", "remove", "-f", container], + node=self.config.node, + ), + check=False, + capture_output=True, + text=True, + timeout=30, + env=safe_srun_env(), + ) except (OSError, RunnerError, subprocess.SubprocessError): logger.warning( "Could not remove Pyxis container %s", @@ -610,6 +1701,8 @@ def cleanup(self) -> None: (completed.stderr or completed.stdout or "").strip()[-500:], ) finally: + if hasattr(self, "_persistent_enabled"): + self._write_persistent_stats() self._tmp.cleanup() def __del__(self) -> None: diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py index 10472e3b4..caff9c8d4 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py @@ -14,7 +14,7 @@ from typing import Any from .artifacts import atomic_write_bytes -from .pyxis_environment import resolve_image, run_srun_step +from .pyxis_environment import load_node_map, resolve_image, run_srun_step from .runner import EVAL_INFRA_FAILURES_FILE, RunnerError _PRINT_LOCK = threading.Lock() @@ -60,12 +60,23 @@ def _run_agent(args: argparse.Namespace) -> None: failure_path = args.output / _INFRASTRUCTURE_FAILURE failure_path.unlink(missing_ok=True) + node_map = load_node_map(getattr(args, "node_map", None)) def get_pyxis_environment(config: dict, instance: dict): environment_config = copy.deepcopy(config.get("environment", {})) environment_config["image"] = resolve_image( - args.image_registry, instance["instance_id"] + args.image_registry, + instance["instance_id"], + image_dir=getattr(args, "image_dir", None), ) + instance_id = instance["instance_id"] + node = node_map.get(instance_id) + if getattr(args, "node_map", None) is not None and node is None: + raise RunnerError( + f"Pyxis node map has no assignment for requested instance: " + f"{instance_id}" + ) + environment_config["node"] = node environment_config["infrastructure_failure_path"] = str(failure_path) return get_environment(environment_config) @@ -116,6 +127,7 @@ def _evaluate_instance( output_dir: Path, run_id: str, timeout_s: int, + node: str | None = None, ) -> None: instance_id = test_spec.instance_id safe_model = prediction["model_name_or_path"].replace("/", "__") @@ -143,6 +155,7 @@ def _evaluate_instance( workdir="/testbed", status_path=status_path, timeout_s=timeout_s + 30, + node=node, stderr=subprocess.PIPE, argv=[ "bash", @@ -194,8 +207,21 @@ def _run_eval(args: argparse.Namespace) -> None: if prediction["instance_id"] in args.instance_ids } rows = load_swebench_dataset(args.dataset_name, args.split, args.instance_ids) + node_map = load_node_map(getattr(args, "node_map", None)) + if getattr(args, "node_map", None) is not None: + missing = sorted(set(args.instance_ids) - node_map.keys()) + if missing: + raise RunnerError( + "Pyxis node map has no assignment for requested instances: " + + ", ".join(missing[:10]) + + (" ..." if len(missing) > 10 else "") + ) images = { - instance_id: resolve_image(args.image_registry, instance_id) + instance_id: resolve_image( + args.image_registry, + instance_id, + image_dir=getattr(args, "image_dir", None), + ) for instance_id in args.instance_ids } payloads = [] @@ -212,6 +238,7 @@ def _run_eval(args: argparse.Namespace) -> None: "output_dir": args.output_dir, "run_id": args.run_id, "timeout_s": args.timeout, + "node": node_map.get(instance_id), } ) @@ -232,8 +259,7 @@ def _run_eval(args: argparse.Namespace) -> None: except Exception as exc: # noqa: BLE001 -- one instance, not the run with _PRINT_LOCK: print( - f"Pyxis evaluation failed for {instance_id} " - f"(non-fatal): {exc}", + f"Pyxis evaluation failed for {instance_id} (non-fatal): {exc}", flush=True, ) failures[instance_id] = f"{type(exc).__name__}: {exc}" @@ -275,7 +301,10 @@ def main(argv: list[str] | None = None) -> None: agent_parser.add_argument("--filter", required=True) agent_parser.add_argument("--workers", type=int, required=True) agent_parser.add_argument("--output", type=Path, required=True) - agent_parser.add_argument("--image-registry", required=True) + agent_images = agent_parser.add_mutually_exclusive_group(required=True) + agent_images.add_argument("--image-registry") + agent_images.add_argument("--image-dir", type=Path) + agent_parser.add_argument("--node-map", type=Path) eval_parser = commands.add_parser("eval") eval_parser.add_argument("--dataset-name", required=True) @@ -283,7 +312,10 @@ def main(argv: list[str] | None = None) -> None: eval_parser.add_argument("--predictions-path", type=Path, required=True) eval_parser.add_argument("--max-workers", type=int, required=True) eval_parser.add_argument("--run-id", required=True) - eval_parser.add_argument("--image-registry", required=True) + eval_images = eval_parser.add_mutually_exclusive_group(required=True) + eval_images.add_argument("--image-registry") + eval_images.add_argument("--image-dir", type=Path) + eval_parser.add_argument("--node-map", type=Path) eval_parser.add_argument("--output-dir", type=Path, required=True) eval_parser.add_argument("--timeout", type=int, default=1800) eval_parser.add_argument("--instance-ids", nargs="+", required=True) diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 7d5991a3e..a39703ea5 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -258,15 +258,14 @@ def run( return self._run(request, run_dir, cancel_token) finally: try: - cleanup_kwargs: dict[str, Any] = {} + cleanup_kwargs: dict[str, Any] = { + "instance_ids": request.evaluated_instance_ids + } eval_run_id_path = run_dir / "swe_bench_eval_run_id.txt" if eval_run_id_path.exists(): eval_run_id = eval_run_id_path.read_text().strip() if eval_run_id: - cleanup_kwargs = { - "eval_run_id": eval_run_id, - "instance_ids": request.evaluated_instance_ids, - } + cleanup_kwargs["eval_run_id"] = eval_run_id self._cleanup_containers(run_dir.name, **cleanup_kwargs) except Exception: logger.warning( @@ -706,13 +705,63 @@ def __init__( *, project_root: Path, subprocess_timeout_s: int, - image_registry: str, + image_registry: str | None = None, + image_dir: Path | None = None, + node_map: Path | None = None, ): super().__init__( project_root=project_root, subprocess_timeout_s=subprocess_timeout_s, ) self.image_registry = image_registry + self.image_dir = image_dir + self.node_map = node_map + self._node_map_snapshot_dir: tempfile.TemporaryDirectory[str] | None = None + self._node_map_snapshot: Path | None = None + if node_map is not None: + # Snapshot routing once. Dispatch, evaluation, and terminal cleanup + # must agree even if an operator later replaces the shared file. + from .pyxis_environment import load_node_map + + self._node_assignments = load_node_map(node_map) + self._node_map_snapshot_dir = tempfile.TemporaryDirectory( + prefix="swebench-node-map-" + ) + self._node_map_snapshot = ( + Path(self._node_map_snapshot_dir.name) / "node-map.txt" + ) + self._node_map_snapshot.write_text( + "".join( + f"{instance_id}\t{node}\n" + for instance_id, node in self._node_assignments.items() + ) + ) + self._node_map_snapshot.chmod(0o600) + else: + self._node_assignments = {} + + def _image_source_args(self) -> list[str]: + if self.image_dir is not None: + args = ["--image-dir", str(self.image_dir)] + elif self.image_registry is not None: + args = ["--image-registry", self.image_registry] + else: # guarded by create_runner; defensive for direct construction + raise RunnerError("Pyxis runtime requires an image registry or directory") + if self._node_map_snapshot is not None: + args.extend(["--node-map", str(self._node_map_snapshot)]) + return args + + def _validate_node_assignments(self, instance_ids: list[str]) -> None: + if self.node_map is None: + return + missing = sorted(set(instance_ids) - self._node_assignments.keys()) + if missing: + preview = ", ".join(missing[:10]) + suffix = " ..." if len(missing) > 10 else "" + raise RunnerError( + "Pyxis node map has no assignment for requested instances: " + f"{preview}{suffix}" + ) def _configure_environment( self, environment_cfg: dict[str, Any], run_id: str @@ -739,6 +788,7 @@ def _run_agent( secret_values: set[str], cancel_token: CancellationToken | None = None, ) -> None: + self._validate_node_assignments(request.evaluated_instance_ids) command = [ sys.executable, "-m", @@ -758,8 +808,7 @@ def _run_agent( str(request.workers), "--output", str(output_dir), - "--image-registry", - self.image_registry, + *self._image_source_args(), ] self._run_logged_subprocess( command, @@ -780,6 +829,7 @@ def _run_eval( secret_values: set[str], cancel_token: CancellationToken | None = None, ) -> Path: + self._validate_node_assignments(request.evaluated_instance_ids) run_id, dataset_name = _prepare_eval(request, run_dir) command = [ sys.executable, @@ -796,8 +846,7 @@ def _run_eval( str(request.max_eval_workers), "--run-id", run_id, - "--image-registry", - self.image_registry, + *self._image_source_args(), "--output-dir", str(output_dir), "--instance-ids", @@ -824,36 +873,74 @@ def _cleanup_containers( instance_ids: list[str] | None = None, ) -> None: # Local import avoids the runner <-> Pyxis environment import cycle. - from .pyxis_environment import build_srun_command, safe_srun_env + from .pyxis_environment import ( + build_srun_command, + safe_srun_env, + srun_step_admission, + ) - del eval_run_id, instance_ids + del eval_run_id safe_run_id = re.sub(r"[^A-Za-z0-9_.-]", "-", run_id)[:24] - prefix = f"pyxis_mswe_{safe_run_id}_" - try: - listed = subprocess.run( - build_srun_command(argv=["enroot", "list", "-f"]), - check=True, - capture_output=True, - text=True, - timeout=30, - env=safe_srun_env(), + suffix_prefix = f"mswe_{safe_run_id}_" + nodes: list[str | None] = [] + if instance_ids and self._node_assignments: + nodes.extend( + sorted( + { + self._node_assignments[instance_id] + for instance_id in instance_ids + if instance_id in self._node_assignments + } + ) ) - for line in listed.stdout.splitlines(): - fields = line.split(maxsplit=1) - name = fields[0] if fields else "" - if name.startswith(prefix): - subprocess.run( - build_srun_command(argv=["enroot", "remove", "-f", name]), + else: + nodes.extend(sorted(set(self._node_assignments.values()))) + if not nodes: + nodes.append(None) + failures: list[str] = [] + for node in nodes: + try: + with srun_step_admission(): + listed = subprocess.run( + build_srun_command(argv=["enroot", "list", "-f"], node=node), check=True, capture_output=True, text=True, timeout=30, env=safe_srun_env(), ) - except (OSError, subprocess.SubprocessError) as exc: + except (OSError, subprocess.SubprocessError) as exc: + failures.append(f"{node or ''}: list failed: {exc}") + continue + for line in listed.stdout.splitlines(): + fields = line.split(maxsplit=1) + name = fields[0] if fields else "" + # Depending on Pyxis/Enroot versions, list output can be + # either pyxis_mswe_* or pyxis__mswe_*. + if not name.startswith("pyxis_") or suffix_prefix not in name: + continue + suffix = name.split(suffix_prefix, 1)[1] + if not suffix or "_" in suffix: + continue + try: + with srun_step_admission(): + subprocess.run( + build_srun_command( + argv=["enroot", "remove", "-f", name], node=node + ), + check=True, + capture_output=True, + text=True, + timeout=30, + env=safe_srun_env(), + ) + except (OSError, subprocess.SubprocessError) as exc: + failures.append(f"{node or ''}: remove {name} failed: {exc}") + if failures: raise RunnerError( - f"failed to clean up Pyxis containers for SWE-bench run {run_id}" - ) from exc + f"failed to clean up Pyxis containers for SWE-bench run {run_id}: " + + "; ".join(failures[:20]) + ) def create_runner( @@ -862,6 +949,8 @@ def create_runner( project_root: Path, subprocess_timeout_s: int, image_registry: str | None, + image_dir: Path | None = None, + node_map: Path | None = None, ) -> RunnerProtocol: if runtime == "docker": return SweBenchRunner( @@ -869,11 +958,15 @@ def create_runner( subprocess_timeout_s=subprocess_timeout_s, ) if runtime == "pyxis": - if image_registry is None: - raise ValueError("Pyxis runtime requires an image registry") + if (image_registry is None) == (image_dir is None): + raise ValueError( + "Pyxis runtime requires exactly one of image_registry or image_dir" + ) return PyxisSweBenchRunner( project_root=project_root, subprocess_timeout_s=subprocess_timeout_s, image_registry=image_registry, + image_dir=image_dir, + node_map=node_map, ) raise ValueError(f"unknown SWE-bench runtime: {runtime}") diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index d1ce15f49..2d5c11122 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -3,6 +3,7 @@ import json import logging +import os import stat import subprocess import sys @@ -19,6 +20,9 @@ from inference_endpoint.evaluation.swebench_service.swebench_service import ( artifacts as artifacts_mod, ) +from inference_endpoint.evaluation.swebench_service.swebench_service import ( + pyxis_environment as pyxis_environment_mod, +) from inference_endpoint.evaluation.swebench_service.swebench_service import ( pyxis_worker as worker_mod, ) @@ -26,10 +30,17 @@ runner as runner_mod, ) from inference_endpoint.evaluation.swebench_service.swebench_service.pyxis_environment import ( + _PERSISTENT_SERVER_SCRIPT, PyxisEnvironment, StepNotLaunched, + _PersistentExecChannel, + _read_sized_file, + _run_srun_step_once, + _StepPacer, build_srun_command, enroot_container_name, + load_node_map, + read_step_retry_log, read_step_sentinel, resolve_image, safe_srun_env, @@ -457,13 +468,19 @@ def fake_run_eval( def test_run_cleans_labeled_containers_after_success(monkeypatch, tmp_path): runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) _stub_successful_run(monkeypatch, runner) - cleaned: list[str] = [] - monkeypatch.setattr(runner, "_cleanup_containers", cleaned.append) + cleaned: list[tuple[str, dict]] = [] + monkeypatch.setattr( + runner, + "_cleanup_containers", + lambda run_id, **kwargs: cleaned.append((run_id, kwargs)), + ) result = runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-1") assert result == {"resolved_instances": 1, "submitted_instances": 1} - assert cleaned == ["run-1"] + assert cleaned == [ + ("run-1", {"instance_ids": ["repo__repo-1"]}), + ] @pytest.mark.parametrize( @@ -481,20 +498,26 @@ def test_run_cleans_labeled_containers_after_failure( monkeypatch, tmp_path, error, raised, match ): runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) - cleaned: list[str] = [] + cleaned: list[tuple[str, dict]] = [] def fail_agent(*args, **kwargs): raise error monkeypatch.setattr(runner, "_run_agent", fail_agent) - monkeypatch.setattr(runner, "_cleanup_containers", cleaned.append) + monkeypatch.setattr( + runner, + "_cleanup_containers", + lambda run_id, **kwargs: cleaned.append((run_id, kwargs)), + ) with pytest.raises(raised, match=match) as exc_info: runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-2") if raised is not RunCancelled: assert exc_info.value.__cause__ is error - assert cleaned == ["run-2"] + assert cleaned == [ + ("run-2", {"instance_ids": ["repo__repo-1"]}), + ] def test_run_scores_predictions_left_behind_by_a_failed_agent_phase( @@ -780,7 +803,420 @@ def _pyxis_request(instance_ids: list[str] | None = None) -> RunRequest: _PYXIS_IMAGE_REGISTRY = "gitlab-master.nvidia.com:5005/hvagadia/swebench-arm64-images" -def _finish_srun_step(command: list[str], returncode: int) -> None: +def _local_persistent_channel( + monkeypatch, + tmp_path, + *, + server_scripts=None, + driver_grace_s=1.0, + shutdown_grace_s=0.1, + capture_stderr_separately=True, + failure_path=None, +): + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + unshare = bin_dir / "unshare" + unshare.write_text('#!/bin/sh\nshift 3\nexec "$@"\n') + unshare.chmod(0o755) + monkeypatch.setenv("PATH", f"{bin_dir}:{os.environ['PATH']}") + protocol_dir = tmp_path / "protocol" + scripts = list(server_scripts or [_PERSISTENT_SERVER_SCRIPT]) + + def command_factory(generation, secret): + script = scripts.pop(0) if len(scripts) > 1 else scripts[0] + return [ + "bash", + "-c", + script, + "persistent-test-server", + str(protocol_dir), + generation, + secret, + "bash", + "-c", + ] + + channel = _PersistentExecChannel( + protocol_dir=protocol_dir, + command_factory=command_factory, + launch_timeout_s=1.0, + driver_grace_s=driver_grace_s, + shutdown_grace_s=shutdown_grace_s, + capture_stderr_separately=capture_stderr_separately, + failure_path=failure_path, + ) + channel.start() + return channel + + +def test_persistent_channel_handles_multiline_large_output_and_nonce_isolation( + monkeypatch, tmp_path +): + channel = _local_persistent_channel(monkeypatch, tmp_path) + try: + first = channel.execute( + command="printf 'first\\nsecond\\n'; printf 'diagnostic\\n' >&2", + cwd=str(tmp_path), + timeout_s=5, + ) + second = channel.execute( + command="head -c 262144 /dev/zero | tr '\\0' x", + cwd=str(tmp_path), + timeout_s=5, + ) + finally: + channel.close() + + assert first.stdout == "first\nsecond\n" + assert first.stderr == "diagnostic\n" + assert second.stdout == "x" * 262144 + assert second.stderr == "" + assert list((tmp_path / "protocol" / "requests").iterdir()) == [] + assert channel.stats["server_starts"] == 1 + assert channel.stats["commands"] == 2 + + +def test_persistent_channel_preserves_merged_stream_order(monkeypatch, tmp_path): + channel = _local_persistent_channel( + monkeypatch, tmp_path, capture_stderr_separately=False + ) + try: + result = channel.execute( + command="printf 'one\\n'; printf 'two\\n' >&2; printf 'three\\n'", + cwd=str(tmp_path), + timeout_s=5, + ) + finally: + channel.close() + + assert result.stdout == "one\ntwo\nthree\n" + assert result.stderr == "" + + +def test_persistent_completion_rejects_a_forged_manifest(monkeypatch, tmp_path): + channel = _local_persistent_channel(monkeypatch, tmp_path) + request = channel.protocol_dir / "requests" / ("a" * 32) + request.mkdir() + (request / "stdout").write_text("forged") + (request / "stderr").write_text("") + (request / "complete").write_text("0 6 0 0 " + "0" * 64 + "\n") + try: + with pytest.raises(RunnerError, match="digest did not verify"): + channel._read_completion(request, time.monotonic() + 1) + finally: + channel.close() + + +def test_persistent_kill_after_is_reported_as_timeout(monkeypatch, tmp_path): + channel = _local_persistent_channel(monkeypatch, tmp_path) + timeout = tmp_path / "bin" / "timeout" + timeout.write_text("#!/bin/sh\nexit 137\n") + timeout.chmod(0o755) + try: + result = channel.execute(command="true", cwd=str(tmp_path), timeout_s=1) + finally: + channel.close() + + assert result.returncode == 137 + assert result.timed_out is True + + +def test_persistent_response_reader_retries_a_partial_filesystem_view(tmp_path): + responses = iter([b"partial", b"complete-response"]) + + result = _read_sized_file( + tmp_path / "stdout", + len(b"complete-response"), + deadline=1.0, + read_bytes=lambda _path: next(responses), + monotonic=lambda: 0.0, + sleep=lambda _seconds: None, + ) + + assert result == b"complete-response" + + +def test_persistent_server_death_does_not_restart_a_pending_request( + monkeypatch, tmp_path +): + dies_pending = r"""root=$1 +generation=$2 +mkdir -p "$root/requests" +printf '%s\n' "$generation" > "$root/ready.tmp" +mv "$root/ready.tmp" "$root/ready" +sleep 0.1 +exit 17 +""" + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "3") + channel = _local_persistent_channel( + monkeypatch, tmp_path, server_scripts=[dies_pending] + ) + try: + with pytest.raises(StepNotLaunched) as exc_info: + channel.execute( + command="printf must-not-replay", + cwd=str(tmp_path), + timeout_s=5, + ) + finally: + channel.close() + + assert exc_info.value.status == "pending" + assert exc_info.value.provable_non_execution is False + assert channel.stats["server_starts"] == 1 + assert channel.stats["safe_restarts"] == 0 + + +def test_idle_persistent_server_death_records_infrastructure_failure( + monkeypatch, tmp_path +): + failure_path = tmp_path / "infra-failure" + channel = _local_persistent_channel( + monkeypatch, tmp_path, failure_path=failure_path + ) + assert channel._process is not None + channel._process.terminate() + channel._process.wait(timeout=2) + try: + with pytest.raises(RunnerError, match="server is not running"): + channel.execute(command="true", cwd=str(tmp_path), timeout_s=1) + finally: + channel.close() + + assert channel.stats["command_failures"] == 1 + assert failure_path.exists() + + +def test_persistent_server_death_never_replays_a_started_request(monkeypatch, tmp_path): + dies_started = r"""root=$1 +generation=$2 +mkdir -p "$root/requests" +printf '%s\n' "$generation" > "$root/ready.tmp" +mv "$root/ready.tmp" "$root/ready" +while :; do + for request in "$root"/requests/*; do + [ -d "$request" ] || continue + printf 'started\n' > "$request/status.tmp" + mv "$request/status.tmp" "$request/status" + exit 19 + done + sleep 0.01 +done +""" + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "3") + channel = _local_persistent_channel( + monkeypatch, tmp_path, server_scripts=[dies_started] + ) + try: + with pytest.raises(StepNotLaunched) as exc_info: + channel.execute( + command="printf must-not-replay", + cwd=str(tmp_path), + timeout_s=5, + ) + finally: + channel.close() + + assert exc_info.value.status == "started" + assert exc_info.value.provable_non_execution is False + assert channel.stats["server_starts"] == 1 + assert channel.stats["safe_restarts"] == 0 + + +def test_persistent_channel_enforces_server_side_timeout(monkeypatch, tmp_path): + channel = _local_persistent_channel(monkeypatch, tmp_path) + try: + result = channel.execute( + command="sleep 2", + cwd=str(tmp_path), + timeout_s=1, + ) + finally: + channel.close() + + assert result.returncode == 124 + assert channel.stats["nonzero_commands"] == 1 + + +def test_persistent_channel_has_a_nonreplayable_driver_deadline(monkeypatch, tmp_path): + ignores_requests = r"""root=$1 +generation=$2 +mkdir -p "$root/requests" +printf '%s\n' "$generation" > "$root/ready.tmp" +mv "$root/ready.tmp" "$root/ready" +while [ ! -f "$root/stop" ]; do sleep 0.01; done +""" + channel = _local_persistent_channel( + monkeypatch, + tmp_path, + server_scripts=[ignores_requests], + driver_grace_s=0.05, + ) + try: + with pytest.raises(StepNotLaunched, match="driver deadline") as exc_info: + channel.execute(command="true", cwd=str(tmp_path), timeout_s=0) + finally: + channel.close() + + assert exc_info.value.status == "pending" + assert exc_info.value.provable_non_execution is False + + +def test_persistent_cleanup_kills_and_reaps_an_unresponsive_server( + monkeypatch, tmp_path +): + ignores_cleanup = r"""root=$1 +generation=$2 +mkdir -p "$root/requests" +printf '%s\n' "$generation" > "$root/ready.tmp" +mv "$root/ready.tmp" "$root/ready" +trap '' TERM +while :; do :; done +""" + channel = _local_persistent_channel( + monkeypatch, + tmp_path, + server_scripts=[ignores_cleanup], + shutdown_grace_s=0.05, + ) + process = channel._process + + channel.close() + + assert process is not None + assert process.poll() is not None + assert channel._process is None + + +def test_pyxis_persistent_exec_remains_opt_in(monkeypatch, tmp_path): + monkeypatch.delenv("SWEBENCH_PYXIS_PERSISTENT_EXEC", raising=False) + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + monkeypatch.setattr( + subprocess, + "Popen", + lambda *args, **kwargs: pytest.fail("persistent server must stay disabled"), + ) + + def fake_run(command, **kwargs): + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="opt-out") + try: + output = environment.execute({"command": "true"}) + finally: + environment.cleanup() + + assert output["returncode"] == 0 + assert environment._persistent_stats()["enabled"] is False + assert environment._persistent_stats()["direct_command_steps"] == 1 + + +def test_persistent_environment_commands_create_no_scheduler_steps( + monkeypatch, tmp_path +): + class FakeChannel: + stats = { + "server_starts": 1, + "commands": 0, + "command_failures": 0, + } + + def execute(self, *, command, cwd, timeout_s): + self.stats["commands"] += 1 + return subprocess.CompletedProcess( + [command], 0, stdout="stdout\nstderr\n", stderr="" + ) + + environment = _bare_environment(tmp_path) + environment._persistent_enabled = True + environment._persistent_channel = FakeChannel() + environment._scheduler_steps = 1 + environment._direct_command_steps = 0 + environment._cleanup_steps = 0 + monkeypatch.setattr( + pyxis_environment_mod, + "run_srun_step", + lambda **kwargs: pytest.fail("persistent execute must not create an srun step"), + ) + + output = environment.execute({"command": "printf output"}) + stats = environment._persistent_stats() + + assert output["output"] == "stdout\nstderr\n" + assert output["extra"]["stderr"] == "" + assert stats["scheduler_steps"] == 2 + assert stats["persistent_commands"] == 1 + assert stats["scheduler_steps_per_persistent_command"] == 2.0 + + +def test_pyxis_persistent_exec_env_starts_one_server_and_reuses_it( + monkeypatch, tmp_path +): + channels = [] + + class FakeChannel: + def __init__(self, **kwargs): + self.stats = { + "server_starts": 0, + "commands": 0, + "command_failures": 0, + } + channels.append(self) + + def start(self): + self.stats["server_starts"] += 1 + + def execute(self, *, command, cwd, timeout_s): + self.stats["commands"] += 1 + return subprocess.CompletedProcess( + [command], 0, stdout=f"{command}\n", stderr="" + ) + + def close(self): + return None + + scheduler_calls = [] + + def fake_srun_step(**kwargs): + scheduler_calls.append(kwargs) + return subprocess.CompletedProcess(["srun"], 0, stdout="", stderr="") + + cleanup_calls = [] + + def fake_subprocess_run(command, **kwargs): + cleanup_calls.append(command) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + monkeypatch.setenv("SWEBENCH_PYXIS_PERSISTENT_EXEC", "true") + monkeypatch.setattr(pyxis_environment_mod, "_PersistentExecChannel", FakeChannel) + monkeypatch.setattr(pyxis_environment_mod, "run_srun_step", fake_srun_step) + monkeypatch.setattr(subprocess, "run", fake_subprocess_run) + + environment = PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="reuse") + try: + first = environment.execute({"command": "first"}) + second = environment.execute({"command": "second"}) + finally: + environment.cleanup() + + assert first["output"] == "first\n" + assert second["output"] == "second\n" + assert len(channels) == 1 + assert len(scheduler_calls) == 1 + assert scheduler_calls[0]["argv"] == ["true"] + assert len(cleanup_calls) == 1 + assert environment._persistent_stats()["scheduler_steps"] == 3 + assert environment._persistent_stats()["persistent_commands"] == 2 + + +def _finish_srun_step( + command: list[str], returncode: int, *, timed_out: bool = False +) -> None: mount_argument = next( ( argument @@ -794,10 +1230,12 @@ def _finish_srun_step(command: list[str], returncode: int) -> None: for mount in mount_argument.removeprefix("--container-mounts=").split(","): source, destination = mount.split(":", 1) if destination == "/tmp/.mlperf_srun_status": - Path(source).write_text(f"finished:{returncode}\n") + Path(source).write_text(f"finished:{returncode}:{int(timed_out)}\n") return if destination == "/tmp": - Path(source, ".mlperf_srun_status").write_text(f"finished:{returncode}\n") + Path(source, ".mlperf_srun_status").write_text( + f"finished:{returncode}:{int(timed_out)}\n" + ) return raise AssertionError("srun command does not mount its status file") @@ -853,7 +1291,7 @@ def test_create_runner_runtime_is_typed(): def test_create_runner_requires_image_registry_for_pyxis(tmp_path): - with pytest.raises(ValueError, match="image registry"): + with pytest.raises(ValueError, match="image_registry or image_dir"): create_runner( "pyxis", project_root=tmp_path, @@ -873,6 +1311,22 @@ def test_create_runner_selects_pyxis(tmp_path): assert isinstance(runner, PyxisSweBenchRunner) +def test_create_runner_accepts_local_pyxis_images(tmp_path): + (tmp_path / "nodes.txt").write_text("repo__repo-1 cpu-0001\n") + runner = create_runner( + "pyxis", + project_root=tmp_path, + subprocess_timeout_s=30, + image_registry=None, + image_dir=tmp_path / "images", + node_map=tmp_path / "nodes.txt", + ) + + assert isinstance(runner, PyxisSweBenchRunner) + assert runner.image_dir == tmp_path / "images" + assert runner.node_map == tmp_path / "nodes.txt" + + def test_pyxis_patch_config_selects_pyxis_environment(tmp_path): runner = PyxisSweBenchRunner( project_root=tmp_path, @@ -909,6 +1363,40 @@ def test_pyxis_normalizes_instance_id_for_registry_image(): ) +def test_pyxis_resolves_staged_local_image(tmp_path): + assert resolve_image(None, "Repo__Repo-1", image_dir=tmp_path) == ( + tmp_path / "Repo__Repo-1.sqsh" + ) + + +def test_pyxis_loads_multi_node_assignment(tmp_path): + node_map = tmp_path / "nodes.txt" + node_map.write_text( + "# staged image locations\nrepo__repo-1 cpu-0001\nrepo__repo-2 cpu-0002\n" + ) + + assert load_node_map(node_map) == { + "repo__repo-1": "cpu-0001", + "repo__repo-2": "cpu-0002", + } + + +def test_pyxis_rejects_conflicting_multi_node_assignment(tmp_path): + node_map = tmp_path / "nodes.txt" + node_map.write_text("repo__repo-1 cpu-0001\nrepo__repo-1 cpu-0002\n") + + with pytest.raises(RunnerError, match="conflicting Pyxis node assignments"): + load_node_map(node_map) + + +def test_pyxis_rejects_invalid_node_name(tmp_path): + node_map = tmp_path / "nodes.txt" + node_map.write_text("repo__repo-1 --cpu-0001\n") + + with pytest.raises(RunnerError, match="invalid Pyxis node name"): + load_node_map(node_map) + + def test_pyxis_image_registry_requires_repository(): with pytest.raises(RunnerError, match="must include a repository"): resolve_image("registry.example.com", "repo__repo-1") @@ -1000,6 +1488,70 @@ def test_pyxis_builds_host_srun_command(monkeypatch): ] +def test_pyxis_builds_srun_command_for_explicit_remote_node(monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + + command = build_srun_command(argv=["true"], node="cpu-worker-02") + + assert "--nodelist=cpu-worker-02" in command + assert "--nodelist=cpu-driver" not in command + + +def test_pyxis_outer_timeout_includes_configured_step_launch_grace( + monkeypatch, tmp_path +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_LAUNCH_GRACE_S", "900") + seen = [] + + def fake_run(command, **kwargs): + seen.append(kwargs["timeout"]) + (tmp_path / ".mlperf_srun_status").write_text("finished:0:0\n") + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + _run_srun_step_once( + argv=["true"], + status_path=tmp_path / ".mlperf_srun_status", + timeout_s=300, + ) + + assert seen == [1230.0] + + +@pytest.mark.parametrize( + ("isolate_pid_namespace", "expects_unshare"), + [(True, True), (False, False)], +) +def test_pyxis_step_can_preserve_host_pid_namespace( + monkeypatch, tmp_path, isolate_pid_namespace, expects_unshare +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + (tmp_path / ".mlperf_srun_status").write_text("finished:0:0\n") + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + _run_srun_step_once( + argv=["enroot", "list", "-f"], + status_path=tmp_path / ".mlperf_srun_status", + timeout_s=60, + isolate_pid_namespace=isolate_pid_namespace, + ) + + script = commands[0][commands[0].index("pyxis-step") - 1] + assert ("unshare --pid --fork --mount-proc" in script) is expects_unshare + assert 'timeout --verbose -k 5 "$timeout_s"' in script + + def test_pyxis_srun_environment_does_not_forward_credentials(monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "model-secret") monkeypatch.setenv("SWEBENCH_SERVICE_AUTH_TOKEN", "service-secret") @@ -1012,6 +1564,18 @@ def test_pyxis_srun_environment_does_not_forward_credentials(monkeypatch): assert "HF_TOKEN" not in environment +def test_pyxis_srun_uses_node_local_tmpdir_without_moving_the_host_mount( + monkeypatch, tmp_path +): + shared_tmp = tmp_path / "shared-driver-tmp" + monkeypatch.setenv("TMPDIR", str(shared_tmp)) + + environment = safe_srun_env() + + assert environment["TMPDIR"] == "/tmp" + assert os.environ["TMPDIR"] == str(shared_tmp) + + @pytest.mark.parametrize( "name", [ @@ -1137,10 +1701,53 @@ def fake_run(command, **kwargs): assert destination == "/tmp" persistent_tmp = Path(source) assert persistent_tmp.is_dir() - assert stat.S_IMODE(persistent_tmp.stat().st_mode) == 0o1777 + assert stat.S_IMODE(persistent_tmp.stat().st_mode) == 0o700 + + environment.cleanup() + + assert not persistent_tmp.exists() + + +def test_pyxis_environment_cleanup_after_nonretryable_outer_timeout( + monkeypatch, tmp_path +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if len(calls) == 1: + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + if len(calls) == 2: + mount = next( + argument.removeprefix("--container-mounts=") + for argument in command + if argument.startswith("--container-mounts=") + ) + source = next( + source + for source, destination in ( + item.split(":", 1) for item in mount.split(",") + ) + if destination == "/tmp" + ) + Path(source, ".mlperf_srun_status").write_text("started\n") + raise subprocess.TimeoutExpired( + command, kwargs["timeout"], output="partial\n" + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="run-1") + persistent_tmp = environment._tmp_dir + + with pytest.raises(StepNotLaunched): + environment.execute({"command": "pytest -q"}) environment.cleanup() + assert calls[-1][-4:-1] == ["enroot", "remove", "-f"] assert not persistent_tmp.exists() @@ -1205,6 +1812,31 @@ def fake_run(command, **kwargs): environment.cleanup() +@pytest.mark.parametrize(("timed_out", "expected"), [(True, -1), (False, 137)]) +def test_pyxis_environment_distinguishes_kill_after_from_voluntary_137( + monkeypatch, tmp_path, timed_out, expected +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + calls = 0 + + def fake_run(command, **kwargs): + nonlocal calls + calls += 1 + returncode = 137 if calls == 2 else 0 + _finish_srun_step(command, returncode, timed_out=timed_out and calls == 2) + return subprocess.CompletedProcess(command, returncode, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + environment = PyxisEnvironment(image=tmp_path / "task.sqsh", run_id="run-1") + try: + output = environment.execute({"command": "kill -KILL $$"}) + finally: + environment.cleanup() + + assert output["returncode"] == expected + + def test_pyxis_environment_raises_when_srun_never_starts_command(monkeypatch, tmp_path): failure_path = tmp_path / ".pyxis_infrastructure_failure" environment = object.__new__(PyxisEnvironment) @@ -1214,6 +1846,7 @@ def test_pyxis_environment_raises_when_srun_never_starts_command(monkeypatch, tm timeout_s=30, interpreter=["bash", "-c"], infrastructure_failure_path=failure_path, + node=None, ) environment.name = "mswe_run-1_abcd1234" environment._tmp_dir = tmp_path @@ -1228,7 +1861,7 @@ def test_pyxis_environment_raises_when_srun_never_starts_command(monkeypatch, tm ), ) - with pytest.raises(RunnerError, match=r"exceeded its 60s deadline"): + with pytest.raises(RunnerError, match=r"exceeded its 60\.0s outer deadline"): environment.execute({"command": "pytest -q"}) assert failure_path.exists() @@ -1400,6 +2033,179 @@ def fake_run(command, **kwargs): assert "failed to start Pyxis container" in str(exc_info.value) +def test_pyxis_failure_carries_separately_captured_scheduler_stderr( + monkeypatch, tmp_path +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "gb-nvl-053-compute04") + scheduler_error = ( + "srun: RPC rate limited 16 time(s). Sleeping then trying again.\n" + "srun: error: Task launch failed: Job credential expired\n" + ) + + monkeypatch.setattr( + subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess( + command, 167, stdout="", stderr=scheduler_error + ), + ) + + with pytest.raises(StepNotLaunched) as exc_info: + _run_srun_step_once( + argv=["true"], + status_path=tmp_path / ".mlperf_srun_status", + timeout_s=30, + stderr=subprocess.PIPE, + ) + + failure = exc_info.value + assert failure.provable_non_execution is True + assert failure.srun_rc == 167 + assert "RPC rate limited" in str(failure) + assert "Job credential expired" in str(failure) + + +def test_outer_timeout_with_pending_status_is_retryable_and_keeps_output( + monkeypatch, tmp_path +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "2") + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if len(calls) == 1: + raise subprocess.TimeoutExpired( + command, + kwargs["timeout"], + output=b"srun: RPC rate limited\n", + stderr=b"srun: Job credential expired\n", + ) + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="recovered\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + output = _bare_environment(tmp_path).execute({"command": "pytest -q"}) + + assert output["output"] == "recovered\n" + assert len(calls) == 2 + + +def test_recovered_outer_timeout_does_not_leave_failure_marker(monkeypatch, tmp_path): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "2") + failure_path = tmp_path / ".pyxis_infrastructure_failure" + calls = 0 + + def fake_run(command, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="recovered\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + output = _bare_environment(tmp_path, failure_path).execute({"command": "pytest -q"}) + + assert output["returncode"] == 0 + assert not failure_path.exists() + + +def test_step_retry_log_reader_preserves_recovered_infrastructure_events(tmp_path): + retry_log = tmp_path / "infra_retries.jsonl" + retry_log.write_text( + '{"target":"mswe_probe","attempt":1,"outcome":"retrying"}\n' + '{"target":"mswe_probe","attempt":2,"outcome":"recovered"}\n' + ) + + records, errors = read_step_retry_log(retry_log) + + assert [record["outcome"] for record in records] == ["retrying", "recovered"] + assert errors == [] + + +def test_step_retry_log_reader_makes_corrupt_accounting_visible(tmp_path): + retry_log = tmp_path / "infra_retries.jsonl" + retry_log.write_text('{"outcome":"exhausted"}\nnot-json\n{"attempt":2}\n') + + records, errors = read_step_retry_log(retry_log) + + assert [record["outcome"] for record in records] == ["exhausted"] + assert errors == [ + "line 2: invalid JSON: Expecting value", + "line 3: expected an object with string outcome", + ] + + +def test_outer_timeout_with_started_status_is_not_replayed_and_keeps_output( + monkeypatch, tmp_path +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "3") + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + (tmp_path / ".mlperf_srun_status").write_text("started\n") + raise subprocess.TimeoutExpired( + command, + kwargs["timeout"], + output="partial command output\n", + stderr="srun diagnostic\n", + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(StepNotLaunched) as exc_info: + _bare_environment(tmp_path).execute({"command": "rm -rf build"}) + + failure = exc_info.value + assert failure.provable_non_execution is False + assert failure.status == "started" + assert failure.srun_rc is None + assert "partial command output" in str(failure) + assert "srun diagnostic" in str(failure) + assert len(calls) == 1 + + +@pytest.mark.parametrize("channel", ["sentinel", "status"]) +def test_outer_timeout_accepts_proven_command_completion( + monkeypatch, tmp_path, channel +): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + + def fake_run(command, **kwargs): + if channel == "sentinel": + nonce = command[command.index("pyxis-step") + 3] + output = f"partial\n__MLPERF_STEP_RC__ {nonce} 7\n" + else: + (tmp_path / ".mlperf_srun_status").write_text("finished:7:0\n") + output = "partial\n" + raise subprocess.TimeoutExpired(command, kwargs["timeout"], output=output) + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = _run_srun_step_once( + argv=["false"], + status_path=tmp_path / ".mlperf_srun_status", + timeout_s=300, + ) + + assert result.returncode == 7 + expected = "partial" if channel == "sentinel" else "partial\n" + assert result.stdout == expected + + def _bare_environment(tmp_path, failure_path=None): environment = object.__new__(PyxisEnvironment) environment.config = types.SimpleNamespace( @@ -1408,6 +2214,7 @@ def _bare_environment(tmp_path, failure_path=None): timeout_s=30, interpreter=["bash", "-c"], infrastructure_failure_path=failure_path, + node=None, ) environment.name = "mswe_run-1_abcd1234" environment._tmp_dir = tmp_path @@ -1423,7 +2230,7 @@ def _bare_environment(tmp_path, failure_path=None): # The step script started; the command may well have executed. ("started\n", False), # A report for some other return code: the command ran. - ("finished:0\n", False), + ("finished:0:0\n", False), ], ) def test_step_failure_reports_whether_non_execution_is_provable( @@ -1491,6 +2298,30 @@ def fake_run(command, **kwargs): assert output["returncode"] == 0 assert len(calls) == 3 + def test_every_retry_attempt_is_globally_paced(self, monkeypatch, tmp_path): + monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "3") + paced = [] + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if len(calls) < 3: + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + _finish_srun_step(command, 0) + return subprocess.CompletedProcess(command, 0, stdout="ok\n", stderr="") + + monkeypatch.setattr( + "inference_endpoint.evaluation.swebench_service.swebench_service" + ".pyxis_environment._pace_srun_step", + lambda: paced.append(True), + ) + monkeypatch.setattr(subprocess, "run", fake_run) + + output = self._environment(tmp_path).execute({"command": "pytest -q"}) + + assert output["returncode"] == 0 + assert len(paced) == len(calls) == 3 + def test_a_step_that_started_is_never_retried(self, monkeypatch, tmp_path): """It may have executed. Another attempt could double-apply it.""" monkeypatch.setenv("SWEBENCH_PYXIS_STEP_RETRIES", "5") @@ -1585,6 +2416,40 @@ def test_step_not_launched_is_a_runner_error(): assert issubclass(StepNotLaunched, RunnerError) +def test_step_pacer_spaces_admissions_without_holding_sleep_lock(): + now = [10.0] + sleeps = [] + + def sleep(seconds): + sleeps.append(seconds) + now[0] += seconds + + pacer = _StepPacer(2.0, monotonic=lambda: now[0], sleep=sleep) + + pacer.wait() + pacer.wait() + pacer.wait() + + assert sleeps == [0.5, 0.5] + + +def test_step_pacer_coordinates_processes_through_shared_state(tmp_path): + sleeps = [] + state_path = tmp_path / "global-step-rate" + kwargs = { + "state_path": state_path, + "monotonic": lambda: 1.0, + "wall_time": lambda: 100.0, + "sleep": sleeps.append, + } + + _StepPacer(2.0, **kwargs).wait() + _StepPacer(2.0, **kwargs).wait() + + assert sleeps == [0.5] + assert float(state_path.read_text()) == 101.0 + + def test_step_reports_its_return_code_in_band(monkeypatch, tmp_path): """The sentinel is authoritative and is stripped from the output. @@ -1843,6 +2708,38 @@ def fake_run(command, **kwargs): ] +def test_pyxis_cleanup_fans_out_to_every_routed_node(monkeypatch, tmp_path): + monkeypatch.setenv("SLURM_JOB_ID", "1738605") + monkeypatch.setenv("SLURMD_NODENAME", "cpu-driver") + node_map = tmp_path / "nodes.txt" + node_map.write_text("repo__repo-1 cpu-0001\nrepo__repo-2 cpu-0002\n") + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + node = next(arg for arg in command if arg.startswith("--nodelist=")) + output = "" + if command[-3:] == ["enroot", "list", "-f"]: + output = f"pyxis_1738605_mswe_run-1_{node[-4:]}\n" + return subprocess.CompletedProcess(command, 0, stdout=output, stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + runner = PyxisSweBenchRunner( + project_root=tmp_path, + subprocess_timeout_s=30, + image_dir=tmp_path / "images", + node_map=node_map, + ) + + runner._cleanup_containers("run-1") + + assert {arg for call in calls for arg in call if arg.startswith("--nodelist=")} == { + "--nodelist=cpu-0001", + "--nodelist=cpu-0002", + } + assert sum(call[-4:-2] == ["enroot", "remove"] for call in calls) == 2 + + def test_pyxis_agent_command_uses_host_model_and_image_registry(monkeypatch, tmp_path): calls: list[tuple[list[str], dict]] = [] @@ -1875,6 +2772,60 @@ def fake_run(command, log_path, **kwargs): assert kwargs["env"]["OPENAI_API_KEY"] == "model-secret" +def test_pyxis_agent_command_uses_staged_images_and_node_map(monkeypatch, tmp_path): + calls = [] + node_map = tmp_path / "nodes.txt" + node_map.write_text("repo__repo-1 cpu-0001\n") + monkeypatch.setattr( + runner_mod, + "_run_subprocess", + lambda command, log_path, **kwargs: calls.append(command), + ) + runner = PyxisSweBenchRunner( + project_root=tmp_path, + subprocess_timeout_s=30, + image_dir=tmp_path / "images", + node_map=node_map, + ) + + runner._run_agent( + _pyxis_request(), tmp_path / "config.yaml", tmp_path, tmp_path, set() + ) + + command = calls[0] + assert command[command.index("--image-dir") + 1] == str(tmp_path / "images") + snapshot = Path(command[command.index("--node-map") + 1]) + assert snapshot != node_map + assert snapshot.read_text() == "repo__repo-1\tcpu-0001\n" + assert "--image-registry" not in command + + node_map.write_text("repo__repo-1 cpu-9999\n") + assert snapshot.read_text() == "repo__repo-1\tcpu-0001\n" + + +def test_pyxis_agent_rejects_missing_node_assignment_before_dispatch( + monkeypatch, tmp_path +): + node_map = tmp_path / "nodes.txt" + node_map.write_text("repo__another-1 cpu-0001\n") + monkeypatch.setattr( + runner_mod, + "_run_subprocess", + lambda *args, **kwargs: pytest.fail("worker must not be dispatched"), + ) + runner = PyxisSweBenchRunner( + project_root=tmp_path, + subprocess_timeout_s=30, + image_dir=tmp_path / "images", + node_map=node_map, + ) + + with pytest.raises(RunnerError, match="no assignment.*repo__repo-1"): + runner._run_agent( + _pyxis_request(), tmp_path / "config.yaml", tmp_path, tmp_path, set() + ) + + def test_pyxis_agent_requires_upstream_environment_hook(monkeypatch, tmp_path): swebench = types.SimpleNamespace(main=lambda **kwargs: None) _install_fake_minisweagent(monkeypatch, swebench) @@ -1896,6 +2847,36 @@ def test_pyxis_agent_restores_upstream_environment_hook(monkeypatch, tmp_path): assert swebench.get_sb_environment is original +def test_pyxis_agent_routes_staged_image_to_assigned_node(monkeypatch, tmp_path): + captured = [] + original = object() + + def fake_main(**kwargs): + captured.append( + swebench.get_sb_environment({}, {"instance_id": "repo__repo-1"}) + ) + + swebench = types.SimpleNamespace( + get_sb_environment=original, + main=fake_main, + ) + _install_fake_minisweagent(monkeypatch, swebench) + node_map = tmp_path / "nodes.txt" + node_map.write_text("repo__repo-1 cpu-0042\n") + args = _pyxis_agent_args(tmp_path) + registry_index = args.index("--image-registry") + args[registry_index : registry_index + 2] = [ + "--image-dir", + str(tmp_path / "images"), + ] + args.extend(["--node-map", str(node_map)]) + + worker_mod.main(args) + + assert captured[0]["image"] == tmp_path / "images/repo__repo-1.sqsh" + assert captured[0]["node"] == "cpu-0042" + + def test_pyxis_agent_propagates_environment_infrastructure_failure( monkeypatch, tmp_path ): From c6b77f43752b13bac3fbba6d53a67a22888b602d Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Fri, 4 Sep 2026 15:31:13 -0700 Subject: [PATCH 25/25] Preserve endpoint credentials for fleet scoring --- .../commands/benchmark/accuracy.py | 5 +- .../evaluation/swe_bench_fleet_scorer.py | 42 ++++++++++---- tests/unit/commands/test_score_accuracy.py | 13 ++++- .../test_fleet_scorer.py | 58 ++++++++++++++++++- 4 files changed, 102 insertions(+), 16 deletions(-) diff --git a/src/inference_endpoint/commands/benchmark/accuracy.py b/src/inference_endpoint/commands/benchmark/accuracy.py index bdfdc9ac4..133a45504 100644 --- a/src/inference_endpoint/commands/benchmark/accuracy.py +++ b/src/inference_endpoint/commands/benchmark/accuracy.py @@ -262,7 +262,10 @@ def score_accuracy( scorer_kwargs = dict(eval_cfg.extras) if ( getattr(eval_cfg.scorer, "SCORER_ID", None) - == ScorerMethod.SWE_BENCH.value + in { + ScorerMethod.SWE_BENCH.value, + ScorerMethod.SWE_BENCH_FLEET.value, + } ): scorer_kwargs.update( model_params=eval_cfg.model_params, diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py index 092fe54df..0483fd5bb 100644 --- a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -28,7 +28,7 @@ import logging import time from pathlib import Path -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urljoin import msgspec @@ -54,6 +54,9 @@ logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from ..config.schema import EndpointConfig, ModelParams + class SWEBenchFleetScorer(Scorer, scorer_id="swe_bench_fleet"): """Distributed SWE-bench scoring across N services.""" @@ -74,6 +77,8 @@ def __init__( report_dir: Any, extractor: type[Extractor] | None = None, ground_truth_column: str | None = "instance_id", + model_params: ModelParams | None = None, + endpoint_config: EndpointConfig | None = None, **extras: Any, ) -> None: super().__init__( @@ -85,6 +90,8 @@ def __init__( ) self.report_dir = self.report_dir.resolve() self.options = self._resolve_options(extras) + self.model_params = model_params + self.endpoint_config = endpoint_config # --------------------------------------------------------------- config -- @@ -219,14 +226,27 @@ def score_single_sample(self, value: str, ground_truth: str) -> float: def score(self) -> tuple[float | None, int]: self.complete = True config = load_benchmark_config(self.report_dir) - model_params = config.get("model_params") or {} - model_name = model_params.get("name") + persisted_model_params = config.get("model_params") or {} + model_name = ( + self.model_params.name + if self.model_params is not None + else persisted_model_params.get("name") + ) if not model_name: raise ValueError("model_params.name is required in the benchmark config") - endpoint_config = config.get("endpoint_config") or {} - endpoint_urls = list(endpoint_config.get("endpoints") or []) + persisted_endpoint_config = config.get("endpoint_config") or {} + endpoint_urls = list( + self.endpoint_config.endpoints + if self.endpoint_config is not None + else persisted_endpoint_config.get("endpoints") or [] + ) if not endpoint_urls: raise SetupError("the benchmark config lists no endpoint URLs") + endpoint_api_key = ( + self.endpoint_config.api_key + if self.endpoint_config is not None + else persisted_endpoint_config.get("api_key") + ) instance_ids = self._instance_ids() if not instance_ids: @@ -238,7 +258,7 @@ def score(self) -> tuple[float | None, int]: expected_model=self.options["expected_model"], tool_call_model=model_name, min_prompt_tokens=self.options["min_prompt_tokens"], - api_key=endpoint_config.get("api_key"), + api_key=endpoint_api_key, ) try: run_gates(gates, endpoint_urls) @@ -254,16 +274,16 @@ def score(self) -> tuple[float | None, int]: self._model_name = model_name self._endpoint_urls = endpoint_urls - self._endpoint_api_key = endpoint_config.get("api_key") + self._endpoint_api_key = endpoint_api_key # load_benchmark_config() yaml.safe_load()s config.yaml, so model_params # is a plain mapping here, while _generation_params() expects the # pydantic ModelParams. Re-validate rather than re-implement the field # selection, so the fleet path and the single-service path agree. - from ..config.schema import ModelParams + if self.model_params is None: + from ..config.schema import ModelParams - self._generation_params = SWEBenchScorer._generation_params( - ModelParams.model_validate(model_params) - ) + self.model_params = ModelParams.model_validate(persisted_model_params) + self._generation_params = SWEBenchScorer._generation_params(self.model_params) self._unit_root = unit_root def fingerprint() -> str | None: diff --git a/tests/unit/commands/test_score_accuracy.py b/tests/unit/commands/test_score_accuracy.py index fd5e23284..95764cef8 100644 --- a/tests/unit/commands/test_score_accuracy.py +++ b/tests/unit/commands/test_score_accuracy.py @@ -88,6 +88,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) +class _FakeSWEBenchFleetScorer(_FakeSWEBenchScorer): + SCORER_ID = ScorerMethod.SWE_BENCH_FLEET.value + + class _FakeBreakdownScorer(_FakeScorer): """Scorer that returns a breakdown (like the composite gpt-oss scorer).""" @@ -210,11 +214,14 @@ def encode_batch(self, texts, add_special_tokens=False): @pytest.mark.unit class TestScoreAccuracy: + @pytest.mark.parametrize( + "scorer_cls", [_FakeSWEBenchScorer, _FakeSWEBenchFleetScorer] + ) @pytest.mark.parametrize( "test_mode", [config_schema.TestMode.ACC, config_schema.TestMode.BOTH] ) def test_swebench_receives_typed_runtime_model_and_endpoint( - self, tmp_path, test_mode + self, tmp_path, test_mode, scorer_cls ): model_params = ModelParams( name="test-model", @@ -228,7 +235,7 @@ def test_swebench_receives_typed_runtime_model_and_endpoint( api_key="runtime-secret", ) cfg = AccuracyConfiguration( - scorer=_FakeSWEBenchScorer, # type: ignore[arg-type] + scorer=scorer_cls, # type: ignore[arg-type] extractor=None, dataset_name="swe_bench", dataset=_FakeDataset(1, 1.0), # type: ignore[arg-type] @@ -242,7 +249,7 @@ def test_swebench_receives_typed_runtime_model_and_endpoint( score_accuracy(_ctx([cfg], test_mode=test_mode), _RESULT) - assert _FakeSWEBenchScorer.received_kwargs == { + assert scorer_cls.received_kwargs == { "extractor": None, "ground_truth_column": None, "swebench_service_auth_token": "service-secret", diff --git a/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py b/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py index b96026227..ccee7b0e0 100644 --- a/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py +++ b/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py @@ -5,9 +5,14 @@ from __future__ import annotations +from unittest.mock import MagicMock + +import pandas as pd import pytest +import yaml -from inference_endpoint.config.schema import ScorerMethod +from inference_endpoint.config.schema import EndpointConfig, ModelParams, ScorerMethod +from inference_endpoint.evaluation import swe_bench_fleet_scorer as fleet_scorer_module from inference_endpoint.evaluation.scoring import Scorer from inference_endpoint.evaluation.swe_bench_fleet_scorer import SWEBenchFleetScorer from inference_endpoint.exceptions import SetupError @@ -76,3 +81,54 @@ def test_a_bad_shard_size_is_rejected(self): SWEBenchFleetScorer._resolve_options( {"swebench_service_urls": URLS, "shard_size": 0} ) + + +def test_runtime_endpoint_secret_wins_over_redacted_report(monkeypatch, tmp_path): + report_dir = tmp_path / "report" + report_dir.mkdir() + (report_dir / "sample_idx_map.json").write_text( + '{"swe_bench":{"sample-uuid":0}}' + ) + (report_dir / "config.yaml").write_text( + yaml.safe_dump( + { + "model_params": {"name": "persisted-model"}, + "endpoint_config": { + "endpoints": ["http://persisted:8000"], + "api_key": "", + }, + } + ) + ) + dataset = MagicMock() + dataset.dataframe = pd.DataFrame( + {"instance_id": ["repo__repo-1"], "prompt": ["prompt"]} + ) + captured = {} + + def build_gates(**kwargs): + captured.update(kwargs) + return [], MagicMock() + + def refuse(_gates, _urls): + raise fleet_scorer_module.GateFailure("stop after credential capture") + + monkeypatch.setattr(fleet_scorer_module, "build_gates", build_gates) + monkeypatch.setattr(fleet_scorer_module, "run_gates", refuse) + scorer = SWEBenchFleetScorer( + dataset_name="swe_bench", + dataset=dataset, + report_dir=report_dir, + swebench_service_urls=URLS, + num_instances=1, + model_params=ModelParams(name="runtime-model"), + endpoint_config=EndpointConfig( + endpoints=["http://runtime:8000"], api_key="runtime-secret" + ), + ) + + with pytest.raises(SetupError, match="credential capture"): + scorer.score() + + assert captured["tool_call_model"] == "runtime-model" + assert captured["api_key"] == "runtime-secret"