From 17c5de2f5dd64d9bd0fb2bbfd5b5528713acc4e8 Mon Sep 17 00:00:00 2001
From: Stanley Phoong <sphoong@nvidia.com>
Date: Wed, 26 Aug 2026 12:53:46 -0700
Subject: [PATCH 1/5] fix(swebench-service): score the predictions a failed
 agent phase left behind

One worker's infrastructure failure discarded the entire eval phase.

The agent phase runs `--workers` trajectories concurrently. `_run_agent`
re-raises whatever the phase raised, so a single worker that could not start
its container -- or whose `srun` step never launched -- took the exception
all the way out of `_run()`. That happens *before* `preds.json` is ever
looked at, so the predictions every other worker had already written were
never scored.

Observed: a 200-instance run with 137 predictions on disk reported as a
total loss, exit non-zero, no accuracy number, and had to be re-scored by
hand from the retained artifacts. The GPU allocation that produced those 137
predictions was gone by then.

Eval is now robust to individual worker failure: whatever predictions exist
are always scored. The failure is not hidden --

* it is written to the new `agent_phase_error.txt` run artifact, with
  secrets redacted, and served through the existing artifact route;
* it is logged at ERROR;
* it is chained as `__cause__` onto the `preds.json` failure when the phase
  genuinely produced nothing, so an empty run still fails loudly.

`RunCancelled` is explicitly not tolerated: a cancelled run is not a
degraded run and must not proceed to eval.

Tests: `test_run_scores_predictions_left_behind_by_a_failed_agent_phase`
(the eval phase runs and the artifact is written),
`test_run_still_fails_when_the_agent_phase_produced_nothing` (no
false pass, cause chained), `test_run_redacts_secrets_from_the_agent_phase_error`,
`test_run_does_not_tolerate_cancellation` and
`test_agent_phase_error_is_a_retrievable_artifact`. The existing
cleanup-after-failure test asserted that the agent error propagated verbatim
and is updated to assert the chained failure instead.
---
 .../evaluation/swe_bench_scorer.py            |   1 +
 .../evaluation/swebench_service/README.md     |  16 ++
 .../swebench_service/artifacts.py             |   1 +
 .../swebench_service/runner.py                |  59 +++++++-
 .../swebench_service/test_runner.py           | 142 +++++++++++++++++-
 5 files changed, 211 insertions(+), 8 deletions(-)

diff --git a/src/inference_endpoint/evaluation/swe_bench_scorer.py b/src/inference_endpoint/evaluation/swe_bench_scorer.py
index a91d63734..98a76d2a1 100644
--- a/src/inference_endpoint/evaluation/swe_bench_scorer.py
+++ b/src/inference_endpoint/evaluation/swe_bench_scorer.py
@@ -67,6 +67,7 @@ class SWEBenchScorer(Scorer, scorer_id="swe_bench_scorer"):
         "artifacts.download",
     }
     SAFE_ARTIFACT_NAMES: ClassVar[set[str]] = {
+        "agent_phase_error.txt",
         "preds.json",
         "swe_bench_agent.log",
         "swe_bench_eval.log",
diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md
index 0e7a501c5..f0751af0c 100644
--- a/src/inference_endpoint/evaluation/swebench_service/README.md
+++ b/src/inference_endpoint/evaluation/swebench_service/README.md
@@ -109,6 +109,22 @@ allocation. `scancel` does not reclaim them either -- ending the job does not
 remove Enroot containers -- which is why the removal has to be both correctly
 named and audible.
 
+### Agent-phase failures
+
+The agent phase runs `--workers` trajectories concurrently. If it fails after
+some workers have already written their patches, the service still evaluates the
+predictions that reached `preds.json` rather than discarding them: the failure is
+recorded in the `agent_phase_error.txt` artifact and logged at ERROR, and the run
+continues into the eval phase. Instances with no prediction are simply absent from
+the results, which the harness reports as unresolved.
+
+A run whose agent phase produced no predictions at all still fails, with the agent
+error chained as the cause. Cancellation is never tolerated this way: it
+propagates immediately and no eval phase runs.
+
+Consumers that need to know a run was degraded should fetch `agent_phase_error.txt`;
+its presence means some instances may be missing from `swe_bench_results.json`.
+
 The benchmark client submits a run to this service only in `ACC` or `BOTH`
 mode; the default `PERF` mode skips external evaluation.
 
diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py
index fd207dc7c..d24d1ee8d 100644
--- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py
+++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py
@@ -42,6 +42,7 @@
     re.compile(r"(://[^:/\s]+:)[^@\s/]+(@)"),
 )
 SAFE_ARTIFACT_NAMES = {
+    "agent_phase_error.txt",
     "preds.json",
     "swe_bench_agent.log",
     "swe_bench_eval.log",
diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py
index 8682b40a7..e54167b2a 100644
--- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py
+++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py
@@ -84,6 +84,8 @@ def detach(self, process: subprocess.Popen[str]) -> None:
 _LOG_TAIL_MAX_BYTES = 64 * 1024
 _LOG_TAIL_MAX_LINES = 50
 _RUN_LABEL = "com.mlcommons.endpoints.swebench-run"
+#: Written when the agent phase failed but predictions were still scored.
+AGENT_PHASE_ERROR_FILE = "agent_phase_error.txt"
 _PROCESS_TERMINATE_TIMEOUT_S = 10
 _SWEBENCH_DATASETS = {
     "verified": "princeton-nlp/SWE-bench_Verified",
@@ -298,7 +300,7 @@ def _run(
                 request,
                 run_id=run_dir.name,
             )
-            self._run_agent(
+            agent_error = self._run_agent_tolerantly(
                 request,
                 patched_config,
                 output_dir,
@@ -309,7 +311,10 @@ def _run(
 
         preds_path = output_dir / "preds.json"
         if not preds_path.exists():
-            raise RunnerError("mini-extra did not produce preds.json")
+            # A genuinely empty agent phase still fails loudly: there is
+            # nothing to score. The agent error, if any, is the cause.
+            error = RunnerError("mini-extra did not produce preds.json")
+            raise error from agent_error
         self._validate_prediction_ids(request, preds_path)
         shutil.copy2(preds_path, run_dir / "preds.json")
 
@@ -319,6 +324,56 @@ def _run(
         shutil.copy2(result_path, run_dir / "swe_bench_results.json")
         return msgspec.json.decode(result_path.read_bytes(), type=dict)
 
+    def _run_agent_tolerantly(
+        self,
+        request: RunRequest,
+        patched_config: Path,
+        output_dir: Path,
+        run_dir: Path,
+        secret_values: set[str],
+        cancel_token: CancellationToken | None,
+    ) -> BaseException | None:
+        """Run the agent phase; record a failure instead of discarding the run.
+
+        The agent phase fans out across many concurrent workers, and a single
+        worker's infrastructure failure (a container that would not start, an
+        `srun` step that never launched) propagates out of the whole phase. The
+        predictions every *other* worker already wrote are on disk and are
+        perfectly scoreable, but the exception reached ``run()`` before
+        ``preds.json`` was ever looked at, so the entire eval phase was skipped
+        and a run with 137 of 200 predictions reported as a total loss.
+
+        The failure is not hidden. It is written to ``agent_phase_error.txt``,
+        served as a run artifact, logged at ERROR, and chained onto the
+        ``preds.json`` failure when the phase really did produce nothing.
+        Cancellation is not a failure to tolerate: it propagates unchanged.
+        """
+        try:
+            self._run_agent(
+                request,
+                patched_config,
+                output_dir,
+                run_dir,
+                secret_values,
+                cancel_token,
+            )
+        except RunCancelled:
+            raise
+        except Exception as exc:
+            atomic_write_bytes(
+                run_dir / AGENT_PHASE_ERROR_FILE,
+                redact_text(f"{type(exc).__name__}: {exc}\n", secret_values).encode(),
+            )
+            logger.error(
+                "SWE-bench agent phase failed for run %s; continuing to eval so "
+                "the predictions already on disk are still scored: %s",
+                run_dir.name,
+                exc,
+                exc_info=True,
+            )
+            return exc
+        return None
+
     def _load_template(self, request: RunRequest) -> dict[str, Any]:
         template_path = self._template_dir / TEMPLATE_FILES[request.template]
         with template_path.open() as f:
diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py
index 7d3f8cac4..81a94473a 100644
--- a/tests/unit/evaluation/swebench_service/test_runner.py
+++ b/tests/unit/evaluation/swebench_service/test_runner.py
@@ -15,6 +15,9 @@
 import msgspec.json
 import pytest
 import yaml
+from inference_endpoint.evaluation.swebench_service.swebench_service import (
+    artifacts as artifacts_mod,
+)
 from inference_endpoint.evaluation.swebench_service.swebench_service import (
     pyxis_worker as worker_mod,
 )
@@ -434,15 +437,18 @@ def test_run_cleans_labeled_containers_after_success(monkeypatch, tmp_path):
 
 
 @pytest.mark.parametrize(
-    ("error", "match"),
+    ("error", "raised", "match"),
     [
-        (RuntimeError("agent failed"), "agent failed"),
-        (RunnerError("subprocess timed out"), "timed out"),
-        (RunCancelled("subprocess cancelled"), "cancelled"),
+        # A non-cancellation agent failure that leaves no prediction behind
+        # surfaces as the empty-predictions failure, with the agent error
+        # chained as its cause.
+        (RuntimeError("agent failed"), RunnerError, "did not produce preds.json"),
+        (RunnerError("subprocess timed out"), RunnerError, "did not produce preds"),
+        (RunCancelled("subprocess cancelled"), RunCancelled, "cancelled"),
     ],
 )
 def test_run_cleans_labeled_containers_after_failure(
-    monkeypatch, tmp_path, error, match
+    monkeypatch, tmp_path, error, raised, match
 ):
     runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30)
     cleaned: list[str] = []
@@ -453,12 +459,136 @@ def fail_agent(*args, **kwargs):
     monkeypatch.setattr(runner, "_run_agent", fail_agent)
     monkeypatch.setattr(runner, "_cleanup_containers", cleaned.append)
 
-    with pytest.raises(type(error), match=match):
+    with pytest.raises(raised, match=match) as exc_info:
         runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-2")
 
+    if raised is not RunCancelled:
+        assert exc_info.value.__cause__ is error
     assert cleaned == ["run-2"]
 
 
+def test_run_scores_predictions_left_behind_by_a_failed_agent_phase(
+    monkeypatch, tmp_path
+):
+    """One worker's infrastructure failure must not discard the eval phase.
+
+    The agent phase fans out across many workers. When one of them dies the
+    exception propagates out of the whole phase, but every prediction the other
+    workers wrote is already on disk. Before this fix a run with predictions for
+    most of its instances was reported as a total loss and never scored at all.
+    """
+    runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30)
+    run_dir = tmp_path / "run-partial"
+    scored: list[Path] = []
+
+    def partial_agent(
+        request, patched_config, output_dir, run_dir, secret_values, cancel_token=None
+    ):
+        (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}')
+        raise RunnerError("Pyxis infrastructure failure before the command completed")
+
+    def fake_run_eval(
+        request, preds_path, output_dir, run_dir, secret_values, cancel_token=None
+    ):
+        scored.append(preds_path)
+        result_path = output_dir / "result.json"
+        result_path.write_text('{"resolved_instances":1,"submitted_instances":1}')
+        return result_path
+
+    monkeypatch.setattr(runner, "_run_agent", partial_agent)
+    monkeypatch.setattr(runner, "_run_eval", fake_run_eval)
+    monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None)
+
+    result = runner.run(_request(["http://endpoint:30000"]), run_dir)
+
+    assert result == {"resolved_instances": 1, "submitted_instances": 1}
+    assert scored, "eval phase never ran"
+    error_text = (run_dir / "agent_phase_error.txt").read_text()
+    assert "RunnerError" in error_text
+    assert "Pyxis infrastructure failure" in error_text
+
+
+def test_agent_phase_error_is_a_retrievable_artifact():
+    assert "agent_phase_error.txt" in artifacts_mod.SAFE_ARTIFACT_NAMES
+
+
+def test_run_redacts_secrets_from_the_agent_phase_error(monkeypatch, tmp_path):
+    runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30)
+    run_dir = tmp_path / "run-secret"
+    request = _request(["http://endpoint:30000"])
+    request.endpoint_api_key = "real-secret"
+
+    def leaky_agent(
+        request, patched_config, output_dir, run_dir, secret_values, cancel_token=None
+    ):
+        (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}')
+        raise RunnerError("connection to http://endpoint:30000 with real-secret failed")
+
+    def fake_run_eval(
+        request, preds_path, output_dir, run_dir, secret_values, cancel_token=None
+    ):
+        result_path = output_dir / "result.json"
+        result_path.write_text("{}")
+        return result_path
+
+    monkeypatch.setattr(runner, "_run_agent", leaky_agent)
+    monkeypatch.setattr(runner, "_run_eval", fake_run_eval)
+    monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None)
+
+    runner.run(request, run_dir)
+
+    error_text = (run_dir / "agent_phase_error.txt").read_text()
+    assert "real-secret" not in error_text
+    assert "<redacted>" in error_text
+
+
+def test_run_still_fails_when_the_agent_phase_produced_nothing(monkeypatch, tmp_path):
+    """Tolerating the failure must not turn an empty run into a pass."""
+    runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30)
+    cause = RunnerError("every worker died")
+
+    def dead_agent(*args, **kwargs):
+        raise cause
+
+    monkeypatch.setattr(runner, "_run_agent", dead_agent)
+    monkeypatch.setattr(
+        runner,
+        "_run_eval",
+        lambda *a, **k: pytest.fail("eval must not run without predictions"),
+    )
+    monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None)
+
+    with pytest.raises(RunnerError, match="did not produce preds.json") as exc_info:
+        runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-empty")
+
+    assert exc_info.value.__cause__ is cause
+
+
+def test_run_does_not_tolerate_cancellation(monkeypatch, tmp_path):
+    """Cancellation is not a worker failure and must propagate unchanged."""
+    runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30)
+    run_dir = tmp_path / "run-cancelled"
+
+    def cancelled_agent(
+        request, patched_config, output_dir, run_dir, secret_values, cancel_token=None
+    ):
+        (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}')
+        raise RunCancelled("subprocess cancelled")
+
+    monkeypatch.setattr(runner, "_run_agent", cancelled_agent)
+    monkeypatch.setattr(
+        runner,
+        "_run_eval",
+        lambda *a, **k: pytest.fail("eval must not run after cancellation"),
+    )
+    monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None)
+
+    with pytest.raises(RunCancelled):
+        runner.run(_request(["http://endpoint:30000"]), run_dir)
+
+    assert not (run_dir / "agent_phase_error.txt").exists()
+
+
 def test_run_cleans_harness_containers_after_eval_cancellation(monkeypatch, tmp_path):
     runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30)
     cleaned: list[tuple[str, dict]] = []

From 502f7577db8e75d5db113feb1143ee66f2301b91 Mon Sep 17 00:00:00 2001
From: Stanley Phoong <sphoong@nvidia.com>
Date: Sat, 8 Aug 2026 17:40:57 -0700
Subject: [PATCH 2/5] feat(swe-bench): eval-phase infra-vs-genuine error
 classifier

The Pyxis sentinel only covers the agent phase; eval-phase error_ids were
counted as real outcomes and never retried, which is what produced 24 of 25
permanently-bad runs on the source cluster. classify.py reads the SWE-bench
report's error_ids and each instance's run_instance.log and classifies them
through an ORDERED rule list, first match wins. The order is load-bearing:
BuildImageError is checked before everything because its message embeds the
other rules' needles, CONMON_EAGAIN and TEST_TIMEOUT precede WEDGE_EVAL, and
PATCH_APPLY_FAILED is last.

Anything unclassifiable is UNKNOWN and UNKNOWN is GENUINE, asserted by a
membership test: a false bad-run costs one redo, a false retry biases the
measurement toward optimism.

Memory-kill markers are consumed by phase - an eval-phase kill is a genuine
failure (an unbounded allocation is a failing patch), an agent-phase kill is
recorded for audit only, since the agent merely gets an error observation
and the instance still reaches a real outcome.
---
 .../swe_bench_distributed/__init__.py         |  14 ++
 .../swe_bench_distributed/classify.py         | 234 ++++++++++++++++++
 .../swe_bench_distributed/test_classify.py    | 171 +++++++++++++
 3 files changed, 419 insertions(+)
 create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/classify.py
 create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_classify.py

diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py
index 7269ab520..8e2ae103e 100644
--- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py
+++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py
@@ -11,6 +11,14 @@
 exactly once.
 """
 
+from .classify import (
+    GENUINE_KINDS,
+    INFRA_KINDS,
+    ErrorKind,
+    UnitClassification,
+    classify_eval_log,
+    classify_unit,
+)
 from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid
 from .infra_retry import (
     InfraRetryLedger,
@@ -38,8 +46,11 @@
 from .units import Unit, UnitPlan, plan_units
 
 __all__ = [
+    "GENUINE_KINDS",
+    "INFRA_KINDS",
     "ClaimError",
     "CompletenessReport",
+    "ErrorKind",
     "HealthTerm",
     "HealthVerdict",
     "InfraRetryLedger",
@@ -53,11 +64,14 @@
     "RunQuality",
     "SlurmStepLiveness",
     "Unit",
+    "UnitClassification",
     "UnitOutcome",
     "UnitPlan",
     "UnitResult",
     "WorkQueue",
     "assess_run",
+    "classify_eval_log",
+    "classify_unit",
     "combine_terms",
     "is_provable_non_execution",
     "kill_by_pid",
diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py
new file mode 100644
index 000000000..b7c06ac25
--- /dev/null
+++ b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py
@@ -0,0 +1,234 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Split a unit's error instances into infrastructure damage and genuine failures.
+
+A SWE-bench run reports three per-instance outcomes: resolved, unresolved, and
+*error*. The agent phase has an infrastructure sentinel (the Pyxis
+``infrastructure_failure_path``), but the eval phase has none: an instance whose
+evaluation container wedged is booked as ``error``, which counts as "accounted
+for", so the unit is published successful and is never retried. Those instances
+silently poison a run that can then never reach a full result.
+
+Classification exists to catch exactly that. It reads each error instance's
+``run_instance.log`` and assigns one kind.
+
+BIAS RULE -- this is the whole design and it is deliberately asymmetric. If an
+error cannot be classified confidently it is treated as GENUINE, never as
+infrastructure. A false bad-run costs one redo; a false retry silently biases
+the measurement toward optimism, and an optimistic accuracy number is worse than
+no number.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class ErrorKind(StrEnum):
+    """One classification of an error instance."""
+
+    # Infrastructure: a defect in the runtime we provided, safe to retry.
+    CONTAINER_EXEC_REFUSED = "container_exec_refused"
+    CONTAINER_FORK_EAGAIN = "container_fork_eagain"
+    RUNTIME_READ_TIMEOUT = "runtime_read_timeout"
+    IMAGE_BUILD_TIMEOUT = "image_build_timeout"
+    IMAGE_BUILD_ERROR = "image_build_error"
+    STEP_INFRASTRUCTURE_FAILURE = "step_infrastructure_failure"
+    ENDPOINT_CHANGED = "endpoint_changed"
+
+    # Genuine: a real outcome of the model's patch, or unreadable. Never retried.
+    TEST_TIMEOUT = "test_timeout"
+    TEST_MEMORY_EXCEEDED = "test_memory_exceeded"
+    PATCH_APPLY_FAILED = "patch_apply_failed"
+    UNKNOWN = "unknown"
+
+
+#: Retryable. Every member is a defect in infrastructure we control.
+INFRA_KINDS: frozenset[ErrorKind] = frozenset(
+    {
+        ErrorKind.CONTAINER_EXEC_REFUSED,
+        ErrorKind.CONTAINER_FORK_EAGAIN,
+        ErrorKind.RUNTIME_READ_TIMEOUT,
+        ErrorKind.IMAGE_BUILD_TIMEOUT,
+        ErrorKind.IMAGE_BUILD_ERROR,
+        ErrorKind.STEP_INFRASTRUCTURE_FAILURE,
+        ErrorKind.ENDPOINT_CHANGED,
+    }
+)
+
+#: Never retried.
+#:
+#: ``TEST_TIMEOUT`` is a plausible model outcome: a patch that makes the suite
+#: loop is a failing patch. ``TEST_MEMORY_EXCEEDED`` is its exact parallel -- a
+#: patch that makes a graded test allocate without bound is a failing patch, and
+#: the alternative to killing it was never "the test passes", it was "the host
+#: OOMs and the instance still never completes". ``PATCH_APPLY_FAILED`` is the
+#: model emitting a diff that does not apply; SWE-bench books it as ``error``
+#: rather than ``unresolved``, but it is model behaviour. ``UNKNOWN`` is the bias
+#: rule.
+GENUINE_KINDS: frozenset[ErrorKind] = frozenset(
+    {
+        ErrorKind.TEST_TIMEOUT,
+        ErrorKind.TEST_MEMORY_EXCEEDED,
+        ErrorKind.PATCH_APPLY_FAILED,
+        ErrorKind.UNKNOWN,
+    }
+)
+
+# ORDERED. First match wins, and the order is load-bearing.
+#
+# CONTAINER_FORK_EAGAIN and TEST_TIMEOUT come BEFORE CONTAINER_EXEC_REFUSED: a
+# timed-out or fork-failed evaluation frequently *also* emits "container state
+# improper" while the harness tears the container down, and reading that as a
+# wedge would retry a genuine model outcome.
+#
+# PATCH_APPLY_FAILED is checked LAST: if a container also wedged, the wedge
+# wins, because a wedged container's verdict is unreliable either way. Only a
+# log with no infrastructure signature at all reaches this rule.
+_RULES: tuple[tuple[ErrorKind, tuple[str, ...]], ...] = (
+    (
+        ErrorKind.CONTAINER_FORK_EAGAIN,
+        ("fork/exec /usr/bin/conmon: resource temporarily unavailable",),
+    ),
+    (ErrorKind.TEST_TIMEOUT, ("Test timed out after",)),
+    (
+        ErrorKind.CONTAINER_EXEC_REFUSED,
+        (
+            "can only create exec sessions on running containers",
+            "container state improper",
+        ),
+    ),
+    (ErrorKind.RUNTIME_READ_TIMEOUT, ("Read timed out. (read timeout=",)),
+    (
+        ErrorKind.PATCH_APPLY_FAILED,
+        (
+            "Reversed (or previously applied) patch detected",
+            ">>>>> Patch Apply Failed",
+            "hunk FAILED",
+            "hunk failed",
+        ),
+    ),
+)
+
+
+def classify_eval_log(text: str) -> ErrorKind:
+    """Classify one instance's evaluation log.
+
+    ``BuildImageError`` is checked before the ordered rules because its message
+    embeds the same "Read timed out" / "500" strings the other rules look for,
+    so any other order misattributes a build failure.
+    """
+    if "BuildImageError" in text:
+        if "Read timed out" in text:
+            return ErrorKind.IMAGE_BUILD_TIMEOUT
+        return ErrorKind.IMAGE_BUILD_ERROR
+    for kind, needles in _RULES:
+        if any(needle in text for needle in needles):
+            return kind
+    return ErrorKind.UNKNOWN
+
+
+@dataclass(slots=True)
+class UnitClassification:
+    """Per-kind counts for one unit's error instances."""
+
+    kinds: dict[ErrorKind, int] = field(default_factory=dict)
+    error_instance_ids: tuple[str, ...] = ()
+    #: False when the run's report could not be read at all. "Not measured" and
+    #: "measured zero" are different, and conflating them once let a damaged
+    #: unit into a clean set.
+    measured: bool = False
+
+    @property
+    def infra_count(self) -> int:
+        return sum(count for kind, count in self.kinds.items() if kind in INFRA_KINDS)
+
+    @property
+    def genuine_count(self) -> int:
+        return sum(count for kind, count in self.kinds.items() if kind in GENUINE_KINDS)
+
+    @property
+    def should_retry(self) -> bool:
+        return self.infra_count > 0
+
+    def as_counts(self) -> dict[str, int]:
+        return {kind.value: count for kind, count in sorted(self.kinds.items())}
+
+
+def _find_instance_log(output_dir: Path, instance_id: str) -> Path | None:
+    patterns = (
+        f"logs/run_evaluation/*/*/{instance_id}/run_instance.log",
+        f"logs/run_evaluation/*/*/*/{instance_id}/run_instance.log",
+    )
+    for pattern in patterns:
+        for match in sorted(output_dir.glob(pattern)):
+            return match
+    return None
+
+
+def memory_kill_markers(killed_dir: Path, instance_id: str) -> bool:
+    """True only for an *eval*-phase memory kill.
+
+    Phase is load-bearing and the two cases must never be collapsed. An eval
+    kill destroyed a graded result, so the instance's error is a genuine
+    failure. An agent kill merely makes one tool call return an error
+    observation and the agent carries on, so the instance still reaches a real
+    outcome; that marker exists for audit and must not influence classification.
+
+    A marker beats any log heuristic: a SIGKILLed test leaves an ambiguous log,
+    but the kill itself is a fact recorded before acting.
+    """
+    return any(killed_dir.glob(f"eval.{instance_id}.*.json"))
+
+
+def classify_unit(
+    output_dir: Path,
+    error_instance_ids: list[str] | tuple[str, ...] | None,
+    *,
+    killed_dir: Path | None = None,
+    infrastructure_failure: bool = False,
+    endpoint_changed: bool = False,
+) -> UnitClassification:
+    """Classify every error instance of one unit.
+
+    ``infrastructure_failure`` carries the Pyxis agent-phase sentinel, and
+    ``endpoint_changed`` carries a mismatch between the inference endpoint
+    fingerprint recorded at claim time and at publish time -- an engine
+    restarted under a live client produces a plausible-looking run that must not
+    be scored.
+    """
+    kinds: dict[ErrorKind, int] = {}
+
+    if infrastructure_failure:
+        kinds[ErrorKind.STEP_INFRASTRUCTURE_FAILURE] = (
+            kinds.get(ErrorKind.STEP_INFRASTRUCTURE_FAILURE, 0) + 1
+        )
+    if endpoint_changed:
+        kinds[ErrorKind.ENDPOINT_CHANGED] = kinds.get(ErrorKind.ENDPOINT_CHANGED, 0) + 1
+
+    if error_instance_ids is None:
+        return UnitClassification(kinds=kinds, error_instance_ids=(), measured=False)
+
+    ids = tuple(str(x) for x in error_instance_ids)
+    for instance_id in ids:
+        if killed_dir is not None and memory_kill_markers(killed_dir, instance_id):
+            kinds[ErrorKind.TEST_MEMORY_EXCEEDED] = (
+                kinds.get(ErrorKind.TEST_MEMORY_EXCEEDED, 0) + 1
+            )
+            continue
+        log_path = _find_instance_log(output_dir, instance_id)
+        kind = ErrorKind.UNKNOWN
+        if log_path is not None:
+            try:
+                kind = classify_eval_log(log_path.read_text(errors="replace"))
+            except OSError:
+                logger.debug("could not read %s", log_path, exc_info=True)
+        kinds[kind] = kinds.get(kind, 0) + 1
+
+    return UnitClassification(kinds=kinds, error_instance_ids=ids, measured=True)
diff --git a/tests/unit/evaluation/swe_bench_distributed/test_classify.py b/tests/unit/evaluation/swe_bench_distributed/test_classify.py
new file mode 100644
index 000000000..2b079b315
--- /dev/null
+++ b/tests/unit/evaluation/swe_bench_distributed/test_classify.py
@@ -0,0 +1,171 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Infrastructure-versus-genuine classification of error instances."""
+
+from __future__ import annotations
+
+import pytest
+
+from inference_endpoint.evaluation.swe_bench_distributed.classify import (
+    GENUINE_KINDS,
+    INFRA_KINDS,
+    ErrorKind,
+    classify_eval_log,
+    classify_unit,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def write_log(output_dir, instance_id: str, text: str) -> None:
+    log_dir = output_dir / "logs" / "run_evaluation" / "run-1" / "model" / instance_id
+    log_dir.mkdir(parents=True, exist_ok=True)
+    (log_dir / "run_instance.log").write_text(text)
+
+
+class TestBiasRule:
+    def test_unknown_is_genuine_never_infra(self):
+        # A false bad-run costs one redo; a false retry biases the measurement
+        # toward optimism. Unclassifiable therefore means "keep the result".
+        assert ErrorKind.UNKNOWN in GENUINE_KINDS
+        assert ErrorKind.UNKNOWN not in INFRA_KINDS
+
+    def test_kinds_are_partitioned(self):
+        assert not (INFRA_KINDS & GENUINE_KINDS)
+        assert INFRA_KINDS | GENUINE_KINDS == set(ErrorKind)
+
+    def test_model_outcomes_are_genuine(self):
+        for kind in (
+            ErrorKind.TEST_TIMEOUT,
+            ErrorKind.TEST_MEMORY_EXCEEDED,
+            ErrorKind.PATCH_APPLY_FAILED,
+        ):
+            assert kind in GENUINE_KINDS
+
+
+class TestLogRules:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            (
+                "fork/exec /usr/bin/conmon: resource temporarily unavailable",
+                ErrorKind.CONTAINER_FORK_EAGAIN,
+            ),
+            ("Test timed out after 1800s", ErrorKind.TEST_TIMEOUT),
+            (
+                "can only create exec sessions on running containers",
+                ErrorKind.CONTAINER_EXEC_REFUSED,
+            ),
+            ("container state improper", ErrorKind.CONTAINER_EXEC_REFUSED),
+            ("Read timed out. (read timeout=60)", ErrorKind.RUNTIME_READ_TIMEOUT),
+            (
+                "Reversed (or previously applied) patch detected",
+                ErrorKind.PATCH_APPLY_FAILED,
+            ),
+            ("1 out of 3 hunk FAILED", ErrorKind.PATCH_APPLY_FAILED),
+            ("nothing recognisable here", ErrorKind.UNKNOWN),
+        ],
+    )
+    def test_each_rule(self, text, expected):
+        assert classify_eval_log(text) is expected
+
+    def test_timeout_wins_over_teardown_noise(self):
+        # A timed-out evaluation also emits "container state improper" while the
+        # harness tears the container down. Reading that as a wedge would retry
+        # a genuine model outcome, so rule order is load-bearing.
+        text = "Test timed out after 1800s\ncontainer state improper\n"
+        assert classify_eval_log(text) is ErrorKind.TEST_TIMEOUT
+
+    def test_fork_failure_wins_over_teardown_noise(self):
+        text = (
+            "fork/exec /usr/bin/conmon: resource temporarily unavailable\n"
+            "can only create exec sessions on running containers\n"
+        )
+        assert classify_eval_log(text) is ErrorKind.CONTAINER_FORK_EAGAIN
+
+    def test_a_wedge_wins_over_patch_apply(self):
+        # A wedged container's verdict is unreliable either way, so the wedge
+        # decides and the unit is retried.
+        text = "container state improper\nhunk FAILED\n"
+        assert classify_eval_log(text) is ErrorKind.CONTAINER_EXEC_REFUSED
+
+    def test_build_error_is_checked_before_every_other_rule(self):
+        # BuildImageError's message embeds the same needles the other rules look
+        # for, so any other ordering misattributes a build failure.
+        assert (
+            classify_eval_log("BuildImageError: Read timed out. (read timeout=60)")
+            is ErrorKind.IMAGE_BUILD_TIMEOUT
+        )
+        assert (
+            classify_eval_log("BuildImageError: 500 Server Error")
+            is ErrorKind.IMAGE_BUILD_ERROR
+        )
+
+
+class TestClassifyUnit:
+    def test_infra_and_genuine_are_counted_separately(self, tmp_path):
+        write_log(tmp_path, "a-1", "container state improper")
+        write_log(tmp_path, "a-2", "Test timed out after 1800s")
+
+        classification = classify_unit(tmp_path, ["a-1", "a-2"])
+
+        assert classification.infra_count == 1
+        assert classification.genuine_count == 1
+        assert classification.should_retry
+
+    def test_only_genuine_errors_do_not_trigger_a_retry(self, tmp_path):
+        write_log(tmp_path, "a-1", "Test timed out after 1800s")
+        assert not classify_unit(tmp_path, ["a-1"]).should_retry
+
+    def test_a_missing_log_is_unknown_and_therefore_genuine(self, tmp_path):
+        classification = classify_unit(tmp_path, ["absent"])
+        assert classification.kinds == {ErrorKind.UNKNOWN: 1}
+        assert not classification.should_retry
+
+    def test_none_error_ids_means_not_measured(self, tmp_path):
+        # "We did not measure" and "we measured zero" are different; conflating
+        # them once let a damaged unit into a clean set.
+        classification = classify_unit(tmp_path, None)
+        assert classification.measured is False
+
+    def test_empty_error_ids_means_measured_zero(self, tmp_path):
+        classification = classify_unit(tmp_path, [])
+        assert classification.measured is True
+        assert classification.infra_count == 0
+
+    def test_eval_memory_kill_marker_is_genuine(self, tmp_path):
+        killed = tmp_path / "killed"
+        killed.mkdir()
+        (killed / "eval.a-1.host.999.json").write_text("{}")
+        write_log(tmp_path, "a-1", "container state improper")
+
+        classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed)
+
+        # The marker beats the log: a SIGKILLed test leaves an ambiguous log,
+        # but the kill is a fact recorded before acting.
+        assert classification.kinds == {ErrorKind.TEST_MEMORY_EXCEEDED: 1}
+        assert not classification.should_retry
+
+    def test_agent_phase_kill_marker_does_not_classify(self, tmp_path):
+        killed = tmp_path / "killed"
+        killed.mkdir()
+        (killed / "agent.a-1.host.999.json").write_text("{}")
+        write_log(tmp_path, "a-1", "Test timed out after 1800s")
+
+        # An agent kill only makes one tool call return an error observation;
+        # the instance still reaches a real outcome, so the marker is audit-only.
+        classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed)
+        assert classification.kinds == {ErrorKind.TEST_TIMEOUT: 1}
+
+    def test_step_infrastructure_failure_forces_a_retry(self, tmp_path):
+        classification = classify_unit(tmp_path, [], infrastructure_failure=True)
+        assert classification.should_retry
+        assert classification.kinds == {ErrorKind.STEP_INFRASTRUCTURE_FAILURE: 1}
+
+    def test_a_changed_endpoint_forces_a_retry(self, tmp_path):
+        # An engine restarted under a live client produces a plausible run that
+        # scores near zero and exits successfully.
+        classification = classify_unit(tmp_path, [], endpoint_changed=True)
+        assert classification.should_retry
+        assert classification.kinds == {ErrorKind.ENDPOINT_CHANGED: 1}

From f2c921145ce9798a83b0863809da35a0958380c7 Mon Sep 17 00:00:00 2001
From: Stanley Phoong <sphoong@nvidia.com>
Date: Wed, 26 Aug 2026 14:15:27 -0700
Subject: [PATCH 3/5] fix(swebench-service): a failed eval container must not
 discard the whole report

The same defect as the agent phase, one phase later.

`pyxis_worker` grades each prediction in its own container concurrently,
collected the per-instance failures, and then raised `RunnerError` *before*
`make_run_report()`. So one wedged evaluation container threw away every
other instance's grade -- the work was done, the reports were on disk, and
nothing was ever written.

The eval phase is now robust to individual instance failure. Whatever was
graded is reported: an instance with no `report.json` is counted as an error
by `make_run_report`, which is the correct and visible outcome, and is
exactly what an operator needs to see. A run in which *no* instance could be
evaluated still fails -- but only after the report has been written, so the
run can be diagnosed from its own artifacts rather than from nothing.

The losses are not hidden and, more importantly, not left as prose in a log:
`eval_infra_failures.txt` lists `instance_id<TAB>error` per lost instance, is
copied beside `swe_bench_results.json` and is served through the existing
artifact route. That distinction is load bearing. An instance the harness
dropped is not an instance the model failed, and a consumer that cannot
separate them reads attrition as an accuracy regression -- which is precisely
the misreport the completeness gate exists to prevent. Leaving the evidence
only in a human-readable log would leave the gate unable to see it.

Found while auditing which cluster-side runtime patches the package had made
redundant: this one had not been, and it would have compromised the very run
intended to validate the package.

Tests: `test_pyxis_worker_reports_the_instances_one_bad_container_did_not_kill`
(the report is produced, only the failed instance is listed),
`test_pyxis_worker_still_fails_when_no_instance_could_be_evaluated` (no false
pass, and the report is written first),
`test_pyxis_worker_records_no_failure_file_for_a_clean_eval`,
`test_eval_infra_failures_are_published_beside_the_results` and
`test_eval_infra_failures_is_a_retrievable_artifact`. Four fail against the
parent commit. The existing propagation test still holds: its single instance
is also every instance.
---
 .../evaluation/swe_bench_scorer.py            |   1 +
 .../evaluation/swebench_service/README.md     |  27 ++-
 .../swebench_service/artifacts.py             |   1 +
 .../swebench_service/pyxis_worker.py          |  47 ++++-
 .../swebench_service/runner.py                |   9 +
 .../swebench_service/test_runner.py           | 173 ++++++++++++++++++
 6 files changed, 244 insertions(+), 14 deletions(-)

diff --git a/src/inference_endpoint/evaluation/swe_bench_scorer.py b/src/inference_endpoint/evaluation/swe_bench_scorer.py
index 98a76d2a1..6cb1399ef 100644
--- a/src/inference_endpoint/evaluation/swe_bench_scorer.py
+++ b/src/inference_endpoint/evaluation/swe_bench_scorer.py
@@ -68,6 +68,7 @@ class SWEBenchScorer(Scorer, scorer_id="swe_bench_scorer"):
     }
     SAFE_ARTIFACT_NAMES: ClassVar[set[str]] = {
         "agent_phase_error.txt",
+        "eval_infra_failures.txt",
         "preds.json",
         "swe_bench_agent.log",
         "swe_bench_eval.log",
diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md
index f0751af0c..93d611b49 100644
--- a/src/inference_endpoint/evaluation/swebench_service/README.md
+++ b/src/inference_endpoint/evaluation/swebench_service/README.md
@@ -97,8 +97,9 @@ compute node. It mounts the patch, SWE-bench evaluation script, and output file
 the task image. It preserves SWE-bench 4.1.0's patch-application order, test timeout,
 captured output, and `get_eval_report` grading. A patch failure or test timeout is an
 unresolved task; an `srun`, Enroot, or container-start failure is an infrastructure
-error that fails the run. The service then aggregates the per-instance reports and
-removes its named Pyxis containers.
+loss, recorded per instance and reported rather than allowed to fail the whole run
+(see *Eval-phase failures*). The service then aggregates the per-instance reports
+and removes its named Pyxis containers.
 
 Pyxis namespaces a named container by its allocation: `--container-name=X` inside
 job `N` is the Enroot container `pyxis_N_X`. `PyxisEnvironment.cleanup()` removes
@@ -122,8 +123,26 @@ A run whose agent phase produced no predictions at all still fails, with the age
 error chained as the cause. Cancellation is never tolerated this way: it
 propagates immediately and no eval phase runs.
 
-Consumers that need to know a run was degraded should fetch `agent_phase_error.txt`;
-its presence means some instances may be missing from `swe_bench_results.json`.
+### Eval-phase failures
+
+The same rule applies one phase later. The eval phase grades each prediction in
+its own container, concurrently. If some of those containers fail, the rest are
+still graded and the run report is still produced: an instance with no
+`report.json` is counted as an error by the harness, which is the correct and
+visible outcome. The instances that were lost are listed in the
+`eval_infra_failures.txt` artifact, one `instance_id<TAB>error` per line.
+
+A run in which *no* instance could be evaluated still fails — but only after the
+report has been written, so the run can be diagnosed from its own artifacts.
+
+### Telling a degraded run apart from a bad one
+
+Both artifacts are machine-readable on purpose. `agent_phase_error.txt` means
+some instances may be missing from `swe_bench_results.json`; `eval_infra_failures.txt`
+names the instances the harness lost during grading. Instances lost this way are
+*infrastructure* losses, not model failures, and a consumer that cannot separate
+the two will read attrition as an accuracy regression. Neither file exists on a
+clean run.
 
 The benchmark client submits a run to this service only in `ACC` or `BOTH`
 mode; the default `PERF` mode skips external evaluation.
diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py
index d24d1ee8d..aa369c032 100644
--- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py
+++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py
@@ -43,6 +43,7 @@
 )
 SAFE_ARTIFACT_NAMES = {
     "agent_phase_error.txt",
+    "eval_infra_failures.txt",
     "preds.json",
     "swe_bench_agent.log",
     "swe_bench_eval.log",
diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py
index efd6a6bf3..10472e3b4 100644
--- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py
+++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py
@@ -15,7 +15,7 @@
 
 from .artifacts import atomic_write_bytes
 from .pyxis_environment import resolve_image, run_srun_step
-from .runner import RunnerError
+from .runner import EVAL_INFRA_FAILURES_FILE, RunnerError
 
 _PRINT_LOCK = threading.Lock()
 _INFRASTRUCTURE_FAILURE = ".pyxis_infrastructure_failure"
@@ -96,6 +96,18 @@ def get_pyxis_environment(config: dict, instance: dict):
         swebench.get_sb_environment = original_get_sb_environment
 
 
+def _record_eval_failures(output_dir: Path, failures: dict[str, str]) -> None:
+    lines = "".join(
+        f"{instance_id}\t{detail}\n" for instance_id, detail in sorted(failures.items())
+    )
+    try:
+        atomic_write_bytes(output_dir / EVAL_INFRA_FAILURES_FILE, lines.encode())
+    except OSError:
+        # Accounting must never be able to take the report down.
+        with _PRINT_LOCK:
+            print("could not record eval infrastructure failures", flush=True)
+
+
 def _evaluate_instance(
     *,
     test_spec: Any,
@@ -212,19 +224,26 @@ def _run_eval(args: argparse.Namespace) -> None:
             ].instance_id
             for payload in payloads
         }
-        failures = []
+        failures: dict[str, str] = {}
         for future in concurrent.futures.as_completed(futures):
+            instance_id = futures[future]
             try:
                 future.result()
-            except Exception as exc:
+            except Exception as exc:  # noqa: BLE001 -- one instance, not the run
                 with _PRINT_LOCK:
-                    print(f"Pyxis evaluation failed: {exc}", flush=True)
-                failures.append(futures[future])
-        if failures:
-            raise RunnerError(
-                "Pyxis infrastructure failure evaluating: "
-                + ", ".join(sorted(failures))
-            )
+                    print(
+                        f"Pyxis evaluation failed for {instance_id} "
+                        f"(non-fatal): {exc}",
+                        flush=True,
+                    )
+                failures[instance_id] = f"{type(exc).__name__}: {exc}"
+
+    # The report is produced even when some instances could not be evaluated.
+    # An instance with no report.json is counted as an error by make_run_report,
+    # which is the correct and visible outcome; raising here instead discarded
+    # every other instance's grade along with it.
+    if failures:
+        _record_eval_failures(args.output_dir, failures)
 
     output_dir = args.output_dir.resolve()
     with contextlib.chdir(output_dir):
@@ -235,6 +254,14 @@ def _run_eval(args: argparse.Namespace) -> None:
             client=None,
         )
 
+    if payloads and len(failures) == len(payloads):
+        # Nothing was evaluated at all. The report is on disk for diagnosis,
+        # but a report in which no instance ran is not a result.
+        raise RunnerError(
+            "Pyxis infrastructure failure evaluating every instance: "
+            + ", ".join(sorted(failures)[:10])
+        )
+
 
 def main(argv: list[str] | None = None) -> None:
     parser = argparse.ArgumentParser()
diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py
index e54167b2a..8dadb9a06 100644
--- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py
+++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py
@@ -86,6 +86,8 @@ def detach(self, process: subprocess.Popen[str]) -> None:
 _RUN_LABEL = "com.mlcommons.endpoints.swebench-run"
 #: Written when the agent phase failed but predictions were still scored.
 AGENT_PHASE_ERROR_FILE = "agent_phase_error.txt"
+#: Written when some instances could not be evaluated but the rest were scored.
+EVAL_INFRA_FAILURES_FILE = "eval_infra_failures.txt"
 _PROCESS_TERMINATE_TIMEOUT_S = 10
 _SWEBENCH_DATASETS = {
     "verified": "princeton-nlp/SWE-bench_Verified",
@@ -321,6 +323,13 @@ def _run(
         result_path = self._run_eval(
             request, preds_path, output_dir, run_dir, secret_values, cancel_token
         )
+        eval_failures = output_dir / EVAL_INFRA_FAILURES_FILE
+        if eval_failures.exists():
+            # Instances the harness lost during eval. Published beside the
+            # results so a consumer can tell them apart from instances the model
+            # genuinely failed; without it a degraded run is indistinguishable
+            # from a bad one.
+            shutil.copy2(eval_failures, run_dir / EVAL_INFRA_FAILURES_FILE)
         shutil.copy2(result_path, run_dir / "swe_bench_results.json")
         return msgspec.json.decode(result_path.read_bytes(), type=dict)
 
diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py
index 81a94473a..121c1f3a7 100644
--- a/tests/unit/evaluation/swebench_service/test_runner.py
+++ b/tests/unit/evaluation/swebench_service/test_runner.py
@@ -2176,3 +2176,176 @@ def test_pyxis_worker_propagates_evaluation_infrastructure_failure(
                 "repo__repo-1",
             ]
         )
+
+
+def _stub_swebench_eval(monkeypatch, predictions, *, make_run_report):
+    """Stand in for the SWE-bench harness modules the eval worker imports."""
+    swebench = types.ModuleType("swebench")
+    harness = types.ModuleType("swebench.harness")
+    reporting = types.ModuleType("swebench.harness.reporting")
+    reporting.make_run_report = make_run_report
+    test_spec = types.ModuleType("swebench.harness.test_spec")
+    test_spec_module = types.ModuleType("swebench.harness.test_spec.test_spec")
+    test_spec_module.make_test_spec = lambda row, arch: types.SimpleNamespace(
+        instance_id=row["instance_id"], eval_script="pytest -q"
+    )
+    utils = types.ModuleType("swebench.harness.utils")
+    utils.get_predictions_from_file = lambda *args: predictions.values()
+    utils.load_swebench_dataset = lambda *args: [
+        {"instance_id": instance_id} for instance_id in predictions
+    ]
+    for name, module in (
+        ("swebench", swebench),
+        ("swebench.harness", harness),
+        ("swebench.harness.reporting", reporting),
+        ("swebench.harness.test_spec", test_spec),
+        ("swebench.harness.test_spec.test_spec", test_spec_module),
+        ("swebench.harness.utils", utils),
+    ):
+        monkeypatch.setitem(sys.modules, name, module)
+
+
+def _eval_argv(output_dir: Path, instance_ids: list[str]) -> list[str]:
+    return [
+        "eval",
+        "--dataset-name",
+        "princeton-nlp/SWE-bench_Verified",
+        "--split",
+        "test",
+        "--predictions-path",
+        str(output_dir / "preds.json"),
+        "--max-workers",
+        "2",
+        "--run-id",
+        "endpoints_test",
+        "--image-registry",
+        _PYXIS_IMAGE_REGISTRY,
+        "--output-dir",
+        str(output_dir),
+        "--instance-ids",
+        *instance_ids,
+    ]
+
+
+def _predictions(*instance_ids: str) -> dict[str, dict[str, str]]:
+    return {
+        instance_id: {
+            "model_name_or_path": "test-model",
+            "instance_id": instance_id,
+            "model_patch": "diff --git a/a b/a",
+        }
+        for instance_id in instance_ids
+    }
+
+
+def test_pyxis_worker_reports_the_instances_one_bad_container_did_not_kill(
+    monkeypatch, tmp_path
+):
+    """One failed eval container must not discard every other instance's grade.
+
+    The per-instance failures were collected and then raised *before*
+    `make_run_report`, so a single wedged evaluation container threw away the
+    whole report -- the same defect as the agent phase, one phase later.
+    """
+    output_dir = tmp_path / "output"
+    output_dir.mkdir()
+    predictions = _predictions("repo__repo-1", "repo__repo-2", "repo__repo-3")
+    reported: list[tuple] = []
+
+    def fake_make_run_report(predictions, dataset, run_id, client):
+        reported.append((predictions, dataset, run_id, client))
+
+    _stub_swebench_eval(monkeypatch, predictions, make_run_report=fake_make_run_report)
+
+    def flaky(**kwargs):
+        if kwargs["test_spec"].instance_id == "repo__repo-2":
+            raise RunnerError("Pyxis infrastructure failure")
+
+    monkeypatch.setattr(worker_mod, "_evaluate_instance", flaky)
+
+    worker_mod.main(_eval_argv(output_dir, list(predictions)))
+
+    assert len(reported) == 1, "the report was never produced"
+    failures = (output_dir / "eval_infra_failures.txt").read_text()
+    assert "repo__repo-2" in failures
+    assert "RunnerError" in failures
+    assert "repo__repo-1" not in failures
+
+
+def test_pyxis_worker_still_fails_when_no_instance_could_be_evaluated(
+    monkeypatch, tmp_path
+):
+    """Tolerating losses must not turn a total loss into a pass."""
+    output_dir = tmp_path / "output"
+    output_dir.mkdir()
+    predictions = _predictions("repo__repo-1", "repo__repo-2")
+    reported: list[tuple] = []
+
+    _stub_swebench_eval(
+        monkeypatch,
+        predictions,
+        make_run_report=lambda *args, **kwargs: reported.append(args),
+    )
+    monkeypatch.setattr(
+        worker_mod,
+        "_evaluate_instance",
+        lambda **kwargs: (_ for _ in ()).throw(RunnerError("no space left")),
+    )
+
+    with pytest.raises(RunnerError, match="every instance"):
+        worker_mod.main(_eval_argv(output_dir, list(predictions)))
+
+    # The report is still written first, so the run can be diagnosed.
+    assert len(reported) == 1
+
+
+def test_pyxis_worker_records_no_failure_file_for_a_clean_eval(monkeypatch, tmp_path):
+    output_dir = tmp_path / "output"
+    output_dir.mkdir()
+    predictions = _predictions("repo__repo-1")
+
+    _stub_swebench_eval(
+        monkeypatch, predictions, make_run_report=lambda *args, **kwargs: None
+    )
+    monkeypatch.setattr(worker_mod, "_evaluate_instance", lambda **kwargs: None)
+
+    worker_mod.main(_eval_argv(output_dir, list(predictions)))
+
+    assert not (output_dir / "eval_infra_failures.txt").exists()
+
+
+def test_eval_infra_failures_are_published_beside_the_results(monkeypatch, tmp_path):
+    """The losses have to reach the caller, not just the service host's disk."""
+    runner = PyxisSweBenchRunner(
+        project_root=tmp_path,
+        subprocess_timeout_s=30,
+        image_registry=_PYXIS_IMAGE_REGISTRY,
+    )
+    run_dir = tmp_path / "run-1"
+
+    def fake_run_agent(
+        request, patched_config, output_dir, run_dir, secret_values, cancel_token=None
+    ):
+        (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}')
+
+    def fake_run_eval(
+        request, preds_path, output_dir, run_dir, secret_values, cancel_token=None
+    ):
+        (output_dir / "eval_infra_failures.txt").write_text(
+            "repo__repo-1\tRunnerError: no space left\n"
+        )
+        result_path = output_dir / "result.json"
+        result_path.write_text("{}")
+        return result_path
+
+    monkeypatch.setattr(runner, "_run_agent", fake_run_agent)
+    monkeypatch.setattr(runner, "_run_eval", fake_run_eval)
+    monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None)
+
+    runner.run(_request(["http://endpoint:30000"]), run_dir)
+
+    assert "no space left" in (run_dir / "eval_infra_failures.txt").read_text()
+
+
+def test_eval_infra_failures_is_a_retrievable_artifact():
+    assert "eval_infra_failures.txt" in artifacts_mod.SAFE_ARTIFACT_NAMES

From 4cec80ec2bc3b8ad5d5ad159afdaf55cb2dd8986 Mon Sep 17 00:00:00 2001
From: Stanley Phoong <sphoong@nvidia.com>
Date: Thu, 27 Aug 2026 08:49:42 -0700
Subject: [PATCH 4/5] style(swe-bench): normalize classifier test imports

---
 tests/unit/evaluation/swe_bench_distributed/test_classify.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/unit/evaluation/swe_bench_distributed/test_classify.py b/tests/unit/evaluation/swe_bench_distributed/test_classify.py
index 2b079b315..09bee8769 100644
--- a/tests/unit/evaluation/swe_bench_distributed/test_classify.py
+++ b/tests/unit/evaluation/swe_bench_distributed/test_classify.py
@@ -6,7 +6,6 @@
 from __future__ import annotations
 
 import pytest
-
 from inference_endpoint.evaluation.swe_bench_distributed.classify import (
     GENUINE_KINDS,
     INFRA_KINDS,

From 513ff4e79700732985b12b1a44c613e7818534e0 Mon Sep 17 00:00:00 2001
From: Stanley Phoong <sphoong@nvidia.com>
Date: Thu, 27 Aug 2026 08:57:25 -0700
Subject: [PATCH 5/5] docs(swe-bench): apply eval README formatting

---
 .../evaluation/swebench_service/README.md                   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md
index 93d611b49..fbdbdc557 100644
--- a/src/inference_endpoint/evaluation/swebench_service/README.md
+++ b/src/inference_endpoint/evaluation/swebench_service/README.md
@@ -98,7 +98,7 @@ the task image. It preserves SWE-bench 4.1.0's patch-application order, test tim
 captured output, and `get_eval_report` grading. A patch failure or test timeout is an
 unresolved task; an `srun`, Enroot, or container-start failure is an infrastructure
 loss, recorded per instance and reported rather than allowed to fail the whole run
-(see *Eval-phase failures*). The service then aggregates the per-instance reports
+(see _Eval-phase failures_). The service then aggregates the per-instance reports
 and removes its named Pyxis containers.
 
 Pyxis namespaces a named container by its allocation: `--container-name=X` inside
@@ -132,7 +132,7 @@ still graded and the run report is still produced: an instance with no
 visible outcome. The instances that were lost are listed in the
 `eval_infra_failures.txt` artifact, one `instance_id<TAB>error` per line.
 
-A run in which *no* instance could be evaluated still fails — but only after the
+A run in which _no_ instance could be evaluated still fails — but only after the
 report has been written, so the run can be diagnosed from its own artifacts.
 
 ### Telling a degraded run apart from a bad one
@@ -140,7 +140,7 @@ report has been written, so the run can be diagnosed from its own artifacts.
 Both artifacts are machine-readable on purpose. `agent_phase_error.txt` means
 some instances may be missing from `swe_bench_results.json`; `eval_infra_failures.txt`
 names the instances the harness lost during grading. Instances lost this way are
-*infrastructure* losses, not model failures, and a consumer that cannot separate
+_infrastructure_ losses, not model failures, and a consumer that cannot separate
 the two will read attrition as an accuracy regression. Neither file exists on a
 clean run.