diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 7269ab520..8e2ae103e 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,6 +11,14 @@ exactly once. """ +from .classify import ( + GENUINE_KINDS, + INFRA_KINDS, + ErrorKind, + UnitClassification, + classify_eval_log, + classify_unit, +) from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .infra_retry import ( InfraRetryLedger, @@ -38,8 +46,11 @@ from .units import Unit, UnitPlan, plan_units __all__ = [ + "GENUINE_KINDS", + "INFRA_KINDS", "ClaimError", "CompletenessReport", + "ErrorKind", "HealthTerm", "HealthVerdict", "InfraRetryLedger", @@ -53,11 +64,14 @@ "RunQuality", "SlurmStepLiveness", "Unit", + "UnitClassification", "UnitOutcome", "UnitPlan", "UnitResult", "WorkQueue", "assess_run", + "classify_eval_log", + "classify_unit", "combine_terms", "is_provable_non_execution", "kill_by_pid", diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py new file mode 100644 index 000000000..b7c06ac25 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Split a unit's error instances into infrastructure damage and genuine failures. + +A SWE-bench run reports three per-instance outcomes: resolved, unresolved, and +*error*. The agent phase has an infrastructure sentinel (the Pyxis +``infrastructure_failure_path``), but the eval phase has none: an instance whose +evaluation container wedged is booked as ``error``, which counts as "accounted +for", so the unit is published successful and is never retried. Those instances +silently poison a run that can then never reach a full result. + +Classification exists to catch exactly that. It reads each error instance's +``run_instance.log`` and assigns one kind. + +BIAS RULE -- this is the whole design and it is deliberately asymmetric. If an +error cannot be classified confidently it is treated as GENUINE, never as +infrastructure. A false bad-run costs one redo; a false retry silently biases +the measurement toward optimism, and an optimistic accuracy number is worse than +no number. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path + +logger = logging.getLogger(__name__) + + +class ErrorKind(StrEnum): + """One classification of an error instance.""" + + # Infrastructure: a defect in the runtime we provided, safe to retry. + CONTAINER_EXEC_REFUSED = "container_exec_refused" + CONTAINER_FORK_EAGAIN = "container_fork_eagain" + RUNTIME_READ_TIMEOUT = "runtime_read_timeout" + IMAGE_BUILD_TIMEOUT = "image_build_timeout" + IMAGE_BUILD_ERROR = "image_build_error" + STEP_INFRASTRUCTURE_FAILURE = "step_infrastructure_failure" + ENDPOINT_CHANGED = "endpoint_changed" + + # Genuine: a real outcome of the model's patch, or unreadable. Never retried. + TEST_TIMEOUT = "test_timeout" + TEST_MEMORY_EXCEEDED = "test_memory_exceeded" + PATCH_APPLY_FAILED = "patch_apply_failed" + UNKNOWN = "unknown" + + +#: Retryable. Every member is a defect in infrastructure we control. +INFRA_KINDS: frozenset[ErrorKind] = frozenset( + { + ErrorKind.CONTAINER_EXEC_REFUSED, + ErrorKind.CONTAINER_FORK_EAGAIN, + ErrorKind.RUNTIME_READ_TIMEOUT, + ErrorKind.IMAGE_BUILD_TIMEOUT, + ErrorKind.IMAGE_BUILD_ERROR, + ErrorKind.STEP_INFRASTRUCTURE_FAILURE, + ErrorKind.ENDPOINT_CHANGED, + } +) + +#: Never retried. +#: +#: ``TEST_TIMEOUT`` is a plausible model outcome: a patch that makes the suite +#: loop is a failing patch. ``TEST_MEMORY_EXCEEDED`` is its exact parallel -- a +#: patch that makes a graded test allocate without bound is a failing patch, and +#: the alternative to killing it was never "the test passes", it was "the host +#: OOMs and the instance still never completes". ``PATCH_APPLY_FAILED`` is the +#: model emitting a diff that does not apply; SWE-bench books it as ``error`` +#: rather than ``unresolved``, but it is model behaviour. ``UNKNOWN`` is the bias +#: rule. +GENUINE_KINDS: frozenset[ErrorKind] = frozenset( + { + ErrorKind.TEST_TIMEOUT, + ErrorKind.TEST_MEMORY_EXCEEDED, + ErrorKind.PATCH_APPLY_FAILED, + ErrorKind.UNKNOWN, + } +) + +# ORDERED. First match wins, and the order is load-bearing. +# +# CONTAINER_FORK_EAGAIN and TEST_TIMEOUT come BEFORE CONTAINER_EXEC_REFUSED: a +# timed-out or fork-failed evaluation frequently *also* emits "container state +# improper" while the harness tears the container down, and reading that as a +# wedge would retry a genuine model outcome. +# +# PATCH_APPLY_FAILED is checked LAST: if a container also wedged, the wedge +# wins, because a wedged container's verdict is unreliable either way. Only a +# log with no infrastructure signature at all reaches this rule. +_RULES: tuple[tuple[ErrorKind, tuple[str, ...]], ...] = ( + ( + ErrorKind.CONTAINER_FORK_EAGAIN, + ("fork/exec /usr/bin/conmon: resource temporarily unavailable",), + ), + (ErrorKind.TEST_TIMEOUT, ("Test timed out after",)), + ( + ErrorKind.CONTAINER_EXEC_REFUSED, + ( + "can only create exec sessions on running containers", + "container state improper", + ), + ), + (ErrorKind.RUNTIME_READ_TIMEOUT, ("Read timed out. (read timeout=",)), + ( + ErrorKind.PATCH_APPLY_FAILED, + ( + "Reversed (or previously applied) patch detected", + ">>>>> Patch Apply Failed", + "hunk FAILED", + "hunk failed", + ), + ), +) + + +def classify_eval_log(text: str) -> ErrorKind: + """Classify one instance's evaluation log. + + ``BuildImageError`` is checked before the ordered rules because its message + embeds the same "Read timed out" / "500" strings the other rules look for, + so any other order misattributes a build failure. + """ + if "BuildImageError" in text: + if "Read timed out" in text: + return ErrorKind.IMAGE_BUILD_TIMEOUT + return ErrorKind.IMAGE_BUILD_ERROR + for kind, needles in _RULES: + if any(needle in text for needle in needles): + return kind + return ErrorKind.UNKNOWN + + +@dataclass(slots=True) +class UnitClassification: + """Per-kind counts for one unit's error instances.""" + + kinds: dict[ErrorKind, int] = field(default_factory=dict) + error_instance_ids: tuple[str, ...] = () + #: False when the run's report could not be read at all. "Not measured" and + #: "measured zero" are different, and conflating them once let a damaged + #: unit into a clean set. + measured: bool = False + + @property + def infra_count(self) -> int: + return sum(count for kind, count in self.kinds.items() if kind in INFRA_KINDS) + + @property + def genuine_count(self) -> int: + return sum(count for kind, count in self.kinds.items() if kind in GENUINE_KINDS) + + @property + def should_retry(self) -> bool: + return self.infra_count > 0 + + def as_counts(self) -> dict[str, int]: + return {kind.value: count for kind, count in sorted(self.kinds.items())} + + +def _find_instance_log(output_dir: Path, instance_id: str) -> Path | None: + patterns = ( + f"logs/run_evaluation/*/*/{instance_id}/run_instance.log", + f"logs/run_evaluation/*/*/*/{instance_id}/run_instance.log", + ) + for pattern in patterns: + for match in sorted(output_dir.glob(pattern)): + return match + return None + + +def memory_kill_markers(killed_dir: Path, instance_id: str) -> bool: + """True only for an *eval*-phase memory kill. + + Phase is load-bearing and the two cases must never be collapsed. An eval + kill destroyed a graded result, so the instance's error is a genuine + failure. An agent kill merely makes one tool call return an error + observation and the agent carries on, so the instance still reaches a real + outcome; that marker exists for audit and must not influence classification. + + A marker beats any log heuristic: a SIGKILLed test leaves an ambiguous log, + but the kill itself is a fact recorded before acting. + """ + return any(killed_dir.glob(f"eval.{instance_id}.*.json")) + + +def classify_unit( + output_dir: Path, + error_instance_ids: list[str] | tuple[str, ...] | None, + *, + killed_dir: Path | None = None, + infrastructure_failure: bool = False, + endpoint_changed: bool = False, +) -> UnitClassification: + """Classify every error instance of one unit. + + ``infrastructure_failure`` carries the Pyxis agent-phase sentinel, and + ``endpoint_changed`` carries a mismatch between the inference endpoint + fingerprint recorded at claim time and at publish time -- an engine + restarted under a live client produces a plausible-looking run that must not + be scored. + """ + kinds: dict[ErrorKind, int] = {} + + if infrastructure_failure: + kinds[ErrorKind.STEP_INFRASTRUCTURE_FAILURE] = ( + kinds.get(ErrorKind.STEP_INFRASTRUCTURE_FAILURE, 0) + 1 + ) + if endpoint_changed: + kinds[ErrorKind.ENDPOINT_CHANGED] = kinds.get(ErrorKind.ENDPOINT_CHANGED, 0) + 1 + + if error_instance_ids is None: + return UnitClassification(kinds=kinds, error_instance_ids=(), measured=False) + + ids = tuple(str(x) for x in error_instance_ids) + for instance_id in ids: + if killed_dir is not None and memory_kill_markers(killed_dir, instance_id): + kinds[ErrorKind.TEST_MEMORY_EXCEEDED] = ( + kinds.get(ErrorKind.TEST_MEMORY_EXCEEDED, 0) + 1 + ) + continue + log_path = _find_instance_log(output_dir, instance_id) + kind = ErrorKind.UNKNOWN + if log_path is not None: + try: + kind = classify_eval_log(log_path.read_text(errors="replace")) + except OSError: + logger.debug("could not read %s", log_path, exc_info=True) + kinds[kind] = kinds.get(kind, 0) + 1 + + return UnitClassification(kinds=kinds, error_instance_ids=ids, measured=True) diff --git a/src/inference_endpoint/evaluation/swe_bench_scorer.py b/src/inference_endpoint/evaluation/swe_bench_scorer.py index a91d63734..6cb1399ef 100644 --- a/src/inference_endpoint/evaluation/swe_bench_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_scorer.py @@ -67,6 +67,8 @@ class SWEBenchScorer(Scorer, scorer_id="swe_bench_scorer"): "artifacts.download", } SAFE_ARTIFACT_NAMES: ClassVar[set[str]] = { + "agent_phase_error.txt", + "eval_infra_failures.txt", "preds.json", "swe_bench_agent.log", "swe_bench_eval.log", diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index 0e7a501c5..fbdbdc557 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -97,8 +97,9 @@ compute node. It mounts the patch, SWE-bench evaluation script, and output file the task image. It preserves SWE-bench 4.1.0's patch-application order, test timeout, captured output, and `get_eval_report` grading. A patch failure or test timeout is an unresolved task; an `srun`, Enroot, or container-start failure is an infrastructure -error that fails the run. The service then aggregates the per-instance reports and -removes its named Pyxis containers. +loss, recorded per instance and reported rather than allowed to fail the whole run +(see _Eval-phase failures_). The service then aggregates the per-instance reports +and removes its named Pyxis containers. Pyxis namespaces a named container by its allocation: `--container-name=X` inside job `N` is the Enroot container `pyxis_N_X`. `PyxisEnvironment.cleanup()` removes @@ -109,6 +110,40 @@ allocation. `scancel` does not reclaim them either -- ending the job does not remove Enroot containers -- which is why the removal has to be both correctly named and audible. +### Agent-phase failures + +The agent phase runs `--workers` trajectories concurrently. If it fails after +some workers have already written their patches, the service still evaluates the +predictions that reached `preds.json` rather than discarding them: the failure is +recorded in the `agent_phase_error.txt` artifact and logged at ERROR, and the run +continues into the eval phase. Instances with no prediction are simply absent from +the results, which the harness reports as unresolved. + +A run whose agent phase produced no predictions at all still fails, with the agent +error chained as the cause. Cancellation is never tolerated this way: it +propagates immediately and no eval phase runs. + +### Eval-phase failures + +The same rule applies one phase later. The eval phase grades each prediction in +its own container, concurrently. If some of those containers fail, the rest are +still graded and the run report is still produced: an instance with no +`report.json` is counted as an error by the harness, which is the correct and +visible outcome. The instances that were lost are listed in the +`eval_infra_failures.txt` artifact, one `instance_iderror` per line. + +A run in which _no_ instance could be evaluated still fails — but only after the +report has been written, so the run can be diagnosed from its own artifacts. + +### Telling a degraded run apart from a bad one + +Both artifacts are machine-readable on purpose. `agent_phase_error.txt` means +some instances may be missing from `swe_bench_results.json`; `eval_infra_failures.txt` +names the instances the harness lost during grading. Instances lost this way are +_infrastructure_ losses, not model failures, and a consumer that cannot separate +the two will read attrition as an accuracy regression. Neither file exists on a +clean run. + The benchmark client submits a run to this service only in `ACC` or `BOTH` mode; the default `PERF` mode skips external evaluation. diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py index fd207dc7c..aa369c032 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/artifacts.py @@ -42,6 +42,8 @@ re.compile(r"(://[^:/\s]+:)[^@\s/]+(@)"), ) SAFE_ARTIFACT_NAMES = { + "agent_phase_error.txt", + "eval_infra_failures.txt", "preds.json", "swe_bench_agent.log", "swe_bench_eval.log", diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py index efd6a6bf3..10472e3b4 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/pyxis_worker.py @@ -15,7 +15,7 @@ from .artifacts import atomic_write_bytes from .pyxis_environment import resolve_image, run_srun_step -from .runner import RunnerError +from .runner import EVAL_INFRA_FAILURES_FILE, RunnerError _PRINT_LOCK = threading.Lock() _INFRASTRUCTURE_FAILURE = ".pyxis_infrastructure_failure" @@ -96,6 +96,18 @@ def get_pyxis_environment(config: dict, instance: dict): swebench.get_sb_environment = original_get_sb_environment +def _record_eval_failures(output_dir: Path, failures: dict[str, str]) -> None: + lines = "".join( + f"{instance_id}\t{detail}\n" for instance_id, detail in sorted(failures.items()) + ) + try: + atomic_write_bytes(output_dir / EVAL_INFRA_FAILURES_FILE, lines.encode()) + except OSError: + # Accounting must never be able to take the report down. + with _PRINT_LOCK: + print("could not record eval infrastructure failures", flush=True) + + def _evaluate_instance( *, test_spec: Any, @@ -212,19 +224,26 @@ def _run_eval(args: argparse.Namespace) -> None: ].instance_id for payload in payloads } - failures = [] + failures: dict[str, str] = {} for future in concurrent.futures.as_completed(futures): + instance_id = futures[future] try: future.result() - except Exception as exc: + except Exception as exc: # noqa: BLE001 -- one instance, not the run with _PRINT_LOCK: - print(f"Pyxis evaluation failed: {exc}", flush=True) - failures.append(futures[future]) - if failures: - raise RunnerError( - "Pyxis infrastructure failure evaluating: " - + ", ".join(sorted(failures)) - ) + print( + f"Pyxis evaluation failed for {instance_id} " + f"(non-fatal): {exc}", + flush=True, + ) + failures[instance_id] = f"{type(exc).__name__}: {exc}" + + # The report is produced even when some instances could not be evaluated. + # An instance with no report.json is counted as an error by make_run_report, + # which is the correct and visible outcome; raising here instead discarded + # every other instance's grade along with it. + if failures: + _record_eval_failures(args.output_dir, failures) output_dir = args.output_dir.resolve() with contextlib.chdir(output_dir): @@ -235,6 +254,14 @@ def _run_eval(args: argparse.Namespace) -> None: client=None, ) + if payloads and len(failures) == len(payloads): + # Nothing was evaluated at all. The report is on disk for diagnosis, + # but a report in which no instance ran is not a result. + raise RunnerError( + "Pyxis infrastructure failure evaluating every instance: " + + ", ".join(sorted(failures)[:10]) + ) + def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser() diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 8682b40a7..8dadb9a06 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -84,6 +84,10 @@ def detach(self, process: subprocess.Popen[str]) -> None: _LOG_TAIL_MAX_BYTES = 64 * 1024 _LOG_TAIL_MAX_LINES = 50 _RUN_LABEL = "com.mlcommons.endpoints.swebench-run" +#: Written when the agent phase failed but predictions were still scored. +AGENT_PHASE_ERROR_FILE = "agent_phase_error.txt" +#: Written when some instances could not be evaluated but the rest were scored. +EVAL_INFRA_FAILURES_FILE = "eval_infra_failures.txt" _PROCESS_TERMINATE_TIMEOUT_S = 10 _SWEBENCH_DATASETS = { "verified": "princeton-nlp/SWE-bench_Verified", @@ -298,7 +302,7 @@ def _run( request, run_id=run_dir.name, ) - self._run_agent( + agent_error = self._run_agent_tolerantly( request, patched_config, output_dir, @@ -309,16 +313,76 @@ def _run( preds_path = output_dir / "preds.json" if not preds_path.exists(): - raise RunnerError("mini-extra did not produce preds.json") + # A genuinely empty agent phase still fails loudly: there is + # nothing to score. The agent error, if any, is the cause. + error = RunnerError("mini-extra did not produce preds.json") + raise error from agent_error self._validate_prediction_ids(request, preds_path) shutil.copy2(preds_path, run_dir / "preds.json") result_path = self._run_eval( request, preds_path, output_dir, run_dir, secret_values, cancel_token ) + eval_failures = output_dir / EVAL_INFRA_FAILURES_FILE + if eval_failures.exists(): + # Instances the harness lost during eval. Published beside the + # results so a consumer can tell them apart from instances the model + # genuinely failed; without it a degraded run is indistinguishable + # from a bad one. + shutil.copy2(eval_failures, run_dir / EVAL_INFRA_FAILURES_FILE) shutil.copy2(result_path, run_dir / "swe_bench_results.json") return msgspec.json.decode(result_path.read_bytes(), type=dict) + def _run_agent_tolerantly( + self, + request: RunRequest, + patched_config: Path, + output_dir: Path, + run_dir: Path, + secret_values: set[str], + cancel_token: CancellationToken | None, + ) -> BaseException | None: + """Run the agent phase; record a failure instead of discarding the run. + + The agent phase fans out across many concurrent workers, and a single + worker's infrastructure failure (a container that would not start, an + `srun` step that never launched) propagates out of the whole phase. The + predictions every *other* worker already wrote are on disk and are + perfectly scoreable, but the exception reached ``run()`` before + ``preds.json`` was ever looked at, so the entire eval phase was skipped + and a run with 137 of 200 predictions reported as a total loss. + + The failure is not hidden. It is written to ``agent_phase_error.txt``, + served as a run artifact, logged at ERROR, and chained onto the + ``preds.json`` failure when the phase really did produce nothing. + Cancellation is not a failure to tolerate: it propagates unchanged. + """ + try: + self._run_agent( + request, + patched_config, + output_dir, + run_dir, + secret_values, + cancel_token, + ) + except RunCancelled: + raise + except Exception as exc: + atomic_write_bytes( + run_dir / AGENT_PHASE_ERROR_FILE, + redact_text(f"{type(exc).__name__}: {exc}\n", secret_values).encode(), + ) + logger.error( + "SWE-bench agent phase failed for run %s; continuing to eval so " + "the predictions already on disk are still scored: %s", + run_dir.name, + exc, + exc_info=True, + ) + return exc + return None + def _load_template(self, request: RunRequest) -> dict[str, Any]: template_path = self._template_dir / TEMPLATE_FILES[request.template] with template_path.open() as f: diff --git a/tests/unit/evaluation/swe_bench_distributed/test_classify.py b/tests/unit/evaluation/swe_bench_distributed/test_classify.py new file mode 100644 index 000000000..09bee8769 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_classify.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Infrastructure-versus-genuine classification of error instances.""" + +from __future__ import annotations + +import pytest +from inference_endpoint.evaluation.swe_bench_distributed.classify import ( + GENUINE_KINDS, + INFRA_KINDS, + ErrorKind, + classify_eval_log, + classify_unit, +) + +pytestmark = pytest.mark.unit + + +def write_log(output_dir, instance_id: str, text: str) -> None: + log_dir = output_dir / "logs" / "run_evaluation" / "run-1" / "model" / instance_id + log_dir.mkdir(parents=True, exist_ok=True) + (log_dir / "run_instance.log").write_text(text) + + +class TestBiasRule: + def test_unknown_is_genuine_never_infra(self): + # A false bad-run costs one redo; a false retry biases the measurement + # toward optimism. Unclassifiable therefore means "keep the result". + assert ErrorKind.UNKNOWN in GENUINE_KINDS + assert ErrorKind.UNKNOWN not in INFRA_KINDS + + def test_kinds_are_partitioned(self): + assert not (INFRA_KINDS & GENUINE_KINDS) + assert INFRA_KINDS | GENUINE_KINDS == set(ErrorKind) + + def test_model_outcomes_are_genuine(self): + for kind in ( + ErrorKind.TEST_TIMEOUT, + ErrorKind.TEST_MEMORY_EXCEEDED, + ErrorKind.PATCH_APPLY_FAILED, + ): + assert kind in GENUINE_KINDS + + +class TestLogRules: + @pytest.mark.parametrize( + ("text", "expected"), + [ + ( + "fork/exec /usr/bin/conmon: resource temporarily unavailable", + ErrorKind.CONTAINER_FORK_EAGAIN, + ), + ("Test timed out after 1800s", ErrorKind.TEST_TIMEOUT), + ( + "can only create exec sessions on running containers", + ErrorKind.CONTAINER_EXEC_REFUSED, + ), + ("container state improper", ErrorKind.CONTAINER_EXEC_REFUSED), + ("Read timed out. (read timeout=60)", ErrorKind.RUNTIME_READ_TIMEOUT), + ( + "Reversed (or previously applied) patch detected", + ErrorKind.PATCH_APPLY_FAILED, + ), + ("1 out of 3 hunk FAILED", ErrorKind.PATCH_APPLY_FAILED), + ("nothing recognisable here", ErrorKind.UNKNOWN), + ], + ) + def test_each_rule(self, text, expected): + assert classify_eval_log(text) is expected + + def test_timeout_wins_over_teardown_noise(self): + # A timed-out evaluation also emits "container state improper" while the + # harness tears the container down. Reading that as a wedge would retry + # a genuine model outcome, so rule order is load-bearing. + text = "Test timed out after 1800s\ncontainer state improper\n" + assert classify_eval_log(text) is ErrorKind.TEST_TIMEOUT + + def test_fork_failure_wins_over_teardown_noise(self): + text = ( + "fork/exec /usr/bin/conmon: resource temporarily unavailable\n" + "can only create exec sessions on running containers\n" + ) + assert classify_eval_log(text) is ErrorKind.CONTAINER_FORK_EAGAIN + + def test_a_wedge_wins_over_patch_apply(self): + # A wedged container's verdict is unreliable either way, so the wedge + # decides and the unit is retried. + text = "container state improper\nhunk FAILED\n" + assert classify_eval_log(text) is ErrorKind.CONTAINER_EXEC_REFUSED + + def test_build_error_is_checked_before_every_other_rule(self): + # BuildImageError's message embeds the same needles the other rules look + # for, so any other ordering misattributes a build failure. + assert ( + classify_eval_log("BuildImageError: Read timed out. (read timeout=60)") + is ErrorKind.IMAGE_BUILD_TIMEOUT + ) + assert ( + classify_eval_log("BuildImageError: 500 Server Error") + is ErrorKind.IMAGE_BUILD_ERROR + ) + + +class TestClassifyUnit: + def test_infra_and_genuine_are_counted_separately(self, tmp_path): + write_log(tmp_path, "a-1", "container state improper") + write_log(tmp_path, "a-2", "Test timed out after 1800s") + + classification = classify_unit(tmp_path, ["a-1", "a-2"]) + + assert classification.infra_count == 1 + assert classification.genuine_count == 1 + assert classification.should_retry + + def test_only_genuine_errors_do_not_trigger_a_retry(self, tmp_path): + write_log(tmp_path, "a-1", "Test timed out after 1800s") + assert not classify_unit(tmp_path, ["a-1"]).should_retry + + def test_a_missing_log_is_unknown_and_therefore_genuine(self, tmp_path): + classification = classify_unit(tmp_path, ["absent"]) + assert classification.kinds == {ErrorKind.UNKNOWN: 1} + assert not classification.should_retry + + def test_none_error_ids_means_not_measured(self, tmp_path): + # "We did not measure" and "we measured zero" are different; conflating + # them once let a damaged unit into a clean set. + classification = classify_unit(tmp_path, None) + assert classification.measured is False + + def test_empty_error_ids_means_measured_zero(self, tmp_path): + classification = classify_unit(tmp_path, []) + assert classification.measured is True + assert classification.infra_count == 0 + + def test_eval_memory_kill_marker_is_genuine(self, tmp_path): + killed = tmp_path / "killed" + killed.mkdir() + (killed / "eval.a-1.host.999.json").write_text("{}") + write_log(tmp_path, "a-1", "container state improper") + + classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed) + + # The marker beats the log: a SIGKILLed test leaves an ambiguous log, + # but the kill is a fact recorded before acting. + assert classification.kinds == {ErrorKind.TEST_MEMORY_EXCEEDED: 1} + assert not classification.should_retry + + def test_agent_phase_kill_marker_does_not_classify(self, tmp_path): + killed = tmp_path / "killed" + killed.mkdir() + (killed / "agent.a-1.host.999.json").write_text("{}") + write_log(tmp_path, "a-1", "Test timed out after 1800s") + + # An agent kill only makes one tool call return an error observation; + # the instance still reaches a real outcome, so the marker is audit-only. + classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed) + assert classification.kinds == {ErrorKind.TEST_TIMEOUT: 1} + + def test_step_infrastructure_failure_forces_a_retry(self, tmp_path): + classification = classify_unit(tmp_path, [], infrastructure_failure=True) + assert classification.should_retry + assert classification.kinds == {ErrorKind.STEP_INFRASTRUCTURE_FAILURE: 1} + + def test_a_changed_endpoint_forces_a_retry(self, tmp_path): + # An engine restarted under a live client produces a plausible run that + # scores near zero and exits successfully. + classification = classify_unit(tmp_path, [], endpoint_changed=True) + assert classification.should_retry + assert classification.kinds == {ErrorKind.ENDPOINT_CHANGED: 1} diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 7d3f8cac4..121c1f3a7 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -15,6 +15,9 @@ import msgspec.json import pytest import yaml +from inference_endpoint.evaluation.swebench_service.swebench_service import ( + artifacts as artifacts_mod, +) from inference_endpoint.evaluation.swebench_service.swebench_service import ( pyxis_worker as worker_mod, ) @@ -434,15 +437,18 @@ def test_run_cleans_labeled_containers_after_success(monkeypatch, tmp_path): @pytest.mark.parametrize( - ("error", "match"), + ("error", "raised", "match"), [ - (RuntimeError("agent failed"), "agent failed"), - (RunnerError("subprocess timed out"), "timed out"), - (RunCancelled("subprocess cancelled"), "cancelled"), + # A non-cancellation agent failure that leaves no prediction behind + # surfaces as the empty-predictions failure, with the agent error + # chained as its cause. + (RuntimeError("agent failed"), RunnerError, "did not produce preds.json"), + (RunnerError("subprocess timed out"), RunnerError, "did not produce preds"), + (RunCancelled("subprocess cancelled"), RunCancelled, "cancelled"), ], ) def test_run_cleans_labeled_containers_after_failure( - monkeypatch, tmp_path, error, match + monkeypatch, tmp_path, error, raised, match ): runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) cleaned: list[str] = [] @@ -453,12 +459,136 @@ def fail_agent(*args, **kwargs): monkeypatch.setattr(runner, "_run_agent", fail_agent) monkeypatch.setattr(runner, "_cleanup_containers", cleaned.append) - with pytest.raises(type(error), match=match): + with pytest.raises(raised, match=match) as exc_info: runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-2") + if raised is not RunCancelled: + assert exc_info.value.__cause__ is error assert cleaned == ["run-2"] +def test_run_scores_predictions_left_behind_by_a_failed_agent_phase( + monkeypatch, tmp_path +): + """One worker's infrastructure failure must not discard the eval phase. + + The agent phase fans out across many workers. When one of them dies the + exception propagates out of the whole phase, but every prediction the other + workers wrote is already on disk. Before this fix a run with predictions for + most of its instances was reported as a total loss and never scored at all. + """ + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + run_dir = tmp_path / "run-partial" + scored: list[Path] = [] + + def partial_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + raise RunnerError("Pyxis infrastructure failure before the command completed") + + def fake_run_eval( + request, preds_path, output_dir, run_dir, secret_values, cancel_token=None + ): + scored.append(preds_path) + result_path = output_dir / "result.json" + result_path.write_text('{"resolved_instances":1,"submitted_instances":1}') + return result_path + + monkeypatch.setattr(runner, "_run_agent", partial_agent) + monkeypatch.setattr(runner, "_run_eval", fake_run_eval) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + result = runner.run(_request(["http://endpoint:30000"]), run_dir) + + assert result == {"resolved_instances": 1, "submitted_instances": 1} + assert scored, "eval phase never ran" + error_text = (run_dir / "agent_phase_error.txt").read_text() + assert "RunnerError" in error_text + assert "Pyxis infrastructure failure" in error_text + + +def test_agent_phase_error_is_a_retrievable_artifact(): + assert "agent_phase_error.txt" in artifacts_mod.SAFE_ARTIFACT_NAMES + + +def test_run_redacts_secrets_from_the_agent_phase_error(monkeypatch, tmp_path): + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + run_dir = tmp_path / "run-secret" + request = _request(["http://endpoint:30000"]) + request.endpoint_api_key = "real-secret" + + def leaky_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + raise RunnerError("connection to http://endpoint:30000 with real-secret failed") + + def fake_run_eval( + request, preds_path, output_dir, run_dir, secret_values, cancel_token=None + ): + result_path = output_dir / "result.json" + result_path.write_text("{}") + return result_path + + monkeypatch.setattr(runner, "_run_agent", leaky_agent) + monkeypatch.setattr(runner, "_run_eval", fake_run_eval) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + runner.run(request, run_dir) + + error_text = (run_dir / "agent_phase_error.txt").read_text() + assert "real-secret" not in error_text + assert "" in error_text + + +def test_run_still_fails_when_the_agent_phase_produced_nothing(monkeypatch, tmp_path): + """Tolerating the failure must not turn an empty run into a pass.""" + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + cause = RunnerError("every worker died") + + def dead_agent(*args, **kwargs): + raise cause + + monkeypatch.setattr(runner, "_run_agent", dead_agent) + monkeypatch.setattr( + runner, + "_run_eval", + lambda *a, **k: pytest.fail("eval must not run without predictions"), + ) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + with pytest.raises(RunnerError, match="did not produce preds.json") as exc_info: + runner.run(_request(["http://endpoint:30000"]), tmp_path / "run-empty") + + assert exc_info.value.__cause__ is cause + + +def test_run_does_not_tolerate_cancellation(monkeypatch, tmp_path): + """Cancellation is not a worker failure and must propagate unchanged.""" + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + run_dir = tmp_path / "run-cancelled" + + def cancelled_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + raise RunCancelled("subprocess cancelled") + + monkeypatch.setattr(runner, "_run_agent", cancelled_agent) + monkeypatch.setattr( + runner, + "_run_eval", + lambda *a, **k: pytest.fail("eval must not run after cancellation"), + ) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + with pytest.raises(RunCancelled): + runner.run(_request(["http://endpoint:30000"]), run_dir) + + assert not (run_dir / "agent_phase_error.txt").exists() + + def test_run_cleans_harness_containers_after_eval_cancellation(monkeypatch, tmp_path): runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) cleaned: list[tuple[str, dict]] = [] @@ -2046,3 +2176,176 @@ def test_pyxis_worker_propagates_evaluation_infrastructure_failure( "repo__repo-1", ] ) + + +def _stub_swebench_eval(monkeypatch, predictions, *, make_run_report): + """Stand in for the SWE-bench harness modules the eval worker imports.""" + swebench = types.ModuleType("swebench") + harness = types.ModuleType("swebench.harness") + reporting = types.ModuleType("swebench.harness.reporting") + reporting.make_run_report = make_run_report + test_spec = types.ModuleType("swebench.harness.test_spec") + test_spec_module = types.ModuleType("swebench.harness.test_spec.test_spec") + test_spec_module.make_test_spec = lambda row, arch: types.SimpleNamespace( + instance_id=row["instance_id"], eval_script="pytest -q" + ) + utils = types.ModuleType("swebench.harness.utils") + utils.get_predictions_from_file = lambda *args: predictions.values() + utils.load_swebench_dataset = lambda *args: [ + {"instance_id": instance_id} for instance_id in predictions + ] + for name, module in ( + ("swebench", swebench), + ("swebench.harness", harness), + ("swebench.harness.reporting", reporting), + ("swebench.harness.test_spec", test_spec), + ("swebench.harness.test_spec.test_spec", test_spec_module), + ("swebench.harness.utils", utils), + ): + monkeypatch.setitem(sys.modules, name, module) + + +def _eval_argv(output_dir: Path, instance_ids: list[str]) -> list[str]: + return [ + "eval", + "--dataset-name", + "princeton-nlp/SWE-bench_Verified", + "--split", + "test", + "--predictions-path", + str(output_dir / "preds.json"), + "--max-workers", + "2", + "--run-id", + "endpoints_test", + "--image-registry", + _PYXIS_IMAGE_REGISTRY, + "--output-dir", + str(output_dir), + "--instance-ids", + *instance_ids, + ] + + +def _predictions(*instance_ids: str) -> dict[str, dict[str, str]]: + return { + instance_id: { + "model_name_or_path": "test-model", + "instance_id": instance_id, + "model_patch": "diff --git a/a b/a", + } + for instance_id in instance_ids + } + + +def test_pyxis_worker_reports_the_instances_one_bad_container_did_not_kill( + monkeypatch, tmp_path +): + """One failed eval container must not discard every other instance's grade. + + The per-instance failures were collected and then raised *before* + `make_run_report`, so a single wedged evaluation container threw away the + whole report -- the same defect as the agent phase, one phase later. + """ + output_dir = tmp_path / "output" + output_dir.mkdir() + predictions = _predictions("repo__repo-1", "repo__repo-2", "repo__repo-3") + reported: list[tuple] = [] + + def fake_make_run_report(predictions, dataset, run_id, client): + reported.append((predictions, dataset, run_id, client)) + + _stub_swebench_eval(monkeypatch, predictions, make_run_report=fake_make_run_report) + + def flaky(**kwargs): + if kwargs["test_spec"].instance_id == "repo__repo-2": + raise RunnerError("Pyxis infrastructure failure") + + monkeypatch.setattr(worker_mod, "_evaluate_instance", flaky) + + worker_mod.main(_eval_argv(output_dir, list(predictions))) + + assert len(reported) == 1, "the report was never produced" + failures = (output_dir / "eval_infra_failures.txt").read_text() + assert "repo__repo-2" in failures + assert "RunnerError" in failures + assert "repo__repo-1" not in failures + + +def test_pyxis_worker_still_fails_when_no_instance_could_be_evaluated( + monkeypatch, tmp_path +): + """Tolerating losses must not turn a total loss into a pass.""" + output_dir = tmp_path / "output" + output_dir.mkdir() + predictions = _predictions("repo__repo-1", "repo__repo-2") + reported: list[tuple] = [] + + _stub_swebench_eval( + monkeypatch, + predictions, + make_run_report=lambda *args, **kwargs: reported.append(args), + ) + monkeypatch.setattr( + worker_mod, + "_evaluate_instance", + lambda **kwargs: (_ for _ in ()).throw(RunnerError("no space left")), + ) + + with pytest.raises(RunnerError, match="every instance"): + worker_mod.main(_eval_argv(output_dir, list(predictions))) + + # The report is still written first, so the run can be diagnosed. + assert len(reported) == 1 + + +def test_pyxis_worker_records_no_failure_file_for_a_clean_eval(monkeypatch, tmp_path): + output_dir = tmp_path / "output" + output_dir.mkdir() + predictions = _predictions("repo__repo-1") + + _stub_swebench_eval( + monkeypatch, predictions, make_run_report=lambda *args, **kwargs: None + ) + monkeypatch.setattr(worker_mod, "_evaluate_instance", lambda **kwargs: None) + + worker_mod.main(_eval_argv(output_dir, list(predictions))) + + assert not (output_dir / "eval_infra_failures.txt").exists() + + +def test_eval_infra_failures_are_published_beside_the_results(monkeypatch, tmp_path): + """The losses have to reach the caller, not just the service host's disk.""" + runner = PyxisSweBenchRunner( + project_root=tmp_path, + subprocess_timeout_s=30, + image_registry=_PYXIS_IMAGE_REGISTRY, + ) + run_dir = tmp_path / "run-1" + + def fake_run_agent( + request, patched_config, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "preds.json").write_text('{"repo__repo-1":"patch"}') + + def fake_run_eval( + request, preds_path, output_dir, run_dir, secret_values, cancel_token=None + ): + (output_dir / "eval_infra_failures.txt").write_text( + "repo__repo-1\tRunnerError: no space left\n" + ) + result_path = output_dir / "result.json" + result_path.write_text("{}") + return result_path + + monkeypatch.setattr(runner, "_run_agent", fake_run_agent) + monkeypatch.setattr(runner, "_run_eval", fake_run_eval) + monkeypatch.setattr(runner, "_cleanup_containers", lambda *a, **k: None) + + runner.run(_request(["http://endpoint:30000"]), run_dir) + + assert "no space left" in (run_dir / "eval_infra_failures.txt").read_text() + + +def test_eval_infra_failures_is_a_retrievable_artifact(): + assert "eval_infra_failures.txt" in artifacts_mod.SAFE_ARTIFACT_NAMES