From 50d369bea58a0185fa819037218e43708e5d3cc7 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Wed, 26 Aug 2026 12:49:27 -0700 Subject: [PATCH 1/6] fix(swebench-service): always give the agent a credential placeholder A run against a remote engine with no endpoint credential makes zero progress and never ends. `SweBenchRunner._base_env()` auto-filled `OPENAI_API_KEY="EMPTY"` only when the endpoint hostname was `localhost`, `127.0.0.1` or `::1`. For any other host with `endpoint_api_key` unset it did the opposite: it *removed* the variable. litellm then refuses to build the request locally -- litellm.AuthenticationError: Missing credentials -- mini-swe-agent classifies that as transient and retries it every 60s, forever. Not one request reaches the engine, nothing is logged at ERROR, the agent processes stay alive, and the run neither progresses nor terminates. Observed on a 20-node run against a remote GB300 engine: 200 workers, 0 requests served, no failure surfaced. The hostname gate is the defect. An unauthenticated OpenAI-compatible server ignores the credential value whether it is reached over loopback or over the network, so the placeholder is correct in both cases and the distinction only ever suppressed it where it was needed most. Replace the pop with the placeholder. The security property that motivated the pop is kept and made explicit: an ambient `OPENAI_API_KEY` inherited from the service host is still never forwarded to the endpoint -- it is overwritten rather than deleted. Tests: `test_base_env_always_supplies_a_credential_placeholder` covers loopback and remote hosts; the existing `test_base_env_supplies_api_key_only_to_agent_subprocess` encoded the old behaviour and is corrected to assert the placeholder while still proving a configured key wins and an ambient key never leaks. --- .../evaluation/swebench_service/README.md | 14 +++++++++ .../swebench_service/runner.py | 22 ++++++++----- .../swebench_service/test_runner.py | 31 ++++++++++++++++++- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index fbdbdc557..1909358e7 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -18,6 +18,20 @@ The endpoint URL in the benchmark config must be reachable from the service host Service mode supports exactly one endpoint URL and follows the LiveCodeBench-style external-service convention for heavyweight evaluation work. +### Endpoint credentials + +`accuracy_config.extras.swebench_service_auth_token` authenticates the *client to +this service*. The credential the agent presents to the *model endpoint* is +separate and comes from the run's endpoint configuration. + +When no endpoint credential is configured, the agent subprocess is given +`OPENAI_API_KEY=EMPTY`, which is what an unauthenticated OpenAI-compatible server +expects. An `OPENAI_API_KEY` inherited from the service host's environment is never +forwarded to the endpoint; it is replaced by the placeholder. The variable is +always set, regardless of whether the endpoint is on loopback or on another host, +because the client library refuses to issue a request with no credential at all and +retries that refusal indefinitely. + ## Runtime workflow ### Common workflow diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 8dadb9a06..7d5991a3e 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -558,17 +558,23 @@ def _base_env(self, request: RunRequest) -> dict[str, str]: no_proxy_value = ",".join(sorted(no_proxy)) env["NO_PROXY"] = no_proxy_value env["no_proxy"] = no_proxy_value - endpoint_host = ( - urlparse(str(request.endpoint_urls[0])).hostname - if request.endpoint_urls - else None - ) if request.endpoint_api_key: env["OPENAI_API_KEY"] = request.endpoint_api_key - elif endpoint_host in {"localhost", "127.0.0.1", "::1"}: - env["OPENAI_API_KEY"] = "EMPTY" else: - env.pop("OPENAI_API_KEY", None) + # No key was configured for this run. An ambient OPENAI_API_KEY + # inherited from the service host must never reach the endpoint, so + # it is replaced -- but it must be replaced, not removed. litellm + # refuses to build a request without a credential and raises + # `Missing credentials` locally; mini-swe-agent treats that as a + # transient error and retries every 60s indefinitely. The result is + # a run in which zero requests ever reach the engine, nothing is + # logged as an error, and the run never terminates. + # + # The placeholder is not host-dependent. An unauthenticated engine + # ignores the value whether it is on loopback or on another node, + # and gating the placeholder on the hostname is what left every + # remote-engine run hanging. + env["OPENAI_API_KEY"] = "EMPTY" return env def _cleanup_containers( diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 121c1f3a7..45d838d18 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -273,7 +273,36 @@ def test_base_env_supplies_api_key_only_to_agent_subprocess(monkeypatch, tmp_pat monkeypatch.setenv("OPENAI_API_KEY", "ambient-secret") unauthenticated = _request(["http://endpoint:30000"]) - assert "OPENAI_API_KEY" not in runner._base_env(unauthenticated) + assert runner._base_env(unauthenticated)["OPENAI_API_KEY"] == "EMPTY" + assert runner._base_env(authenticated)["OPENAI_API_KEY"] == "real-secret" + + +@pytest.mark.parametrize( + "endpoint", + [ + "http://localhost:30000", + "http://127.0.0.1:30000", + "http://[::1]:30000", + "http://swebench-host:30000", + "https://engine.example.com:8443", + ], +) +def test_base_env_always_supplies_a_credential_placeholder( + monkeypatch, tmp_path, endpoint +): + """A keyless run must never leave OPENAI_API_KEY unset. + + litellm raises ``Missing credentials`` before issuing anything and the agent + retries it forever, so a remote endpoint with no key made zero progress + while the run looked healthy. The placeholder must not depend on whether the + endpoint happens to be loopback. + """ + monkeypatch.setenv("OPENAI_API_KEY", "ambient-secret") + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + + env = runner._base_env(_request([endpoint])) + + assert env["OPENAI_API_KEY"] == "EMPTY" def test_run_agent_filters_exact_instance_ids(monkeypatch, tmp_path): From e853eb342fb073f3c3762541227e143cf0ebce8f Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:41:31 -0700 Subject: [PATCH 2/6] feat(swe-bench): pre-dispatch gates that must prove their own scale run_gates() calls assert_scale() before check() and treats GateScaleError as a gate FAILURE, never a skip. This is the code-level form of the most expensive lesson available: a tool-call gate that exercised the right operation at a 278-token prompt passed, while prompts over 2k tokens silently returned empty, and the run scored 0/80. - CheckpointIdentityGate probes /get_model_info then /v1/models and compares the served model path with == , never startswith or in: the bf16 path is a strict prefix of the fp8 path, so any substring test passes an FP8 engine as bf16. Unidentifiable or ambiguous endpoints fail closed. - ToolCallGate requires a well-formed bash tool call at a prompt of at least min_prompt_tokens measured with the server's own /tokenize, not estimated from characters. No tokenizer means the gate cannot prove its scale, so it fails. - EndpointFingerprintGate records a per-endpoint identity the dispatcher re-checks at publish time, so an engine restarted under a live client cannot yield a 0%-accuracy run that still exits rc=0. --- .../swe_bench_distributed/__init__.py | 22 +- .../evaluation/swe_bench_distributed/gates.py | 423 ++++++++++++++++++ .../swe_bench_distributed/test_gates.py | 224 ++++++++++ 3 files changed, 667 insertions(+), 2 deletions(-) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/gates.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_gates.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 8e2ae103e..58310bf12 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -19,6 +19,16 @@ classify_eval_log, classify_unit, ) +from .gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + Gate, + GateFailure, + GateReport, + GateScaleError, + ToolCallGate, + run_gates, +) from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .infra_retry import ( InfraRetryLedger, @@ -46,13 +56,19 @@ from .units import Unit, UnitPlan, plan_units __all__ = [ - "GENUINE_KINDS", - "INFRA_KINDS", + "CheckpointIdentityGate", "ClaimError", "CompletenessReport", + "EndpointFingerprintGate", "ErrorKind", + "GENUINE_KINDS", + "Gate", + "GateFailure", + "GateReport", + "GateScaleError", "HealthTerm", "HealthVerdict", + "INFRA_KINDS", "InfraRetryLedger", "LocalProcessLiveness", "MemoryGuard", @@ -63,6 +79,7 @@ "RetryRecord", "RunQuality", "SlurmStepLiveness", + "ToolCallGate", "Unit", "UnitClassification", "UnitOutcome", @@ -79,5 +96,6 @@ "plan_units", "reap", "retry_on_provable_non_execution", + "run_gates", "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py new file mode 100644 index 000000000..9330fe012 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -0,0 +1,423 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates on the inference endpoints. + +A gate proves, before a single instance is dispatched, that the endpoints under +test can actually do the thing the benchmark requires. Gates fail closed: an +endpoint that cannot be identified or reached is a failure, never a pass. + +THE SCALE RULE. Every gate must first prove it is testing at the scale it +claims, via :meth:`Gate.assert_scale`, and a scale failure is a *gate failure*, +not a skip. This is not defensive programming; it is the most expensive lesson +in this codebase's history. A tool-call gate that exercised exactly the right +operation with a 278-token prompt passed cleanly while every prompt above 2000 +tokens silently returned an empty completion -- and SWE-bench prompts are all +far larger than 2000 tokens. The gate was green and the run scored zero. A gate +that cannot prove its scale is not a gate. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Protocol +from urllib import error as urllib_error +from urllib import request as urllib_request + +logger = logging.getLogger(__name__) + +_DEFAULT_TIMEOUT_S = 60.0 +#: SWE-bench prompts are far larger than this; the threshold is a floor, not a +#: target. +DEFAULT_MIN_PROMPT_TOKENS = 2000 + + +class GateFailure(RuntimeError): + """A gate refused to let the run start.""" + + +class GateScaleError(GateFailure): + """A gate could not prove it was testing at the scale it claims.""" + + +@dataclass(slots=True) +class GateReport: + name: str + passed: bool + checked: int = 0 + failures: list[tuple[str, str]] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + def summary(self) -> str: + head = ( + f"{self.name}: {'pass' if self.passed else 'FAIL'} ({self.checked} checked)" + ) + detail = "".join( + f"\n {target} -> {reason}" for target, reason in self.failures[:8] + ) + notes = "".join(f"\n note: {note}" for note in self.notes) + return head + detail + notes + + +class Gate(Protocol): + name: str + + def assert_scale(self, targets: list[str]) -> None: + """Prove this gate tests what it claims. Raise :class:`GateScaleError`.""" + ... + + def check(self, targets: list[str]) -> GateReport: ... + + +def _http_json( + url: str, + payload: dict[str, Any] | None = None, + *, + timeout_s: float = _DEFAULT_TIMEOUT_S, + api_key: str | None = None, +) -> dict[str, Any]: + data = None + headers = {"Accept": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + if payload is not None: + data = json.dumps(payload).encode() + headers["Content-Type"] = "application/json" + request = urllib_request.Request(url, data=data, headers=headers) + with urllib_request.urlopen(request, timeout=timeout_s) as response: + return json.loads(response.read()) + + +def run_gates(gates: list[Gate], targets: list[str]) -> list[GateReport]: + """Run every gate; raise :class:`GateFailure` if any refused. + + Every gate runs even after one fails, so one preflight reports every problem + rather than sending the operator round the loop once per endpoint. + """ + reports: list[GateReport] = [] + for gate in gates: + try: + gate.assert_scale(targets) + except GateScaleError as exc: + reports.append( + GateReport( + name=gate.name, + passed=False, + failures=[("", str(exc))], + notes=[ + "a gate that cannot prove its scale is a failing gate, " + "not a skipped one" + ], + ) + ) + continue + reports.append(gate.check(targets)) + + failed = [report for report in reports if not report.passed] + if failed: + raise GateFailure( + "pre-dispatch gate(s) refused:\n" + + "\n".join(report.summary() for report in failed) + ) + return reports + + +class CheckpointIdentityGate: + """Every endpoint must serve exactly the expected checkpoint. + + Two traps, both of which produced silently contaminated results: + + 1. ``/v1/models`` echoes ``--served-model-name``, which operators routinely + set identically for two different checkpoints (e.g. an FP8 and a BF16 + build of the same model). ``/get_model_info`` reports the real model + path, so it is tried first. + 2. Checkpoint names nest: ``Org/Model`` is a strict prefix of + ``Org/Model-FP8``. Any ``startswith``/``in`` test therefore accepts an + FP8 endpoint as BF16. Comparison is ``==`` and nothing else. + """ + + name = "checkpoint_identity" + + def __init__( + self, + expected_model: str, + *, + timeout_s: float = 10.0, + api_key: str | None = None, + ) -> None: + if not expected_model: + raise ValueError("expected_model is required") + self.expected_model = expected_model + self.timeout_s = timeout_s + self.api_key = api_key + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to identify") + + def probe(self, url: str) -> tuple[str | None, str]: + base = url.rstrip("/") + try: + info = _http_json( + f"{base}/get_model_info", + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + model_path = info.get("model_path") + if model_path: + return str(model_path), "get_model_info" + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + pass # not an SGLang endpoint; fall through to the OpenAI route + try: + listing = _http_json( + f"{base}/v1/models", timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + return None, f"unreachable: {type(exc).__name__}" + ids = [ + entry.get("id") for entry in listing.get("data") or [] if entry.get("id") + ] + if len(ids) == 1: + return str(ids[0]), "v1/models" + if len(ids) > 1: + return None, f"ambiguous /v1/models: {ids!r}" + return None, "no model id from either endpoint" + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + sources: set[str] = set() + for url in targets: + identity, source = self.probe(url) + if identity is None: + report.failures.append((url, source)) + elif identity != self.expected_model: # EXACT; never startswith/in + report.failures.append( + (url, f"serves {identity!r}, expected {self.expected_model!r}") + ) + else: + sources.add(source) + if "v1/models" in sources: + report.notes.append( + "identity came from /v1/models, which echoes --served-model-name; " + "that string can be identical across checkpoints, so it cannot " + "separate two builds that share a served name" + ) + report.passed = not report.failures + report.data["expected_model"] = self.expected_model + return report + + +class ToolCallGate: + """Every endpoint must return a well-formed tool call at SWE-bench scale. + + The prompt is measured with the *server's own* tokenizer (``/tokenize``), + never estimated from character count, and a prompt that measures below + ``min_prompt_tokens`` fails the scale assertion rather than passing the + gate. + """ + + name = "tool_call" + + def __init__( + self, + model: str, + *, + min_prompt_tokens: int = DEFAULT_MIN_PROMPT_TOKENS, + prompt: str | None = None, + timeout_s: float = 180.0, + api_key: str | None = None, + tool_name: str = "bash", + ) -> None: + self.model = model + self.min_prompt_tokens = min_prompt_tokens + self.prompt = prompt if prompt is not None else build_scale_prompt() + self.timeout_s = timeout_s + self.api_key = api_key + self.tool_name = tool_name + self._measured: dict[str, int] = {} + + @property + def tools(self) -> list[dict[str, Any]]: + return [ + { + "type": "function", + "function": { + "name": self.tool_name, + "description": "Run a shell command", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + }, + }, + } + ] + + def count_tokens(self, url: str) -> int | None: + try: + response = _http_json( + f"{url.rstrip('/')}/tokenize", + {"model": self.model, "prompt": self.prompt}, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + return None + count = response.get("count") + if count is None: + tokens = response.get("tokens") + count = len(tokens) if isinstance(tokens, list) else None + try: + return int(count) if count is not None else None + except (TypeError, ValueError): + return None + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to gate") + measured = False + for url in targets: + count = self.count_tokens(url) + if count is None: + continue + self._measured[url] = count + measured = True + if count < self.min_prompt_tokens: + raise GateScaleError( + f"{url}: gate prompt measures {count} tokens, below the " + f"{self.min_prompt_tokens}-token floor this gate claims to " + "test. A tool-call gate that passes at a small prompt says " + "nothing about SWE-bench-sized prompts." + ) + if not measured and self.min_prompt_tokens > 0: + raise GateScaleError( + "no endpoint exposed /tokenize, so the gate cannot prove the " + f"prompt reaches {self.min_prompt_tokens} tokens. Serve a " + "tokenizer endpoint or set min_prompt_tokens=0 to accept an " + "unverified prompt size." + ) + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + tokens = self._measured.get(url) + try: + response = _http_json( + f"{url.rstrip('/')}/v1/chat/completions", + { + "model": self.model, + "messages": [{"role": "user", "content": self.prompt}], + "tools": self.tools, + "tool_choice": "auto", + "max_tokens": 256, + "temperature": 0.0, + }, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + report.failures.append((url, f"{type(exc).__name__}: {exc}")) + continue + failure = self._validate(response) + if failure is not None: + report.failures.append((url, f"tokens={tokens}: {failure}")) + report.passed = not report.failures + report.data["measured_tokens"] = dict(self._measured) + return report + + def _validate(self, response: dict[str, Any]) -> str | None: + try: + message = response["choices"][0]["message"] + except (KeyError, IndexError, TypeError): + return "malformed chat completion response" + tool_calls = message.get("tool_calls") + if not tool_calls: + content = (message.get("content") or "")[:120] + return f"no tool_calls; content={content!r}" + function = tool_calls[0].get("function") or {} + if function.get("name") != self.tool_name: + return f"wrong tool {function.get('name')!r}" + try: + arguments = json.loads(function.get("arguments") or "") + except (TypeError, ValueError): + return f"arguments are not valid JSON: {function.get('arguments')!r}" + command = arguments.get("command") + if not isinstance(command, str) or not command.strip(): + return f"malformed arguments {function.get('arguments')!r}" + return None + + +def build_scale_prompt(repetitions: int = 120) -> str: + """A prompt long enough to exercise the large-context path.""" + filler = "\n".join( + f"def helper_{index}(path, flags=None):\n" + f" # legacy shim retained for compatibility with the v{index} api\n" + " result = compute_checksum(path, flags or DEFAULT_FLAGS)\n" + " return normalise(result), path, flags\n" + for index in range(repetitions) + ) + return ( + "You are working in a Python repository checked out at /testbed.\n" + "Below is the current content of /testbed/legacy/helpers.py.\n\n" + "\n" + filler + "\n\n" + "Before proposing any change you must inspect the repository.\n" + "List the files in /testbed using the shell tool. Call the tool; do not " + "answer in prose." + ) + + +class EndpointFingerprintGate: + """Record a per-endpoint fingerprint for later comparison. + + An engine restarted under a live client yields a run that scores near zero + and still exits successfully -- nothing in the result distinguishes it from + a genuinely bad model. The dispatcher therefore records each endpoint's + fingerprint when a unit is claimed and re-reads it when the unit is + published; a change means the unit was scored against something other than + what it was dispatched to, and the unit is requeued rather than counted. + """ + + name = "endpoint_fingerprint" + + def __init__(self, *, timeout_s: float = 10.0, api_key: str | None = None) -> None: + self.timeout_s = timeout_s + self.api_key = api_key + self.fingerprints: dict[str, str] = {} + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to fingerprint") + + def fingerprint(self, url: str) -> str | None: + base = url.rstrip("/") + parts: list[str] = [] + for path in ("/get_model_info", "/v1/models"): + try: + payload = _http_json( + base + path, timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + continue + parts.append(json.dumps(payload, sort_keys=True, default=str)) + if not parts: + return None + import hashlib + + return hashlib.sha256("|".join(parts).encode()).hexdigest()[:16] + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + value = self.fingerprint(url) + if value is None: + report.failures.append( + (url, "could not read an identity to fingerprint") + ) + continue + self.fingerprints[url] = value + report.passed = not report.failures + report.data["fingerprints"] = dict(self.fingerprints) + return report diff --git a/tests/unit/evaluation/swe_bench_distributed/test_gates.py b/tests/unit/evaluation/swe_bench_distributed/test_gates.py new file mode 100644 index 000000000..607b04d3f --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_gates.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates, including the scale rule.""" + +from __future__ import annotations + +import json + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed import gates as gates_mod +from inference_endpoint.evaluation.swe_bench_distributed.gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + GateFailure, + GateScaleError, + ToolCallGate, + build_scale_prompt, + run_gates, +) + +pytestmark = pytest.mark.unit + +ENDPOINT = "http://engine-1:8000" + + +def install_http(monkeypatch, routes): + """Route ``_http_json`` by URL suffix; a missing route raises like a network error.""" + + def fake(url, payload=None, *, timeout_s=60.0, api_key=None): + for suffix, response in routes.items(): + if url.endswith(suffix): + if isinstance(response, Exception): + raise response + if callable(response): + return response(payload) + return response + raise OSError(f"no route for {url}") + + monkeypatch.setattr(gates_mod, "_http_json", fake) + + +def tool_call_response(command="ls /testbed", name="bash", arguments=None): + return { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": name, + "arguments": ( + arguments + if arguments is not None + else json.dumps({"command": command}) + ), + } + } + ] + } + } + ] + } + + +class TestCheckpointIdentity: + def test_exact_match_passes(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]) + assert report.passed + + def test_a_prefix_is_not_a_match(self, monkeypatch): + # "Org/Model" is a strict prefix of "Org/Model-FP8", so any + # startswith/in test would accept an FP8 engine as the BF16 build. + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert not report.passed + assert "Org/Model-FP8" in report.failures[0][1] + + def test_get_model_info_is_preferred_over_v1_models(self, monkeypatch): + # /v1/models echoes --served-model-name, which operators routinely set + # identically for two different checkpoints. + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Org/Model-FP8"}, + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + assert CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]).passed + + def test_v1_models_fallback_warns_about_its_own_ambiguity(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert report.passed + assert any("served-model-name" in note for note in report.notes) + + def test_an_unreachable_endpoint_fails_closed(self, monkeypatch): + install_http( + monkeypatch, + {"/get_model_info": OSError("down"), "/v1/models": OSError("down")}, + ) + assert not CheckpointIdentityGate("Org/Model").check([ENDPOINT]).passed + + def test_ambiguous_model_listing_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "a"}, {"id": "b"}]}, + }, + ) + report = CheckpointIdentityGate("a").check([ENDPOINT]) + assert not report.passed + assert "ambiguous" in report.failures[0][1] + + +class TestToolCallScale: + def test_a_small_prompt_fails_the_scale_assertion(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 278}}) + gate = ToolCallGate("Org/Model", prompt="tiny", min_prompt_tokens=2000) + # A gate exercising the right operation at a 278-token prompt passed + # while every prompt above 2000 tokens silently returned nothing. + with pytest.raises(GateScaleError, match="278 tokens"): + gate.assert_scale([ENDPOINT]) + + def test_a_scale_failure_is_a_gate_failure_not_a_skip(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 100}}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateFailure, match="failing gate"): + run_gates([gate], [ENDPOINT]) + + def test_no_tokenizer_means_the_gate_cannot_prove_its_scale(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateScaleError, match="cannot prove"): + gate.assert_scale([ENDPOINT]) + + def test_scale_can_be_waived_explicitly(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + ToolCallGate("Org/Model", min_prompt_tokens=0).assert_scale([ENDPOINT]) + + def test_the_default_prompt_is_large(self): + assert len(build_scale_prompt()) > 20_000 + + +class TestToolCallCheck: + def _gate(self, monkeypatch, chat_response): + install_http( + monkeypatch, + {"/tokenize": {"count": 4096}, "/v1/chat/completions": chat_response}, + ) + gate = ToolCallGate("Org/Model") + gate.assert_scale([ENDPOINT]) + return gate + + def test_a_well_formed_call_passes(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response()) + report = gate.check([ENDPOINT]) + assert report.passed + assert report.data["measured_tokens"][ENDPOINT] == 4096 + + def test_an_empty_completion_fails(self, monkeypatch): + gate = self._gate(monkeypatch, {"choices": [{"message": {"content": ""}}]}) + report = gate.check([ENDPOINT]) + assert not report.passed + assert "no tool_calls" in report.failures[0][1] + + def test_the_wrong_tool_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(name="python")) + assert not gate.check([ENDPOINT]).passed + + def test_unparseable_arguments_fail(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(arguments="{not json")) + report = gate.check([ENDPOINT]) + assert "not valid JSON" in report.failures[0][1] + + def test_an_empty_command_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(command=" ")) + assert not gate.check([ENDPOINT]).passed + + +class TestFingerprint: + def test_the_fingerprint_changes_with_the_served_model(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "A"}}) + first = EndpointFingerprintGate().fingerprint(ENDPOINT) + install_http(monkeypatch, {"/get_model_info": {"model_path": "B"}}) + second = EndpointFingerprintGate().fingerprint(ENDPOINT) + assert first is not None and first != second + + def test_an_unidentifiable_endpoint_fails(self, monkeypatch): + install_http(monkeypatch, {}) + assert not EndpointFingerprintGate().check([ENDPOINT]).passed + + +class TestRunGates: + def test_every_gate_runs_even_after_one_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Wrong/Model"}, + "/tokenize": {"count": 4096}, + "/v1/chat/completions": {"choices": [{"message": {}}]}, + }, + ) + with pytest.raises(GateFailure) as excinfo: + run_gates( + [CheckpointIdentityGate("Org/Model"), ToolCallGate("Org/Model")], + [ENDPOINT], + ) + message = str(excinfo.value) + assert "checkpoint_identity" in message + assert "tool_call" in message + + def test_no_targets_is_a_failure_not_a_pass(self, monkeypatch): + with pytest.raises(GateFailure): + run_gates([CheckpointIdentityGate("Org/Model")], []) From bf2a58beafeaaa1a9675861f5c1cc36248b1e469 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:04 -0700 Subject: [PATCH 3/6] fix(swe-bench): fingerprint endpoint identity, not the time of asking EndpointFingerprintGate hashed the whole /v1/models payload. vLLM stamps that response with a request-time `created` field and mints a fresh `permission[].id` on every call, so two reads of one healthy, untouched engine produce two different fingerprints -- four calls, four values. The dispatcher records a fingerprint when a unit is claimed and re-reads it when the unit is published, and treats any difference as `endpoint_changed`: an infrastructure fault, which requeues the unit. With an unstable fingerprint that comparison is always true, so every unit is retried until it exhausts max_attempts, is published as abandoned, and the merge gate refuses the run. The failure costs the full agent and evaluation time of every attempt first, and reports itself as infrastructure damage rather than as a bug here. Hash only the identity-bearing fields by dropping the per-request ones. The gate still fails closed on an endpoint whose identity cannot be read at all, which is the property it exists to provide. --- .../evaluation/swe_bench_distributed/gates.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py index 9330fe012..302ce3d3e 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -369,6 +369,30 @@ def build_scale_prompt(repetitions: int = 120) -> str: ) +#: Response fields that change on every request and carry no checkpoint +#: identity. vLLM's ``/v1/models`` stamps ``created`` with the request time and +#: mints a fresh ``permission[].id`` per call, so hashing the raw payload makes +#: the fingerprint differ between any two reads of a perfectly healthy engine. +#: The dispatcher compares the claim-time and publish-time fingerprints and +#: treats a difference as ``endpoint_changed`` -- an infrastructure fault -- so +#: an unstable fingerprint retries and then abandons every unit, and the merge +#: gate can never produce a number. +_VOLATILE_IDENTITY_KEYS = frozenset({"created", "created_at", "permission"}) + + +def _strip_volatile(value: Any) -> Any: + """Drop per-request fields so a fingerprint reflects identity, not time.""" + if isinstance(value, dict): + return { + key: _strip_volatile(item) + for key, item in value.items() + if key not in _VOLATILE_IDENTITY_KEYS + } + if isinstance(value, list): + return [_strip_volatile(item) for item in value] + return value + + class EndpointFingerprintGate: """Record a per-endpoint fingerprint for later comparison. @@ -401,7 +425,9 @@ def fingerprint(self, url: str) -> str | None: ) except (urllib_error.URLError, OSError, ValueError, TimeoutError): continue - parts.append(json.dumps(payload, sort_keys=True, default=str)) + parts.append( + json.dumps(_strip_volatile(payload), sort_keys=True, default=str) + ) if not parts: return None import hashlib From 8db87bdba58fa1acca9b2dc298b519f5c9e0c46c Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Thu, 27 Aug 2026 08:29:31 -0700 Subject: [PATCH 4/6] fix(swe-bench): make gate protocol stubs explicit --- .../evaluation/swe_bench_distributed/gates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py index 302ce3d3e..bd9c3a7d3 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -67,9 +67,10 @@ class Gate(Protocol): def assert_scale(self, targets: list[str]) -> None: """Prove this gate tests what it claims. Raise :class:`GateScaleError`.""" - ... + pass - def check(self, targets: list[str]) -> GateReport: ... + def check(self, targets: list[str]) -> GateReport: + pass def _http_json( From d2c3d2ab992c8b99862d0aedf9be5cb9065052ef Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Thu, 27 Aug 2026 08:50:09 -0700 Subject: [PATCH 5/6] style(swe-bench): normalize gate test imports --- tests/unit/evaluation/swe_bench_distributed/test_gates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/evaluation/swe_bench_distributed/test_gates.py b/tests/unit/evaluation/swe_bench_distributed/test_gates.py index 607b04d3f..578c5036e 100644 --- a/tests/unit/evaluation/swe_bench_distributed/test_gates.py +++ b/tests/unit/evaluation/swe_bench_distributed/test_gates.py @@ -8,7 +8,6 @@ import json import pytest - from inference_endpoint.evaluation.swe_bench_distributed import gates as gates_mod from inference_endpoint.evaluation.swe_bench_distributed.gates import ( CheckpointIdentityGate, From ad7777c5a7dfd3f27a54a84bc5dd560ae446d300 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Thu, 27 Aug 2026 08:57:48 -0700 Subject: [PATCH 6/6] docs(swe-bench): format credential guidance --- src/inference_endpoint/evaluation/swebench_service/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index 1909358e7..59f0bfb18 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -20,8 +20,8 @@ external-service convention for heavyweight evaluation work. ### Endpoint credentials -`accuracy_config.extras.swebench_service_auth_token` authenticates the *client to -this service*. The credential the agent presents to the *model endpoint* is +`accuracy_config.extras.swebench_service_auth_token` authenticates the _client to +this service_. The credential the agent presents to the _model endpoint_ is separate and comes from the run's endpoint configuration. When no endpoint credential is configured, the agent subprocess is given