diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 8e2ae103e..58310bf12 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -19,6 +19,16 @@ classify_eval_log, classify_unit, ) +from .gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + Gate, + GateFailure, + GateReport, + GateScaleError, + ToolCallGate, + run_gates, +) from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .infra_retry import ( InfraRetryLedger, @@ -46,13 +56,19 @@ from .units import Unit, UnitPlan, plan_units __all__ = [ - "GENUINE_KINDS", - "INFRA_KINDS", + "CheckpointIdentityGate", "ClaimError", "CompletenessReport", + "EndpointFingerprintGate", "ErrorKind", + "GENUINE_KINDS", + "Gate", + "GateFailure", + "GateReport", + "GateScaleError", "HealthTerm", "HealthVerdict", + "INFRA_KINDS", "InfraRetryLedger", "LocalProcessLiveness", "MemoryGuard", @@ -63,6 +79,7 @@ "RetryRecord", "RunQuality", "SlurmStepLiveness", + "ToolCallGate", "Unit", "UnitClassification", "UnitOutcome", @@ -79,5 +96,6 @@ "plan_units", "reap", "retry_on_provable_non_execution", + "run_gates", "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py new file mode 100644 index 000000000..bd9c3a7d3 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -0,0 +1,450 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates on the inference endpoints. + +A gate proves, before a single instance is dispatched, that the endpoints under +test can actually do the thing the benchmark requires. Gates fail closed: an +endpoint that cannot be identified or reached is a failure, never a pass. + +THE SCALE RULE. Every gate must first prove it is testing at the scale it +claims, via :meth:`Gate.assert_scale`, and a scale failure is a *gate failure*, +not a skip. This is not defensive programming; it is the most expensive lesson +in this codebase's history. A tool-call gate that exercised exactly the right +operation with a 278-token prompt passed cleanly while every prompt above 2000 +tokens silently returned an empty completion -- and SWE-bench prompts are all +far larger than 2000 tokens. The gate was green and the run scored zero. A gate +that cannot prove its scale is not a gate. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Protocol +from urllib import error as urllib_error +from urllib import request as urllib_request + +logger = logging.getLogger(__name__) + +_DEFAULT_TIMEOUT_S = 60.0 +#: SWE-bench prompts are far larger than this; the threshold is a floor, not a +#: target. +DEFAULT_MIN_PROMPT_TOKENS = 2000 + + +class GateFailure(RuntimeError): + """A gate refused to let the run start.""" + + +class GateScaleError(GateFailure): + """A gate could not prove it was testing at the scale it claims.""" + + +@dataclass(slots=True) +class GateReport: + name: str + passed: bool + checked: int = 0 + failures: list[tuple[str, str]] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + def summary(self) -> str: + head = ( + f"{self.name}: {'pass' if self.passed else 'FAIL'} ({self.checked} checked)" + ) + detail = "".join( + f"\n {target} -> {reason}" for target, reason in self.failures[:8] + ) + notes = "".join(f"\n note: {note}" for note in self.notes) + return head + detail + notes + + +class Gate(Protocol): + name: str + + def assert_scale(self, targets: list[str]) -> None: + """Prove this gate tests what it claims. Raise :class:`GateScaleError`.""" + pass + + def check(self, targets: list[str]) -> GateReport: + pass + + +def _http_json( + url: str, + payload: dict[str, Any] | None = None, + *, + timeout_s: float = _DEFAULT_TIMEOUT_S, + api_key: str | None = None, +) -> dict[str, Any]: + data = None + headers = {"Accept": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + if payload is not None: + data = json.dumps(payload).encode() + headers["Content-Type"] = "application/json" + request = urllib_request.Request(url, data=data, headers=headers) + with urllib_request.urlopen(request, timeout=timeout_s) as response: + return json.loads(response.read()) + + +def run_gates(gates: list[Gate], targets: list[str]) -> list[GateReport]: + """Run every gate; raise :class:`GateFailure` if any refused. + + Every gate runs even after one fails, so one preflight reports every problem + rather than sending the operator round the loop once per endpoint. + """ + reports: list[GateReport] = [] + for gate in gates: + try: + gate.assert_scale(targets) + except GateScaleError as exc: + reports.append( + GateReport( + name=gate.name, + passed=False, + failures=[("", str(exc))], + notes=[ + "a gate that cannot prove its scale is a failing gate, " + "not a skipped one" + ], + ) + ) + continue + reports.append(gate.check(targets)) + + failed = [report for report in reports if not report.passed] + if failed: + raise GateFailure( + "pre-dispatch gate(s) refused:\n" + + "\n".join(report.summary() for report in failed) + ) + return reports + + +class CheckpointIdentityGate: + """Every endpoint must serve exactly the expected checkpoint. + + Two traps, both of which produced silently contaminated results: + + 1. ``/v1/models`` echoes ``--served-model-name``, which operators routinely + set identically for two different checkpoints (e.g. an FP8 and a BF16 + build of the same model). ``/get_model_info`` reports the real model + path, so it is tried first. + 2. Checkpoint names nest: ``Org/Model`` is a strict prefix of + ``Org/Model-FP8``. Any ``startswith``/``in`` test therefore accepts an + FP8 endpoint as BF16. Comparison is ``==`` and nothing else. + """ + + name = "checkpoint_identity" + + def __init__( + self, + expected_model: str, + *, + timeout_s: float = 10.0, + api_key: str | None = None, + ) -> None: + if not expected_model: + raise ValueError("expected_model is required") + self.expected_model = expected_model + self.timeout_s = timeout_s + self.api_key = api_key + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to identify") + + def probe(self, url: str) -> tuple[str | None, str]: + base = url.rstrip("/") + try: + info = _http_json( + f"{base}/get_model_info", + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + model_path = info.get("model_path") + if model_path: + return str(model_path), "get_model_info" + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + pass # not an SGLang endpoint; fall through to the OpenAI route + try: + listing = _http_json( + f"{base}/v1/models", timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + return None, f"unreachable: {type(exc).__name__}" + ids = [ + entry.get("id") for entry in listing.get("data") or [] if entry.get("id") + ] + if len(ids) == 1: + return str(ids[0]), "v1/models" + if len(ids) > 1: + return None, f"ambiguous /v1/models: {ids!r}" + return None, "no model id from either endpoint" + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + sources: set[str] = set() + for url in targets: + identity, source = self.probe(url) + if identity is None: + report.failures.append((url, source)) + elif identity != self.expected_model: # EXACT; never startswith/in + report.failures.append( + (url, f"serves {identity!r}, expected {self.expected_model!r}") + ) + else: + sources.add(source) + if "v1/models" in sources: + report.notes.append( + "identity came from /v1/models, which echoes --served-model-name; " + "that string can be identical across checkpoints, so it cannot " + "separate two builds that share a served name" + ) + report.passed = not report.failures + report.data["expected_model"] = self.expected_model + return report + + +class ToolCallGate: + """Every endpoint must return a well-formed tool call at SWE-bench scale. + + The prompt is measured with the *server's own* tokenizer (``/tokenize``), + never estimated from character count, and a prompt that measures below + ``min_prompt_tokens`` fails the scale assertion rather than passing the + gate. + """ + + name = "tool_call" + + def __init__( + self, + model: str, + *, + min_prompt_tokens: int = DEFAULT_MIN_PROMPT_TOKENS, + prompt: str | None = None, + timeout_s: float = 180.0, + api_key: str | None = None, + tool_name: str = "bash", + ) -> None: + self.model = model + self.min_prompt_tokens = min_prompt_tokens + self.prompt = prompt if prompt is not None else build_scale_prompt() + self.timeout_s = timeout_s + self.api_key = api_key + self.tool_name = tool_name + self._measured: dict[str, int] = {} + + @property + def tools(self) -> list[dict[str, Any]]: + return [ + { + "type": "function", + "function": { + "name": self.tool_name, + "description": "Run a shell command", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + }, + }, + } + ] + + def count_tokens(self, url: str) -> int | None: + try: + response = _http_json( + f"{url.rstrip('/')}/tokenize", + {"model": self.model, "prompt": self.prompt}, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + return None + count = response.get("count") + if count is None: + tokens = response.get("tokens") + count = len(tokens) if isinstance(tokens, list) else None + try: + return int(count) if count is not None else None + except (TypeError, ValueError): + return None + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to gate") + measured = False + for url in targets: + count = self.count_tokens(url) + if count is None: + continue + self._measured[url] = count + measured = True + if count < self.min_prompt_tokens: + raise GateScaleError( + f"{url}: gate prompt measures {count} tokens, below the " + f"{self.min_prompt_tokens}-token floor this gate claims to " + "test. A tool-call gate that passes at a small prompt says " + "nothing about SWE-bench-sized prompts." + ) + if not measured and self.min_prompt_tokens > 0: + raise GateScaleError( + "no endpoint exposed /tokenize, so the gate cannot prove the " + f"prompt reaches {self.min_prompt_tokens} tokens. Serve a " + "tokenizer endpoint or set min_prompt_tokens=0 to accept an " + "unverified prompt size." + ) + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + tokens = self._measured.get(url) + try: + response = _http_json( + f"{url.rstrip('/')}/v1/chat/completions", + { + "model": self.model, + "messages": [{"role": "user", "content": self.prompt}], + "tools": self.tools, + "tool_choice": "auto", + "max_tokens": 256, + "temperature": 0.0, + }, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + report.failures.append((url, f"{type(exc).__name__}: {exc}")) + continue + failure = self._validate(response) + if failure is not None: + report.failures.append((url, f"tokens={tokens}: {failure}")) + report.passed = not report.failures + report.data["measured_tokens"] = dict(self._measured) + return report + + def _validate(self, response: dict[str, Any]) -> str | None: + try: + message = response["choices"][0]["message"] + except (KeyError, IndexError, TypeError): + return "malformed chat completion response" + tool_calls = message.get("tool_calls") + if not tool_calls: + content = (message.get("content") or "")[:120] + return f"no tool_calls; content={content!r}" + function = tool_calls[0].get("function") or {} + if function.get("name") != self.tool_name: + return f"wrong tool {function.get('name')!r}" + try: + arguments = json.loads(function.get("arguments") or "") + except (TypeError, ValueError): + return f"arguments are not valid JSON: {function.get('arguments')!r}" + command = arguments.get("command") + if not isinstance(command, str) or not command.strip(): + return f"malformed arguments {function.get('arguments')!r}" + return None + + +def build_scale_prompt(repetitions: int = 120) -> str: + """A prompt long enough to exercise the large-context path.""" + filler = "\n".join( + f"def helper_{index}(path, flags=None):\n" + f" # legacy shim retained for compatibility with the v{index} api\n" + " result = compute_checksum(path, flags or DEFAULT_FLAGS)\n" + " return normalise(result), path, flags\n" + for index in range(repetitions) + ) + return ( + "You are working in a Python repository checked out at /testbed.\n" + "Below is the current content of /testbed/legacy/helpers.py.\n\n" + "\n" + filler + "\n\n" + "Before proposing any change you must inspect the repository.\n" + "List the files in /testbed using the shell tool. Call the tool; do not " + "answer in prose." + ) + + +#: Response fields that change on every request and carry no checkpoint +#: identity. vLLM's ``/v1/models`` stamps ``created`` with the request time and +#: mints a fresh ``permission[].id`` per call, so hashing the raw payload makes +#: the fingerprint differ between any two reads of a perfectly healthy engine. +#: The dispatcher compares the claim-time and publish-time fingerprints and +#: treats a difference as ``endpoint_changed`` -- an infrastructure fault -- so +#: an unstable fingerprint retries and then abandons every unit, and the merge +#: gate can never produce a number. +_VOLATILE_IDENTITY_KEYS = frozenset({"created", "created_at", "permission"}) + + +def _strip_volatile(value: Any) -> Any: + """Drop per-request fields so a fingerprint reflects identity, not time.""" + if isinstance(value, dict): + return { + key: _strip_volatile(item) + for key, item in value.items() + if key not in _VOLATILE_IDENTITY_KEYS + } + if isinstance(value, list): + return [_strip_volatile(item) for item in value] + return value + + +class EndpointFingerprintGate: + """Record a per-endpoint fingerprint for later comparison. + + An engine restarted under a live client yields a run that scores near zero + and still exits successfully -- nothing in the result distinguishes it from + a genuinely bad model. The dispatcher therefore records each endpoint's + fingerprint when a unit is claimed and re-reads it when the unit is + published; a change means the unit was scored against something other than + what it was dispatched to, and the unit is requeued rather than counted. + """ + + name = "endpoint_fingerprint" + + def __init__(self, *, timeout_s: float = 10.0, api_key: str | None = None) -> None: + self.timeout_s = timeout_s + self.api_key = api_key + self.fingerprints: dict[str, str] = {} + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to fingerprint") + + def fingerprint(self, url: str) -> str | None: + base = url.rstrip("/") + parts: list[str] = [] + for path in ("/get_model_info", "/v1/models"): + try: + payload = _http_json( + base + path, timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + continue + parts.append( + json.dumps(_strip_volatile(payload), sort_keys=True, default=str) + ) + if not parts: + return None + import hashlib + + return hashlib.sha256("|".join(parts).encode()).hexdigest()[:16] + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + value = self.fingerprint(url) + if value is None: + report.failures.append( + (url, "could not read an identity to fingerprint") + ) + continue + self.fingerprints[url] = value + report.passed = not report.failures + report.data["fingerprints"] = dict(self.fingerprints) + return report diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index fbdbdc557..59f0bfb18 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -18,6 +18,20 @@ The endpoint URL in the benchmark config must be reachable from the service host Service mode supports exactly one endpoint URL and follows the LiveCodeBench-style external-service convention for heavyweight evaluation work. +### Endpoint credentials + +`accuracy_config.extras.swebench_service_auth_token` authenticates the _client to +this service_. The credential the agent presents to the _model endpoint_ is +separate and comes from the run's endpoint configuration. + +When no endpoint credential is configured, the agent subprocess is given +`OPENAI_API_KEY=EMPTY`, which is what an unauthenticated OpenAI-compatible server +expects. An `OPENAI_API_KEY` inherited from the service host's environment is never +forwarded to the endpoint; it is replaced by the placeholder. The variable is +always set, regardless of whether the endpoint is on loopback or on another host, +because the client library refuses to issue a request with no credential at all and +retries that refusal indefinitely. + ## Runtime workflow ### Common workflow diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 8dadb9a06..7d5991a3e 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -558,17 +558,23 @@ def _base_env(self, request: RunRequest) -> dict[str, str]: no_proxy_value = ",".join(sorted(no_proxy)) env["NO_PROXY"] = no_proxy_value env["no_proxy"] = no_proxy_value - endpoint_host = ( - urlparse(str(request.endpoint_urls[0])).hostname - if request.endpoint_urls - else None - ) if request.endpoint_api_key: env["OPENAI_API_KEY"] = request.endpoint_api_key - elif endpoint_host in {"localhost", "127.0.0.1", "::1"}: - env["OPENAI_API_KEY"] = "EMPTY" else: - env.pop("OPENAI_API_KEY", None) + # No key was configured for this run. An ambient OPENAI_API_KEY + # inherited from the service host must never reach the endpoint, so + # it is replaced -- but it must be replaced, not removed. litellm + # refuses to build a request without a credential and raises + # `Missing credentials` locally; mini-swe-agent treats that as a + # transient error and retries every 60s indefinitely. The result is + # a run in which zero requests ever reach the engine, nothing is + # logged as an error, and the run never terminates. + # + # The placeholder is not host-dependent. An unauthenticated engine + # ignores the value whether it is on loopback or on another node, + # and gating the placeholder on the hostname is what left every + # remote-engine run hanging. + env["OPENAI_API_KEY"] = "EMPTY" return env def _cleanup_containers( diff --git a/tests/unit/evaluation/swe_bench_distributed/test_gates.py b/tests/unit/evaluation/swe_bench_distributed/test_gates.py new file mode 100644 index 000000000..578c5036e --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_gates.py @@ -0,0 +1,223 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates, including the scale rule.""" + +from __future__ import annotations + +import json + +import pytest +from inference_endpoint.evaluation.swe_bench_distributed import gates as gates_mod +from inference_endpoint.evaluation.swe_bench_distributed.gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + GateFailure, + GateScaleError, + ToolCallGate, + build_scale_prompt, + run_gates, +) + +pytestmark = pytest.mark.unit + +ENDPOINT = "http://engine-1:8000" + + +def install_http(monkeypatch, routes): + """Route ``_http_json`` by URL suffix; a missing route raises like a network error.""" + + def fake(url, payload=None, *, timeout_s=60.0, api_key=None): + for suffix, response in routes.items(): + if url.endswith(suffix): + if isinstance(response, Exception): + raise response + if callable(response): + return response(payload) + return response + raise OSError(f"no route for {url}") + + monkeypatch.setattr(gates_mod, "_http_json", fake) + + +def tool_call_response(command="ls /testbed", name="bash", arguments=None): + return { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": name, + "arguments": ( + arguments + if arguments is not None + else json.dumps({"command": command}) + ), + } + } + ] + } + } + ] + } + + +class TestCheckpointIdentity: + def test_exact_match_passes(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]) + assert report.passed + + def test_a_prefix_is_not_a_match(self, monkeypatch): + # "Org/Model" is a strict prefix of "Org/Model-FP8", so any + # startswith/in test would accept an FP8 engine as the BF16 build. + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert not report.passed + assert "Org/Model-FP8" in report.failures[0][1] + + def test_get_model_info_is_preferred_over_v1_models(self, monkeypatch): + # /v1/models echoes --served-model-name, which operators routinely set + # identically for two different checkpoints. + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Org/Model-FP8"}, + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + assert CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]).passed + + def test_v1_models_fallback_warns_about_its_own_ambiguity(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert report.passed + assert any("served-model-name" in note for note in report.notes) + + def test_an_unreachable_endpoint_fails_closed(self, monkeypatch): + install_http( + monkeypatch, + {"/get_model_info": OSError("down"), "/v1/models": OSError("down")}, + ) + assert not CheckpointIdentityGate("Org/Model").check([ENDPOINT]).passed + + def test_ambiguous_model_listing_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "a"}, {"id": "b"}]}, + }, + ) + report = CheckpointIdentityGate("a").check([ENDPOINT]) + assert not report.passed + assert "ambiguous" in report.failures[0][1] + + +class TestToolCallScale: + def test_a_small_prompt_fails_the_scale_assertion(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 278}}) + gate = ToolCallGate("Org/Model", prompt="tiny", min_prompt_tokens=2000) + # A gate exercising the right operation at a 278-token prompt passed + # while every prompt above 2000 tokens silently returned nothing. + with pytest.raises(GateScaleError, match="278 tokens"): + gate.assert_scale([ENDPOINT]) + + def test_a_scale_failure_is_a_gate_failure_not_a_skip(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 100}}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateFailure, match="failing gate"): + run_gates([gate], [ENDPOINT]) + + def test_no_tokenizer_means_the_gate_cannot_prove_its_scale(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateScaleError, match="cannot prove"): + gate.assert_scale([ENDPOINT]) + + def test_scale_can_be_waived_explicitly(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + ToolCallGate("Org/Model", min_prompt_tokens=0).assert_scale([ENDPOINT]) + + def test_the_default_prompt_is_large(self): + assert len(build_scale_prompt()) > 20_000 + + +class TestToolCallCheck: + def _gate(self, monkeypatch, chat_response): + install_http( + monkeypatch, + {"/tokenize": {"count": 4096}, "/v1/chat/completions": chat_response}, + ) + gate = ToolCallGate("Org/Model") + gate.assert_scale([ENDPOINT]) + return gate + + def test_a_well_formed_call_passes(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response()) + report = gate.check([ENDPOINT]) + assert report.passed + assert report.data["measured_tokens"][ENDPOINT] == 4096 + + def test_an_empty_completion_fails(self, monkeypatch): + gate = self._gate(monkeypatch, {"choices": [{"message": {"content": ""}}]}) + report = gate.check([ENDPOINT]) + assert not report.passed + assert "no tool_calls" in report.failures[0][1] + + def test_the_wrong_tool_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(name="python")) + assert not gate.check([ENDPOINT]).passed + + def test_unparseable_arguments_fail(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(arguments="{not json")) + report = gate.check([ENDPOINT]) + assert "not valid JSON" in report.failures[0][1] + + def test_an_empty_command_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(command=" ")) + assert not gate.check([ENDPOINT]).passed + + +class TestFingerprint: + def test_the_fingerprint_changes_with_the_served_model(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "A"}}) + first = EndpointFingerprintGate().fingerprint(ENDPOINT) + install_http(monkeypatch, {"/get_model_info": {"model_path": "B"}}) + second = EndpointFingerprintGate().fingerprint(ENDPOINT) + assert first is not None and first != second + + def test_an_unidentifiable_endpoint_fails(self, monkeypatch): + install_http(monkeypatch, {}) + assert not EndpointFingerprintGate().check([ENDPOINT]).passed + + +class TestRunGates: + def test_every_gate_runs_even_after_one_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Wrong/Model"}, + "/tokenize": {"count": 4096}, + "/v1/chat/completions": {"choices": [{"message": {}}]}, + }, + ) + with pytest.raises(GateFailure) as excinfo: + run_gates( + [CheckpointIdentityGate("Org/Model"), ToolCallGate("Org/Model")], + [ENDPOINT], + ) + message = str(excinfo.value) + assert "checkpoint_identity" in message + assert "tool_call" in message + + def test_no_targets_is_a_failure_not_a_pass(self, monkeypatch): + with pytest.raises(GateFailure): + run_gates([CheckpointIdentityGate("Org/Model")], []) diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 121c1f3a7..45d838d18 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -273,7 +273,36 @@ def test_base_env_supplies_api_key_only_to_agent_subprocess(monkeypatch, tmp_pat monkeypatch.setenv("OPENAI_API_KEY", "ambient-secret") unauthenticated = _request(["http://endpoint:30000"]) - assert "OPENAI_API_KEY" not in runner._base_env(unauthenticated) + assert runner._base_env(unauthenticated)["OPENAI_API_KEY"] == "EMPTY" + assert runner._base_env(authenticated)["OPENAI_API_KEY"] == "real-secret" + + +@pytest.mark.parametrize( + "endpoint", + [ + "http://localhost:30000", + "http://127.0.0.1:30000", + "http://[::1]:30000", + "http://swebench-host:30000", + "https://engine.example.com:8443", + ], +) +def test_base_env_always_supplies_a_credential_placeholder( + monkeypatch, tmp_path, endpoint +): + """A keyless run must never leave OPENAI_API_KEY unset. + + litellm raises ``Missing credentials`` before issuing anything and the agent + retries it forever, so a remote endpoint with no key made zero progress + while the run looked healthy. The placeholder must not depend on whether the + endpoint happens to be loopback. + """ + monkeypatch.setenv("OPENAI_API_KEY", "ambient-secret") + runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) + + env = runner._base_env(_request([endpoint])) + + assert env["OPENAI_API_KEY"] == "EMPTY" def test_run_agent_filters_exact_instance_ids(monkeypatch, tmp_path):