From 6f9900704f84a85e12a7c61dd7ed489697a1769c Mon Sep 17 00:00:00 2001
From: Stanley Phoong <sphoong@nvidia.com>
Date: Mon, 10 Aug 2026 03:51:04 -0700
Subject: [PATCH] fix(swe-bench): fingerprint endpoint identity, not the time
 of asking

EndpointFingerprintGate hashed the whole /v1/models payload. vLLM stamps that
response with a request-time `created` field and mints a fresh
`permission[].id` on every call, so two reads of one healthy, untouched engine
produce two different fingerprints -- four calls, four values.

The dispatcher records a fingerprint when a unit is claimed and re-reads it
when the unit is published, and treats any difference as `endpoint_changed`:
an infrastructure fault, which requeues the unit. With an unstable fingerprint
that comparison is always true, so every unit is retried until it exhausts
max_attempts, is published as abandoned, and the merge gate refuses the run.
The failure costs the full agent and evaluation time of every attempt first,
and reports itself as infrastructure damage rather than as a bug here.

Hash only the identity-bearing fields by dropping the per-request ones. The
gate still fails closed on an endpoint whose identity cannot be read at all,
which is the property it exists to provide.
---
 .../evaluation/swe_bench_distributed/gates.py | 28 ++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py
index 9330fe012..302ce3d3e 100644
--- a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py
+++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py
@@ -369,6 +369,30 @@ def build_scale_prompt(repetitions: int = 120) -> str:
     )
 
 
+#: Response fields that change on every request and carry no checkpoint
+#: identity. vLLM's ``/v1/models`` stamps ``created`` with the request time and
+#: mints a fresh ``permission[].id`` per call, so hashing the raw payload makes
+#: the fingerprint differ between any two reads of a perfectly healthy engine.
+#: The dispatcher compares the claim-time and publish-time fingerprints and
+#: treats a difference as ``endpoint_changed`` -- an infrastructure fault -- so
+#: an unstable fingerprint retries and then abandons every unit, and the merge
+#: gate can never produce a number.
+_VOLATILE_IDENTITY_KEYS = frozenset({"created", "created_at", "permission"})
+
+
+def _strip_volatile(value: Any) -> Any:
+    """Drop per-request fields so a fingerprint reflects identity, not time."""
+    if isinstance(value, dict):
+        return {
+            key: _strip_volatile(item)
+            for key, item in value.items()
+            if key not in _VOLATILE_IDENTITY_KEYS
+        }
+    if isinstance(value, list):
+        return [_strip_volatile(item) for item in value]
+    return value
+
+
 class EndpointFingerprintGate:
     """Record a per-endpoint fingerprint for later comparison.
 
@@ -401,7 +425,9 @@ def fingerprint(self, url: str) -> str | None:
                 )
             except (urllib_error.URLError, OSError, ValueError, TimeoutError):
                 continue
-            parts.append(json.dumps(payload, sort_keys=True, default=str))
+            parts.append(
+                json.dumps(_strip_volatile(payload), sort_keys=True, default=str)
+            )
         if not parts:
             return None
         import hashlib