fix(benchmark): don't reject an accuracy-only run for listing many endpoints

docs/evaluation/DESIGN.md

@@ -93,6 +93,21 @@ Code execution cannot be done safely in-process. The evaluation server runs in a
 with resource limits. This is a deliberate architecture choice — not a shortcut — and is
 documented prominently in the dataset README.
 
+**An externally-scored accuracy-only run builds an idle issuer**
+
+A scorer with `SKIP_ENDPOINT_PHASE` evaluates through its own service, so the run
+issues no samples and `total_samples` is zero. `BenchmarkSession` still requires a
+non-None issuer, so one is built and never used. Accuracy-only runs are pinned to
+`num_workers=1` for deterministic single-stream ordering, and `HTTPClientConfig`
+requires `num_workers` to divide the endpoint count, so that idle client is handed a
+single endpoint. This is scoped strictly to the zero-sample case: a run that will
+issue keeps the full endpoint list and the divisibility invariant applies to it
+unchanged.
+
+Scorers that themselves fan work across several endpoints read the endpoint list from
+the run's `config.yaml`, not from this client, so the narrowing does not narrow the
+run.
+
 ## Integration Points
 
 | Component                        | Role                                                   |

src/inference_endpoint/commands/benchmark/execute.py

@@ -705,6 +705,25 @@ async def _create_issuer(
     """Create the HTTP endpoint client + sample issuer, or raise SetupError."""
     config = ctx.config
     endpoints = config.endpoint_config.endpoints
+    if ctx.accuracy_only and ctx.total_samples == 0:
+        # This client will not issue a single sample: every accuracy dataset is
+        # scored externally (Scorer.SKIP_ENDPOINT_PHASE), which is what makes
+        # total_samples zero. It is built only so the session has an issuer.
+        #
+        # It still has to satisfy HTTPClientConfig, which requires num_workers
+        # to divide the endpoint count -- and accuracy-only runs are forced to
+        # num_workers=1 for deterministic ordering, so any run listing more than
+        # one endpoint is rejected before it starts. Hand the idle client a
+        # single endpoint so the invariant holds; it drives none of them.
+        #
+        # Scorers that fan out across endpoints themselves (swe_bench_fleet)
+        # read the real endpoint list from the run's config.yaml, not from this
+        # client, so narrowing it here does not narrow the run.
+        #
+        # The proper fix is to not build an issuer at all when nothing will be
+        # issued; that needs a null issuer, because BenchmarkSession requires a
+        # non-None one. This is the narrow version of that change.
+        endpoints = endpoints[:1]
     logger.info(f"Connecting: {endpoints}")
     try:
         api_type: APIType = config.endpoint_config.api_type

tests/unit/commands/test_benchmark.py

@@ -3559,3 +3559,85 @@ def _run_audit(cfg, base_report_dir):
 
         benchmark_spy.assert_not_called()
         assert audit_calls == [tmp_path / "audit"]
+
+
+class TestAccuracyOnlyIdleIssuer:
+    """An accuracy-only run whose datasets are all scored externally.
+
+    `setup_benchmark` pins num_workers=1 and max_connections=1 for every
+    TestMode.ACC run so the compliance gate's single_stream assertion holds, and
+    `HTTPClientConfig` separately requires num_workers to divide the endpoint
+    count. One worker cannot divide four endpoints, so listing several endpoints
+    took the whole run down during setup -- for a client that issues nothing.
+    """
+
+    def _ctx(self, tmp_path, endpoints: list[str], *, total_samples: int = 0):
+        config = OfflineConfig(
+            endpoint_config={"endpoints": endpoints},
+            model_params={"name": "test-model"},
+            datasets=[{"path": "test.jsonl"}],
+            settings=OfflineSettings(client=HTTPClientConfig(num_workers=1)),
+        )
+        ctx = _make_benchmark_context(config, tmp_path, test_mode=TestMode.ACC)
+        return dataclasses.replace(ctx, total_samples=total_samples)
+
+    async def _capture_endpoints(self, ctx) -> list[str]:
+        captured: list[str] = []
+
+        async def _create(http_config, loop):
+            captured.extend(http_config.endpoint_urls)
+            return MagicMock()
+
+        with (
+            patch.object(execute_mod.HTTPEndpointClient, "create", new=_create),
+            patch.object(execute_mod, "HttpClientSampleIssuer", MagicMock()),
+        ):
+            await execute_mod._create_issuer(ctx, asyncio.get_event_loop())
+        return captured
+
+    @pytest.mark.unit
+    @pytest.mark.asyncio
+    async def test_many_endpoints_do_not_reject_an_idle_accuracy_run(self, tmp_path):
+        ctx = self._ctx(
+            tmp_path,
+            [f"http://engine-{i}:8000" for i in range(4)],
+            total_samples=0,
+        )
+
+        captured = await self._capture_endpoints(ctx)
+
+        assert len(captured) == 1
+        assert captured[0].startswith("http://engine-0:8000")
+
+    @pytest.mark.unit
+    @pytest.mark.asyncio
+    async def test_a_single_endpoint_run_is_unchanged(self, tmp_path):
+        ctx = self._ctx(tmp_path, ["http://engine-0:8000"], total_samples=0)
+
+        captured = await self._capture_endpoints(ctx)
+
+        assert len(captured) == 1
+
+    @pytest.mark.unit
+    @pytest.mark.asyncio
+    async def test_an_issuing_accuracy_run_still_gets_every_endpoint(self, tmp_path):
+        """The narrowing is scoped to a client that issues nothing.
+
+        A run that will issue samples keeps the full endpoint list, so the
+        divisibility invariant still means what it says where it matters.
+        """
+        endpoints = [f"http://engine-{i}:8000" for i in range(2)]
+        config = OfflineConfig(
+            endpoint_config={"endpoints": endpoints},
+            model_params={"name": "test-model"},
+            datasets=[{"path": "test.jsonl"}],
+            settings=OfflineSettings(client=HTTPClientConfig(num_workers=2)),
+        )
+        ctx = dataclasses.replace(
+            _make_benchmark_context(config, tmp_path, test_mode=TestMode.ACC),
+            total_samples=8,
+        )
+
+        captured = await self._capture_endpoints(ctx)
+
+        assert len(captured) == 2