From 4aee88dd10f0d5e7c36ee233f72e81c234dfc811 Mon Sep 17 00:00:00 2001 From: "Li, Tianmu" Date: Thu, 10 Sep 2026 08:57:43 -0700 Subject: [PATCH 1/3] feat(swebench): add trajectory routing headers --- examples/10_Agentic_Inference/README.md | 8 +++ .../10_Agentic_Inference/accuracy/RUNBOOK.md | 7 +++ .../commands/benchmark/accuracy.py | 23 ++++++++- .../evaluation/swe_bench_scorer.py | 4 ++ .../evaluation/swebench_service/README.md | 17 ++++--- .../swebench_service/__init__.py | 1 + .../swebench_service/qwen_tools_model.py | 4 +- .../swebench_service/routing_model.py | 35 +++++++++++++ .../swebench_service/runner.py | 5 +- .../swebench_service/schemas.py | 1 + tests/unit/commands/test_score_accuracy.py | 31 +++++++++++ .../swebench_service/test_runner.py | 51 ++++++++++++++++++- .../swebench_service/test_server.py | 1 + .../unit/evaluation/test_swe_bench_scorer.py | 30 +++++++++++ 14 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 src/inference_endpoint/evaluation/swebench_service/swebench_service/routing_model.py diff --git a/examples/10_Agentic_Inference/README.md b/examples/10_Agentic_Inference/README.md index 7f0f84b6c..e8d3892ea 100644 --- a/examples/10_Agentic_Inference/README.md +++ b/examples/10_Agentic_Inference/README.md @@ -108,6 +108,14 @@ Keep `accuracy_config.num_repeats: 1`: the scorer performs one external evaluati `accuracy_config.extras.workers` sets the agent run's parallelism (`--workers`). If unset, it defaults to the load pattern's `target_concurrency` (for `concurrency`/`agentic_inference` patterns), else 10. `max_eval_workers` (default 10, `--max_workers`) sets the eval harness's parallelism. +SWE-bench endpoint requests use the same routing-header names as the performance +dataset's `agentic_inference.routing_headers` setting. If that setting is +unavailable, the scorer defaults to `X-Session-ID`. The service creates one opaque +session ID per SWE-bench trajectory, reuses it for every model turn in that +trajectory, and uses a different ID for each concurrent trajectory. Setting +`routing_headers: []` on the agentic performance dataset disables these headers in +both phases. + Qwen tool-call runs should set `accuracy_config.extras.swebench_template: qwen_tools`. The selected packaged template also activates the service's `QwenToolsModel` through mini-swe-agent's `model_class` hook. If SWE-bench evaluation is needed, start the service with the following command on a host that has Docker: diff --git a/examples/10_Agentic_Inference/accuracy/RUNBOOK.md b/examples/10_Agentic_Inference/accuracy/RUNBOOK.md index 2988d4c0c..dcb6a4564 100644 --- a/examples/10_Agentic_Inference/accuracy/RUNBOOK.md +++ b/examples/10_Agentic_Inference/accuracy/RUNBOOK.md @@ -66,6 +66,13 @@ Qwen SWE-bench configs opt in with packaged Qwen template and activates `QwenToolsModel` through mini-swe-agent's `model_class` hook. Omit this setting for Kimi and other non-Qwen runs. +The scorer forwards the performance dataset's +`agentic_inference.routing_headers` names to the service, defaulting to +`X-Session-ID` when no agentic performance config is present. The service assigns +one routing ID per SWE-bench trajectory and reuses it across that trajectory's +model turns, so consistent-hash routers keep each agent on one backend without +pinning all concurrent agents to the same backend. + ## Common failure modes | Symptom | Likely cause | Fix | diff --git a/src/inference_endpoint/commands/benchmark/accuracy.py b/src/inference_endpoint/commands/benchmark/accuracy.py index e709e8a73..6c6811b42 100644 --- a/src/inference_endpoint/commands/benchmark/accuracy.py +++ b/src/inference_endpoint/commands/benchmark/accuracy.py @@ -38,7 +38,12 @@ encode_lengths, load_reference_backend, ) -from inference_endpoint.config.schema import DatasetType, ScorerMethod, TestMode +from inference_endpoint.config.schema import ( + AgenticInferenceConfig, + DatasetType, + ScorerMethod, + TestMode, +) from inference_endpoint.dataset_manager.dataset import Dataset from inference_endpoint.evaluation import Extractor from inference_endpoint.evaluation.accuracy_results import ( @@ -57,6 +62,21 @@ logger = logging.getLogger(__name__) +def _swebench_routing_headers(ctx: BenchmarkContext) -> tuple[str, ...]: + """Match SWE-bench routing to the agentic performance conversation contract.""" + perf_dataset = next( + ( + dataset + for dataset in ctx.config.datasets + if dataset.type == DatasetType.PERFORMANCE + ), + None, + ) + if perf_dataset is not None and perf_dataset.agentic_inference is not None: + return perf_dataset.agentic_inference.routing_headers + return AgenticInferenceConfig().routing_headers + + @dataclass(frozen=True) class AccuracyConfiguration: scorer: type[Scorer] @@ -269,6 +289,7 @@ def score_accuracy( scorer_kwargs.update( model_params=eval_cfg.model_params, endpoint_config=eval_cfg.endpoint_config, + routing_headers=_swebench_routing_headers(ctx), ) scorer_instance = eval_cfg.scorer( eval_cfg.dataset_name, diff --git a/src/inference_endpoint/evaluation/swe_bench_scorer.py b/src/inference_endpoint/evaluation/swe_bench_scorer.py index a91d63734..f894fb5aa 100644 --- a/src/inference_endpoint/evaluation/swe_bench_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_scorer.py @@ -65,6 +65,7 @@ class SWEBenchScorer(Scorer, scorer_id="swe_bench_scorer"): "swebench.run", "swebench.cancel", "artifacts.download", + "swebench.routing_headers", } SAFE_ARTIFACT_NAMES: ClassVar[set[str]] = { "preds.json", @@ -95,6 +96,7 @@ def __init__( poll_interval_s: float | None = None, model_params: ModelParams | None = None, endpoint_config: EndpointConfig | None = None, + routing_headers: tuple[str, ...] = ("X-Session-ID",), ): ground_truth_column = ground_truth_column or "instance_id" super().__init__( @@ -131,6 +133,7 @@ def __init__( self.poll_interval_s = options["poll_interval_s"] self.model_params = model_params self.endpoint_config = endpoint_config + self.routing_headers = tuple(routing_headers) @classmethod def _normalize_service_url(cls, value: Any) -> str: @@ -629,6 +632,7 @@ def score(self) -> tuple[float | None, int]: "model_name": model_name, "endpoint_urls": endpoint_urls, "endpoint_api_key": self.endpoint_config.api_key, + "routing_headers": self.routing_headers, "generation_params": self._generation_params(self.model_params), "subset": self.subset, "split": self.split, diff --git a/src/inference_endpoint/evaluation/swebench_service/README.md b/src/inference_endpoint/evaluation/swebench_service/README.md index cd11904b4..6ccda6dc4 100644 --- a/src/inference_endpoint/evaluation/swebench_service/README.md +++ b/src/inference_endpoint/evaluation/swebench_service/README.md @@ -23,12 +23,17 @@ external-service convention for heavyweight evaluation work. ### Common workflow The benchmark client sends the selected SWE-bench instances, model configuration, -and endpoint URL to the service. The service first runs mini-swe-agent to generate -one patch per instance and writes the patches to `preds.json`. It then evaluates -those predictions with the SWE-bench harness and returns the aggregate result and -retained run artifacts. The selected runtime changes where and how the task -containers execute; it does not change the benchmark client configuration or the -model endpoint request path. +endpoint URL, and routing-header names to the service. The routing names come from +the performance dataset's `agentic_inference.routing_headers` setting and default +to `X-Session-ID`. The service creates one opaque routing ID for each mini-swe-agent +trajectory and sends it in every configured header on every model turn. IDs are +stable within a trajectory and distinct across concurrent trajectories. + +The service first runs mini-swe-agent to generate one patch per instance and +writes the patches to `preds.json`. It then evaluates those predictions with the +SWE-bench harness and returns the aggregate result and retained run artifacts. The +selected runtime changes where and how the task containers execute; it does not +change the benchmark client configuration or the model endpoint request path. ### Docker runtime diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/__init__.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/__init__.py index 6873cbd37..18c1bba98 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/__init__.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/__init__.py @@ -21,4 +21,5 @@ "swebench.cancel", "artifacts.download", "swebench.progress", + "swebench.routing_headers", ] diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/qwen_tools_model.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/qwen_tools_model.py index 7bc7a64ee..77d55987a 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/qwen_tools_model.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/qwen_tools_model.py @@ -16,16 +16,16 @@ """mini-swe-agent model extension for the Qwen SWE-bench tool contract.""" import litellm -from minisweagent.models.litellm_model import LitellmModel from .qwen_tools import ( TOOL_SCHEMAS, format_toolcall_observation_messages, parse_toolcall_actions, ) +from .routing_model import SessionRoutingLitellmModel -class QwenToolsModel(LitellmModel): +class QwenToolsModel(SessionRoutingLitellmModel): """Expose the Qwen tool behavior through mini-swe-agent's model hook.""" def _query(self, messages: list[dict[str, str]], **kwargs): diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/routing_model.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/routing_model.py new file mode 100644 index 000000000..103f996b9 --- /dev/null +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/routing_model.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""mini-swe-agent model wrapper for trajectory-stable endpoint routing.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from minisweagent.models.litellm_model import LitellmModel, LitellmModelConfig + + +class SessionRoutingModelConfig(LitellmModelConfig): + """Add endpoint routing header names to mini-swe-agent's model config.""" + + routing_headers: tuple[str, ...] = () + + +class SessionRoutingLitellmModel(LitellmModel): + """Attach one opaque routing ID to every request in this model trajectory.""" + + def __init__(self, **kwargs: Any): + super().__init__(config_class=SessionRoutingModelConfig, **kwargs) + self.routing_session_id = uuid.uuid4().hex + + if not self.config.routing_headers: + return + model_kwargs = dict(self.config.model_kwargs) + extra_headers = dict(model_kwargs.get("extra_headers") or {}) + extra_headers.update( + dict.fromkeys(self.config.routing_headers, self.routing_session_id) + ) + model_kwargs["extra_headers"] = extra_headers + self.config.model_kwargs = model_kwargs diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py index 62f414a82..7ea9a6659 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/runner.py @@ -344,12 +344,15 @@ def _patch_config( model_kwargs = model_cfg["model_kwargs"] model_cfg["model_name"] = request.model_name + model_cfg["routing_headers"] = list(request.routing_headers) if request.template == "qwen_tools": model_cfg["model_class"] = ( "swebench_service.qwen_tools_model.QwenToolsModel" ) else: - model_cfg.pop("model_class", None) + model_cfg["model_class"] = ( + "swebench_service.routing_model.SessionRoutingLitellmModel" + ) if request.endpoint_urls: base = _normalize_endpoint_base(str(request.endpoint_urls[0])) model_kwargs["api_base"] = base + "/v1" diff --git a/src/inference_endpoint/evaluation/swebench_service/swebench_service/schemas.py b/src/inference_endpoint/evaluation/swebench_service/swebench_service/schemas.py index 256992a9e..292f689a0 100644 --- a/src/inference_endpoint/evaluation/swebench_service/swebench_service/schemas.py +++ b/src/inference_endpoint/evaluation/swebench_service/swebench_service/schemas.py @@ -39,6 +39,7 @@ class RunRequest(BaseModel): model_name: str = Field(min_length=1) endpoint_urls: list[str] = Field(min_length=1, max_length=1) endpoint_api_key: str | None = None + routing_headers: tuple[str, ...] = ("X-Session-ID",) generation_params: dict[str, Any] = Field(default_factory=dict) subset: SWEBenchSubset = "verified" split: str = "test" diff --git a/tests/unit/commands/test_score_accuracy.py b/tests/unit/commands/test_score_accuracy.py index 835de4a95..aaa4f583c 100644 --- a/tests/unit/commands/test_score_accuracy.py +++ b/tests/unit/commands/test_score_accuracy.py @@ -188,6 +188,7 @@ def _ctx( tokenizer_name=None, report_dir=None, test_mode: config_schema.TestMode = config_schema.TestMode.ACC, + datasets=None, ): # tokenizer_name None => OSL is skipped (fake scorers have no get_raw_outputs). # report_dir None => the uuid bound falls back to an unbounded read (no map). @@ -196,6 +197,7 @@ def _ctx( tokenizer_name=tokenizer_name, report_dir=report_dir, test_mode=test_mode, + config=SimpleNamespace(datasets=datasets or []), ) @@ -249,8 +251,37 @@ def test_swebench_receives_typed_runtime_model_and_endpoint( "swebench_service_auth_token": "service-secret", "model_params": model_params, "endpoint_config": endpoint_config, + "routing_headers": ("X-Session-ID",), } + @pytest.mark.parametrize( + "routing_headers", + [ + ("X-Session-ID", "X-SMG-Routing-Key"), + (), + ], + ) + def test_swebench_inherits_agentic_performance_routing_headers( + self, tmp_path, routing_headers + ): + cfg = AccuracyConfiguration( + scorer=_FakeSWEBenchScorer, # type: ignore[arg-type] + extractor=None, + dataset_name="swe_bench", + dataset=_FakeDataset(1, 1.0), # type: ignore[arg-type] + report_dir=tmp_path, + ground_truth_column=None, + num_repeats=1, + ) + perf_dataset = SimpleNamespace( + type=DatasetType.PERFORMANCE, + agentic_inference=SimpleNamespace(routing_headers=routing_headers), + ) + + score_accuracy(_ctx([cfg], datasets=[perf_dataset]), _RESULT) + + assert _FakeSWEBenchScorer.received_kwargs["routing_headers"] == routing_headers + def test_each_dataset_gets_its_own_entry(self, tmp_path): cfgs = [ _cfg("aime25::gptoss", 30, 0.8, tmp_path, repeats=8), diff --git a/tests/unit/evaluation/swebench_service/test_runner.py b/tests/unit/evaluation/swebench_service/test_runner.py index 944d22bbf..38ee08386 100644 --- a/tests/unit/evaluation/swebench_service/test_runner.py +++ b/tests/unit/evaluation/swebench_service/test_runner.py @@ -214,7 +214,10 @@ def test_patch_config_normalizes_api_base(tmp_path, endpoint, expected_api_base) assert "user:pass" not in text assert "token=secret" not in text assert "fragment" not in text - assert "model_class" not in cfg["model"] + assert cfg["model"]["model_class"] == ( + "swebench_service.routing_model.SessionRoutingLitellmModel" + ) + assert cfg["model"]["routing_headers"] == ["X-Session-ID"] assert "api_key" not in cfg["model"]["model_kwargs"] assert cfg["environment"]["run_args"] == [ "--rm", @@ -247,6 +250,51 @@ def test_patch_config_keeps_api_key_out_of_yaml_and_forwards_generation(tmp_path assert model_kwargs["chat_template_kwargs"] == {"enable_thinking": False} +def test_routing_model_reuses_one_id_per_trajectory_and_separates_instances(): + routing_model = pytest.importorskip( + "inference_endpoint.evaluation.swebench_service.swebench_service.routing_model" + ) + SessionRoutingLitellmModel = routing_model.SessionRoutingLitellmModel + headers = ("X-Session-ID", "X-SMG-Routing-Key") + + first = SessionRoutingLitellmModel( + model_name="test-model", + routing_headers=headers, + model_kwargs={"extra_headers": {"X-Existing": "keep"}}, + ) + second = SessionRoutingLitellmModel( + model_name="test-model", + routing_headers=headers, + ) + + first_headers = first.config.model_kwargs["extra_headers"] + second_headers = second.config.model_kwargs["extra_headers"] + assert first_headers == { + "X-Existing": "keep", + "X-Session-ID": first.routing_session_id, + "X-SMG-Routing-Key": first.routing_session_id, + } + assert second_headers == { + "X-Session-ID": second.routing_session_id, + "X-SMG-Routing-Key": second.routing_session_id, + } + assert first.routing_session_id != second.routing_session_id + + +def test_routing_model_allows_headers_to_be_disabled(): + routing_model = pytest.importorskip( + "inference_endpoint.evaluation.swebench_service.swebench_service.routing_model" + ) + SessionRoutingLitellmModel = routing_model.SessionRoutingLitellmModel + model = SessionRoutingLitellmModel( + model_name="test-model", + routing_headers=(), + model_kwargs={"temperature": 0.2}, + ) + + assert "extra_headers" not in model.config.model_kwargs + + def test_base_env_supplies_api_key_only_to_agent_subprocess(monkeypatch, tmp_path): runner = SweBenchRunner(project_root=tmp_path, subprocess_timeout_s=30) authenticated = _request(["http://endpoint:30000"]) @@ -305,6 +353,7 @@ def fake_run_subprocess(cmd, log_path, *, env, **kwargs): assert cfg["model"]["model_class"] == ( "swebench_service.qwen_tools_model.QwenToolsModel" ) + assert cfg["model"]["routing_headers"] == ["X-Session-ID"] assert envs[0]["PYTHONPATH"] == "/existing/path" diff --git a/tests/unit/evaluation/swebench_service/test_server.py b/tests/unit/evaluation/swebench_service/test_server.py index 284e2ff33..29303892a 100644 --- a/tests/unit/evaluation/swebench_service/test_server.py +++ b/tests/unit/evaluation/swebench_service/test_server.py @@ -188,6 +188,7 @@ async def test_health_response_schema(tmp_path): assert "swebench.cancel" in body["capabilities"] assert "artifacts.download" in body["capabilities"] assert "swebench.progress" in body["capabilities"] + assert "swebench.routing_headers" in body["capabilities"] @pytest.mark.asyncio diff --git a/tests/unit/evaluation/test_swe_bench_scorer.py b/tests/unit/evaluation/test_swe_bench_scorer.py index de5cedbf5..79e917c8b 100644 --- a/tests/unit/evaluation/test_swe_bench_scorer.py +++ b/tests/unit/evaluation/test_swe_bench_scorer.py @@ -156,6 +156,7 @@ def fake_http_json(url, *, method="GET", **kwargs): "swebench.run", "swebench.cancel", "artifacts.download", + "swebench.routing_headers", ], } @@ -183,6 +184,7 @@ def test_preflight_never_calls_docker_or_subprocess(self, monkeypatch): "swebench.run", "swebench.cancel", "artifacts.download", + "swebench.routing_headers", ], } ), @@ -315,6 +317,7 @@ def fake_http_json(url, *, method="GET", payload=None, **kwargs): assert "benchmark_config" not in payloads[0] assert payloads[0]["endpoint_urls"] == ["http://endpoint-host:30000"] assert payloads[0]["endpoint_api_key"] == "secret-key" + assert payloads[0]["routing_headers"] == ("X-Session-ID",) assert payloads[0]["generation_params"] == { "temperature": 0.25, "seed": 17, @@ -324,6 +327,33 @@ def fake_http_json(url, *, method="GET", payload=None, **kwargs): assert payloads[0]["template"] == "default" assert (report_dir / "swe_bench_results.json").exists() + def test_score_submits_custom_routing_headers(self, report_dir, monkeypatch): + payloads: list[dict] = [] + + def fake_http_json(url, *, method="GET", payload=None, **kwargs): + if method == "POST": + payloads.append(payload) + return { + "run_id": "run-1", + "status": "succeeded", + "result": {"resolved_instances": 1, "submitted_instances": 1}, + "artifacts": [], + } + raise AssertionError(f"unexpected GET {url}") + + monkeypatch.setattr(SWEBenchScorer, "_http_json", fake_http_json) + scorer = _make_scorer( + report_dir, + routing_headers=("X-Session-ID", "X-SMG-Routing-Key"), + ) + + scorer.score() + + assert payloads[0]["routing_headers"] == ( + "X-Session-ID", + "X-SMG-Routing-Key", + ) + def test_score_polls_until_terminal(self, report_dir, monkeypatch): calls: list[str] = [] From 19ec956880e7f3d043632713ec649b3b10c1e0e8 Mon Sep 17 00:00:00 2001 From: "Li, Tianmu" Date: Thu, 10 Sep 2026 09:32:16 -0700 Subject: [PATCH 2/3] test(swebench): update Qwen routing model stub --- tests/unit/evaluation/test_actions_toolcall.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/evaluation/test_actions_toolcall.py b/tests/unit/evaluation/test_actions_toolcall.py index 36af14a76..63c85fa0b 100644 --- a/tests/unit/evaluation/test_actions_toolcall.py +++ b/tests/unit/evaluation/test_actions_toolcall.py @@ -28,15 +28,20 @@ def _install_minisweagent_stubs(monkeypatch): class FormatError(Exception): pass - class LitellmModel: + class LitellmModelConfig: def __init__(self, **kwargs): defaults = { "format_error_template": "{{ error }}", "observation_template": "{{ output.output }}", "multimodal_regex": "", "model_kwargs": {}, + "routing_headers": (), } - self.config = SimpleNamespace(**(defaults | kwargs)) + self.__dict__.update(defaults | kwargs) + + class LitellmModel: + def __init__(self, *, config_class=LitellmModelConfig, **kwargs): + self.config = config_class(**kwargs) litellm = types.ModuleType("litellm") litellm.completion = lambda **kwargs: kwargs @@ -50,6 +55,7 @@ def __init__(self, **kwargs): } litellm_model = types.ModuleType("minisweagent.models.litellm_model") litellm_model.LitellmModel = LitellmModel + litellm_model.LitellmModelConfig = LitellmModelConfig modules = { "litellm": litellm, From 030ea0b791c395c8aad81281db00be1966292ec3 Mon Sep 17 00:00:00 2001 From: "Li, Tianmu" Date: Thu, 10 Sep 2026 10:16:29 -0700 Subject: [PATCH 3/3] test(swebench): cover Qwen routing headers --- tests/unit/evaluation/test_actions_toolcall.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/evaluation/test_actions_toolcall.py b/tests/unit/evaluation/test_actions_toolcall.py index 63c85fa0b..d62121e57 100644 --- a/tests/unit/evaluation/test_actions_toolcall.py +++ b/tests/unit/evaluation/test_actions_toolcall.py @@ -223,7 +223,12 @@ def test_qwen_model_query_sends_custom_tool_request(monkeypatch): ) model = model_mod.QwenToolsModel( model_name="openai/test-model", - model_kwargs={"api_base": "http://endpoint/v1", "temperature": 0.2}, + model_kwargs={ + "api_base": "http://endpoint/v1", + "temperature": 0.2, + "extra_headers": {"X-Existing": "keep"}, + }, + routing_headers=("X-Session-ID",), ) response = model._query([{"role": "user", "content": "task"}], temperature=0.7) @@ -236,6 +241,10 @@ def test_qwen_model_query_sends_custom_tool_request(monkeypatch): "tools": tools.TOOL_SCHEMAS, "api_base": "http://endpoint/v1", "temperature": 0.7, + "extra_headers": { + "X-Existing": "keep", + "X-Session-ID": model.routing_session_id, + }, } ]