From 365b98dcc96daacb8c96eedbbaf35c71c3cc99a0 Mon Sep 17 00:00:00 2001 From: "Michael J. Jabbour" Date: Mon, 10 Aug 2026 12:12:16 -0400 Subject: [PATCH] fix: treat gateway-instant-408 responses as retryable timeouts The vLLM provider pins max_retries=0 and retries via its own backoff wrapper, but the OpenAI SDK maps a *received* HTTP 408 to a generic APIStatusError, so a gateway-fabricated 408 bypassed retry entirely and surfaced to the caller after ~0.2s. The LiteLLM gateway emits 408s with 'time taken=0.0 seconds' in the body when its own transport fails instantly; a genuine client inactivity 408 carries real elapsed time. Distinguish on that marker: instant 408 -> retryable LLMTimeoutError (handled by the existing retry_with_backoff path); genuine 408 -> non-retryable LLMError. --- amplifier_module_provider_vllm/__init__.py | 15 +++++++ tests/test_error_translation.py | 48 ++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/amplifier_module_provider_vllm/__init__.py b/amplifier_module_provider_vllm/__init__.py index 450a6ab..d4f109e 100644 --- a/amplifier_module_provider_vllm/__init__.py +++ b/amplifier_module_provider_vllm/__init__.py @@ -1507,6 +1507,21 @@ async def _do_complete(): status_code=status, retryable=True, ) from e + if status == 408: + if "time taken=0.0" in error_msg: + raise kernel_errors.LLMTimeoutError( + "Gateway reported an instant timeout (408 with " + "'time taken=0.0'): the request never reached the " + "backend. Retrying.", + provider=self.name, + retryable=True, + ) from e + raise kernel_errors.LLMError( + error_msg, + provider=self.name, + status_code=status, + retryable=False, + ) from e if status == 404: raise kernel_errors.NotFoundError( error_msg, diff --git a/tests/test_error_translation.py b/tests/test_error_translation.py index c92ef10..3b5de87 100644 --- a/tests/test_error_translation.py +++ b/tests/test_error_translation.py @@ -178,6 +178,54 @@ def test_api_status_error_5xx_translated(): assert err.__cause__ is native +def test_instant_timeout_408_retryable(): + """Gateway 408 with 'time taken=0.0' (request never reached backend) -> retryable LLMTimeoutError.""" + provider = _make_provider() + native = openai.APIStatusError( + "Error code: 408", + response=_mock_httpx_response(408), + body={ + "message": "Request timed out. error_type=APITimeoutError, " + "timeout value=55.0, time taken=0.0 seconds", + "code": 408, + }, + ) + provider.client.responses.create = AsyncMock(side_effect=native) + + with pytest.raises(kernel_errors.LLMTimeoutError) as exc_info: + asyncio.run(provider.complete(_simple_request())) + + err = exc_info.value + assert err.provider == "vllm" + assert err.retryable is True + assert err.__cause__ is native + + +def test_genuine_timeout_408_not_retryable(): + """Gateway 408 with elapsed time (real upstream timeout) -> non-retryable LLMError.""" + provider = _make_provider() + native = openai.APIStatusError( + "Error code: 408", + response=_mock_httpx_response(408), + body={ + "message": "Request timed out. error_type=APITimeoutError, " + "timeout value=55.0, time taken=55.02 seconds", + "code": 408, + }, + ) + provider.client.responses.create = AsyncMock(side_effect=native) + + with pytest.raises(kernel_errors.LLMError) as exc_info: + asyncio.run(provider.complete(_simple_request())) + + err = exc_info.value + assert not isinstance(err, kernel_errors.LLMTimeoutError) + assert err.provider == "vllm" + assert err.status_code == 408 + assert err.retryable is False + assert err.__cause__ is native + + def test_timeout_error_translated(): """asyncio.TimeoutError -> kernel LLMTimeoutError (retryable=True).""" provider = _make_provider()