diff --git a/CHANGELOG.md b/CHANGELOG.md index de77bd8..b9c16a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,47 @@ # Changelog +## 0.9.1 — 2026-08-07 + +### Added + +- **Token-detail passthrough on streamed usage.** The final `include_usage` + frame's `prompt_tokens_details` / `completion_tokens_details` (reasoning and + cached-token breakdowns) and the Anthropic `cache_read_input_tokens` / + `cache_creation_input_tokens` split now survive LiteLLM stream aggregation + instead of being flattened to three bare counts. Forwarding + `cache_read_input_tokens` also lets LiteLLM's estimate-path cost math apply + the cache-read discount (the gateway folds cache reads into `prompt_tokens`, + matching LiteLLM's own convention, so the subtraction is always safe). + Both `_to_generic_chunk` branches — the choice-less usage frame and a + choices-bearing chunk carrying usage inline — go through one shared + `_usage_dict` builder, so the two paths cannot disagree about which detail + fields survive. + +- **Spec-correct Responses API usage.** `/v1/responses` (non-stream and SSE) + now always emits `input_tokens_details.cached_tokens` and + `output_tokens_details.reasoning_tokens` — both REQUIRED by + `openai.types.responses.ResponseUsage`; typed openai-python clients crashed + on their absence. Values default to 0, only spec keys are projected (no + leaked `audio_tokens` / prediction-token fields), non-dict upstream garbage + coerces to 0, and Anthropic's top-level `cache_read_input_tokens` maps into + `cached_tokens` (those models carry no `prompt_tokens_details`). On the SSE + path a later bare usage frame merges instead of wiping detail captured + earlier. + +### Fixed + +- **Streamed calls no longer die on usage frames without detail blocks.** + `prompt_tokens_details` / `completion_tokens_details` are pydantic extras on + `ChatUsage` (`extra="allow"`), not declared fields — attribute access on an + absent extra raises `AttributeError`. Any streamed call whose usage frame + omitted them (Anthropic-shaped, or bare three-count) crashed at the final + frame; LiteLLM surfaced it as `APIConnectionError`, which retry-on-connection + clients treated as transient and retried an already-settled call — paying + twice for one completion. Details are now read from `model_extra`, where + extras actually live. Regression-locked for the bare, Anthropic, and + OpenAI-shaped frames end-to-end through `CustomStreamWrapper` + + `stream_chunk_builder`. + ## 0.9.0 — 2026-07-24 ### Added diff --git a/blockrun_litellm/__init__.py b/blockrun_litellm/__init__.py index 9c4b880..8e3a6e3 100644 --- a/blockrun_litellm/__init__.py +++ b/blockrun_litellm/__init__.py @@ -40,4 +40,4 @@ "model_ids", "register", ] -__version__ = "0.9.0" +__version__ = "0.9.1" diff --git a/blockrun_litellm/provider.py b/blockrun_litellm/provider.py index e9235b9..7a8135d 100644 --- a/blockrun_litellm/provider.py +++ b/blockrun_litellm/provider.py @@ -42,7 +42,7 @@ from litellm.types.utils import GenericStreamingChunk from blockrun_llm.types import APIError as BlockRunAPIError -from blockrun_llm.types import ChatCompletionChunk +from blockrun_llm.types import ChatCompletionChunk, ChatUsage from blockrun_litellm import _adapter @@ -249,12 +249,53 @@ def _native_extras(chunk: ChatCompletionChunk) -> Dict[str, Any]: # leaking a stray ``cost_usd`` field into ``provider_specific_fields``. extras.pop("cost_usd", None) if chunk.usage is not None: - usage_extra = chunk.usage.model_extra or {} - if usage_extra: - extras.setdefault("usage_details", {}).update(usage_extra) + # NB: ``prompt_tokens_details`` / ``completion_tokens_details`` are NOT + # declared fields on ChatUsage — they are pydantic extras (``extra = + # "allow"``), so they already live in ``model_extra`` and are picked up + # by the dict() copy below. Never read them by attribute: access to an + # absent pydantic extra raises AttributeError, it does not return None. + usage_details: Dict[str, Any] = dict(chunk.usage.model_extra or {}) + if chunk.usage.cache_read_input_tokens is not None: + usage_details["cache_read_input_tokens"] = chunk.usage.cache_read_input_tokens + if chunk.usage.cache_creation_input_tokens is not None: + usage_details["cache_creation_input_tokens"] = chunk.usage.cache_creation_input_tokens + if usage_details: + extras.setdefault("usage_details", {}).update(usage_details) return extras +def _usage_dict(usage: ChatUsage) -> Dict[str, Any]: + """Map a :class:`ChatUsage` → the plain usage dict LiteLLM's streaming + handler feeds into ``litellm.Usage``. + + Carries the token-detail passthrough: ``prompt_tokens_details`` / + ``completion_tokens_details`` (pydantic extras — read from ``model_extra``, + see :func:`_native_extras`) and the Anthropic cache split. The gateway + folds cache reads INTO ``prompt_tokens`` (matching LiteLLM's own + convention), so forwarding ``cache_read_input_tokens`` lets LiteLLM apply + the cache-read discount instead of billing the full prompt rate. + + Used by BOTH ``_to_generic_chunk`` branches: the choice-less + ``include_usage`` final frame AND a choices-bearing chunk that carries + usage inline. Keeping one builder means the two paths can never disagree + about which detail fields survive. + """ + out: Dict[str, Any] = { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + } + extra = usage.model_extra or {} + for key in ("prompt_tokens_details", "completion_tokens_details"): + if extra.get(key) is not None: + out[key] = extra[key] + if usage.cache_read_input_tokens is not None: + out["cache_read_input_tokens"] = usage.cache_read_input_tokens + if usage.cache_creation_input_tokens is not None: + out["cache_creation_input_tokens"] = usage.cache_creation_input_tokens + return out + + def _to_generic_chunk(chunk: ChatCompletionChunk) -> GenericStreamingChunk: """Map a BlockRun :class:`ChatCompletionChunk` → LiteLLM :class:`GenericStreamingChunk` (a ``TypedDict``). @@ -275,13 +316,7 @@ def _to_generic_chunk(chunk: ChatCompletionChunk) -> GenericStreamingChunk: # off them instead of re-estimating the prompt with its own tokenizer # (tiktoken drifts ~37% vs the gateway's real upstream count). Older # gateways that never send this frame still hit the usage=None path. - usage = None - if chunk.usage is not None: - usage = { - "prompt_tokens": chunk.usage.prompt_tokens, - "completion_tokens": chunk.usage.completion_tokens, - "total_tokens": chunk.usage.total_tokens, - } + usage = _usage_dict(chunk.usage) if chunk.usage is not None else None # NOTE: deliberately do NOT set tool_use here. This is the post-finish # usage frame; adding the key changes LiteLLM's CustomStreamWrapper # post-finish guard and lets the frame survive even without @@ -318,13 +353,14 @@ def _to_generic_chunk(chunk: ChatCompletionChunk) -> GenericStreamingChunk: # Anthropic adapter — see _iter_stream_chunks(), which wraps this function. # BlockRun's per-chunk usage is rarely populated; LiteLLM tolerates None. - usage = None - if chunk.usage is not None: - usage = { - "prompt_tokens": chunk.usage.prompt_tokens, - "completion_tokens": chunk.usage.completion_tokens, - "total_tokens": chunk.usage.total_tokens, - } + # When a provider DOES attach usage to a choices-bearing chunk (instead of + # a separate include_usage frame), it must carry the same detail fields as + # the choice-less branch — a bare three-count dict here would contradict + # the full detail riding on provider_specific_fields, and LiteLLM's chunk + # builder resets prompt_tokens_details unconditionally from the LAST + # usage-bearing chunk (streaming_chunk_builder_utils), so an asymmetric + # frame can null out detail a previous frame delivered. + usage = _usage_dict(chunk.usage) if chunk.usage is not None else None return GenericStreamingChunk( text=text, diff --git a/blockrun_litellm/proxy.py b/blockrun_litellm/proxy.py index 298d0de..7d9e879 100644 --- a/blockrun_litellm/proxy.py +++ b/blockrun_litellm/proxy.py @@ -1735,6 +1735,57 @@ def _responses_to_chat( return model, messages, openai_kwargs, bool(body.get("stream")) +def _int_or_zero(value: Any) -> int: + """Coerce an upstream token count to a non-negative int (0 on garbage). + + The detail blocks arrive as untyped pydantic extras — whatever JSON the + upstream sent. This is the guard between that and our typed public + Responses surface (LiteLLM has the same guard in ``Usage.__init__``). + """ + if isinstance(value, bool): # bool is an int subclass; reject explicitly + return 0 + if isinstance(value, int): + return max(value, 0) + if isinstance(value, float) and value == int(value): + return max(int(value), 0) + return 0 + + +def _usage_to_responses(usage: Dict[str, Any]) -> Dict[str, Any]: + """chat.completion ``usage`` dict → Responses API ``usage`` object. + + Spec-correct per ``openai.types.responses.ResponseUsage``: BOTH + ``input_tokens_details`` and ``output_tokens_details`` are REQUIRED, as are + ``cached_tokens`` / ``reasoning_tokens`` inside them — so they are always + emitted, defaulting to 0, never conditionally omitted. Only the spec keys + are projected: splatting the chat-shaped dicts verbatim would leak + ``audio_tokens`` / ``accepted_prediction_tokens`` etc. into a surface that + doesn't define them, and would pass through non-dict garbage unchecked. + + Anthropic models carry cache reads ONLY in the top-level + ``cache_read_input_tokens`` (no ``prompt_tokens_details``), so that field + is the fallback for ``cached_tokens``. Safe to equate: the gateway folds + cache reads into ``prompt_tokens``, matching what OpenAI's own + ``cached_tokens`` is a subset of. + """ + ptd = usage.get("prompt_tokens_details") + ctd = usage.get("completion_tokens_details") + ptd = ptd if isinstance(ptd, dict) else {} + ctd = ctd if isinstance(ctd, dict) else {} + cached = _int_or_zero(ptd.get("cached_tokens")) + if cached == 0: + cached = _int_or_zero(usage.get("cache_read_input_tokens")) + return { + "input_tokens": _int_or_zero(usage.get("prompt_tokens")), + "output_tokens": _int_or_zero(usage.get("completion_tokens")), + "total_tokens": _int_or_zero(usage.get("total_tokens")), + "input_tokens_details": {"cached_tokens": cached}, + "output_tokens_details": { + "reasoning_tokens": _int_or_zero(ctd.get("reasoning_tokens")) + }, + } + + def _chat_payload_to_response(payload: Dict[str, Any], model: str) -> Dict[str, Any]: """chat.completion dict → Responses API ``response`` object (non-streaming).""" choice = (payload.get("choices") or [{}])[0] @@ -1758,11 +1809,7 @@ def _chat_payload_to_response(payload: Dict[str, Any], model: str) -> Dict[str, } ], "output_text": text, - "usage": { - "input_tokens": usage.get("prompt_tokens", 0), - "output_tokens": usage.get("completion_tokens", 0), - "total_tokens": usage.get("total_tokens", 0), - }, + "usage": _usage_to_responses(usage), } @@ -1824,7 +1871,7 @@ def base( seq += 1 parts: List[str] = [] - usage_out = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0} + usage_out: Dict[str, Any] = _usage_to_responses({}) async with _get_semaphore(): try: @@ -1849,11 +1896,19 @@ def base( seq += 1 u = cd.get("usage") if u: - usage_out = { - "input_tokens": u.get("prompt_tokens", 0), - "output_tokens": u.get("completion_tokens", 0), - "total_tokens": u.get("total_tokens", 0), - } + # MERGE, don't replace: the gateway emits one canonical + # include_usage final frame, but if a provider ever sends a + # later bare usage frame (three counts, no details), a + # wholesale reassignment would wipe detail captured + # earlier. Scalar counts take the latest value; the two + # detail blocks only advance, never regress to zero. + fresh = _usage_to_responses(u) + for block in ("input_tokens_details", "output_tokens_details"): + prev = usage_out.get(block) or {} + if any(fresh[block].values()) or not any(prev.values()): + continue + fresh[block] = prev + usage_out = fresh except PaymentError as exc: yield _responses_event( seq, diff --git a/pyproject.toml b/pyproject.toml index 8e2b4aa..d159035 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "blockrun-litellm" -version = "0.9.0" +version = "0.9.1" description = "LiteLLM adapter for BlockRun — call x402-paid AI models via LiteLLM (custom provider or local OpenAI-compatible proxy)" readme = "README.md" license = "MIT" diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py index 56a3e46..2f32540 100644 --- a/tests/test_fingerprint.py +++ b/tests/test_fingerprint.py @@ -64,6 +64,7 @@ def _fingerprinted_response() -> ChatResponse: cache_read_input_tokens=4, # OpenAI-native nested breakdown (extra → must survive) prompt_tokens_details={"cached_tokens": 4}, + completion_tokens_details={"reasoning_tokens": 3}, ), ) @@ -113,6 +114,7 @@ def test_build_response_preserves_usage_cache_details() -> None: usage = dumped["usage"] assert usage["cache_read_input_tokens"] == 4 assert usage["prompt_tokens_details"]["cached_tokens"] == 4 + assert usage["completion_tokens_details"]["reasoning_tokens"] == 3 def test_build_response_preserves_reasoning_content() -> None: @@ -173,4 +175,5 @@ def test_proxy_dump_preserves_fingerprint() -> None: assert dumped["service_tier"] == "default" assert dumped["usage"]["cache_read_input_tokens"] == 4 assert dumped["usage"]["prompt_tokens_details"]["cached_tokens"] == 4 + assert dumped["usage"]["completion_tokens_details"]["reasoning_tokens"] == 3 assert dumped["choices"][0]["message"]["reasoning_content"] == "because reasons" diff --git a/tests/test_responses.py b/tests/test_responses.py index be01a0c..4fddeac 100644 --- a/tests/test_responses.py +++ b/tests/test_responses.py @@ -76,7 +76,15 @@ def test_responses_non_streaming_shape() -> None: assert j["output"][0]["type"] == "message" assert j["output"][0]["content"][0] == {"type": "output_text", "text": "我是助手", "annotations": []} assert j["output_text"] == "我是助手" - assert j["usage"] == {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8} + # input/output_tokens_details are REQUIRED by the Responses spec + # (openai.types.responses.ResponseUsage) — always present, zero-defaulted. + assert j["usage"] == { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + } assert j["id"].startswith("resp_") diff --git a/tests/test_responses_usage.py b/tests/test_responses_usage.py new file mode 100644 index 0000000..2e35150 --- /dev/null +++ b/tests/test_responses_usage.py @@ -0,0 +1,233 @@ +"""Responses API usage conversion — token detail passthrough. + +The Responses spec (``openai.types.responses.ResponseUsage``) makes BOTH +``input_tokens_details`` and ``output_tokens_details`` REQUIRED, as are +``cached_tokens`` / ``reasoning_tokens`` inside them. These tests lock in: + +- details present → carried through (spec keys only, nothing leaked) +- details absent → blocks still emitted, zero-defaulted (openai-python + crashes on a usage object missing them: ``'NoneType' has no attribute + 'cached_tokens'``) +- Anthropic shape → ``cache_read_input_tokens`` (top-level, no + prompt_tokens_details) lands in ``cached_tokens`` +- garbage extras → non-dict details / non-int counts coerced to 0, and + chat-only keys (``audio_tokens`` …) are NOT leaked into the Responses + surface +- the SSE stream path emits the same detail on ``response.completed`` +""" + +from __future__ import annotations + +import json + +import pytest + +pytest.importorskip("fastapi") + +from unittest.mock import patch # noqa: E402 + +from fastapi.testclient import TestClient # noqa: E402 + +from blockrun_llm.types import ChatCompletionChunk # noqa: E402 + +import blockrun_litellm.proxy as P # noqa: E402 +from blockrun_litellm.proxy import _chat_payload_to_response, _usage_to_responses # noqa: E402 + +client = TestClient(P.app) + + +def _payload(usage: dict) -> dict: + return { + "id": "chatcmpl-1", + "model": "openai/gpt-5.5", + "choices": [{"message": {"content": "ok"}}], + "usage": usage, + } + + +def test_chat_to_responses_preserves_token_details() -> None: + response = _chat_payload_to_response( + _payload( + { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "prompt_tokens_details": {"cached_tokens": 4}, + "completion_tokens_details": {"reasoning_tokens": 12}, + } + ), + "openai/gpt-5.5", + ) + + assert response["usage"] == { + "input_tokens": 10, + "output_tokens": 20, + "total_tokens": 30, + "input_tokens_details": {"cached_tokens": 4}, + "output_tokens_details": {"reasoning_tokens": 12}, + } + + +def test_detail_blocks_always_present_even_when_gateway_omits_them() -> None: + """openai-python's ResponseUsage requires both blocks; omitting them breaks + typed clients. Absent upstream detail → zero-defaulted, never missing.""" + response = _chat_payload_to_response( + _payload({"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}), + "openai/gpt-5.5", + ) + assert response["usage"]["input_tokens_details"] == {"cached_tokens": 0} + assert response["usage"]["output_tokens_details"] == {"reasoning_tokens": 0} + + +def test_anthropic_cache_read_maps_to_cached_tokens() -> None: + """Anthropic models carry cache reads ONLY top-level (no + prompt_tokens_details); the gateway folds them into prompt_tokens, so + cached_tokens can safely equal cache_read_input_tokens.""" + usage = _usage_to_responses( + { + "prompt_tokens": 2160, + "completion_tokens": 20, + "total_tokens": 2180, + "cache_read_input_tokens": 2048, + "cache_creation_input_tokens": 100, + } + ) + assert usage["input_tokens"] == 2160 + assert usage["input_tokens_details"] == {"cached_tokens": 2048} + + +def test_explicit_cached_tokens_wins_over_cache_read_fallback() -> None: + usage = _usage_to_responses( + { + "prompt_tokens": 100, + "prompt_tokens_details": {"cached_tokens": 40}, + "cache_read_input_tokens": 99, # OpenAI-style detail is authoritative + } + ) + assert usage["input_tokens_details"] == {"cached_tokens": 40} + + +def test_only_spec_keys_are_projected() -> None: + """Chat-shaped detail dicts carry keys the Responses spec doesn't define + (audio_tokens, accepted/rejected_prediction_tokens) — they must not leak.""" + usage = _usage_to_responses( + { + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120, + "prompt_tokens_details": {"cached_tokens": 40, "audio_tokens": 7}, + "completion_tokens_details": { + "reasoning_tokens": 12, + "accepted_prediction_tokens": 3, + "rejected_prediction_tokens": 1, + }, + } + ) + assert usage["input_tokens_details"] == {"cached_tokens": 40} + assert usage["output_tokens_details"] == {"reasoning_tokens": 12} + + +def test_garbage_details_are_coerced_not_crashed() -> None: + """The detail fields are untyped pydantic extras — whatever JSON the + upstream sent. Lists, strings, bools, negatives all coerce to 0.""" + usage = _usage_to_responses( + { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "prompt_tokens_details": ["not", "a", "dict"], + "completion_tokens_details": {"reasoning_tokens": "twelve"}, + "cache_read_input_tokens": -3, + } + ) + assert usage["input_tokens_details"] == {"cached_tokens": 0} + assert usage["output_tokens_details"] == {"reasoning_tokens": 0} + + +# --------------------------------------------------------------------------- +# SSE stream path — the same detail must reach response.completed +# --------------------------------------------------------------------------- + +async def _stream_with_details(model, messages, **kw): + for t in ["ok"]: + yield ChatCompletionChunk( + id="c", object="chat.completion.chunk", created=1, model=model, + choices=[{"index": 0, "delta": {"content": t}, "finish_reason": None}], + ) + yield ChatCompletionChunk( + id="c", object="chat.completion.chunk", created=1, model=model, + choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}], + ) + # The include_usage final frame (choices:[] + usage) with full detail. + yield ChatCompletionChunk( + id="c", object="chat.completion.chunk", created=1, model=model, + choices=[], + usage={ + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120, + "prompt_tokens_details": {"cached_tokens": 40}, + "completion_tokens_details": {"reasoning_tokens": 12}, + }, + ) + + +def _completed_usage(raw_lines: list[str]) -> dict: + for ln in raw_lines: + if ln.startswith("data: "): + data = json.loads(ln[len("data: "):]) + if data.get("type") == "response.completed": + return data["response"]["usage"] + raise AssertionError("no response.completed event seen") + + +def test_responses_sse_stream_preserves_token_details() -> None: + with patch.object(P._adapter, "chat_completion_stream_async", _stream_with_details): + with client.stream( + "POST", "/v1/responses", + json={"model": "gpt-5.5", "input": "hi", "stream": True}, + ) as r: + assert r.status_code == 200 + usage = _completed_usage(list(r.iter_lines())) + + assert usage == { + "input_tokens": 100, + "output_tokens": 20, + "total_tokens": 120, + "input_tokens_details": {"cached_tokens": 40}, + "output_tokens_details": {"reasoning_tokens": 12}, + } + + +async def _stream_detail_then_bare(model, messages, **kw): + """Detail-bearing usage frame followed by a bare one — the bare frame must + not wipe the detail (merge, not replace).""" + yield ChatCompletionChunk( + id="c", object="chat.completion.chunk", created=1, model=model, + choices=[{"index": 0, "delta": {"content": "ok"}, "finish_reason": "stop"}], + ) + yield ChatCompletionChunk( + id="c", object="chat.completion.chunk", created=1, model=model, + choices=[], + usage={ + "prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, + "completion_tokens_details": {"reasoning_tokens": 12}, + }, + ) + yield ChatCompletionChunk( + id="c", object="chat.completion.chunk", created=1, model=model, + choices=[], + usage={"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}, + ) + + +def test_responses_sse_later_bare_usage_frame_does_not_wipe_details() -> None: + with patch.object(P._adapter, "chat_completion_stream_async", _stream_detail_then_bare): + with client.stream( + "POST", "/v1/responses", + json={"model": "gpt-5.5", "input": "hi", "stream": True}, + ) as r: + usage = _completed_usage(list(r.iter_lines())) + + assert usage["output_tokens_details"] == {"reasoning_tokens": 12} + assert usage["total_tokens"] == 120 diff --git a/tests/test_stream_usage.py b/tests/test_stream_usage.py index 0936f76..bb504f3 100644 --- a/tests/test_stream_usage.py +++ b/tests/test_stream_usage.py @@ -40,7 +40,13 @@ from blockrun_litellm.provider import _to_generic_chunk, register -GATEWAY_USAGE = ChatUsage(prompt_tokens=100, completion_tokens=20, total_tokens=120) +GATEWAY_USAGE = ChatUsage( + prompt_tokens=100, + completion_tokens=20, + total_tokens=120, + cache_read_input_tokens=40, + completion_tokens_details={"reasoning_tokens": 12}, +) def _chunk( @@ -71,6 +77,8 @@ def test_usage_frame_forwards_token_counts() -> None: "prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, + "completion_tokens_details": {"reasoning_tokens": 12}, + "cache_read_input_tokens": 40, } # The frame must not terminate or mutate the stream state. assert gchunk["text"] == "" @@ -151,6 +159,9 @@ def test_usage_survives_custom_stream_wrapper() -> None: assert built.usage.prompt_tokens == 100 assert built.usage.completion_tokens == 20 assert built.usage.total_tokens == 120 + dumped_usage = built.usage.model_dump() + assert dumped_usage["completion_tokens_details"]["reasoning_tokens"] == 12 + assert dumped_usage["cache_read_input_tokens"] == 40 def test_usage_dropped_without_provider_specific_fields_key() -> None: @@ -224,3 +235,56 @@ def _stripped(c: ChatCompletionChunk): "_to_generic_chunk may no longer need to emit the key. Re-check " "streaming_handler.py before relaxing the contract." ) + + +# --------------------------------------------------------------------------- +# Detail-field passthrough: both usage branches, and the absent-details case. +# --------------------------------------------------------------------------- + +def test_usage_frame_without_details_does_not_crash() -> None: + """Regression: prompt_tokens_details / completion_tokens_details are + pydantic EXTRAS on ChatUsage (extra="allow"), not declared fields. + Attribute access on an absent extra raises AttributeError — it does not + return None. A gateway usage frame carrying only the three counts (or only + the Anthropic cache fields) must convert cleanly; before the model_extra + fix this raised, LiteLLM wrapped it into APIConnectionError, and clients + retried an already-settled stream.""" + bare = ChatUsage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + gchunk = _to_generic_chunk(_chunk([], usage=bare)) + assert gchunk["usage"] == { + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120, + } + + anthropic_shape = ChatUsage( + prompt_tokens=2160, + completion_tokens=20, + total_tokens=2180, + cache_read_input_tokens=2048, + cache_creation_input_tokens=100, + ) + gchunk = _to_generic_chunk(_chunk([], usage=anthropic_shape)) + assert gchunk["usage"]["cache_read_input_tokens"] == 2048 + assert gchunk["usage"]["cache_creation_input_tokens"] == 100 + assert "prompt_tokens_details" not in gchunk["usage"] + + +def test_choices_bearing_chunk_carries_same_usage_details() -> None: + """A provider that attaches usage to a choices-bearing chunk (instead of a + separate include_usage frame) must emit the SAME detail fields as the + choice-less branch. LiteLLM's chunk builder takes prompt_tokens_details + unconditionally from the LAST usage-bearing chunk, so an asymmetric bare + dict here could null out detail delivered earlier.""" + chunk = _chunk( + [ + ChatChunkChoice( + index=0, delta=ChatChunkDelta(), finish_reason="stop" + ) + ], + usage=GATEWAY_USAGE, + ) + gchunk = _to_generic_chunk(chunk) + assert gchunk["usage"]["completion_tokens_details"] == {"reasoning_tokens": 12} + assert gchunk["usage"]["cache_read_input_tokens"] == 40 + assert gchunk["is_finished"] is True