From 09fa2038827e1fd4a61be4f5f40a0aaa5c40570d Mon Sep 17 00:00:00 2001 From: sadlilas <11658960+sadlilas@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:14:39 -0700 Subject: [PATCH] fix(http): prompt_tokens double-counted cache reads on the OpenAI wire extract_usage() computed prompt_tokens as inputTokens + cacheReadTokens + cacheWriteTokens, on the premise that the three are disjoint buckets. They are not. amplifier-core's docs/contracts/PROVIDER_CONTRACT.md specifies a provider's input_tokens as the "gross total (fresh + cache_read combined)", and providers normalize to that shape -- the Anthropic module adds cache_read_input_tokens into input_tokens, and the OpenAI module subtracts only cache_write out of the vendor total, leaving cache reads inside. cacheReadTokens is therefore a reported SUBSET of the gross figure, surfaced for visibility, not an addend. Adding it a second time roughly doubles prompt_tokens on a cache-heavy turn, which is the normal case in an agent loop. The value flows to the terminal SSE chunk's usage block, so any OpenAI-compatible client doing its own cost arithmetic from prompt_tokens sees close to twice the real prompt. prompt_tokens is now gross input + cache writes -- cache writes being the one bucket billed on top of the gross total. That matches _compute_total_input in amplifier-module-hooks-streaming-ui, which consumes the same event. cached_tokens is unchanged and stays correct: it is a subset of prompt_tokens under both the old and the new formula, but only the new one gives it a correct whole to be a subset of. Renames the local new_input to gross_input so the name stops asserting the thing that was wrong, and corrects the matching comment on the accumulator in routes/chat_completions.py. No behavior change beyond the corrected total. cost_usd is unaffected: it is summed from the per-call cost the provider computed from the raw buckets, and was already right. Generated with [Amplifier](https://github.com/microsoft/amplifier) Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> --- src/amplifier_agent_http/_event_translator.py | 36 +++++++++++-------- .../routes/chat_completions.py | 5 +-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/amplifier_agent_http/_event_translator.py b/src/amplifier_agent_http/_event_translator.py index 9a21db5c..d6dd506f 100644 --- a/src/amplifier_agent_http/_event_translator.py +++ b/src/amplifier_agent_http/_event_translator.py @@ -140,18 +140,25 @@ def extract_usage(event: DisplayEvent) -> dict[str, Any] | None: Returns ``None`` for non-usage events so the caller can use a simple accumulator: ``if (u := extract_usage(ev)): usage_block = u``. - Anthropic's prompt caching splits input tokens across three buckets: - - - ``inputTokens`` : new tokens, billed at the full input rate - - ``cacheWriteTokens`` : tokens being written to the cache (billed at ~1.25x) - - ``cacheReadTokens`` : tokens read from the cache (billed at ~0.1x) - - The model sees **all three** as input -- the cache distinction is purely - a billing optimization. opencode's @ai-sdk/openai-compatible adapter - (and any OpenAI-compatible client) expects ``prompt_tokens`` to be the - total token count, not just the uncached portion. We were previously - only forwarding ``inputTokens``, which made every cached turn look 1000-2000x - cheaper than it actually was. + Input tokens arrive in two ADDITIVE parts, not three: + + - ``inputTokens`` : the GROSS input total. Per amplifier-core's + ``docs/contracts/PROVIDER_CONTRACT.md``, a provider's + ``input_tokens`` is the "gross total (fresh + + cache_read combined)", so this ALREADY contains the + cache reads. + - ``cacheWriteTokens`` : cache creation, billed on top of the gross total and + NOT included in ``inputTokens``. + - ``cacheReadTokens`` : a REPORTED SUBSET of ``inputTokens``, surfaced for + visibility. Adding it to ``inputTokens`` counts it + twice. + + opencode's @ai-sdk/openai-compatible adapter (and any OpenAI-compatible + client) expects ``prompt_tokens`` to be the total charged token count, which + is therefore ``inputTokens + cacheWriteTokens``. Forwarding ``inputTokens`` + alone understates a turn by the cache-write portion; adding ``cacheReadTokens`` + on top overstates it by the whole cached portion, which on a cache-heavy turn + is most of the prompt. OpenAI's wire extension for cache visibility is ``prompt_tokens_details.cached_tokens`` (the portion that was a cache hit). @@ -178,12 +185,13 @@ def _to_int(value: Any) -> int: except (TypeError, ValueError): return 0 - new_input = _to_int(event.get("inputTokens")) + gross_input = _to_int(event.get("inputTokens")) cache_read = _to_int(event.get("cacheReadTokens")) cache_write = _to_int(event.get("cacheWriteTokens")) output = _to_int(event.get("outputTokens")) - prompt_total = new_input + cache_read + cache_write + # cache_read is deliberately absent: it is already inside gross_input. + prompt_total = gross_input + cache_write result: dict[str, Any] = { "prompt_tokens": prompt_total, "completion_tokens": output, diff --git a/src/amplifier_agent_http/routes/chat_completions.py b/src/amplifier_agent_http/routes/chat_completions.py index 8442184d..997122a0 100644 --- a/src/amplifier_agent_http/routes/chat_completions.py +++ b/src/amplifier_agent_http/routes/chat_completions.py @@ -464,8 +464,9 @@ async def _stream_chat_completion( # Summing in the POC is a reasonable approximation; per-call breakdown is # in the v2 backlog. # - # ``usage_prompt`` is the TOTAL input tokens (new + cache_read + cache_write), - # not just the uncached portion. ``usage_cached`` is surfaced separately via + # ``usage_prompt`` is the TOTAL charged input tokens (gross input + cache + # writes; cache reads are already inside the gross figure and must not be + # added again). ``usage_cached`` is surfaced separately via # ``prompt_tokens_details.cached_tokens`` on the terminal chunk so # the client's cost tracking sees the cache hit rate accurately. usage_prompt: int = 0