From fe73bc8ac657fc8729c3b3bde244730bab8285f6 Mon Sep 17 00:00:00 2001 From: Flo H Date: Mon, 17 Aug 2026 15:15:42 +1000 Subject: [PATCH] feat(usage): surface prompt-cache tokens in usage maps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI-compatible providers report prompt-cache hits in usage.prompt_tokens_details.cached_tokens (OpenAI, z.ai, Kimi, DeepSeek); Anthropic reports cache_read_input_tokens / cache_creation_input_tokens. All were dropped — callers cannot measure cache hit rates or true input cost, and cost estimators re-bill cached tokens at full price. - openai.ex: usage_from_openai/1 adds :cached_tokens when prompt_tokens_details.cached_tokens is present (covers both the non-streaming response and the streamed {:usage, _} event, which shares this parser). Non-reporting providers are unaffected (key omitted). - anthropic.ex: usage map adds :cached_tokens (cache reads) and :cache_creation_input_tokens (cache writes). - response.ex: document the extended usage shape. Measured motivation (gc_daemon#3): a 5-turn agent conversation on z.ai shows 8,384/8,437 cached on steady-state turns — ~99% hit rate invisible without this field, and billed at 100% by naive estimators. --- lib/llm_core/llm/anthropic.ex | 5 +++++ lib/llm_core/llm/openai.ex | 12 ++++++++++ lib/llm_core/llm/response.ex | 5 ++++- test/llm_core/llm/openai_test.exs | 37 ++++++++++++++++++++++++++++++- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/lib/llm_core/llm/anthropic.ex b/lib/llm_core/llm/anthropic.ex index 25da023..b45c52f 100644 --- a/lib/llm_core/llm/anthropic.ex +++ b/lib/llm_core/llm/anthropic.ex @@ -295,6 +295,11 @@ defmodule LlmCore.LLM.Anthropic do |> maybe_put(:prompt_tokens, get_in(body, ["usage", "input_tokens"])) |> maybe_put(:completion_tokens, get_in(body, ["usage", "output_tokens"])) |> maybe_put(:total_tokens, total_tokens(body)) + # Anthropic prompt-caching usage (cache reads billed ~0.1x input, writes + # 1.25x) — surfaced so callers can measure hit rates and true input cost + # (gc_daemon#3). + |> maybe_put(:cached_tokens, get_in(body, ["usage", "cache_read_input_tokens"])) + |> maybe_put(:cache_creation_input_tokens, get_in(body, ["usage", "cache_creation_input_tokens"])) tool_calls = if stop_reason == "tool_use" do diff --git a/lib/llm_core/llm/openai.ex b/lib/llm_core/llm/openai.ex index ddf9d0b..0605324 100644 --- a/lib/llm_core/llm/openai.ex +++ b/lib/llm_core/llm/openai.ex @@ -254,6 +254,7 @@ defmodule LlmCore.LLM.OpenAI do completion_tokens: completion, total_tokens: total } + |> maybe_put_cached_tokens(usage) end defp usage_from_openai(%{"total_tokens" => total}) do @@ -262,6 +263,17 @@ defmodule LlmCore.LLM.OpenAI do defp usage_from_openai(_), do: %{} + # OpenAI-compatible providers report prompt-cache hits in + # usage.prompt_tokens_details.cached_tokens (OpenAI, z.ai, Kimi, DeepSeek...). + # cached_tokens is a subset of prompt_tokens, already billed by the provider + # at its (much lower) cached rate — surfaced so callers can measure hit rates + # and true input cost instead of re-billing cache at full price (gc_daemon#3). + defp maybe_put_cached_tokens(acc, %{"prompt_tokens_details" => %{"cached_tokens" => cached}}) + when is_integer(cached), + do: Map.put(acc, :cached_tokens, cached) + + defp maybe_put_cached_tokens(acc, _usage), do: acc + @spec maybe_put_tools(map(), [LlmToolkit.Tool.t()] | nil) :: map() defp maybe_put_tools(body, nil), do: body defp maybe_put_tools(body, []), do: body diff --git a/lib/llm_core/llm/response.ex b/lib/llm_core/llm/response.ex index 7427bff..5bf51cc 100644 --- a/lib/llm_core/llm/response.ex +++ b/lib/llm_core/llm/response.ex @@ -10,7 +10,10 @@ defmodule LlmCore.LLM.Response do * `content` - The main text content of the response * `provider` - Atom identifying the provider (e.g., `:claude_code`, `:openai`) * `model` - String identifying the model used (e.g., "claude-3-opus", "gpt-4") - * `usage` - Map with token usage info (prompt_tokens, completion_tokens, total_tokens) + * `usage` - Map with token usage info (prompt_tokens, completion_tokens, total_tokens; + plus `cached_tokens` when the provider reports prompt-cache hits — OpenAI-compatible + `prompt_tokens_details.cached_tokens`, or Anthropic `cache_read_input_tokens` — and + `cache_creation_input_tokens` for Anthropic cache writes) * `raw` - The raw response from the provider for debugging/passthrough * `metadata` - Additional provider-specific metadata (latency, request_id, etc.) * `structured` - Parsed/validated structured output (when requested) diff --git a/test/llm_core/llm/openai_test.exs b/test/llm_core/llm/openai_test.exs index 509cb66..ddda8eb 100644 --- a/test/llm_core/llm/openai_test.exs +++ b/test/llm_core/llm/openai_test.exs @@ -36,13 +36,48 @@ defmodule LlmCore.LLM.OpenAITest do ] end + test "surfaces prompt_tokens_details.cached_tokens in the final usage event" do + # OpenAI-compatible providers (OpenAI, z.ai, Kimi, DeepSeek) report + # prompt-cache hits here; cached_tokens is a subset of prompt_tokens. + chunk = """ + data: {\"choices\":[],\"usage\":{\"prompt_tokens\":8437,\"completion_tokens\":28,\"total_tokens\":8465,\"prompt_tokens_details\":{\"cached_tokens\":8384}}} + + data: [DONE] + """ + + assert {events, true, true} = OpenAI.decode_stream_chunk(chunk) + + assert events == [ + {:usage, + %{ + prompt_tokens: 8437, + completion_tokens: 28, + total_tokens: 8465, + cached_tokens: 8384 + }} + ] + end + + test "usage without prompt_tokens_details omits cached_tokens" do + chunk = """ + data: {\"choices\":[],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":5,\"total_tokens\":12}} + + data: [DONE] + """ + + assert {events, true, true} = OpenAI.decode_stream_chunk(chunk) + + assert events == [ + {:usage, %{prompt_tokens: 7, completion_tokens: 5, total_tokens: 12}} + ] + end + test "emits empty usage when the stream ends without provider usage" do chunk = """ data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]} data: [DONE] """ - assert {["Hi", {:usage, %{}}], true, false} = OpenAI.decode_stream_chunk(chunk) end