diff --git a/agent_core/core/impl/llm/interface.py b/agent_core/core/impl/llm/interface.py index 945cb82a..99938e01 100644 --- a/agent_core/core/impl/llm/interface.py +++ b/agent_core/core/impl/llm/interface.py @@ -2774,7 +2774,6 @@ def _generate_bedrock( usage = response.get("usage", {}) or {} token_count_input = int(usage.get("inputTokens", 0) or 0) token_count_output = int(usage.get("outputTokens", 0) or 0) - total_tokens = token_count_input + token_count_output if self._bedrock_model_supports_caching(): # Official Converse response uses `cacheReadInputTokens` / @@ -2791,7 +2790,13 @@ def _generate_bedrock( or usage.get("cacheWriteInputTokenCount") or 0 ) - cached_tokens = cache_read + cache_write + # Bedrock's `inputTokens` EXCLUDES cache activity, unlike the + # Anthropic API where input covers the full prompt. Normalize + # to the Anthropic shape — input = full prompt, cached = reads + # only — so downstream `input - cached` display math holds for + # every provider. + token_count_input += cache_read + cache_write + cached_tokens = cache_read metrics = get_cache_metrics() if cache_read > 0: @@ -2818,6 +2823,8 @@ def _generate_bedrock( "bedrock", cache_type, total_tokens=token_count_input ) + total_tokens = token_count_input + token_count_output + status = "success" except Exception as exc: # pragma: no cover diff --git a/agent_core/core/impl/vlm/interface.py b/agent_core/core/impl/vlm/interface.py index 34dde4cf..00b29de5 100644 --- a/agent_core/core/impl/vlm/interface.py +++ b/agent_core/core/impl/vlm/interface.py @@ -922,7 +922,6 @@ def _bedrock_describe_bytes( usage = response.get("usage", {}) or {} token_count_input = int(usage.get("inputTokens", 0) or 0) token_count_output = int(usage.get("outputTokens", 0) or 0) - total_tokens = token_count_input + token_count_output cached_tokens = 0 if self._bedrock_model_supports_caching(): @@ -940,7 +939,13 @@ def _bedrock_describe_bytes( or usage.get("cacheWriteInputTokenCount") or 0 ) - cached_tokens = cache_read + cache_write + # Bedrock's `inputTokens` EXCLUDES cache activity, unlike the + # Anthropic API where input covers the full prompt. Normalize to + # the Anthropic shape — input = full prompt, cached = reads only — + # so downstream `input - cached` display math holds for every + # provider. + token_count_input += cache_read + cache_write + cached_tokens = cache_read metrics = get_cache_metrics() if cache_read > 0: @@ -965,6 +970,8 @@ def _bedrock_describe_bytes( "bedrock", "cachepoint_vlm", total_tokens=token_count_input ) + total_tokens = token_count_input + token_count_output + self._report_usage_async( "vlm_bedrock", "bedrock", diff --git a/agent_core/core/session/session.py b/agent_core/core/session/session.py index 00ec9c4b..444e8c83 100644 --- a/agent_core/core/session/session.py +++ b/agent_core/core/session/session.py @@ -58,6 +58,8 @@ class Session: (reset when a new run starts). input_tokens/output_tokens/cache_tokens: LLM usage breakdown for the current run. + total_input_tokens/total_output_tokens/total_cache_tokens: the same + breakdown accumulated across every run in this session. """ id: str @@ -83,6 +85,11 @@ class Session: input_tokens: int = 0 output_tokens: int = 0 cache_tokens: int = 0 + # Cumulative session totals — deliberately NOT cleared by + # reset_run_counters(); they span every run in this session. + total_input_tokens: int = 0 + total_output_tokens: int = 0 + total_cache_tokens: int = 0 def touch(self) -> None: """Update last_active_at to now.""" @@ -138,6 +145,9 @@ def to_dict(self) -> Dict[str, Any]: "input_tokens": self.input_tokens, "output_tokens": self.output_tokens, "cache_tokens": self.cache_tokens, + "total_input_tokens": self.total_input_tokens, + "total_output_tokens": self.total_output_tokens, + "total_cache_tokens": self.total_cache_tokens, } @classmethod @@ -165,4 +175,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Session": input_tokens=data.get("input_tokens", 0), output_tokens=data.get("output_tokens", 0), cache_tokens=data.get("cache_tokens", 0), + total_input_tokens=data.get("total_input_tokens", 0), + total_output_tokens=data.get("total_output_tokens", 0), + total_cache_tokens=data.get("total_cache_tokens", 0), ) diff --git a/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.module.css b/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.module.css index dd4ce5d7..4c82ca58 100644 --- a/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.module.css +++ b/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.module.css @@ -235,6 +235,11 @@ transition: width 0.3s ease; } +.tokenCachedBar { + background: var(--color-success); + transition: width 0.3s ease; +} + .tokenRatioLabels { display: flex; justify-content: center; diff --git a/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.tsx b/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.tsx index 3506b521..ca97b8c4 100644 --- a/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.tsx +++ b/app/ui_layer/browser/frontend/src/pages/Dashboard/DashboardPage.tsx @@ -153,15 +153,26 @@ export function DashboardPage() { // Token metrics - use cached filtered metrics for all periods (including 'total') const tokenFilteredData = filteredMetricsCache[tokenPeriod] - const inputTokens = tokenFilteredData?.token.input ?? (metrics?.token.input ?? 0) + const rawInputTokens = tokenFilteredData?.token.input ?? (metrics?.token.input ?? 0) const outputTokens = tokenFilteredData?.token.output ?? (metrics?.token.output ?? 0) - const totalTokens = tokenFilteredData?.token.total ?? (metrics?.token.total ?? 0) const cachedTokens = tokenFilteredData?.token.cached ?? (metrics?.token.cached ?? 0) + // `token.input` from the API is the full prompt size — cache reads included. + // The Input tile shows only the genuinely new tokens; Cached shows the rest. + const inputTokens = Math.max(0, rawInputTokens - cachedTokens) + // Total counts new tokens only. `token.total` from the API is + // rawInput + output, so it double-counts cache reads — derive instead. + const totalTokens = inputTokens + outputTokens + const totalTokensRatio = inputTokens + outputTokens + cachedTokens + // Calculate token ratios - const inputRatio = totalTokens > 0 ? Math.round((inputTokens / totalTokens) * 100) : 0 - const outputRatio = totalTokens > 0 ? Math.round((outputTokens / totalTokens) * 100) : 0 - const cachedRatio = inputTokens > 0 ? Math.min(100, Math.round((cachedTokens / inputTokens) * 100)) : 0 + const inputRatio = totalTokensRatio > 0 ? Math.round((inputTokens / totalTokensRatio) * 100) : 0 + const outputRatio = totalTokensRatio > 0 ? Math.round((outputTokens / totalTokensRatio) * 100) : 0 + // Cache hit rate — share of the prompt served from cache. Denominator is the + // full prompt, not totalTokens: output tokens are never cacheable. + //const cachedRatio = rawInputTokens > 0 ? Math.min(100, Math.round((cachedTokens / rawInputTokens) * 100)) : 0 <--- STORAGE ONLY, ABLE TO BE DELETED IF NEEDED + const cachedRatio = totalTokensRatio > 0 ? Math.min(100, Math.round((cachedTokens / totalTokensRatio) * 100)) : 0 + const cpuPercent = metrics?.system.cpuPercent ?? 0 const memoryPercent = metrics?.system.memoryPercent ?? 0 @@ -304,6 +315,10 @@ export function DashboardPage() { className={styles.tokenOutputBar} style={{ width: `${outputRatio}%` }} /> +
diff --git a/app/ui_layer/commands/builtin/__init__.py b/app/ui_layer/commands/builtin/__init__.py index 75a79eed..787d2c44 100644 --- a/app/ui_layer/commands/builtin/__init__.py +++ b/app/ui_layer/commands/builtin/__init__.py @@ -11,6 +11,7 @@ from app.ui_layer.commands.builtin.cred import CredCommand from app.ui_layer.commands.builtin.integrations import IntegrationCommand from app.ui_layer.commands.builtin.update import UpdateCommand +from app.ui_layer.commands.builtin.tokens import TokensCommand from app.ui_layer.commands.builtin.agent_command import AgentCommandWrapper from app.ui_layer.commands.builtin.skill_invoke import SkillInvokeCommand @@ -26,6 +27,7 @@ "CredCommand", "IntegrationCommand", "UpdateCommand", + "TokensCommand", "AgentCommandWrapper", "SkillInvokeCommand", ] diff --git a/app/ui_layer/commands/builtin/tokens.py b/app/ui_layer/commands/builtin/tokens.py new file mode 100644 index 00000000..4f3d2172 --- /dev/null +++ b/app/ui_layer/commands/builtin/tokens.py @@ -0,0 +1,61 @@ +"""Tokens command implementation — shows this session's token usage.""" + +from __future__ import annotations + +from typing import List + +from agent_core.core.session import MAIN_SESSION_ID + +from app.ui_layer.commands.base import Command, CommandResult + + +class TokensCommand(Command): + """Show cumulative token usage for the session it was typed in.""" + + @property + def name(self) -> str: + return "/tokens" + + @property + def description(self) -> str: + return "Show this session's token usage" + + async def execute( + self, + args: List[str], + adapter_id: str = "", + session_id: str | None = None, + ) -> CommandResult: + """Report this session's token totals as a chat message.""" + target = session_id or MAIN_SESSION_ID + # Sessions are created lazily on first message, so a brand-new chat + # has no session object yet — that reads as zero usage, not an error. + session = self._controller.agent.session_manager.get(target) + + # Providers report input as the full prompt, cache reads included, so + # `cached` is a subset of `total_input_tokens` — never a sibling. + raw_input = getattr(session, "total_input_tokens", 0) or 0 + cached = getattr(session, "total_cache_tokens", 0) or 0 + new_input = max(0, raw_input - cached) + output = getattr(session, "total_output_tokens", 0) or 0 + total = new_input + output + + message = ( + "Session token usage\n" + f" Input: {new_input:,}\n" + f" Cached: {cached:,}\n" + f" Output: {output:,}\n" + f" Total: {total:,}" + ) + return CommandResult( + success=True, + message=message, + data={ + "session_id": target, + "input": new_input, + "raw_input": raw_input, + "cached": cached, + "output": output, + "total": total, + }, + ) diff --git a/app/ui_layer/controller/ui_controller.py b/app/ui_layer/controller/ui_controller.py index 87152add..7cdddf16 100644 --- a/app/ui_layer/controller/ui_controller.py +++ b/app/ui_layer/controller/ui_controller.py @@ -433,6 +433,7 @@ def _register_builtin_commands(self) -> None: SkillCommand, CredCommand, UpdateCommand, + TokensCommand, ) self._command_registry.register(HelpCommand(self)) @@ -445,6 +446,7 @@ def _register_builtin_commands(self) -> None: self._command_registry.register(SkillCommand(self)) self._command_registry.register(CredCommand(self)) self._command_registry.register(UpdateCommand(self)) + self._command_registry.register(TokensCommand(self)) # Register integration commands self._register_integration_commands() diff --git a/app/usage/task_attribution.py b/app/usage/task_attribution.py index 93dc0e43..028a8196 100644 --- a/app/usage/task_attribution.py +++ b/app/usage/task_attribution.py @@ -44,11 +44,25 @@ def attribute_usage_to_current_task(event: UsageEventData) -> None: event.cached_tokens or 0 ) + # Cumulative session totals — survive reset_run_counters(), which + # clears the per-run counters above at the start of every run. + session.total_input_tokens = (session.total_input_tokens or 0) + int( + event.input_tokens or 0 + ) + session.total_output_tokens = (session.total_output_tokens or 0) + int( + event.output_tokens or 0 + ) + session.total_cache_tokens = (session.total_cache_tokens or 0) + int( + event.cached_tokens or 0 + ) + logger.info( f"[TOKEN_ATTR] session={session.id} +in={event.input_tokens} " f"+out={event.output_tokens} +cached={event.cached_tokens} " - f"-> totals: in={session.input_tokens} out={session.output_tokens} " - f"cache={session.cache_tokens}" + f"-> run: in={session.input_tokens} out={session.output_tokens} " + f"cache={session.cache_tokens} " + f"| session: in={session.total_input_tokens} " + f"out={session.total_output_tokens} cache={session.total_cache_tokens}" ) bus = STATE.event_bus diff --git a/mkdocs/docs/core/commands/builtin.md b/mkdocs/docs/core/commands/builtin.md index 7f2e5b49..ec1e1f89 100644 --- a/mkdocs/docs/core/commands/builtin.md +++ b/mkdocs/docs/core/commands/builtin.md @@ -17,6 +17,7 @@ At a glance: | [`/skill `](#skill) | — | Manage skills | | [`/cred `](#cred) | — | Credentials and integration status | | [`/update [--check]`](#update) | `/upgrade` | Check for and install updates | +| [`/tokens`](#tokens) | — | Show this session's token usage | Beyond these, the registry also holds [integration commands](#integration-commands) (`/gmail`, `/slack`, ...) and [skill commands](#skill-commands) (`/pdf`, `/docx`, ...), covered at the end. @@ -141,6 +142,20 @@ Read-only overview of credentials and integrations. Connecting happens with the An update pulls the latest code, installs dependencies, and restarts CraftBot automatically, streaming progress as system messages. If you're already current, it says so. +## /tokens + +Prints the cumulative token usage of the chat it's typed in, as a system message: + +``` +Session token usage + Input: 79,022 + Cached: 312,455 + Output: 5,770 + Total: 84,792 +``` + +**Input** is genuinely new prompt tokens (cache reads excluded), **Cached** is prompt tokens served from the provider's cache, and **Total** is Input + Output. Totals accumulate across every run in the session and survive restarts and [`/clear`](#clear) (clearing wipes the conversation, not the session's lifetime counters). A brand-new chat that hasn't sent a message yet reports zeros. Sessions created before this command existed start counting from their next run. + ## Integration commands Every available [integration](../../integrations/index.md) registers its own command named after itself: `/gmail`, `/slack`, `/discord`, `/telegram_bot`, `/notion`, and so on (run `/cred integrations` for the live list). Each supports: diff --git a/tests/test_bedrock_token_normalization.py b/tests/test_bedrock_token_normalization.py new file mode 100644 index 00000000..0f1690e2 --- /dev/null +++ b/tests/test_bedrock_token_normalization.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +"""Bedrock usage extraction must match the Anthropic shape. + +Bedrock's Converse API reports `inputTokens` EXCLUSIVE of cache activity, +while the Anthropic API path reassembles input as base + creation + read. +The display layer (dashboard, /tokens) assumes `cached` is a subset of +`input` for every provider, so the Bedrock paths normalize: +input = inputTokens + cacheRead + cacheWrite, cached = cacheRead only. +""" + +from typing import Any, Dict + +from agent_core.core.impl.llm.interface import LLMInterface +from agent_core.core.impl.vlm.interface import VLMInterface + + +BEDROCK_USAGE = { + "inputTokens": 100, # new tokens only — AWS excludes cache activity + "outputTokens": 50, + "cacheReadInputTokens": 900, + "cacheWriteInputTokens": 30, +} + + +class _StubBedrockClient: + def converse(self, **kwargs) -> Dict[str, Any]: + return { + "output": {"message": {"content": [{"text": "hello"}]}}, + "usage": dict(BEDROCK_USAGE), + } + + +def _stub_common(iface) -> Dict[str, Any]: + """Set the attributes the bedrock call paths touch; capture usage reports.""" + reported = {} + + def _capture(call_kind, provider, model, input_tokens, output_tokens, cached, *a, **kw): + reported.update( + input=input_tokens, output=output_tokens, cached=cached + ) + + iface._bedrock_client = _StubBedrockClient() + iface.model = "anthropic.claude-3-5-sonnet-20241022-v2:0" # caching-capable + iface.provider = "bedrock" + iface.temperature = 0.0 + iface.max_tokens = 1024 + iface._report_usage_async = _capture + return reported + + +def test_llm_bedrock_input_includes_cache_and_cached_is_reads_only(): + iface = LLMInterface.__new__(LLMInterface) + reported = _stub_common(iface) + iface._call_log_to_db = lambda *a, **kw: None + + result = iface._generate_bedrock(None, "hi") + + assert "error" not in result + # input = 100 + 900 + 30, the full prompt; cached = reads only + assert reported == {"input": 1030, "output": 50, "cached": 900} + assert result["cached_tokens"] == 900 + assert result["tokens_used"] == 1080 # 1030 + 50 + # the display invariant the dashboard and /tokens rely on + assert reported["cached"] <= reported["input"] + + +def test_vlm_bedrock_input_includes_cache_and_cached_is_reads_only(): + iface = VLMInterface.__new__(VLMInterface) + reported = _stub_common(iface) + + png = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 + result = iface._bedrock_describe_bytes(png, None, "describe") + + assert reported == {"input": 1030, "output": 50, "cached": 900} + assert result["cached_tokens"] == 900 + assert result["tokens_used"] == 1080 + assert reported["cached"] <= reported["input"] diff --git a/tests/test_tokens_command.py b/tests/test_tokens_command.py new file mode 100644 index 00000000..26c19dfd --- /dev/null +++ b/tests/test_tokens_command.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +"""Checks for /tokens: cumulative counters and the display math.""" + +import asyncio +from types import SimpleNamespace + +from agent_core.core.session import Session +from app.ui_layer.commands.builtin.tokens import TokensCommand + + +def _run(session): + """Execute /tokens against a stub controller holding `session`.""" + controller = SimpleNamespace( + agent=SimpleNamespace( + session_manager=SimpleNamespace(get=lambda _id: session) + ) + ) + return asyncio.run(TokensCommand(controller).execute([], session_id="s1")) + + +def test_totals_survive_run_reset(): + """reset_run_counters() clears per-run counters but not the totals.""" + s = Session(id="s1") + s.input_tokens, s.output_tokens, s.cache_tokens = 100, 20, 80 + s.total_input_tokens, s.total_output_tokens, s.total_cache_tokens = 100, 20, 80 + + s.reset_run_counters() + + assert s.input_tokens == 0 + assert (s.total_input_tokens, s.total_output_tokens, s.total_cache_tokens) == ( + 100, + 20, + 80, + ) + + +def test_totals_round_trip_through_dict(): + """Persistence goes through session_json, so to_dict/from_dict must carry them.""" + s = Session(id="s1") + s.total_input_tokens, s.total_output_tokens, s.total_cache_tokens = 100, 20, 80 + + restored = Session.from_dict(s.to_dict()) + + assert restored.total_input_tokens == 100 + assert restored.total_output_tokens == 20 + assert restored.total_cache_tokens == 80 + + +def test_old_sessions_load_as_zero(): + """Sessions persisted before this feature have no totals key.""" + restored = Session.from_dict({"id": "s1"}) + + assert restored.total_input_tokens == 0 + + +def test_input_excludes_cached_and_total_excludes_both(): + """cached is a subset of input; total counts new tokens only.""" + s = Session(id="s1") + s.total_input_tokens, s.total_output_tokens, s.total_cache_tokens = 100, 20, 80 + + d = _run(s).data + + assert d["input"] == 20 # 100 - 80 + assert d["raw_input"] == 100 # full prompt, cache reads included + assert d["cached"] == 80 + assert d["output"] == 20 + assert d["total"] == 40 # 20 + 20, cached excluded + + +def test_input_clamps_when_cached_exceeds_input(): + """A provider over-reporting cache reads must not yield a negative count.""" + s = Session(id="s1") + s.total_input_tokens, s.total_cache_tokens = 50, 80 + + d = _run(s).data + + assert d["input"] == 0 + assert d["total"] == 0 + + +def test_unsaved_session_reads_as_zero(): + """A brand-new chat has no session yet (id is "new") — report zeros.""" + result = _run(None) + + assert result.success is True + assert result.data["input"] == 0 + assert result.data["cached"] == 0 + assert result.data["output"] == 0 + assert result.data["total"] == 0