From 24b8752a5c530f295666757027011c5e20164ffc Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:14:54 -0400 Subject: [PATCH 1/9] docs: spec and plan for the dollar-denominated budget gate --- .../plans/2026-08-18-dollar-budget-gate.md | 1179 +++++++++++++++++ .../2026-08-18-dollar-budget-gate-design.md | 127 ++ 2 files changed, 1306 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-18-dollar-budget-gate.md create mode 100644 docs/superpowers/specs/2026-08-18-dollar-budget-gate-design.md diff --git a/docs/superpowers/plans/2026-08-18-dollar-budget-gate.md b/docs/superpowers/plans/2026-08-18-dollar-budget-gate.md new file mode 100644 index 0000000..d37699a --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-dollar-budget-gate.md @@ -0,0 +1,1179 @@ +# Dollar-Denominated Budget Gate Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let each brain optionally cap on real USD spend (`DAILY_USD_`), layered on top of +the existing daily token cap rather than replacing it. + +**Architecture:** Four new `Settings` fields feed a second, independent check in +`LLM.complete()` — the token cap keeps enforcing unconditionally exactly as today; if a brain also +has a `$` cap configured (`> 0`), a second read of `store.cost_today(brain)` gates the call too. +Whichever trips first wins. The same layered read feeds the existing ops-alert threshold and the +`/status` readout, so all three budget-aware surfaces (gate, alert, status) stay consistent. + +**Tech Stack:** Python 3.14, pydantic-settings, pytest/pytest-asyncio, prometheus_client. No new +dependencies. + +## Global Constraints + +- `DAILY_USD_` defaults to `0.0` (disabled/opt-in) — matches the `gigabrain_interval_days = 0` + convention already in `roger/config.py`. No behavior change on deploy until explicitly set. +- Token cap always enforces, unconditionally, regardless of whether a `$` cap is set. This is the + approved design (see `docs/superpowers/specs/2026-08-18-dollar-budget-gate-design.md`) — never + make the `$` cap replace the token cap. +- Match existing code style exactly: docstrings in the file's existing voice, `f"..."` formatting + conventions already used nearby, no new abstractions. +- Every task must leave `pytest` and `ruff check .` green before its commit. + +--- + +### Task 1: `DAILY_USD_` config + +**Files:** +- Modify: `roger/config.py:39-43` +- Modify: `roger.env.example:27-31` +- Test: `tests/test_config.py` + +**Interfaces:** +- Produces: `Settings.daily_usd_admin`, `Settings.daily_usd_ambient`, `Settings.daily_usd_digest`, + `Settings.daily_usd_gigabrain` — all `float`, default `0.0`. Consumed by Tasks 2, 3, 5, 6. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_config.py`: + +```python +def test_daily_usd_defaults_to_disabled(monkeypatch): + _set_required(monkeypatch) + settings = Settings() + assert settings.daily_usd_admin == 0.0 + assert settings.daily_usd_ambient == 0.0 + assert settings.daily_usd_digest == 0.0 + assert settings.daily_usd_gigabrain == 0.0 + + +def test_daily_usd_parses_from_env(monkeypatch): + _set_required(monkeypatch) + monkeypatch.setenv("DAILY_USD_ADMIN", "2.5") + assert Settings().daily_usd_admin == 2.5 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/test_config.py -v -k daily_usd` +Expected: FAIL — `AttributeError: 'Settings' object has no attribute 'daily_usd_admin'` + +- [ ] **Step 3: Add the settings fields** + +In `roger/config.py`, find: + +```python + # --- budgets (daily in+out tokens per brain) --- + daily_tokens_admin: int = 150_000 + daily_tokens_ambient: int = 40_000 + daily_tokens_digest: int = 30_000 + daily_tokens_gigabrain: int = 100_000 +``` + +Replace with: + +```python + # --- budgets (daily in+out tokens per brain) --- + daily_tokens_admin: int = 150_000 + daily_tokens_ambient: int = 40_000 + daily_tokens_digest: int = 30_000 + daily_tokens_gigabrain: int = 100_000 + + # --- budgets (daily USD, layered on top of the token caps above) --- + # 0 = disabled (opt-in); set to a real figure once OpenRouter cost data looks right for your + # model mix. The token cap above keeps enforcing regardless — this is an additional, tighter + # trip wire, not a replacement (a provider that never reports cost would otherwise leave the + # brain with no effective cap at all). + daily_usd_admin: float = 0.0 + daily_usd_ambient: float = 0.0 + daily_usd_digest: float = 0.0 + daily_usd_gigabrain: float = 0.0 +``` + +- [ ] **Step 4: Add the env template block** + +In `roger.env.example`, find: + +``` +# --- budgets (daily in+out tokens per brain) --- +DAILY_TOKENS_ADMIN=150000 +DAILY_TOKENS_AMBIENT=40000 +DAILY_TOKENS_DIGEST=30000 +DAILY_TOKENS_GIGABRAIN=100000 +``` + +Replace with: + +``` +# --- budgets (daily in+out tokens per brain) --- +DAILY_TOKENS_ADMIN=150000 +DAILY_TOKENS_AMBIENT=40000 +DAILY_TOKENS_DIGEST=30000 +DAILY_TOKENS_GIGABRAIN=100000 + +# --- budgets (daily USD, layered on top of the token caps above; 0 = disabled/opt-in) --- +# the token cap keeps enforcing regardless of these — set a real figure per brain once you know +# what your model mix actually costs. +DAILY_USD_ADMIN=0 +DAILY_USD_AMBIENT=0 +DAILY_USD_DIGEST=0 +DAILY_USD_GIGABRAIN=0 +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pytest tests/test_config.py -v` +Expected: PASS (all tests in the file) + +- [ ] **Step 6: Commit** + +```bash +git add roger/config.py roger.env.example tests/test_config.py +git commit -m "feat: add DAILY_USD_* config for the dollar budget gate" +``` + +--- + +### Task 2: budget metrics — USD cap gauge + unit-aware counter + +**Files:** +- Modify: `roger/metrics.py` +- Test: `tests/test_metrics.py` + +**Interfaces:** +- Consumes: `Settings.daily_usd_*` (Task 1). +- Produces: `metrics.COST_USD_CAP` gauge (labels: `brain`); `metrics.LLM_BUDGET_EXCEEDED` counter now + takes two labels, `.labels(brain, reason)` where `reason` is `"tokens"` or `"usd"` — Task 3's call + sites depend on this exact label arity. + +- [ ] **Step 1: Write the failing tests** + +In `tests/test_metrics.py`, find: + +```python +def _settings(): + return SimpleNamespace( + daily_tokens_admin=150_000, + daily_tokens_ambient=40_000, + daily_tokens_digest=30_000, + daily_tokens_gigabrain=100_000, + ) +``` + +Replace with: + +```python +def _settings(): + return SimpleNamespace( + daily_tokens_admin=150_000, + daily_tokens_ambient=40_000, + daily_tokens_digest=30_000, + daily_tokens_gigabrain=100_000, + daily_usd_admin=0.0, + daily_usd_ambient=0.0, + daily_usd_digest=0.0, + daily_usd_gigabrain=0.0, + ) +``` + +Then append a new test at the end of the file: + +```python +async def test_refresh_populates_usd_cap_gauge(tmp_path): + store = await Store(str(tmp_path / "m.db")).open() + try: + settings = _settings() + settings.daily_usd_admin = 2.5 + + await metrics.refresh(store, settings, "sha-test") + + get = REGISTRY.get_sample_value + assert get("roger_cost_usd_cap", {"brain": "admin"}) == 2.5 + assert get("roger_cost_usd_cap", {"brain": "ambient"}) == 0.0 + finally: + await store.close() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/test_metrics.py -v -k usd_cap` +Expected: FAIL — `AssertionError` (metric `roger_cost_usd_cap` doesn't exist yet, `get_sample_value` +returns `None`) + +- [ ] **Step 3: Add the gauge and widen the counter's labels** + +In `roger/metrics.py`, find: + +```python +LLM_BUDGET_EXCEEDED = Counter( + "roger_llm_budget_exceeded_total", "Calls refused by the daily token budget", ["brain"] +) + +# --- state gauges (refreshed from SQLite on a timer) --- +TOKENS_TODAY = Gauge("roger_tokens_today", "Tokens spent today", ["brain"]) +TOKENS_CAP = Gauge("roger_tokens_cap", "Daily token cap", ["brain"]) +COST_USD_TODAY = Gauge("roger_cost_usd_today", "USD spent today (OpenRouter-reported)", ["brain"]) +``` + +Replace with: + +```python +LLM_BUDGET_EXCEEDED = Counter( + "roger_llm_budget_exceeded_total", + "Calls refused by the daily budget", + ["brain", "reason"], # reason: "tokens" or "usd" +) + +# --- state gauges (refreshed from SQLite on a timer) --- +TOKENS_TODAY = Gauge("roger_tokens_today", "Tokens spent today", ["brain"]) +TOKENS_CAP = Gauge("roger_tokens_cap", "Daily token cap", ["brain"]) +COST_USD_TODAY = Gauge("roger_cost_usd_today", "USD spent today (OpenRouter-reported)", ["brain"]) +COST_USD_CAP = Gauge("roger_cost_usd_cap", "Daily USD cap (0 = disabled)", ["brain"]) +``` + +- [ ] **Step 4: Refresh the new gauge** + +In `roger/metrics.py`, find: + +```python +async def refresh(store: Any, settings: Any, version: str) -> None: + """Repopulate the SQLite-sourced gauges. Cheap; called once at startup and then on a timer.""" + caps = { + "admin": settings.daily_tokens_admin, + "ambient": settings.daily_tokens_ambient, + "digest": settings.daily_tokens_digest, + "gigabrain": settings.daily_tokens_gigabrain, + } + for brain in _BRAINS: + TOKENS_TODAY.labels(brain).set(await store.usage_today(brain)) + COST_USD_TODAY.labels(brain).set(await store.cost_today(brain)) + TOKENS_CAP.labels(brain).set(caps[brain]) +``` + +Replace with: + +```python +async def refresh(store: Any, settings: Any, version: str) -> None: + """Repopulate the SQLite-sourced gauges. Cheap; called once at startup and then on a timer.""" + caps = { + "admin": settings.daily_tokens_admin, + "ambient": settings.daily_tokens_ambient, + "digest": settings.daily_tokens_digest, + "gigabrain": settings.daily_tokens_gigabrain, + } + usd_caps = { + "admin": settings.daily_usd_admin, + "ambient": settings.daily_usd_ambient, + "digest": settings.daily_usd_digest, + "gigabrain": settings.daily_usd_gigabrain, + } + for brain in _BRAINS: + TOKENS_TODAY.labels(brain).set(await store.usage_today(brain)) + COST_USD_TODAY.labels(brain).set(await store.cost_today(brain)) + TOKENS_CAP.labels(brain).set(caps[brain]) + COST_USD_CAP.labels(brain).set(usd_caps[brain]) +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pytest tests/test_metrics.py -v` +Expected: PASS (all tests in the file — note `test_completion_increments_the_request_counter` and +the other pre-existing tests never call `.labels()` on `LLM_BUDGET_EXCEEDED` directly, so the wider +label set doesn't touch them; that call site is fixed in Task 3) + +- [ ] **Step 6: Commit** + +```bash +git add roger/metrics.py tests/test_metrics.py +git commit -m "feat: add USD-cap gauge and reason label to budget metrics" +``` + +--- + +### Task 3: layer the dollar cap into `LLM.complete` + +**Files:** +- Modify: `roger/llm.py` +- Test: `tests/test_llm.py` + +**Interfaces:** +- Consumes: `Settings.daily_usd_*` (Task 1), `metrics.LLM_BUDGET_EXCEEDED.labels(brain, reason)` + (Task 2), `Store.cost_today(brain) -> float` (already exists in `roger/store.py`). +- Produces: `BudgetExceeded(brain, used, cap, *, unit="tokens")` — `unit` is `"tokens"` or `"usd"`, + stored as `.unit` on the instance. Consumed by Task 4. + +- [ ] **Step 1: Write the failing tests** + +In `tests/test_llm.py`, find: + +```python +async def test_budget_exceeded_before_network(monkeypatch, tmp_path): + _env(monkeypatch, MODEL_ADMIN="a/b", DAILY_TOKENS_ADMIN="10") + store = await Store(str(tmp_path / "l.db")).open() + try: + await store.add_usage("admin", 8, 5) # 13 >= 10 + llm = LLM(Settings(), store) + with pytest.raises(BudgetExceeded): + await llm.complete("admin", [{"role": "user", "content": "hi"}]) + finally: + await store.close() +``` + +Replace with (adds a `unit` assertion to the existing test, then two new tests directly after it): + +```python +async def test_budget_exceeded_before_network(monkeypatch, tmp_path): + _env(monkeypatch, MODEL_ADMIN="a/b", DAILY_TOKENS_ADMIN="10") + store = await Store(str(tmp_path / "l.db")).open() + try: + await store.add_usage("admin", 8, 5) # 13 >= 10 + llm = LLM(Settings(), store) + with pytest.raises(BudgetExceeded) as exc_info: + await llm.complete("admin", [{"role": "user", "content": "hi"}]) + assert exc_info.value.unit == "tokens" + finally: + await store.close() + + +async def test_usd_budget_exceeded_before_network(monkeypatch, tmp_path): + # Token cap (default 150k) is nowhere near tripped — only the $ cap should fire. + _env(monkeypatch, MODEL_ADMIN="a/b", DAILY_USD_ADMIN="1.0") + store = await Store(str(tmp_path / "l.db")).open() + try: + await store.add_usage("admin", 1, 1, cost_usd=1.5) # $1.50 >= $1.00 cap + llm = LLM(Settings(), store) + with pytest.raises(BudgetExceeded) as exc_info: + await llm.complete("admin", [{"role": "user", "content": "hi"}]) + assert exc_info.value.unit == "usd" + finally: + await store.close() + + +async def test_usd_cap_does_not_trip_below_threshold(monkeypatch, tmp_path): + _env(monkeypatch, MODEL_ADMIN="a/b", DAILY_USD_ADMIN="1.0") + store = await Store(str(tmp_path / "l.db")).open() + try: + await store.add_usage("admin", 1, 1, cost_usd=0.5) # $0.50 < $1.00 cap + llm = LLM(Settings(), store) + + async def fake_create(**kwargs): + return SimpleNamespace( + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, cost=0.1) + ) + + monkeypatch.setattr(llm._client.chat.completions, "create", fake_create) + await llm.complete("admin", [{"role": "user", "content": "hi"}]) # does not raise + finally: + await store.close() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/test_llm.py -v -k "budget or usd_cap"` +Expected: FAIL — `exc_info.value.unit` raises `AttributeError` (no `unit` on `BudgetExceeded` yet); +the new `$`-cap tests never trip since `LLM.complete` doesn't check `cost_today` yet +(`test_usd_budget_exceeded_before_network` fails because no exception is raised at all) + +- [ ] **Step 3: Make `BudgetExceeded` unit-aware** + +In `roger/llm.py`, find: + +```python +class BudgetExceeded(RuntimeError): + def __init__(self, brain: str, used: int, cap: int) -> None: + super().__init__(f"{brain} daily token budget exceeded ({used} >= {cap})") + self.brain = brain + self.used = used + self.cap = cap +``` + +Replace with: + +```python +class BudgetExceeded(RuntimeError): + def __init__(self, brain: str, used: float, cap: float, *, unit: str = "tokens") -> None: + if unit == "usd": + message = f"{brain} daily $ budget exceeded (${used:.4f} >= ${cap:.4f})" + else: + message = f"{brain} daily token budget exceeded ({used} >= {cap})" + super().__init__(message) + self.brain = brain + self.used = used + self.cap = cap + self.unit = unit +``` + +- [ ] **Step 4: Track the USD caps in `LLM.__init__`** + +In `roger/llm.py`, find: + +```python + self._caps = { + "admin": settings.daily_tokens_admin, + "ambient": settings.daily_tokens_ambient, + "digest": settings.daily_tokens_digest, + "gigabrain": settings.daily_tokens_gigabrain, + } + # Opt-in OpenRouter `reasoning.effort` passthrough — only gigabrain ever sets this today. +``` + +Replace with: + +```python + self._caps = { + "admin": settings.daily_tokens_admin, + "ambient": settings.daily_tokens_ambient, + "digest": settings.daily_tokens_digest, + "gigabrain": settings.daily_tokens_gigabrain, + } + self._usd_caps = { + "admin": settings.daily_usd_admin, + "ambient": settings.daily_usd_ambient, + "digest": settings.daily_usd_digest, + "gigabrain": settings.daily_usd_gigabrain, + } + # Opt-in OpenRouter `reasoning.effort` passthrough — only gigabrain ever sets this today. +``` + +- [ ] **Step 5: Add the layered `$` check to `complete`** + +In `roger/llm.py`, find: + +```python + used = await self._store.usage_today(brain) + cap = self._caps[brain] + if used >= cap: + metrics.LLM_BUDGET_EXCEEDED.labels(brain).inc() + raise BudgetExceeded(brain, used, cap) +``` + +Replace with: + +```python + used = await self._store.usage_today(brain) + cap = self._caps[brain] + if used >= cap: + metrics.LLM_BUDGET_EXCEEDED.labels(brain, "tokens").inc() + raise BudgetExceeded(brain, used, cap, unit="tokens") + + usd_cap = self._usd_caps[brain] + if usd_cap > 0: + spent = await self._store.cost_today(brain) + if spent >= usd_cap: + metrics.LLM_BUDGET_EXCEEDED.labels(brain, "usd").inc() + raise BudgetExceeded(brain, spent, usd_cap, unit="usd") +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `pytest tests/test_llm.py -v` +Expected: PASS (all tests in the file) + +- [ ] **Step 7: Commit** + +```bash +git add roger/llm.py tests/test_llm.py +git commit -m "feat: layer a dollar cap on top of the token budget gate" +``` + +--- + +### Task 4: unit-aware budget messages in the four brains + +**Files:** +- Modify: `roger/brains/admin.py:167-176` +- Modify: `roger/brains/gigabrain.py:170-179` +- Modify: `roger/brains/digest.py:99-101` +- Test: `tests/test_admin.py`, `tests/test_gigabrain.py` + +**Interfaces:** +- Consumes: `BudgetExceeded.unit` (Task 3). + +`roger/brains/ambient.py`'s `BUDGET_LINE` ("I'm out of words for today.") is already unit-agnostic — +no change needed there. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_admin.py`: + +```python +async def test_budget_exceeded_audit_detail_reflects_unit(tmp_path): + store = await _open_store(tmp_path) + try: + llm = FakeLLM([BudgetExceeded("admin", 2.5, 2.0, unit="usd")]) + await admin.handle_admin_request( + request="anything", guild=object(), actor_id=1, llm=llm, store=store + ) + rows = await store.fetch_audit() + assert any(r["detail"] == "daily usd cap" for r in rows) + finally: + await store.close() +``` + +Append to `tests/test_gigabrain.py`: + +```python +async def test_budget_exceeded_audit_detail_reflects_unit(tmp_path): + store = await _open_store(tmp_path) + try: + llm = FakeLLM([BudgetExceeded("gigabrain", 2.5, 2.0, unit="usd")]) + await gigabrain.handle_gigabrain_request( + request="anything", guild=object(), actor_id=1, llm=llm, store=store + ) + rows = await store.fetch_audit() + assert any(r["detail"] == "daily usd cap" for r in rows) + finally: + await store.close() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/test_admin.py tests/test_gigabrain.py -v -k unit` +Expected: FAIL — `AssertionError` (audit `detail` is still the hardcoded `"daily token cap"`) + +- [ ] **Step 3: Fix `admin.py`** + +In `roger/brains/admin.py`, find: + +```python + except BudgetExceeded: + await store.record_audit( + actor_id=actor_id, + brain="admin", + tool=None, + args={"request": request}, + status=AuditStatus.ERROR, + detail="daily token cap", + ) + return "I've hit my daily token budget for admin work. Try again tomorrow." +``` + +Replace with: + +```python + except BudgetExceeded as exc: + await store.record_audit( + actor_id=actor_id, + brain="admin", + tool=None, + args={"request": request}, + status=AuditStatus.ERROR, + detail=f"daily {exc.unit} cap", + ) + return "I've hit my daily budget for admin work. Try again tomorrow." +``` + +- [ ] **Step 4: Fix `gigabrain.py`** + +In `roger/brains/gigabrain.py`, find: + +```python + except BudgetExceeded: + await store.record_audit( + actor_id=actor_id, + brain="gigabrain", + tool=None, + args={"request": request}, + status=AuditStatus.ERROR, + detail="daily token cap", + ) + return "I've hit my daily token budget for gigabrain work. Try again tomorrow." +``` + +Replace with: + +```python + except BudgetExceeded as exc: + await store.record_audit( + actor_id=actor_id, + brain="gigabrain", + tool=None, + args={"request": request}, + status=AuditStatus.ERROR, + detail=f"daily {exc.unit} cap", + ) + return "I've hit my daily budget for gigabrain work. Try again tomorrow." +``` + +- [ ] **Step 5: Fix `digest.py`** + +In `roger/brains/digest.py`, find: + +```python + try: + summary = await _summarize(entries, llm) + except BudgetExceeded: + log.warning("digest skipped: daily token budget hit") + return {"status": "budget exceeded; skipped"} +``` + +Replace with: + +```python + try: + summary = await _summarize(entries, llm) + except BudgetExceeded as exc: + log.warning("digest skipped: daily %s budget hit", exc.unit) + return {"status": "budget exceeded; skipped"} +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `pytest tests/test_admin.py tests/test_gigabrain.py tests/test_digest.py tests/test_ambient.py -v` +Expected: PASS (all tests in these four files — the pre-existing +`test_budget_exceeded_returns_polite_refusal` tests only assert `"budget" in out.lower()`, which +still holds with "daily budget" in place of "daily token budget") + +- [ ] **Step 7: Commit** + +```bash +git add roger/brains/admin.py roger/brains/gigabrain.py roger/brains/digest.py \ + tests/test_admin.py tests/test_gigabrain.py +git commit -m "fix: make budget-exceeded messages unit-aware, not token-only" +``` + +--- + +### Task 5: ops alert watches the dollar cap + +**Files:** +- Modify: `roger/bot.py:261-268` (add `_daily_usd_caps`) +- Modify: `roger/bot.py:397-407` (`_budget_alert`) +- Modify: `roger/bot.py:714-724` (watchdog loop) +- Test: `tests/test_ops.py` + +**Interfaces:** +- Consumes: `Settings.daily_usd_*` (Task 1). +- Produces: `_daily_usd_caps(settings) -> dict[str, float]` (mirrors `_daily_caps`, used by Task 6 + too). `_budget_alert` gains a keyword-only `usd_cap: float = 0.0` param — default preserves every + existing call site and test unchanged. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_ops.py`: + +```python +def test_budget_alert_fires_from_usd_cap_alone(): + # tokens are nowhere near their cap (10%); the $ cap (84%) is what should trip this. + msg = _budget_alert("gigabrain", 10_000, 100_000, 4.2, usd_cap=5.0) + assert msg is not None + assert "84%" in msg and "$4.2000 / $5.0000" in msg + + +def test_budget_alert_usd_exhausted_reads_as_exhausted(): + msg = _budget_alert("gigabrain", 1_000, 100_000, 6.0, usd_cap=5.0) + assert msg is not None and "exhausted" in msg + + +def test_budget_alert_silent_when_both_caps_disabled(): + assert _budget_alert("admin", 1_000_000, 0, 999.0, usd_cap=0.0) is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/test_ops.py -v -k usd_cap` +Expected: FAIL — `TypeError: _budget_alert() got an unexpected keyword argument 'usd_cap'` + +- [ ] **Step 3: Add `_daily_usd_caps`** + +In `roger/bot.py`, find: + +```python +def _daily_caps(settings: Settings) -> dict[str, int]: + """Per-brain daily token caps, keyed by brain (shared by /status and the watchdog).""" + return { + "admin": settings.daily_tokens_admin, + "ambient": settings.daily_tokens_ambient, + "digest": settings.daily_tokens_digest, + "gigabrain": settings.daily_tokens_gigabrain, + } +``` + +Replace with: + +```python +def _daily_caps(settings: Settings) -> dict[str, int]: + """Per-brain daily token caps, keyed by brain (shared by /status and the watchdog).""" + return { + "admin": settings.daily_tokens_admin, + "ambient": settings.daily_tokens_ambient, + "digest": settings.daily_tokens_digest, + "gigabrain": settings.daily_tokens_gigabrain, + } + + +def _daily_usd_caps(settings: Settings) -> dict[str, float]: + """Per-brain daily USD caps, keyed by brain (0 = disabled). Mirrors `_daily_caps`.""" + return { + "admin": settings.daily_usd_admin, + "ambient": settings.daily_usd_ambient, + "digest": settings.daily_usd_digest, + "gigabrain": settings.daily_usd_gigabrain, + } +``` + +- [ ] **Step 4: Make `_budget_alert` watch both caps** + +In `roger/bot.py`, find: + +```python +def _budget_alert( + brain: str, used: int, cap: int, cost: float, *, fraction: float = BUDGET_ALERT_FRACTION +) -> str | None: + """Alert text when ``brain`` crosses ``fraction`` of its daily token cap, else None (pure).""" + if cap <= 0 or used < fraction * cap: + return None + tokens = f"{used:,} / {cap:,} tokens today (${cost:.4f})" + if used >= cap: + return f"⚠️ **{brain} budget exhausted** — {tokens}. Calls refused until the daily reset." + pct = round(100 * used / cap) + return f"⚠️ **{brain} budget {pct}%** — {tokens}. Approaching the daily cap." +``` + +Replace with: + +```python +def _budget_alert( + brain: str, + used: int, + cap: int, + cost: float, + *, + usd_cap: float = 0.0, + fraction: float = BUDGET_ALERT_FRACTION, +) -> str | None: + """Alert text once ``brain`` crosses ``fraction`` of its daily token or $ cap, else None (pure). + + The two caps are independent — whichever fraction is worse decides both whether this fires and + the wording (exhausted vs. approaching). A disabled cap (``cap`` or ``usd_cap`` <= 0) contributes + 0 to that comparison, so it can never itself trigger an alert. + """ + token_frac = used / cap if cap > 0 else 0.0 + usd_frac = cost / usd_cap if usd_cap > 0 else 0.0 + worst = max(token_frac, usd_frac) + if worst < fraction: + return None + detail = f"{used:,} / {cap:,} tokens today" if cap > 0 else f"{used:,} tokens today" + if usd_cap > 0: + detail += f" · ${cost:.4f} / ${usd_cap:.4f}" + else: + detail += f" (${cost:.4f})" + if worst >= 1.0: + return f"⚠️ **{brain} budget exhausted** — {detail}. Calls refused until the daily reset." + pct = round(100 * worst) + return f"⚠️ **{brain} budget {pct}%** — {detail}. Approaching the daily cap." +``` + +- [ ] **Step 5: Wire the watchdog loop to pass the USD cap through** + +In `roger/bot.py`, find: + +```python + caps = _daily_caps(self.settings) + today = time.strftime("%Y-%m-%d") + for brain in _BRAINS: + message = _budget_alert( + brain, + await self.store.usage_today(brain), + caps[brain], + await self.store.cost_today(brain), + ) + if message: +``` + +Replace with: + +```python + caps = _daily_caps(self.settings) + usd_caps = _daily_usd_caps(self.settings) + today = time.strftime("%Y-%m-%d") + for brain in _BRAINS: + message = _budget_alert( + brain, + await self.store.usage_today(brain), + caps[brain], + await self.store.cost_today(brain), + usd_cap=usd_caps[brain], + ) + if message: +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `pytest tests/test_ops.py -v` +Expected: PASS (all tests in the file — the 4 pre-existing `_budget_alert` tests never pass +`usd_cap`, so it defaults to `0.0`; verify by hand: `max(token_frac, 0.0) == token_frac`, so their +behavior is byte-for-byte unchanged) + +- [ ] **Step 7: Commit** + +```bash +git add roger/bot.py tests/test_ops.py +git commit -m "feat: extend the ops budget alert to watch the dollar cap" +``` + +--- + +### Task 6: show the dollar cap in `/status` + +**Files:** +- Modify: `roger/bot.py:297-338` (`_format_status`) +- Modify: `roger/bot.py:341-362` (`gather_status`) +- Test: `tests/test_status.py` + +**Interfaces:** +- Consumes: `_daily_usd_caps` (Task 5), `Settings.daily_usd_*` (Task 1). +- Produces: `_format_status(..., usd_caps: dict[str, float] | None = None)` — default `None` (treated + as `{}`) keeps every existing call site working unchanged. + +- [ ] **Step 1: Write the failing tests** + +In `tests/test_status.py`, find: + +```python +def _settings(**over): + base = dict( + guild_id=9, + daily_tokens_admin=150000, + daily_tokens_ambient=40000, + daily_tokens_digest=30000, + daily_tokens_gigabrain=100000, + digest_hour=8, + digest_channel_id=42, + tz="UTC", + ) + base.update(over) + return SimpleNamespace(**base) +``` + +Replace with: + +```python +def _settings(**over): + base = dict( + guild_id=9, + daily_tokens_admin=150000, + daily_tokens_ambient=40000, + daily_tokens_digest=30000, + daily_tokens_gigabrain=100000, + daily_usd_admin=0.0, + daily_usd_ambient=0.0, + daily_usd_digest=0.0, + daily_usd_gigabrain=0.0, + digest_hour=8, + digest_channel_id=42, + tz="UTC", + ) + base.update(over) + return SimpleNamespace(**base) +``` + +Then append two new tests at the end of the file: + +```python +def test_format_status_shows_usd_cap_when_configured(): + body = _format_status( + guild_name="G", + missing_perms=[], + channel_problems=[], + usage={"admin": 1000}, + caps={"admin": 150000}, + cost={"admin": 0.5}, + usd_caps={"admin": 2.0}, + feeds_count=0, + recent_audit=[], + digest_hour=8, + digest_configured=True, + tz="UTC", + ) + assert "$0.5000 / $2.0000" in body + + +async def test_gather_status_shows_usd_cap_from_settings(tmp_path): + store = await Store(str(tmp_path / "s.db")).open() + try: + await store.add_usage("admin", 10, 10, cost_usd=0.25) + guild = _fake_guild(channels={42: _FakeChannel()}) + settings = _settings(daily_usd_admin=1.0) + body = await gather_status(store=store, settings=settings, guild=guild) + assert "$0.2500 / $1.0000" in body + finally: + await store.close() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/test_status.py -v -k usd_cap` +Expected: FAIL — `TypeError: _format_status() got an unexpected keyword argument 'usd_caps'` + +- [ ] **Step 3: Add `usd_caps` to `_format_status`** + +In `roger/bot.py`, find: + +```python +def _format_status( + *, + guild_name: str, + missing_perms: list[str], + channel_problems: list[str], + usage: dict[str, int], + caps: dict[str, int], + cost: dict[str, float], + feeds_count: int, + recent_audit: list[dict[str, Any]], + digest_hour: int, + digest_configured: bool, + tz: str, +) -> str: + """Render the /status readout body (pure). The caller wraps it in a code block.""" + perms = "OK" if not missing_perms else "MISSING: " + ", ".join(missing_perms) + channels = "OK" if not channel_problems else "; ".join(channel_problems) + lines = [ + f"roger status — {guild_name}", + f"permissions: {perms}", + f"channels: {channels}", + "spend today (tokens used / cap · cost):", + ] + total_cost = 0.0 + for brain in _BRAINS: + spent = cost.get(brain, 0.0) + total_cost += spent + lines.append( + f" {brain:<10}{usage.get(brain, 0):>8,} / {caps.get(brain, 0):<8,} ${spent:.4f}" + ) + lines.append(f" {'total':<29} ${total_cost:.4f}") +``` + +Replace with: + +```python +def _format_status( + *, + guild_name: str, + missing_perms: list[str], + channel_problems: list[str], + usage: dict[str, int], + caps: dict[str, int], + cost: dict[str, float], + feeds_count: int, + recent_audit: list[dict[str, Any]], + digest_hour: int, + digest_configured: bool, + tz: str, + usd_caps: dict[str, float] | None = None, +) -> str: + """Render the /status readout body (pure). The caller wraps it in a code block.""" + usd_caps = usd_caps or {} + perms = "OK" if not missing_perms else "MISSING: " + ", ".join(missing_perms) + channels = "OK" if not channel_problems else "; ".join(channel_problems) + lines = [ + f"roger status — {guild_name}", + f"permissions: {perms}", + f"channels: {channels}", + "spend today (tokens used / cap · cost):", + ] + total_cost = 0.0 + for brain in _BRAINS: + spent = cost.get(brain, 0.0) + total_cost += spent + cost_str = f"${spent:.4f}" + usd_cap = usd_caps.get(brain, 0.0) + if usd_cap > 0: + cost_str += f" / ${usd_cap:.4f}" + lines.append( + f" {brain:<10}{usage.get(brain, 0):>8,} / {caps.get(brain, 0):<8,} {cost_str}" + ) + lines.append(f" {'total':<29} ${total_cost:.4f}") +``` + +- [ ] **Step 4: Pass `usd_caps` through `gather_status`** + +In `roger/bot.py`, find: + +```python + usage = {brain: await store.usage_today(brain) for brain in _BRAINS} + cost = {brain: await store.cost_today(brain) for brain in _BRAINS} + caps = _daily_caps(settings) + return _format_status( + guild_name=guild_name, + missing_perms=missing, + channel_problems=channel_problems, + usage=usage, + caps=caps, + cost=cost, + feeds_count=await store.count_feeds(), +``` + +Replace with: + +```python + usage = {brain: await store.usage_today(brain) for brain in _BRAINS} + cost = {brain: await store.cost_today(brain) for brain in _BRAINS} + caps = _daily_caps(settings) + usd_caps = _daily_usd_caps(settings) + return _format_status( + guild_name=guild_name, + missing_perms=missing, + channel_problems=channel_problems, + usage=usage, + caps=caps, + cost=cost, + usd_caps=usd_caps, + feeds_count=await store.count_feeds(), +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pytest tests/test_status.py -v` +Expected: PASS (all tests in the file) + +- [ ] **Step 6: Commit** + +```bash +git add roger/bot.py tests/test_status.py +git commit -m "feat: show the dollar cap in /status when configured" +``` + +--- + +### Task 7: docs + full verification + +**Files:** +- Modify: `ARCHITECTURE.md` (§2.9, §11) +- Modify: `BACKLOG.md` (1.1) + +**Interfaces:** None — documentation only, no code. + +- [ ] **Step 1: Update §2.9** + +In `ARCHITECTURE.md`, find: + +``` +- **§2.9 Budgets.** A hard cap of **10 tool calls per request** (`ADMIN_MAX_TOOL_CALLS`) and **14 + model round-trips** (`ADMIN_MAX_TURNS`), plus per-brain **daily token caps** (§11) checked before + every call. Both caps are env-overridable per deployment; hitting the tool-call cap mid-request + logs a warning and posts once to the ops channel (if configured), and the model is told to say so + plainly — the cap resets on the next request, not on a timer. +``` + +Replace with: + +``` +- **§2.9 Budgets.** A hard cap of **10 tool calls per request** (`ADMIN_MAX_TOOL_CALLS`) and **14 + model round-trips** (`ADMIN_MAX_TURNS`), plus per-brain **daily token caps**, optionally layered + with a **daily USD cap** (§11), checked before every call. All caps are env-overridable per + deployment; hitting the tool-call cap mid-request logs a warning and posts once to the ops channel + (if configured), and the model is told to say so plainly — the cap resets on the next request, not + on a timer. +``` + +- [ ] **Step 2: Update §11** + +In `ARCHITECTURE.md`, find: + +``` +## §11 LLM layer & budgets + +`roger/llm.py` wraps the OpenAI SDK pointed at OpenRouter. Per call: pick the brain's model chain +(§3), **check the daily token cap before spending** (raises `BudgetExceeded` if over), call with +automatic fallback down the chain, then **record actual usage** to `usage`. A missing/empty model +chain raises `LLMConfigError`, which callers turn into a plain "not configured" reply rather than a +crash. Real spend is additionally bounded off-box by the OpenRouter key's own credit limit. + +Limits at a glance (defaults; all env-overridable): + +| Control | Default | +|---|---| +| Daily tokens — admin / ambient / digest / gigabrain | 150k / 40k / 30k / 100k | +| Tool calls per admin request | 10 | +| Model round-trips per admin request | 14 | +| Tool calls / round-trips per gigabrain request | 10 / 14 | +| Gigabrain periodic check-in interval | off (0 days) | +| Ambient — per user / window / global hourly | 5 / 600s / 30 | +``` + +Replace with: + +``` +## §11 LLM layer & budgets + +`roger/llm.py` wraps the OpenAI SDK pointed at OpenRouter. Per call: pick the brain's model chain +(§3), **check the daily token cap before spending** (raises `BudgetExceeded` if over), then — if a +`DAILY_USD_` cap is also set — check accumulated USD spend the same way. The two caps are +layered, not either/or: the token cap always enforces, and the USD cap is an additional, optional +trip wire on top of it. That's deliberate — a provider that never reports cost +(`OPENROUTER_BASE_URL` pointed elsewhere, ADR-0009) would otherwise leave the USD cap permanently +silent, so the token cap stays the real backstop in that case. Once both checks pass, the call +proceeds with automatic fallback down the chain, then **records actual usage** to `usage`. A +missing/empty model chain raises `LLMConfigError`, which callers turn into a plain "not configured" +reply rather than a crash. Real spend is additionally bounded off-box by the OpenRouter key's own +credit limit. + +Limits at a glance (defaults; all env-overridable): + +| Control | Default | +|---|---| +| Daily tokens — admin / ambient / digest / gigabrain | 150k / 40k / 30k / 100k | +| Daily USD — admin / ambient / digest / gigabrain | off / off / off / off (0 = disabled) | +| Tool calls per admin request | 10 | +| Model round-trips per admin request | 14 | +| Tool calls / round-trips per gigabrain request | 10 / 14 | +| Gigabrain periodic check-in interval | off (0 days) | +| Ambient — per user / window / global hourly | 5 / 600s / 30 | +``` + +- [ ] **Step 3: Close out BACKLOG.md 1.1** + +In `BACKLOG.md`, find: + +``` +### 1.1 Track spend in dollars, not just tokens — **M** — *visibility shipped; gate remains* +`llm.py` records `prompt_tokens` / `completion_tokens` per brain (`add_usage`) and the daily cap is a +raw token count. But a brain's model chain mixes models at very different prices, so a token budget +is a weak proxy for the thing that actually costs money. OpenRouter returns the real generation cost +(a `cost` field on the response `usage` object, always included now). + +- [x] Add a `cost_usd` column to the `usage` table (with an idempotent migration for live DBs); + capture the OpenRouter-reported cost per call in `LLM.complete`. *(a2689b5)* +- [x] Surface per-brain and total `$ today` in `/status`. *(a2689b5)* +- [ ] Make the daily gate dollar-denominated (env: `DAILY_USD_*`) with the token cap as the fallback + when a provider doesn't report cost. Deferred: enforcement is a semantic change, kept out of the + visibility commit. +``` + +Replace with: + +``` +### 1.1 Track spend in dollars, not just tokens — **M** — *shipped* +`llm.py` records `prompt_tokens` / `completion_tokens` per brain (`add_usage`) and the daily cap is a +raw token count. But a brain's model chain mixes models at very different prices, so a token budget +is a weak proxy for the thing that actually costs money. OpenRouter returns the real generation cost +(a `cost` field on the response `usage` object, always included now). + +- [x] Add a `cost_usd` column to the `usage` table (with an idempotent migration for live DBs); + capture the OpenRouter-reported cost per call in `LLM.complete`. *(a2689b5)* +- [x] Surface per-brain and total `$ today` in `/status`. *(a2689b5)* +- [x] Make the daily gate dollar-denominated (env: `DAILY_USD_*`), layered on top of the token cap + rather than replacing it — a provider that never reports cost leaves the token cap as the real + backstop, so nothing regresses for a non-OpenRouter host. +``` + +- [ ] **Step 4: Full verification** + +Run: `pytest --cov=roger --cov-report=term-missing --cov-fail-under=75` +Expected: PASS, all tests green, coverage still at/above 75% + +Run: `ruff check .` +Expected: `All checks passed!` + +- [ ] **Step 5: Commit** + +```bash +git add ARCHITECTURE.md BACKLOG.md +git commit -m "docs: record the dollar budget gate in ARCHITECTURE.md and BACKLOG.md" +``` diff --git a/docs/superpowers/specs/2026-08-18-dollar-budget-gate-design.md b/docs/superpowers/specs/2026-08-18-dollar-budget-gate-design.md new file mode 100644 index 0000000..5755a7c --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-dollar-budget-gate-design.md @@ -0,0 +1,127 @@ +# Dollar-denominated budget gate — design + +Tracks BACKLOG.md 1.1. Visibility (`cost_usd` tracked per brain/day, surfaced in `/status`) already +shipped; this closes the gap — the daily gate itself still only enforces on raw tokens, which is a +weak proxy once a brain's model chain mixes models at very different prices. + +## Goal + +Let each brain optionally cap on real USD spend instead of (or alongside) token count, without +losing the safety the token cap already provides when a provider doesn't report cost. + +## Semantics: layered, not replacing + +Token cap keeps enforcing unconditionally, exactly as today. If `DAILY_USD_` is also set for +a brain, a second, independent check runs against `cost_today(brain)`. Whichever trips first wins. + +This was chosen over "$ cap replaces token cap when set" because a provider that never reports cost +(e.g. `OPENROUTER_BASE_URL` pointed at a local host, per ADR-0009) would otherwise leave that brain +with no effective cap at all. Layering means the $ check simply never trips in that case, and the +token cap remains the real backstop — no code needs to detect "is cost data flowing." + +## Config + +`roger/config.py`, four new settings: + +``` +daily_usd_admin: float = 0.0 +daily_usd_ambient: float = 0.0 +daily_usd_digest: float = 0.0 +daily_usd_gigabrain: float = 0.0 +``` + +`0.0` = disabled/opt-in, matching the existing `gigabrain_interval_days = 0` convention. Defaults +stay 0 rather than shipping real dollar figures — no behavior change on the live host until the +owner sets a number that matches their actual OpenRouter spend. + +`roger.env.example` gets a `DAILY_USD_*` block next to the existing `DAILY_TOKENS_*` block, commented +to explain the opt-in default. + +## `BudgetExceeded` + +Gains a keyword-only `unit: str = "tokens"` field (`"tokens"` or `"usd"`), so callers can report +which cap actually tripped instead of assuming it was always tokens. Existing 2-positional-arg call +sites (tests, all four brain modules) stay valid unchanged since `unit` defaults. + +## Gate (`LLM.complete`) + +``` +used = await self._store.usage_today(brain) +cap = self._caps[brain] +if used >= cap: + metrics.LLM_BUDGET_EXCEEDED.labels(brain, "tokens").inc() + raise BudgetExceeded(brain, used, cap, unit="tokens") + +usd_cap = self._usd_caps[brain] +if usd_cap > 0: + spent = await self._store.cost_today(brain) + if spent >= usd_cap: + metrics.LLM_BUDGET_EXCEEDED.labels(brain, "usd").inc() + raise BudgetExceeded(brain, spent, usd_cap, unit="usd") +``` + +The extra `cost_today` read only happens when a $ cap is configured for that brain — one cheap local +SQLite read, not on the hot path otherwise. + +## Message / audit text + +Three spots currently hardcode "token" and would mislead once a $ cap can be the trigger: + +- `roger/brains/admin.py` audit `detail="daily token cap"` → `f"daily {exc.unit} cap"` +- `roger/brains/gigabrain.py` audit detail: same fix; reply text drops "token", becomes generic + ("I've hit my daily budget for gigabrain work...") +- `roger/brains/digest.py` log line: `"digest skipped: daily token budget hit"` → includes + `exc.unit` + +`roger/brains/ambient.py`'s `BUDGET_LINE` ("I'm out of words for today.") is already unit-agnostic — +no change. + +## Ops alert (`bot.py:_budget_alert`, backlog 1.2) + +Currently only watches token fraction; extending now so it doesn't go blind once a $ cap is set — +otherwise the alerting infra that already exists for exactly this would silently miss it. + +Gains a keyword-only `usd_cap: float = 0.0` param. Computes both fractions +(`used/cap`, `cost/usd_cap`), alerts on `max()` of the two once it crosses `BUDGET_ALERT_FRACTION` +(0.8), message body shows both figures when both caps are configured. + +Verified against the 4 existing `test_ops.py` cases — all pass unchanged with the new param +defaulting to 0 (disabled), since `max(token_frac, 0.0) == token_frac`. + +The watchdog loop (`_watchdog`) passes a new `_daily_usd_caps(settings)` helper through, mirroring +`_daily_caps`. + +## `/status` + +Per-brain cost line grows a `/ $cap` suffix when a $ cap is configured for that brain: + +``` +$0.0842 / $2.0000 +``` + +vs. today's bare `$0.0842` when no cap is set. `_format_status` and `gather_status` take a new +`usd_caps` dict, defaulting to `{}` so existing call sites in tests keep working. + +## Metrics + +- New gauge `roger_cost_usd_cap` (mirrors `roger_tokens_cap`), refreshed the same way. +- `roger_llm_budget_exceeded_total` gains a `reason` label (`tokens`/`usd`) so Grafana can show which + cap is actually biting, not just that budget rejections are happening. + +## Testing + +Extends existing suites, no new test files: + +- `test_llm.py` — $ cap trips when configured and cost is at/over it; token cap still works + unconditionally when $ cap is unset; $ cap never trips when cost stays 0 (non-reporting provider). +- `test_ops.py` — new `_budget_alert` cases with `usd_cap` set (warn, exhausted, both-configured). +- `test_status.py` — usd-cap-column case. +- `test_config.py` — new settings default to 0.0. +- `test_metrics.py` — new gauge present; `reason` label on the counter. + +## Out of scope + +- Making the $ cap the *only* mechanism (see Semantics above — deliberately layered, not a + replacement). +- Backlog 1.7 (on-demand tool-call budget override) — unrelated axis (blast-radius bound, not spend). +- Any change to how OpenRouter reports `cost` on the usage object — already handled, unchanged. From 56608d73d7032165c053680748f3cd74925fa7a8 Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:17:19 -0400 Subject: [PATCH 2/9] feat: add DAILY_USD_* config for the dollar budget gate --- compose.yaml | 4 ++++ roger.env.example | 8 ++++++++ roger/config.py | 10 ++++++++++ tests/test_config.py | 15 +++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/compose.yaml b/compose.yaml index 13d2236..458fa4e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -29,6 +29,10 @@ services: DAILY_TOKENS_AMBIENT: ${DAILY_TOKENS_AMBIENT:-40000} DAILY_TOKENS_DIGEST: ${DAILY_TOKENS_DIGEST:-30000} DAILY_TOKENS_GIGABRAIN: ${DAILY_TOKENS_GIGABRAIN:-100000} + DAILY_USD_ADMIN: ${DAILY_USD_ADMIN:-0} + DAILY_USD_AMBIENT: ${DAILY_USD_AMBIENT:-0} + DAILY_USD_DIGEST: ${DAILY_USD_DIGEST:-0} + DAILY_USD_GIGABRAIN: ${DAILY_USD_GIGABRAIN:-0} ADMIN_MAX_TOOL_CALLS: ${ADMIN_MAX_TOOL_CALLS:-10} ADMIN_MAX_TURNS: ${ADMIN_MAX_TURNS:-14} GIGABRAIN_MAX_TOOL_CALLS: ${GIGABRAIN_MAX_TOOL_CALLS:-10} diff --git a/roger.env.example b/roger.env.example index 43afb97..70012bc 100644 --- a/roger.env.example +++ b/roger.env.example @@ -30,6 +30,14 @@ DAILY_TOKENS_AMBIENT=40000 DAILY_TOKENS_DIGEST=30000 DAILY_TOKENS_GIGABRAIN=100000 +# --- budgets (daily USD, layered on top of the token caps above; 0 = disabled/opt-in) --- +# the token cap keeps enforcing regardless of these — set a real figure per brain once you know +# what your model mix actually costs. +DAILY_USD_ADMIN=0 +DAILY_USD_AMBIENT=0 +DAILY_USD_DIGEST=0 +DAILY_USD_GIGABRAIN=0 + # --- admin tool loop bounds (§2.9) --- # hard cap on tool calls / model round-trips within a single admin request. Raise these if # legitimate multi-step requests (e.g. remediating permissions across many categories) keep diff --git a/roger/config.py b/roger/config.py index 56ebeec..f7c1976 100644 --- a/roger/config.py +++ b/roger/config.py @@ -42,6 +42,16 @@ class Settings(BaseSettings): daily_tokens_digest: int = 30_000 daily_tokens_gigabrain: int = 100_000 + # --- budgets (daily USD, layered on top of the token caps above) --- + # 0 = disabled (opt-in); set to a real figure once OpenRouter cost data looks right for your + # model mix. The token cap above keeps enforcing regardless — this is an additional, tighter + # trip wire, not a replacement (a provider that never reports cost would otherwise leave the + # brain with no effective cap at all). + daily_usd_admin: float = 0.0 + daily_usd_ambient: float = 0.0 + daily_usd_digest: float = 0.0 + daily_usd_gigabrain: float = 0.0 + # --- admin tool loop bounds (§2.9) --- admin_max_tool_calls: int = 10 admin_max_turns: int = 14 diff --git a/tests/test_config.py b/tests/test_config.py index 6ae6125..487903e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -67,3 +67,18 @@ def test_missing_required_field_raises(monkeypatch): monkeypatch.delenv(key, raising=False) with pytest.raises(ValidationError): Settings() + + +def test_daily_usd_defaults_to_disabled(monkeypatch): + _set_required(monkeypatch) + settings = Settings() + assert settings.daily_usd_admin == 0.0 + assert settings.daily_usd_ambient == 0.0 + assert settings.daily_usd_digest == 0.0 + assert settings.daily_usd_gigabrain == 0.0 + + +def test_daily_usd_parses_from_env(monkeypatch): + _set_required(monkeypatch) + monkeypatch.setenv("DAILY_USD_ADMIN", "2.5") + assert Settings().daily_usd_admin == 2.5 From 8d2a3855b19d1eaf88987537233fdee4a48ecf40 Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:27:13 -0400 Subject: [PATCH 3/9] feat: add USD-cap gauge and reason label to budget metrics --- roger/llm.py | 2 +- roger/metrics.py | 12 +++++++++++- tests/test_metrics.py | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/roger/llm.py b/roger/llm.py index 7d3f630..e4a4899 100644 --- a/roger/llm.py +++ b/roger/llm.py @@ -103,7 +103,7 @@ async def complete( used = await self._store.usage_today(brain) cap = self._caps[brain] if used >= cap: - metrics.LLM_BUDGET_EXCEEDED.labels(brain).inc() + metrics.LLM_BUDGET_EXCEEDED.labels(brain, "tokens").inc() raise BudgetExceeded(brain, used, cap) extra_body: dict[str, Any] = {"models": chain} diff --git a/roger/metrics.py b/roger/metrics.py index e6bc383..b141021 100644 --- a/roger/metrics.py +++ b/roger/metrics.py @@ -29,13 +29,16 @@ "roger_llm_errors_total", "LLM calls that failed after retries", ["brain", "type"] ) LLM_BUDGET_EXCEEDED = Counter( - "roger_llm_budget_exceeded_total", "Calls refused by the daily token budget", ["brain"] + "roger_llm_budget_exceeded_total", + "Calls refused by the daily budget", + ["brain", "reason"], # reason: "tokens" or "usd" ) # --- state gauges (refreshed from SQLite on a timer) --- TOKENS_TODAY = Gauge("roger_tokens_today", "Tokens spent today", ["brain"]) TOKENS_CAP = Gauge("roger_tokens_cap", "Daily token cap", ["brain"]) COST_USD_TODAY = Gauge("roger_cost_usd_today", "USD spent today (OpenRouter-reported)", ["brain"]) +COST_USD_CAP = Gauge("roger_cost_usd_cap", "Daily USD cap (0 = disabled)", ["brain"]) FEEDS = Gauge("roger_feeds", "Curated digest feeds") AUDIT_EVENTS = Gauge("roger_audit_events", "Audit rows in the retention window", ["tool", "status"]) BUILD_INFO = Gauge("roger_build_info", "Deployed build; value is always 1", ["version"]) @@ -49,10 +52,17 @@ async def refresh(store: Any, settings: Any, version: str) -> None: "digest": settings.daily_tokens_digest, "gigabrain": settings.daily_tokens_gigabrain, } + usd_caps = { + "admin": settings.daily_usd_admin, + "ambient": settings.daily_usd_ambient, + "digest": settings.daily_usd_digest, + "gigabrain": settings.daily_usd_gigabrain, + } for brain in _BRAINS: TOKENS_TODAY.labels(brain).set(await store.usage_today(brain)) COST_USD_TODAY.labels(brain).set(await store.cost_today(brain)) TOKENS_CAP.labels(brain).set(caps[brain]) + COST_USD_CAP.labels(brain).set(usd_caps[brain]) FEEDS.set(await store.count_feeds()) # Rebuild the audit series from scratch so a (tool, status) combo that drops to zero after a # prune doesn't linger as a stale sample. diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 40b89dc..5aa7d63 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -18,6 +18,10 @@ def _settings(): daily_tokens_ambient=40_000, daily_tokens_digest=30_000, daily_tokens_gigabrain=100_000, + daily_usd_admin=0.0, + daily_usd_ambient=0.0, + daily_usd_digest=0.0, + daily_usd_gigabrain=0.0, ) @@ -84,3 +88,18 @@ async def fake_create(**_kwargs): assert after == before + 1 finally: await store.close() + + +async def test_refresh_populates_usd_cap_gauge(tmp_path): + store = await Store(str(tmp_path / "m.db")).open() + try: + settings = _settings() + settings.daily_usd_admin = 2.5 + + await metrics.refresh(store, settings, "sha-test") + + get = REGISTRY.get_sample_value + assert get("roger_cost_usd_cap", {"brain": "admin"}) == 2.5 + assert get("roger_cost_usd_cap", {"brain": "ambient"}) == 0.0 + finally: + await store.close() From 2392dc702d84040aaf204e3e94e6aa740fcb713e Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:30:35 -0400 Subject: [PATCH 4/9] feat: layer a dollar cap on top of the token budget gate --- roger/llm.py | 24 +++++++++++++++++++++--- tests/test_llm.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/roger/llm.py b/roger/llm.py index e4a4899..cac55d7 100644 --- a/roger/llm.py +++ b/roger/llm.py @@ -58,11 +58,16 @@ class LLMConfigError(RuntimeError): class BudgetExceeded(RuntimeError): - def __init__(self, brain: str, used: int, cap: int) -> None: - super().__init__(f"{brain} daily token budget exceeded ({used} >= {cap})") + def __init__(self, brain: str, used: float, cap: float, *, unit: str = "tokens") -> None: + if unit == "usd": + message = f"{brain} daily $ budget exceeded (${used:.4f} >= ${cap:.4f})" + else: + message = f"{brain} daily token budget exceeded ({used} >= {cap})" + super().__init__(message) self.brain = brain self.used = used self.cap = cap + self.unit = unit class LLM: @@ -87,6 +92,12 @@ def __init__(self, settings: Settings, store: Store) -> None: "digest": settings.daily_tokens_digest, "gigabrain": settings.daily_tokens_gigabrain, } + self._usd_caps = { + "admin": settings.daily_usd_admin, + "ambient": settings.daily_usd_ambient, + "digest": settings.daily_usd_digest, + "gigabrain": settings.daily_usd_gigabrain, + } # Opt-in OpenRouter `reasoning.effort` passthrough — only gigabrain ever sets this today. self._reasoning_effort = {"gigabrain": settings.gigabrain_reasoning_effort or None} @@ -104,7 +115,14 @@ async def complete( cap = self._caps[brain] if used >= cap: metrics.LLM_BUDGET_EXCEEDED.labels(brain, "tokens").inc() - raise BudgetExceeded(brain, used, cap) + raise BudgetExceeded(brain, used, cap, unit="tokens") + + usd_cap = self._usd_caps[brain] + if usd_cap > 0: + spent = await self._store.cost_today(brain) + if spent >= usd_cap: + metrics.LLM_BUDGET_EXCEEDED.labels(brain, "usd").inc() + raise BudgetExceeded(brain, spent, usd_cap, unit="usd") extra_body: dict[str, Any] = {"models": chain} if tools: diff --git a/tests/test_llm.py b/tests/test_llm.py index ec2976f..b547b25 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -48,8 +48,41 @@ async def test_budget_exceeded_before_network(monkeypatch, tmp_path): try: await store.add_usage("admin", 8, 5) # 13 >= 10 llm = LLM(Settings(), store) - with pytest.raises(BudgetExceeded): + with pytest.raises(BudgetExceeded) as exc_info: await llm.complete("admin", [{"role": "user", "content": "hi"}]) + assert exc_info.value.unit == "tokens" + finally: + await store.close() + + +async def test_usd_budget_exceeded_before_network(monkeypatch, tmp_path): + # Token cap (default 150k) is nowhere near tripped — only the $ cap should fire. + _env(monkeypatch, MODEL_ADMIN="a/b", DAILY_USD_ADMIN="1.0") + store = await Store(str(tmp_path / "l.db")).open() + try: + await store.add_usage("admin", 1, 1, cost_usd=1.5) # $1.50 >= $1.00 cap + llm = LLM(Settings(), store) + with pytest.raises(BudgetExceeded) as exc_info: + await llm.complete("admin", [{"role": "user", "content": "hi"}]) + assert exc_info.value.unit == "usd" + finally: + await store.close() + + +async def test_usd_cap_does_not_trip_below_threshold(monkeypatch, tmp_path): + _env(monkeypatch, MODEL_ADMIN="a/b", DAILY_USD_ADMIN="1.0") + store = await Store(str(tmp_path / "l.db")).open() + try: + await store.add_usage("admin", 1, 1, cost_usd=0.5) # $0.50 < $1.00 cap + llm = LLM(Settings(), store) + + async def fake_create(**kwargs): + return SimpleNamespace( + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, cost=0.1) + ) + + monkeypatch.setattr(llm._client.chat.completions, "create", fake_create) + await llm.complete("admin", [{"role": "user", "content": "hi"}]) # does not raise finally: await store.close() From 2ee5e9d6d2ca3683ef828a71a54ec8759bbeaabf Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:34:59 -0400 Subject: [PATCH 5/9] fix: make budget-exceeded messages unit-aware, not token-only --- roger/brains/admin.py | 6 +++--- roger/brains/digest.py | 4 ++-- roger/brains/gigabrain.py | 6 +++--- tests/test_admin.py | 13 +++++++++++++ tests/test_gigabrain.py | 13 +++++++++++++ 5 files changed, 34 insertions(+), 8 deletions(-) diff --git a/roger/brains/admin.py b/roger/brains/admin.py index c90ac37..412eb22 100644 --- a/roger/brains/admin.py +++ b/roger/brains/admin.py @@ -165,16 +165,16 @@ async def handle_admin_request( detail=detail, ) messages.append(_tool_message(call.id, result)) - except BudgetExceeded: + except BudgetExceeded as exc: await store.record_audit( actor_id=actor_id, brain="admin", tool=None, args={"request": request}, status=AuditStatus.ERROR, - detail="daily token cap", + detail=f"daily {exc.unit} cap", ) - return "I've hit my daily token budget for admin work. Try again tomorrow." + return "I've hit my daily budget for admin work. Try again tomorrow." except LLMConfigError as exc: return f"The admin brain isn't configured yet ({exc})." diff --git a/roger/brains/digest.py b/roger/brains/digest.py index 1068a4e..6cf1117 100644 --- a/roger/brains/digest.py +++ b/roger/brains/digest.py @@ -97,8 +97,8 @@ async def run_digest_job(*, client: Any, settings: Any, llm: LLM, store: Store) try: summary = await _summarize(entries, llm) - except BudgetExceeded: - log.warning("digest skipped: daily token budget hit") + except BudgetExceeded as exc: + log.warning("digest skipped: daily %s budget hit", exc.unit) return {"status": "budget exceeded; skipped"} except LLMConfigError as exc: return {"status": f"digest brain not configured ({exc})"} diff --git a/roger/brains/gigabrain.py b/roger/brains/gigabrain.py index ead5f87..4c65f9c 100644 --- a/roger/brains/gigabrain.py +++ b/roger/brains/gigabrain.py @@ -168,16 +168,16 @@ async def handle_gigabrain_request( detail=detail, ) messages.append(_tool_message(call.id, result)) - except BudgetExceeded: + except BudgetExceeded as exc: await store.record_audit( actor_id=actor_id, brain="gigabrain", tool=None, args={"request": request}, status=AuditStatus.ERROR, - detail="daily token cap", + detail=f"daily {exc.unit} cap", ) - return "I've hit my daily token budget for gigabrain work. Try again tomorrow." + return "I've hit my daily budget for gigabrain work. Try again tomorrow." except LLMConfigError as exc: return f"Gigabrain isn't configured yet ({exc})." diff --git a/tests/test_admin.py b/tests/test_admin.py index 9307a31..f8d34a0 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -320,3 +320,16 @@ async def test_admin_memory_is_scoped_per_channel(tmp_path): assert not any("channel A request" in c for c in contents) # no cross-channel bleed finally: await store.close() + + +async def test_budget_exceeded_audit_detail_reflects_unit(tmp_path): + store = await _open_store(tmp_path) + try: + llm = FakeLLM([BudgetExceeded("admin", 2.5, 2.0, unit="usd")]) + await admin.handle_admin_request( + request="anything", guild=object(), actor_id=1, llm=llm, store=store + ) + rows = await store.fetch_audit() + assert any(r["detail"] == "daily usd cap" for r in rows) + finally: + await store.close() diff --git a/tests/test_gigabrain.py b/tests/test_gigabrain.py index ef293ad..c01852d 100644 --- a/tests/test_gigabrain.py +++ b/tests/test_gigabrain.py @@ -524,3 +524,16 @@ async def test_periodic_suggestion_remembers_the_previous_check_in(tmp_path): assert any("Consider a rules channel." in c for c in contents) finally: await store.close() + + +async def test_budget_exceeded_audit_detail_reflects_unit(tmp_path): + store = await _open_store(tmp_path) + try: + llm = FakeLLM([BudgetExceeded("gigabrain", 2.5, 2.0, unit="usd")]) + await gigabrain.handle_gigabrain_request( + request="anything", guild=object(), actor_id=1, llm=llm, store=store + ) + rows = await store.fetch_audit() + assert any(r["detail"] == "daily usd cap" for r in rows) + finally: + await store.close() From 93196adec6f83d7895ac99e883323b2af8a4e5e7 Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:38:31 -0400 Subject: [PATCH 6/9] feat: extend the ops budget alert to watch the dollar cap --- roger/bot.py | 46 ++++++++++++++++++++++++++++++++++++++-------- tests/test_ops.py | 16 ++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/roger/bot.py b/roger/bot.py index 1c8095f..96dd81d 100644 --- a/roger/bot.py +++ b/roger/bot.py @@ -268,6 +268,16 @@ def _daily_caps(settings: Settings) -> dict[str, int]: } +def _daily_usd_caps(settings: Settings) -> dict[str, float]: + """Per-brain daily USD caps, keyed by brain (0 = disabled). Mirrors `_daily_caps`.""" + return { + "admin": settings.daily_usd_admin, + "ambient": settings.daily_usd_ambient, + "digest": settings.daily_usd_digest, + "gigabrain": settings.daily_usd_gigabrain, + } + + def _boot_header(version: str, missing: list[str], channel_problems: list[str]) -> str: """Header of the boot self-report (pure): health glyph + the deployed build. @@ -395,16 +405,34 @@ async def alert(self, key: str, message: str, *, cooldown_s: float) -> bool: def _budget_alert( - brain: str, used: int, cap: int, cost: float, *, fraction: float = BUDGET_ALERT_FRACTION + brain: str, + used: int, + cap: int, + cost: float, + *, + usd_cap: float = 0.0, + fraction: float = BUDGET_ALERT_FRACTION, ) -> str | None: - """Alert text when ``brain`` crosses ``fraction`` of its daily token cap, else None (pure).""" - if cap <= 0 or used < fraction * cap: + """Alert text once ``brain`` crosses ``fraction`` of its daily token or $ cap, else None (pure). + + The two caps are independent — whichever fraction is worse decides both whether this fires + and the wording (exhausted vs. approaching). A disabled cap (``cap`` or ``usd_cap`` <= 0) + contributes 0 to that comparison, so it can never itself trigger an alert. + """ + token_frac = used / cap if cap > 0 else 0.0 + usd_frac = cost / usd_cap if usd_cap > 0 else 0.0 + worst = max(token_frac, usd_frac) + if worst < fraction: return None - tokens = f"{used:,} / {cap:,} tokens today (${cost:.4f})" - if used >= cap: - return f"⚠️ **{brain} budget exhausted** — {tokens}. Calls refused until the daily reset." - pct = round(100 * used / cap) - return f"⚠️ **{brain} budget {pct}%** — {tokens}. Approaching the daily cap." + detail = f"{used:,} / {cap:,} tokens today" if cap > 0 else f"{used:,} tokens today" + if usd_cap > 0: + detail += f" · ${cost:.4f} / ${usd_cap:.4f}" + else: + detail += f" (${cost:.4f})" + if worst >= 1.0: + return f"⚠️ **{brain} budget exhausted** — {detail}. Calls refused until the daily reset." + pct = round(100 * worst) + return f"⚠️ **{brain} budget {pct}%** — {detail}. Approaching the daily cap." # Digest statuses that mean "ran fine, nothing to flag"; anything else is worth an ops ping. @@ -712,6 +740,7 @@ async def _watchdog(self) -> None: cooldown_s=_PERM_ALERT_COOLDOWN_S, ) caps = _daily_caps(self.settings) + usd_caps = _daily_usd_caps(self.settings) today = time.strftime("%Y-%m-%d") for brain in _BRAINS: message = _budget_alert( @@ -719,6 +748,7 @@ async def _watchdog(self) -> None: await self.store.usage_today(brain), caps[brain], await self.store.cost_today(brain), + usd_cap=usd_caps[brain], ) if message: await self._ops.alert(f"budget:{brain}:{today}", message, cooldown_s=_DAY_S) diff --git a/tests/test_ops.py b/tests/test_ops.py index 9c24579..221720d 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -60,6 +60,22 @@ def test_budget_alert_ignores_zero_or_negative_cap(): assert _budget_alert("admin", 5, 0, 0.0) is None +def test_budget_alert_fires_from_usd_cap_alone(): + # tokens are nowhere near their cap (10%); the $ cap (84%) is what should trip this. + msg = _budget_alert("gigabrain", 10_000, 100_000, 4.2, usd_cap=5.0) + assert msg is not None + assert "84%" in msg and "$4.2000 / $5.0000" in msg + + +def test_budget_alert_usd_exhausted_reads_as_exhausted(): + msg = _budget_alert("gigabrain", 1_000, 100_000, 6.0, usd_cap=5.0) + assert msg is not None and "exhausted" in msg + + +def test_budget_alert_silent_when_both_caps_disabled(): + assert _budget_alert("admin", 1_000_000, 0, 999.0, usd_cap=0.0) is None + + def test_digest_problem_none_for_success_statuses(): assert _digest_problem("posted") is None assert _digest_problem("no new items") is None From 2eb3ca60faf7c8ca9fdea94ea5a6551c1a5e65b5 Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:42:17 -0400 Subject: [PATCH 7/9] feat: show the dollar cap in /status when configured --- roger/bot.py | 10 +++++++++- tests/test_status.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/roger/bot.py b/roger/bot.py index 96dd81d..e5ce5c1 100644 --- a/roger/bot.py +++ b/roger/bot.py @@ -317,8 +317,10 @@ def _format_status( digest_hour: int, digest_configured: bool, tz: str, + usd_caps: dict[str, float] | None = None, ) -> str: """Render the /status readout body (pure). The caller wraps it in a code block.""" + usd_caps = usd_caps or {} perms = "OK" if not missing_perms else "MISSING: " + ", ".join(missing_perms) channels = "OK" if not channel_problems else "; ".join(channel_problems) lines = [ @@ -331,8 +333,12 @@ def _format_status( for brain in _BRAINS: spent = cost.get(brain, 0.0) total_cost += spent + cost_str = f"${spent:.4f}" + usd_cap = usd_caps.get(brain, 0.0) + if usd_cap > 0: + cost_str += f" / ${usd_cap:.4f}" lines.append( - f" {brain:<10}{usage.get(brain, 0):>8,} / {caps.get(brain, 0):<8,} ${spent:.4f}" + f" {brain:<10}{usage.get(brain, 0):>8,} / {caps.get(brain, 0):<8,} {cost_str}" ) lines.append(f" {'total':<29} ${total_cost:.4f}") digest = f"{digest_hour:02d}:00 {tz}" if digest_configured else "unconfigured" @@ -357,6 +363,7 @@ async def gather_status(*, store: Store, settings: Settings, guild: Any) -> str: usage = {brain: await store.usage_today(brain) for brain in _BRAINS} cost = {brain: await store.cost_today(brain) for brain in _BRAINS} caps = _daily_caps(settings) + usd_caps = _daily_usd_caps(settings) return _format_status( guild_name=guild_name, missing_perms=missing, @@ -364,6 +371,7 @@ async def gather_status(*, store: Store, settings: Settings, guild: Any) -> str: usage=usage, caps=caps, cost=cost, + usd_caps=usd_caps, feeds_count=await store.count_feeds(), recent_audit=await store.fetch_audit(limit=8), digest_hour=settings.digest_hour, diff --git a/tests/test_status.py b/tests/test_status.py index 5dc8664..d730ad8 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -152,6 +152,10 @@ def _settings(**over): daily_tokens_ambient=40000, daily_tokens_digest=30000, daily_tokens_gigabrain=100000, + daily_usd_admin=0.0, + daily_usd_ambient=0.0, + daily_usd_digest=0.0, + daily_usd_gigabrain=0.0, digest_hour=8, digest_channel_id=42, tz="UTC", @@ -204,3 +208,33 @@ async def test_gather_status_without_a_visible_guild(tmp_path): assert "digest: unconfigured" in body finally: await store.close() + + +def test_format_status_shows_usd_cap_when_configured(): + body = _format_status( + guild_name="G", + missing_perms=[], + channel_problems=[], + usage={"admin": 1000}, + caps={"admin": 150000}, + cost={"admin": 0.5}, + usd_caps={"admin": 2.0}, + feeds_count=0, + recent_audit=[], + digest_hour=8, + digest_configured=True, + tz="UTC", + ) + assert "$0.5000 / $2.0000" in body + + +async def test_gather_status_shows_usd_cap_from_settings(tmp_path): + store = await Store(str(tmp_path / "s.db")).open() + try: + await store.add_usage("admin", 10, 10, cost_usd=0.25) + guild = _fake_guild(channels={42: _FakeChannel()}) + settings = _settings(daily_usd_admin=1.0) + body = await gather_status(store=store, settings=settings, guild=guild) + assert "$0.2500 / $1.0000" in body + finally: + await store.close() From a0a6ebf8e73f3989393f25c59415c3d53ddde846 Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 00:45:56 -0400 Subject: [PATCH 8/9] docs: record the dollar budget gate in ARCHITECTURE.md and BACKLOG.md --- ARCHITECTURE.md | 24 ++++++++++++++++-------- BACKLOG.md | 8 ++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ec45a3b..8a29a51 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -80,10 +80,11 @@ These hold regardless of what any model outputs. They are the load-bearing part be *static* (a tool always confirms) or *conditional on the args* (`create_channel` confirms only when `private`), via `ToolSpec.needs_confirm`. - **§2.9 Budgets.** A hard cap of **10 tool calls per request** (`ADMIN_MAX_TOOL_CALLS`) and **14 - model round-trips** (`ADMIN_MAX_TURNS`), plus per-brain **daily token caps** (§11) checked before - every call. Both caps are env-overridable per deployment; hitting the tool-call cap mid-request - logs a warning and posts once to the ops channel (if configured), and the model is told to say so - plainly — the cap resets on the next request, not on a timer. + model round-trips** (`ADMIN_MAX_TURNS`), plus per-brain **daily token caps**, optionally layered + with a **daily USD cap** (§11), checked before every call. All caps are env-overridable per + deployment; hitting the tool-call cap mid-request logs a warning and posts once to the ops channel + (if configured), and the model is told to say so plainly — the cap resets on the next request, not + on a timer. - **§2.10 No secrets in git — ever, not even encrypted.** Secrets live only in a `sops`+`age` encrypted `roger.env` on the host. The repo carries `.sops.yaml` (the public recipient) and `roger.env.example`. See [`deploy/`](deploy/README.md). @@ -277,16 +278,23 @@ behaviour adds rows, not migrations. ## §11 LLM layer & budgets `roger/llm.py` wraps the OpenAI SDK pointed at OpenRouter. Per call: pick the brain's model chain -(§3), **check the daily token cap before spending** (raises `BudgetExceeded` if over), call with -automatic fallback down the chain, then **record actual usage** to `usage`. A missing/empty model -chain raises `LLMConfigError`, which callers turn into a plain "not configured" reply rather than a -crash. Real spend is additionally bounded off-box by the OpenRouter key's own credit limit. +(§3), **check the daily token cap before spending** (raises `BudgetExceeded` if over), then — if a +`DAILY_USD_` cap is also set — check accumulated USD spend the same way. The two caps are +layered, not either/or: the token cap always enforces, and the USD cap is an additional, optional +trip wire on top of it. That's deliberate — a provider that never reports cost +(`OPENROUTER_BASE_URL` pointed elsewhere, ADR-0009) would otherwise leave the USD cap permanently +silent, so the token cap stays the real backstop in that case. Once both checks pass, the call +proceeds with automatic fallback down the chain, then **records actual usage** to `usage`. A +missing/empty model chain raises `LLMConfigError`, which callers turn into a plain "not configured" +reply rather than a crash. Real spend is additionally bounded off-box by the OpenRouter key's own +credit limit. Limits at a glance (defaults; all env-overridable): | Control | Default | |---|---| | Daily tokens — admin / ambient / digest / gigabrain | 150k / 40k / 30k / 100k | +| Daily USD — admin / ambient / digest / gigabrain | off / off / off / off (0 = disabled) | | Tool calls per admin request | 10 | | Model round-trips per admin request | 14 | | Tool calls / round-trips per gigabrain request | 10 / 14 | diff --git a/BACKLOG.md b/BACKLOG.md index e56a1f7..858af36 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -17,7 +17,7 @@ Effort key: **S** ≈ an afternoon, **M** ≈ a day or two, **L** ≈ multi-day. Gaps that matter for a bot that's actually live. These are the ones I'd do first. -### 1.1 Track spend in dollars, not just tokens — **M** — *visibility shipped; gate remains* +### 1.1 Track spend in dollars, not just tokens — **M** — *shipped* `llm.py` records `prompt_tokens` / `completion_tokens` per brain (`add_usage`) and the daily cap is a raw token count. But a brain's model chain mixes models at very different prices, so a token budget is a weak proxy for the thing that actually costs money. OpenRouter returns the real generation cost @@ -26,9 +26,9 @@ is a weak proxy for the thing that actually costs money. OpenRouter returns the - [x] Add a `cost_usd` column to the `usage` table (with an idempotent migration for live DBs); capture the OpenRouter-reported cost per call in `LLM.complete`. *(a2689b5)* - [x] Surface per-brain and total `$ today` in `/status`. *(a2689b5)* -- [ ] Make the daily gate dollar-denominated (env: `DAILY_USD_*`) with the token cap as the fallback - when a provider doesn't report cost. Deferred: enforcement is a semantic change, kept out of the - visibility commit. +- [x] Make the daily gate dollar-denominated (env: `DAILY_USD_*`), layered on top of the token cap + rather than replacing it — a provider that never reports cost leaves the token cap as the real + backstop, so nothing regresses for a non-OpenRouter host. *Why:* the single most portfolio-differentiating item here — real LLM cost governance is exactly the infra+AI bridge the portfolio is aiming at, and it's the honest version of the budget the code From 31c00641c05ca29164dd3a63a26420859363f6cb Mon Sep 17 00:00:00 2001 From: Ross Tomsic Date: Tue, 18 Aug 2026 14:58:18 -0400 Subject: [PATCH 9/9] =?UTF-8?q?fix:=20address=20final=20review=20findings?= =?UTF-8?q?=20=E2=80=94=20supersede=20ADR-0001,=20sync=20README=20metrics,?= =?UTF-8?q?=20fix=20budget-message=20wording?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- .../0001-dollar-cost-tracking-token-gate.md | 2 +- ...r-the-dollar-budget-gate-not-flip-to-it.md | 40 +++++++++++++++++++ roger/bot.py | 2 +- roger/brains/admin.py | 2 +- roger/brains/digest.py | 2 +- roger/brains/gigabrain.py | 2 +- tests/test_admin.py | 2 +- tests/test_gigabrain.py | 2 +- tests/test_status.py | 1 + 10 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 docs/decisions/0010-layer-the-dollar-budget-gate-not-flip-to-it.md diff --git a/README.md b/README.md index 8117e2e..52ae4ab 100644 --- a/README.md +++ b/README.md @@ -81,9 +81,9 @@ SQLite so they survive restarts. Key series: | Metric | Type | Labels | |---|---|---| | `roger_tokens_today` / `roger_tokens_cap` | gauge | `brain` | -| `roger_cost_usd_today` | gauge | `brain` | +| `roger_cost_usd_today` / `roger_cost_usd_cap` | gauge | `brain` | | `roger_llm_requests_total` / `roger_llm_errors_total` | counter | `brain` (`type`) | -| `roger_llm_budget_exceeded_total` | counter | `brain` | +| `roger_llm_budget_exceeded_total` | counter | `brain`, `reason` | | `roger_audit_events` | gauge | `tool`, `status` | | `roger_feeds`, `roger_build_info` | gauge | — (`version`) | diff --git a/docs/decisions/0001-dollar-cost-tracking-token-gate.md b/docs/decisions/0001-dollar-cost-tracking-token-gate.md index 3bd7149..48c7931 100644 --- a/docs/decisions/0001-dollar-cost-tracking-token-gate.md +++ b/docs/decisions/0001-dollar-cost-tracking-token-gate.md @@ -1,6 +1,6 @@ # ADR-0001: Track spend in dollars, keep enforcement on tokens -- **Status:** Accepted +- **Status:** Accepted; enforcement mechanism superseded by [ADR-0010](0010-layer-the-dollar-budget-gate-not-flip-to-it.md) - **Date:** 2026-07-23 ## Context diff --git a/docs/decisions/0010-layer-the-dollar-budget-gate-not-flip-to-it.md b/docs/decisions/0010-layer-the-dollar-budget-gate-not-flip-to-it.md new file mode 100644 index 0000000..77479d4 --- /dev/null +++ b/docs/decisions/0010-layer-the-dollar-budget-gate-not-flip-to-it.md @@ -0,0 +1,40 @@ +# ADR-0010: Layer the dollar budget gate on the token gate, don't flip to it + +- **Status:** Accepted +- **Date:** 2026-08-18 + +## Context + +ADR-0001 split dollar-cost visibility from enforcement and predicted the natural follow-up: "flip +the gate to dollars with tokens as fallback." That follow-up landed differently. `DAILY_USD_` +is optional and, when set, enforces *alongside* the token cap rather than replacing it — the token +cap still runs unconditionally on every call. + +The reason is `cost_today()`. OpenRouter always returns a real `usage.cost`, but the field is an +OpenRouter extension, not a `usage` guarantee. ADR-0009 documents `OPENROUTER_BASE_URL` staying +pointed-but-configurable at a non-OpenRouter host. If that ever happens, `cost_today()` for that +brain sits at `0.0` forever. A gate that used dollars as the *primary* enforcement (tokens only as +fallback, per ADR-0001's plan) would need to detect "is cost data actually flowing" before it could +fall back — one more thing to get right, and wrong in exactly the case (a broken or misconfigured +cost feed) where the gate matters most. + +Layering sidesteps that detection problem. The token check is unconditional and always runs first; +the dollar check is a second, independent, optional trip wire that runs after it, only when +`DAILY_USD_ > 0`. If dollar data never arrives, the dollar check simply never fires — no false +permissiveness, no code path that has to notice. + +## Decision + +Both caps enforce when both are configured. Whichever trips first wins. The token cap is never +replaced, only ever added to. + +## Consequences + +- A live host with a working OpenRouter key gets governance in the currency that actually matters — + real spend — without losing the safety net a broken cost feed would otherwise remove. +- Two numbers to reason about per brain instead of one, in `/status`, the ops alert, and + `roger.env.example`. Both default off, so this stays opt-in, not a forced complication. +- Supersedes ADR-0001's Consequences section, which predicted "flip... with tokens as fallback" — + that mechanism was designed but not built, once the non-reporting-provider case above ruled it out. + ADR-0009's cross-reference to "the existing dollar-cost tracking and token gate (ADR-0001)" should + now be read alongside this record. diff --git a/roger/bot.py b/roger/bot.py index e5ce5c1..e83a750 100644 --- a/roger/bot.py +++ b/roger/bot.py @@ -327,7 +327,7 @@ def _format_status( f"roger status — {guild_name}", f"permissions: {perms}", f"channels: {channels}", - "spend today (tokens used / cap · cost):", + "spend today (tokens used / cap · cost / $ cap if set):", ] total_cost = 0.0 for brain in _BRAINS: diff --git a/roger/brains/admin.py b/roger/brains/admin.py index 412eb22..f5852ae 100644 --- a/roger/brains/admin.py +++ b/roger/brains/admin.py @@ -172,7 +172,7 @@ async def handle_admin_request( tool=None, args={"request": request}, status=AuditStatus.ERROR, - detail=f"daily {exc.unit} cap", + detail=f"daily {'$' if exc.unit == 'usd' else 'token'} cap", ) return "I've hit my daily budget for admin work. Try again tomorrow." except LLMConfigError as exc: diff --git a/roger/brains/digest.py b/roger/brains/digest.py index 6cf1117..50f5203 100644 --- a/roger/brains/digest.py +++ b/roger/brains/digest.py @@ -98,7 +98,7 @@ async def run_digest_job(*, client: Any, settings: Any, llm: LLM, store: Store) try: summary = await _summarize(entries, llm) except BudgetExceeded as exc: - log.warning("digest skipped: daily %s budget hit", exc.unit) + log.warning("digest skipped: daily %s budget hit", "$" if exc.unit == "usd" else "token") return {"status": "budget exceeded; skipped"} except LLMConfigError as exc: return {"status": f"digest brain not configured ({exc})"} diff --git a/roger/brains/gigabrain.py b/roger/brains/gigabrain.py index 4c65f9c..b3fa3ff 100644 --- a/roger/brains/gigabrain.py +++ b/roger/brains/gigabrain.py @@ -175,7 +175,7 @@ async def handle_gigabrain_request( tool=None, args={"request": request}, status=AuditStatus.ERROR, - detail=f"daily {exc.unit} cap", + detail=f"daily {'$' if exc.unit == 'usd' else 'token'} cap", ) return "I've hit my daily budget for gigabrain work. Try again tomorrow." except LLMConfigError as exc: diff --git a/tests/test_admin.py b/tests/test_admin.py index f8d34a0..53f4ceb 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -330,6 +330,6 @@ async def test_budget_exceeded_audit_detail_reflects_unit(tmp_path): request="anything", guild=object(), actor_id=1, llm=llm, store=store ) rows = await store.fetch_audit() - assert any(r["detail"] == "daily usd cap" for r in rows) + assert any(r["detail"] == "daily $ cap" for r in rows) finally: await store.close() diff --git a/tests/test_gigabrain.py b/tests/test_gigabrain.py index c01852d..1241efc 100644 --- a/tests/test_gigabrain.py +++ b/tests/test_gigabrain.py @@ -534,6 +534,6 @@ async def test_budget_exceeded_audit_detail_reflects_unit(tmp_path): request="anything", guild=object(), actor_id=1, llm=llm, store=store ) rows = await store.fetch_audit() - assert any(r["detail"] == "daily usd cap" for r in rows) + assert any(r["detail"] == "daily $ cap" for r in rows) finally: await store.close() diff --git a/tests/test_status.py b/tests/test_status.py index d730ad8..ae0bb60 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -226,6 +226,7 @@ def test_format_status_shows_usd_cap_when_configured(): tz="UTC", ) assert "$0.5000 / $2.0000" in body + assert "ambient" in body and "$0.0000 / $" not in body # unconfigured brain: no cap suffix async def test_gather_status_shows_usd_cap_from_settings(tmp_path):