From b684a682e07ceec168af4c79209f6a845061a278 Mon Sep 17 00:00:00 2001 From: Osher Elhadad Date: Wed, 2 Sep 2026 22:25:33 +0000 Subject: [PATCH 1/2] feat(grafana): fill the two empty rows, graph the metrics nobody was watching, and alert on the observability path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The main dashboard advertised two sections it never filled — "Is storage and cold storage healthy?" and "Is anything failing?" were rows with no panels under them, so the two questions an operator asks during an incident had a heading and nothing else. The alerting directory shipped with no rules at all, so nothing paged. context-guru: 27 -> 45 panels. The two empty rows now hold what they promise: local database against cold storage, filesystem in use, cold storage reachability, sessions archived, dropped capture events; and compaction-model failures, cache-write churn, tokens wasted recovering offloaded content, buffered streams, tenants disabled, expand bounces over time, buffered against streamed responses, requests refused by reason, refusals per tenant, expand recovery failures by cause. Four value panels join the savings row — net saved this month, extraction net value, cache-frozen headroom, and the prefix-cache and total-avoided figures that previously sat orphaned below the last row. service SLO: 16 -> 19 panels, all three in "Correctness of the observability path itself", which is exactly where this week's outage would have shown up and did not. cg_metrics_age_seconds is the series that matters: /metrics had a cache TTL shorter than the scrape interval, so it could not answer a single scrape and the target sat down for hours while every cg_* panel read "No data" — a state indistinguishable, from the dashboard, from the service being dead. An exposition age that stops advancing says which one it is in one number. alerting: three rules, each for a failure that was previously silent — the service being down, more than 10% of requests refused, and the metrics exposition ageing past its refresh interval. Every panel's PromQL was run against this deployment's Prometheus before being committed: 83 of 86 Prometheus queries return series on live data, and the three that do not are the two new cg_metrics_* series plus their timeseries, which only exist once the proxy carrying them is deployed. The 17 Loki queries were checked against Loki, which is healthy and ingesting. All three alert expressions return series. Both dashboard uids are unchanged, so provisioning and every existing link still resolve. Colours are untouched: blue #3987e5 actual, orange #d95926 comparison, aqua #199e70 savings, validated CVD-safe on Grafana dark at all-pairs deltaE 9.4. No fourth colour was needed. Note on drift: the deployed copy of context-guru.json came from d9e2f24, which is not on main — 41 panels that only ever existed on the box. This change is a strict superset of it, verified panel by panel, so landing it brings that work into the repo rather than reverting it. context-guru-logs.json and context-guru-host.json are deliberately untouched. Signed-off-by: Osher Elhadad --- .../grafana/dashboards/context-guru-slo.json | 221 +- deploy/grafana/dashboards/context-guru.json | 2739 +++++++++-------- .../provisioning/alerting/context-guru.yml | 61 +- 3 files changed, 1792 insertions(+), 1229 deletions(-) diff --git a/deploy/grafana/dashboards/context-guru-slo.json b/deploy/grafana/dashboards/context-guru-slo.json index 52cab6f0..c3c24613 100644 --- a/deploy/grafana/dashboards/context-guru-slo.json +++ b/deploy/grafana/dashboards/context-guru-slo.json @@ -727,6 +727,219 @@ } } }, + { + "id": 17, + "type": "stat", + "title": "Metrics exposition age", + "description": "How old the /metrics body being served is. Rises above the refresh interval (60s) only if refreshes have stopped \u2014 the outage this row exists to catch: a stale-but-served body reads as healthy on every other panel here. Alert: cg-metrics-stale.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 21 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "cg_metrics_age_seconds", + "editorMode": "code", + "legendFormat": "age" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#199e70", + "value": null + }, + { + "color": "#c98500", + "value": 120 + }, + { + "color": "#e66767", + "value": 300 + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 18, + "type": "stat", + "title": "Metrics render duration", + "description": "How long the last exposition render took. Approaching the 15s scrape interval is the early warning that the per-tenant queries need attention, before a render slow enough to matter ever causes a gap.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 21 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "cg_metrics_render_seconds", + "editorMode": "code", + "legendFormat": "render" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#199e70", + "value": null + }, + { + "color": "#c98500", + "value": 8 + }, + { + "color": "#e66767", + "value": 15 + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 19, + "type": "timeseries", + "title": "Metrics exposition age over time", + "description": "A sawtooth under ~65s is healthy: the cache refreshes on a scrape once it turns 60s old. A climb with no reset means refreshes have stopped and every dashboard reading these series is going stale, silently.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 21 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "cg_metrics_age_seconds", + "editorMode": "code", + "legendFormat": "age" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, { "id": 13, "type": "row", @@ -736,7 +949,7 @@ "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 27 }, "panels": [] }, @@ -753,7 +966,7 @@ "h": 6, "w": 6, "x": 0, - "y": 22 + "y": 28 }, "targets": [ { @@ -823,7 +1036,7 @@ "h": 6, "w": 6, "x": 6, - "y": 22 + "y": 28 }, "targets": [ { @@ -884,7 +1097,7 @@ "h": 6, "w": 12, "x": 12, - "y": 22 + "y": 28 }, "targets": [ { diff --git a/deploy/grafana/dashboards/context-guru.json b/deploy/grafana/dashboards/context-guru.json index b2287a4e..3913da08 100644 --- a/deploy/grafana/dashboards/context-guru.json +++ b/deploy/grafana/dashboards/context-guru.json @@ -74,7 +74,7 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 0 }, "panels": [] }, @@ -91,7 +91,7 @@ "h": 5, "w": 4, "x": 0, - "y": 23 + "y": 1 }, "targets": [ { @@ -171,7 +171,7 @@ "h": 5, "w": 6, "x": 4, - "y": 23 + "y": 1 }, "targets": [ { @@ -234,7 +234,7 @@ "h": 5, "w": 5, "x": 10, - "y": 23 + "y": 1 }, "targets": [ { @@ -296,7 +296,7 @@ "h": 5, "w": 5, "x": 15, - "y": 23 + "y": 1 }, "targets": [ { @@ -364,7 +364,7 @@ "h": 5, "w": 4, "x": 20, - "y": 23 + "y": 1 }, "targets": [ { @@ -426,7 +426,7 @@ "h": 1, "w": 24, "x": 0, - "y": 0 + "y": 6 }, "panels": [] }, @@ -443,7 +443,7 @@ "h": 5, "w": 3, "x": 12, - "y": 1 + "y": 7 }, "targets": [ { @@ -506,7 +506,7 @@ "h": 5, "w": 3, "x": 6, - "y": 1 + "y": 7 }, "targets": [ { @@ -570,7 +570,7 @@ "h": 5, "w": 3, "x": 15, - "y": 1 + "y": 7 }, "targets": [ { @@ -649,7 +649,7 @@ "h": 5, "w": 3, "x": 18, - "y": 1 + "y": 7 }, "targets": [ { @@ -702,6 +702,188 @@ "justifyMode": "auto" } }, + { + "id": 43, + "type": "stat", + "title": "Total cost of all requests", + "description": "What every request through this proxy cost IN TOTAL: the provider's billed cost for the traffic plus context-guru's own compaction-model spend. The bill, with no counterfactual in it \u2014 the savings panels beside this one are all differences against a baseline, this is the number on the invoice. cg_tenant_cost_usd is the provider half and cg_tenant_cg_llm_cost_usd is ours; a deployment where the second is a large fraction of the first is paying too much to compact.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "sum(cg_tenant_cost_usd) + sum(cg_tenant_cg_llm_cost_usd)", + "editorMode": "code", + "legendFormat": "Total cost" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#199e70", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 41, + "type": "stat", + "title": "Prefix-cache saved this month", + "description": "What context-guru's volatile-tail split saved: counted only where the split ran, the environment snapshot had MOVED since that session's previous request, and the provider then read at least the stable half from cache while writing less than it. The amount is the stable half, not the request's whole cache_read \u2014 a cachesplit-off control arm still read 45,805 tokens, because the agent's other breakpoints match regardless. Priced against a cache MISS (creation rate, 1.25x fresh) because those tokens carry cache_control. EXPECT IT TO BE SMALL on interactive traffic: Claude Code captures its environment block once per session (measured: one distinct tail hash across nine turns and four commits), and a new session finds the previous prefix already expired unless it started inside the provider's 5-minute TTL (measured: 1,105 of 1,127 session starts read zero from cache). The regime where this pays is back-to-back sessions \u2014 which is what the SWE-bench A/B measured at -34.1% cost. The provider's own total cache saving is cg_tenant_cache_saved_usd, a diagnostic. This stat ADDS cg_tenant_cachesplit_historical_usd: the same saving on requests written before it could be measured per request, valued on read from the model's measured stable half and never stored. Only the session's FIRST request of each of those, because crediting a mid-session turn needs a tail hash those rows do not carry \u2014 so it is a floor on a floor.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "sum(cg_tenant_cachesplit_saved_usd) + sum(cg_tenant_cachesplit_historical_usd)", + "editorMode": "code", + "legendFormat": "Prefix-cache saved", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#199e70", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 42, + "type": "stat", + "title": "Total avoided this month", + "description": "The dashboard's own total_saved_usd, matched term for term: compaction (baseline minus actual, minus what our own model calls cost), plus the prefix-cache saving, plus the idle keep-alive's net (credit minus ping spend), plus the declaration filter's saving. Four disjoint sources, all ours, added rather than nested \u2014 the compaction baseline already assumes the cache behaved as it did. It is NOT the cost of an uncached world: the provider's own cache saved far more on the same traffic (cg_tenant_cache_saved_usd) and none of that is credited here. NEGATIVE is the bad value and is painted red: spending more on our own model or pings than we save is exactly the state this tile exists to show, and a green -$10.19 congratulates the operator for being underwater.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "sum(cg_tenant_baseline_cost_usd) - sum(cg_tenant_cost_usd) - sum(cg_tenant_cg_llm_cost_usd) + sum(cg_tenant_cachesplit_saved_usd) + sum(cg_tenant_cachesplit_historical_usd) + sum(cg_tenant_keepalive_net_usd) + sum(cg_tenant_decl_filter_usd)", + "editorMode": "code", + "legendFormat": "Total avoided this month", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#e66767", + "value": null + }, + { + "color": "#3987e5", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, { "id": 12, "type": "timeseries", @@ -715,7 +897,7 @@ "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 12 }, "targets": [ { @@ -837,7 +1019,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 12 }, "targets": [ { @@ -959,7 +1141,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 20 }, "targets": [ { @@ -1101,7 +1283,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 20 }, "targets": [ { @@ -1171,32 +1353,19 @@ } }, { - "id": 16, - "type": "row", - "title": "Which components earn their place?", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "panels": [] - }, - { - "id": 17, - "type": "bargauge", - "title": "Tokens removed by component", - "description": "The benefit side of each component. A bar at zero is dead weight: it costs latency on every request and removes nothing, so it belongs out of the preset.", + "id": 44, + "type": "stat", + "title": "Net saved this month", + "description": "This month's provider spend avoided, minus what context-guru's own compaction models cost to run. NEGATIVE means this deployment's configuration spends more than it saves \u2014 the number to alert on. Process-wide, in-process since the proxy started (see cg_tenant_* for the persistent per-tenant figures).", "datasource": { "type": "prometheus", "uid": "cg-prom" }, "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 29 + "h": 5, + "w": 3, + "x": 21, + "y": 7 }, "targets": [ { @@ -1205,68 +1374,273 @@ "type": "prometheus", "uid": "cg-prom" }, - "expr": "topk(12, sum by (component) (cg_component_saved_tokens_unique_total))", + "expr": "cg_net_saved_usd", "editorMode": "code", - "legendFormat": "{{component}}", + "legendFormat": "net saved", "instant": true, "range": false } ], "fieldConfig": { "defaults": { - "unit": "short", + "unit": "currencyUSD", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "text", + "color": "#e66767", "value": null + }, + { + "color": "#3987e5", + "value": 0 } ] }, - "min": 0, - "color": { - "mode": "fixed", - "fixedColor": "#3987e5" - } + "decimals": 2 }, "overrides": [] }, "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "color", - "namePlacement": "left", - "minVizHeight": 16, - "maxVizHeight": 44, - "minVizWidth": 8, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - } + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" } }, { - "id": 18, - "type": "bargauge", - "title": "Activity rate by component", - "description": "How often a component that ran actually CHANGED the request (mutated), not just how often it removed content tokens (acted). The distinction is the whole panel: cachesplit is mutated-never-acted by design \u2014 it moves tokens out of the hashed prefix rather than removing any \u2014 so acted/ran painted the component with the largest measured cost effect (-34.1%) as a dead red bar. 0% here means it genuinely never did anything; check cg_component_gate_declines_total for which gate turned every candidate away.", + "id": 45, + "type": "stat", + "title": "Extraction: net value", + "description": "What extract_llm's own saved tokens are worth at the rate they would have been billed, minus what its model calls cost. NEGATIVE means the component is underwater and should be turned off.", "datasource": { "type": "prometheus", "uid": "cg-prom" }, "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 29 - }, - "targets": [ + "h": 5, + "w": 12, + "x": 0, + "y": 28 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "cg_extract_net_value_usd", + "editorMode": "code", + "legendFormat": "net value", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#e66767", + "value": null + }, + { + "color": "#3987e5", + "value": 0 + } + ] + }, + "decimals": 4 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 46, + "type": "stat", + "title": "Cache-frozen headroom", + "description": "Content tokens deliberately left alone this month because they sat inside the provider's already-cached prefix \u2014 rewriting them would cost more than the tokens removed. The usual explanation for a small \"Net saved\": most of the possible saving is being left on the table on purpose.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 28 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "cg_frozen_tokens_total", + "editorMode": "code", + "legendFormat": "frozen tokens", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 16, + "type": "row", + "title": "Which components earn their place?", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "panels": [] + }, + { + "id": 17, + "type": "bargauge", + "title": "Tokens removed by component", + "description": "The benefit side of each component. A bar at zero is dead weight: it costs latency on every request and removes nothing, so it belongs out of the preset.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 34 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "topk(12, sum by (component) (cg_component_saved_tokens_unique_total))", + "editorMode": "code", + "legendFormat": "{{component}}", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "min": 0, + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "color", + "namePlacement": "left", + "minVizHeight": 16, + "maxVizHeight": 44, + "minVizWidth": 8, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 18, + "type": "bargauge", + "title": "Activity rate by component", + "description": "How often a component that ran actually CHANGED the request (mutated), not just how often it removed content tokens (acted). The distinction is the whole panel: cachesplit is mutated-never-acted by design \u2014 it moves tokens out of the hashed prefix rather than removing any \u2014 so acted/ran painted the component with the largest measured cost effect (-34.1%) as a dead red bar. 0% here means it genuinely never did anything; check cg_component_gate_declines_total for which gate turned every candidate away.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 34 + }, + "targets": [ { "refId": "A", "datasource": { @@ -1334,7 +1708,7 @@ "h": 9, "w": 8, "x": 16, - "y": 29 + "y": 34 }, "targets": [ { @@ -1399,7 +1773,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 43 }, "panels": [] }, @@ -1416,7 +1790,7 @@ "h": 6, "w": 12, "x": 12, - "y": 39 + "y": 44 }, "targets": [ { @@ -1487,7 +1861,7 @@ "id": 23, "type": "table", "title": "Per-tenant month to date", - "description": "One row per account. A table rather than more coloured series: past a handful of tenants, hues stop being distinguishable. Bad rows are a non-zero 'Disabled', or added latency far above the fleet mean. The Disabled column is intersected with this month's request rows: it is built from the REGISTRY (13 accounts) while every other column comes from tenants with traffic this month (11), so the outer join minted two rows with every cell blank but 'Disabled: no'. Fleet-wide disabled count is the 'Tenants disabled' tile, which is registry-scoped on purpose.", + "description": "One row per account. A table rather than more coloured series: past a handful of tenants, hues stop being distinguishable. Bad rows are a non-zero 'Disabled', or added latency far above the fleet mean. The Disabled column is intersected with this month's request rows: it is built from the REGISTRY (13 accounts) while every other column comes from tenants with traffic this month (11), so the outer join minted two rows with every cell blank but 'Disabled: no'. Fleet-wide disabled count is the 'Tenants disabled' tile, which is registry-scoped on purpose. \"Archived\" is sessions in cold storage; \"Archived bytes\" is how much.", "datasource": { "type": "prometheus", "uid": "cg-prom" @@ -1496,7 +1870,7 @@ "h": 6, "w": 24, "x": 0, - "y": 45 + "y": 50 }, "targets": [ { @@ -1582,6 +1956,18 @@ "instant": true, "range": false, "format": "table" + }, + { + "refId": "H", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "expr": "cg_tenant_archived_bytes{label=~\"$tenant\"}", + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" } ], "transformations": [ @@ -1611,7 +1997,8 @@ "Value #D": "Tokens removed", "Value #E": "Added latency", "Value #F": "Archived", - "Value #G": "Disabled" + "Value #G": "Disabled", + "Value #H": "Archived bytes" } } } @@ -1702,6 +2089,18 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Archived bytes" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] } ] }, @@ -1726,1236 +2125,1072 @@ "id": 24, "type": "row", "title": "Is storage and cold storage healthy?", - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 56 + }, + "panels": [] + }, + { + "id": 25, + "type": "timeseries", + "title": "Local database against cold storage", + "description": "Blue is what is still on this box, orange is what moved to Box. Blue climbing while orange stays flat is the bad shape \u2014 tiering has stopped and the disk fills.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 }, - "panels": [ + "targets": [ { - "id": 25, - "type": "timeseries", - "title": "Local database against cold storage", - "description": "Blue is what is still on this box, orange is what moved to Box. Blue climbing while orange stays flat is the bad shape \u2014 tiering has stopped and the disk fills.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 52 + "expr": "cg_dash_db_bytes", + "editorMode": "code", + "legendFormat": "local database" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "cg_dash_db_bytes", - "editorMode": "code", - "legendFormat": "local database" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "cg_archive_bytes_total", - "editorMode": "code", - "legendFormat": "cold storage" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 8, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "fixed", - "fixedColor": "#3987e5" - } + "expr": "cg_archive_bytes_total", + "editorMode": "code", + "legendFormat": "cold storage" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "local database" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#3987e5" - } - } - ] - }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "cold storage" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#d95926" - } - } - ] + "color": "text", + "value": null } ] }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" } }, - { - "id": 26, - "type": "stat", - "title": "Filesystem in use", - "description": "Crossing 90% starts evicting the oldest sessions. With cold storage ready that is a migration; with it unreachable it is deletion.", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 52 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "cg_dash_disk_used_ratio", - "editorMode": "code", - "legendFormat": "used" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#199e70", - "value": null - }, - { - "color": "#c98500", - "value": 0.85 - }, - { - "color": "#e66767", - "value": 0.9 - } - ] - }, - "decimals": 1, - "min": 0, - "max": 1 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "local database" }, - "overrides": [] + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + } + ] }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "matcher": { + "id": "byName", + "options": "cold storage" }, - "textMode": "value", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#d95926" + } + } + ] } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 26, + "type": "stat", + "title": "Filesystem in use", + "description": "Crossing 90% starts evicting the oldest sessions. With cold storage ready that is a migration; with it unreachable it is deletion.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 57 + }, + "targets": [ { - "id": 27, - "type": "stat", - "title": "Cold storage", - "description": "NOT CONFIGURED is the bad value: while it reads that, disk pressure deletes sessions instead of archiving them.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 52 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" + "expr": "cg_dash_disk_used_ratio", + "editorMode": "code", + "legendFormat": "used" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#199e70", + "value": null }, - "expr": "cg_archive_configured", - "editorMode": "code", - "legendFormat": "configured", - "instant": true, - "range": false - } - ], - "fieldConfig": { - "defaults": { - "unit": "none", - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "NOT CONFIGURED", - "color": "#e66767", - "index": 0 - }, - "1": { - "text": "ready", - "color": "#199e70", - "index": 1 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#e66767", - "value": null - }, - { - "color": "#199e70", - "value": 1 - } - ] + { + "color": "#c98500", + "value": 0.85 }, - "decimals": 0 - }, - "overrides": [] + { + "color": "#e66767", + "value": 0.9 + } + ] }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } + "decimals": 1, + "min": 0, + "max": 1 }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 27, + "type": "stat", + "title": "Cold storage", + "description": "NOT CONFIGURED is the bad value: while it reads that, disk pressure deletes sessions instead of archiving them.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 57 + }, + "targets": [ { - "id": 28, - "type": "stat", - "title": "Sessions archived", - "description": "Cumulative sessions moved to cold storage. Stuck at zero while the filesystem gauge climbs means eviction is deleting, not tiering.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 52 - }, - "targets": [ + "expr": "cg_archive_configured", + "editorMode": "code", + "legendFormat": "configured", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(cg_archive_sessions_total)", - "editorMode": "code", - "legendFormat": "archived", - "instant": true, - "range": false + "type": "value", + "options": { + "0": { + "text": "NOT CONFIGURED", + "color": "#e66767", + "index": 0 + }, + "1": { + "text": "ready", + "color": "#199e70", + "index": 1 + } + } } ], - "fieldConfig": { - "defaults": { - "unit": "short", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#e66767", + "value": null }, - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "#3987e5" + { + "color": "#199e70", + "value": 1 } - }, - "overrides": [] + ] }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 28, + "type": "stat", + "title": "Sessions archived", + "description": "Cumulative sessions moved to cold storage. Stuck at zero while the filesystem gauge climbs means eviction is deleting, not tiering.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 57 + }, + "targets": [ { - "id": 29, - "type": "timeseries", - "title": "Dropped capture events", - "description": "The capture queue filled and events were discarded. ANY non-zero bar means the dashboard is losing history exactly when load makes it most wanted.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 56 + "expr": "sum(cg_archive_sessions_total)", + "editorMode": "code", + "legendFormat": "archived", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(increase(cg_dash_events_total{disposition=\"dropped\"}[5m]))", - "editorMode": "code", - "legendFormat": "dropped" - } + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 70, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "fixed", - "fixedColor": "#e66767" - }, - "decimals": 0 - }, - "overrides": [] + "fields": "", + "values": false + }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 29, + "type": "timeseries", + "title": "Dropped capture events", + "description": "The capture queue filled and events were discarded. ANY non-zero bar means the dashboard is losing history exactly when load makes it most wanted.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] + "expr": "sum(increase(cg_dash_events_total{disposition=\"dropped\"}[5m]))", + "editorMode": "code", + "legendFormat": "dropped" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisSoftMax": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "#e66767" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ] + } }, { "id": 30, "type": "row", "title": "Is anything failing?", - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 65 }, - "panels": [ + "panels": [], + "description": "Rate-limit (429) REJECTIONS are not on this row because the proxy exports no counter for them \u2014 proxy/limits.go returns the status without recording it. Disabled accounts below are the closest available proxy." + }, + { + "id": 31, + "type": "timeseries", + "title": "Compaction-model failures", + "description": "Timeouts and errors from context-guru's own compaction model. A run of timeouts means the compactor silently did nothing: the arm looks fast because it stopped working.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "targets": [ { - "id": 31, - "type": "timeseries", - "title": "Compaction-model failures", - "description": "Timeouts and errors from context-guru's own compaction model. A run of timeouts means the compactor silently did nothing: the arm looks fast because it stopped working.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 61 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum by (kind) (increase(cg_llm_failures_total[5m]))", - "editorMode": "code", - "legendFormat": "{{kind}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 70, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "fixed", - "fixedColor": "#3987e5" - }, - "decimals": 0 + "expr": "sum by (kind) (increase(cg_llm_failures_total[5m]))", + "editorMode": "code", + "legendFormat": "{{kind}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" }, - "overrides": [ + "axisSoftMax": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "timeout" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#d95926" - } - } - ] - }, + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" + }, + "decimals": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "timeout" + }, + "properties": [ { - "matcher": { - "id": "byName", - "options": "error" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#e66767" - } - } - ] + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#d95926" + } } ] }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] + { + "matcher": { + "id": "byName", + "options": "error" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e66767" + } + } + ] } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 32, + "type": "timeseries", + "title": "Cache-write churn (frozen decisions flipped)", + "description": "Frozen decisions the store dropped and did not re-derive. Each one flips an already-cached message and forces the provider to re-write the suffix at ~12x the read price. Should sit at zero; anything above it is money.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "targets": [ { - "id": 32, - "type": "timeseries", - "title": "Cache-write churn (frozen decisions flipped)", - "description": "Frozen decisions the store dropped and did not re-derive. Each one flips an already-cached message and forces the provider to re-write the suffix at ~12x the read price. Should sit at zero; anything above it is money.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 61 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(increase(cg_frozen_decisions_total{outcome=\"dropped\"}[15m])) - sum(increase(cg_frozen_decisions_total{outcome=\"repaired\"}[15m]))", - "editorMode": "code", - "legendFormat": "flips" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 70, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "fixed", - "fixedColor": "#d95926" - }, - "decimals": 0 + "expr": "sum(increase(cg_frozen_decisions_total{outcome=\"dropped\"}[15m])) - sum(increase(cg_frozen_decisions_total{outcome=\"repaired\"}[15m]))", + "editorMode": "code", + "legendFormat": "flips" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] + "stacking": { + "mode": "none", + "group": "A" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - } - }, - { - "id": 33, - "type": "timeseries", - "title": "Tokens wasted recovering offloaded content", - "description": "Tokens spent re-serving something compaction removed. This is savings handed back \u2014 a rising line means the pipeline is too aggressive for this traffic.", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" + "axisSoftMax": 1 }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 69 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(rate(cg_wasted_tokens_total[5m])) * 60", - "editorMode": "code", - "legendFormat": "wasted" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 70, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "fixed", - "fixedColor": "#e66767" - }, - "decimals": 0 - }, - "overrides": [] + "color": { + "mode": "fixed", + "fixedColor": "#d95926" }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - } + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, + { + "id": 33, + "type": "timeseries", + "title": "Tokens wasted recovering offloaded content", + "description": "Tokens spent re-serving something compaction removed. This is savings handed back \u2014 a rising line means the pipeline is too aggressive for this traffic.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 74 + }, + "targets": [ { - "id": 34, - "type": "stat", - "title": "Buffered streams", - "description": "Share of responses context-guru had to buffer in full before the client saw a byte. High on traffic that never expands is a straight latency regression.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 69 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(cg_sse_streams_total{path=\"buffered\"}) / clamp_min(sum(cg_sse_streams_total), 1)", - "editorMode": "code", - "legendFormat": "buffered" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#199e70", - "value": null - }, - { - "color": "#c98500", - "value": 0.2 - }, - { - "color": "#e66767", - "value": 0.5 - } - ] - }, - "decimals": 0, - "min": 0, - "max": 1 + "expr": "sum(rate(cg_wasted_tokens_total[5m])) * 60", + "editorMode": "code", + "legendFormat": "wasted" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "stacking": { + "mode": "none", + "group": "A" }, - "textMode": "value", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - } + "axisSoftMax": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "#e66767" + }, + "decimals": 0 }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, + { + "id": 34, + "type": "stat", + "title": "Buffered streams", + "description": "Share of responses context-guru had to buffer in full before the client saw a byte. High on traffic that never expands is a straight latency regression.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 74 + }, + "targets": [ { - "id": 35, - "type": "stat", - "title": "Tenants disabled", - "description": "Accounts the proxy is currently refusing. Anything above zero is somebody's agent failing right now.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 69 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" + "expr": "sum(cg_sse_streams_total{path=\"buffered\"}) / clamp_min(sum(cg_sse_streams_total), 1)", + "editorMode": "code", + "legendFormat": "buffered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#199e70", + "value": null }, - "expr": "sum(cg_tenant_disabled)", - "editorMode": "code", - "legendFormat": "disabled", - "instant": true, - "range": false - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#199e70", - "value": null - }, - { - "color": "#e66767", - "value": 1 - } - ] + { + "color": "#c98500", + "value": 0.2 }, - "decimals": 0 - }, - "overrides": [] + { + "color": "#e66767", + "value": 0.5 + } + ] }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } + "decimals": 0, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "textMode": "value", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 35, + "type": "stat", + "title": "Tenants disabled", + "description": "Accounts the proxy is currently refusing. Anything above zero is somebody's agent failing right now.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 74 + }, + "targets": [ { - "id": 37, - "type": "timeseries", - "title": "Expand bounces over time", - "description": "The same recoveries as the stat above, as a trend. A step up right after a config change means the new preset removes something the agent needs.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 73 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(increase(cg_expand_bounces_total[5m]))", - "editorMode": "code", - "legendFormat": "bounces" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 70, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "fixed", - "fixedColor": "#e66767" + "expr": "sum(cg_tenant_disabled)", + "editorMode": "code", + "legendFormat": "disabled", + "instant": true, + "range": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#199e70", + "value": null }, - "decimals": 0 - }, - "overrides": [] + { + "color": "#e66767", + "value": 1 + } + ] }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - } + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "textMode": "value", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 37, + "type": "timeseries", + "title": "Expand bounces over time", + "description": "The same recoveries as the stat above, as a trend. A step up right after a config change means the new preset removes something the agent needs.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 78 + }, + "targets": [ { - "id": 38, - "type": "timeseries", - "title": "Buffered against streamed responses", - "description": "Orange is responses context-guru had to read in full before the client saw a byte. Orange growing faster than blue is a straight latency regression.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 73 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(cg_sse_streams_total{path=\"streamed\"})", - "editorMode": "code", - "legendFormat": "streamed" + "expr": "sum(increase(cg_expand_bounces_total[5m]))", + "editorMode": "code", + "legendFormat": "bounces" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum(cg_sse_streams_total{path=\"buffered\"})", - "editorMode": "code", - "legendFormat": "buffered" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 8, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "fixed", - "fixedColor": "#3987e5" - }, - "decimals": 0 + "stacking": { + "mode": "none", + "group": "A" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "streamed" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#3987e5" - } - } - ] - }, + "axisSoftMax": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "buffered" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "#d95926" - } - } - ] + "color": "text", + "value": null } ] }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - } + "color": { + "mode": "fixed", + "fixedColor": "#e66767" + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, + { + "id": 38, + "type": "timeseries", + "title": "Buffered against streamed responses", + "description": "Orange is responses context-guru had to read in full before the client saw a byte. Orange growing faster than blue is a straight latency regression.", + "datasource": { + "type": "prometheus", + "uid": "cg-prom" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 78 + }, + "targets": [ { - "id": 39, - "type": "timeseries", - "title": "Requests refused, by reason", - "description": "Requests turned away before an upstream was called. ANY sustained bar is somebody's agent failing right now. rate_limit and concurrency are both 429 with different fixes (raise TENANT_RPM, raise the in-flight cap), auth/forbidden is a bad or disabled token, no_upstream is our own misconfiguration and upstream_error is the provider.", + "refId": "A", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 77 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum by (reason) (increase(cg_refused_requests_total[5m]))", - "editorMode": "code", - "legendFormat": "{{reason}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 70, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic-by-name" - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - } + "expr": "sum(cg_sse_streams_total{path=\"streamed\"})", + "editorMode": "code", + "legendFormat": "streamed" }, { - "id": 40, - "type": "timeseries", - "title": "Refusals per tenant", - "description": "Which account is being refused, and for what. This is the panel that tells a quiet tenant apart from a blocked one: a tenant absent from 'Requests per tenant' AND present here is not idle, it is being turned away. Zero everywhere is healthy.", + "refId": "B", "datasource": { "type": "prometheus", "uid": "cg-prom" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 77 + "expr": "sum(cg_sse_streams_total{path=\"buffered\"})", + "editorMode": "code", + "legendFormat": "buffered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisSoftMax": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "cg-prom" - }, - "expr": "sum by (label, reason) (increase(cg_tenant_refused_requests_total{label=~\"$tenant\"}[5m]))", - "editorMode": "code", - "legendFormat": "{{label}} / {{reason}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 70, - "showPoints": "never", - "pointSize": 8, - "spanNulls": true, - "axisSoftMin": 0, - "axisGridShow": true, - "axisBorderShow": false, - "lineInterpolation": "linear", - "gradientMode": "none", - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisSoftMax": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic-by-name" - }, - "decimals": 0 + "decimals": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "streamed" }, - "overrides": [] + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + } + ] }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] + { + "matcher": { + "id": "byName", + "options": "buffered" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#d95926" + } + } + ] } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - ], - "description": "Rate-limit (429) REJECTIONS are not on this row because the proxy exports no counter for them \u2014 proxy/limits.go returns the status without recording it. Disabled accounts below are the closest available proxy." + } }, { - "id": 43, - "type": "stat", - "title": "Total cost of all requests", - "description": "What every request through this proxy cost IN TOTAL: the provider's billed cost for the traffic plus context-guru's own compaction-model spend. The bill, with no counterfactual in it \u2014 the savings panels beside this one are all differences against a baseline, this is the number on the invoice. cg_tenant_cost_usd is the provider half and cg_tenant_cg_llm_cost_usd is ours; a deployment where the second is a large fraction of the first is paying too much to compact.", + "id": 39, + "type": "timeseries", + "title": "Requests refused, by reason", + "description": "Requests turned away before an upstream was called. ANY sustained bar is somebody's agent failing right now. rate_limit and concurrency are both 429 with different fixes (raise TENANT_RPM, raise the in-flight cap), auth/forbidden is a bad or disabled token, no_upstream is our own misconfiguration and upstream_error is the provider.", "datasource": { "type": "prometheus", "uid": "cg-prom" }, "gridPos": { - "h": 5, - "w": 3, + "h": 8, + "w": 12, "x": 0, - "y": 1 + "y": 82 }, "targets": [ { @@ -2964,56 +3199,79 @@ "type": "prometheus", "uid": "cg-prom" }, - "expr": "sum(cg_tenant_cost_usd) + sum(cg_tenant_cg_llm_cost_usd)", + "expr": "sum by (reason) (increase(cg_refused_requests_total[5m]))", "editorMode": "code", - "legendFormat": "Total cost" + "legendFormat": "{{reason}}" } ], "fieldConfig": { "defaults": { - "unit": "currencyUSD", - "decimals": 2, + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisSoftMax": 1 + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "#199e70", + "color": "text", "value": null } ] - } + }, + "color": { + "mode": "palette-classic-by-name" + }, + "decimals": 0 }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] }, - "textMode": "value", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" + "tooltip": { + "mode": "multi", + "sort": "none" + } } }, { - "id": 41, - "type": "stat", - "title": "Prefix-cache saved this month", - "description": "What context-guru's volatile-tail split saved: counted only where the split ran, the environment snapshot had MOVED since that session's previous request, and the provider then read at least the stable half from cache while writing less than it. The amount is the stable half, not the request's whole cache_read \u2014 a cachesplit-off control arm still read 45,805 tokens, because the agent's other breakpoints match regardless. Priced against a cache MISS (creation rate, 1.25x fresh) because those tokens carry cache_control. EXPECT IT TO BE SMALL on interactive traffic: Claude Code captures its environment block once per session (measured: one distinct tail hash across nine turns and four commits), and a new session finds the previous prefix already expired unless it started inside the provider's 5-minute TTL (measured: 1,105 of 1,127 session starts read zero from cache). The regime where this pays is back-to-back sessions \u2014 which is what the SWE-bench A/B measured at -34.1% cost. The provider's own total cache saving is cg_tenant_cache_saved_usd, a diagnostic. This stat ADDS cg_tenant_cachesplit_historical_usd: the same saving on requests written before it could be measured per request, valued on read from the model's measured stable half and never stored. Only the session's FIRST request of each of those, because crediting a mid-session turn needs a tail hash those rows do not carry \u2014 so it is a floor on a floor.", + "id": 40, + "type": "timeseries", + "title": "Refusals per tenant", + "description": "Which account is being refused, and for what. This is the panel that tells a quiet tenant apart from a blocked one: a tenant absent from 'Requests per tenant' AND present here is not idle, it is being turned away. Zero everywhere is healthy.", "datasource": { "type": "prometheus", "uid": "cg-prom" }, "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 + "h": 8, + "w": 12, + "x": 12, + "y": 82 }, "targets": [ { @@ -3022,58 +3280,79 @@ "type": "prometheus", "uid": "cg-prom" }, - "expr": "sum(cg_tenant_cachesplit_saved_usd) + sum(cg_tenant_cachesplit_historical_usd)", + "expr": "sum by (label, reason) (increase(cg_tenant_refused_requests_total{label=~\"$tenant\"}[5m]))", "editorMode": "code", - "legendFormat": "Prefix-cache saved", - "instant": true, - "range": false + "legendFormat": "{{label}} / {{reason}}" } ], "fieldConfig": { "defaults": { - "unit": "currencyUSD", - "decimals": 2, + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisSoftMax": 1 + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "#199e70", + "color": "text", "value": null } ] - } + }, + "color": { + "mode": "palette-classic-by-name" + }, + "decimals": 0 }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] }, - "textMode": "value", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" + "tooltip": { + "mode": "multi", + "sort": "none" + } } }, { - "id": 42, - "type": "stat", - "title": "Total avoided this month", - "description": "The dashboard's own total_saved_usd, matched term for term: compaction (baseline minus actual, minus what our own model calls cost), plus the prefix-cache saving, plus the idle keep-alive's net (credit minus ping spend), plus the declaration filter's saving. Four disjoint sources, all ours, added rather than nested \u2014 the compaction baseline already assumes the cache behaved as it did. It is NOT the cost of an uncached world: the provider's own cache saved far more on the same traffic (cg_tenant_cache_saved_usd) and none of that is credited here. NEGATIVE is the bad value and is painted red: spending more on our own model or pings than we save is exactly the state this tile exists to show, and a green -$10.19 congratulates the operator for being underwater.", + "id": 47, + "type": "timeseries", + "title": "Expand recovery failures, by cause", + "description": "Expand attempts that recovered nothing. reason=\"missing\" is the ALERTABLE one: an id this proxy could have minted resolved to no stash, so a cut advertised as reversible was not and content the agent asked back for is gone. reason=\"malformed\" means the model invented a marker id. Neither shows up in Expand bounces, which counts only successful re-serves.", "datasource": { "type": "prometheus", "uid": "cg-prom" }, "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 1 + "h": 4, + "w": 24, + "x": 0, + "y": 90 }, "targets": [ { @@ -3082,47 +3361,63 @@ "type": "prometheus", "uid": "cg-prom" }, - "expr": "sum(cg_tenant_baseline_cost_usd) - sum(cg_tenant_cost_usd) - sum(cg_tenant_cg_llm_cost_usd) + sum(cg_tenant_cachesplit_saved_usd) + sum(cg_tenant_cachesplit_historical_usd) + sum(cg_tenant_keepalive_net_usd) + sum(cg_tenant_decl_filter_usd)", + "expr": "sum by (reason) (increase(cg_expand_unresolved_total[5m]))", "editorMode": "code", - "legendFormat": "Total avoided this month", - "instant": true, - "range": false + "legendFormat": "{{reason}}" } ], "fieldConfig": { "defaults": { - "unit": "currencyUSD", - "decimals": 2, + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "spanNulls": true, + "axisSoftMin": 0, + "axisGridShow": true, + "axisBorderShow": false, + "lineInterpolation": "linear", + "gradientMode": "none", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "#e66767", + "color": "text", "value": null - }, - { - "color": "#199e70", - "value": 0 } ] - } + }, + "color": { + "mode": "palette-classic-by-name" + }, + "decimals": 0 }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] }, - "textMode": "value", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" + "tooltip": { + "mode": "multi", + "sort": "none" + } } } ] -} \ No newline at end of file +} diff --git a/deploy/grafana/provisioning/alerting/context-guru.yml b/deploy/grafana/provisioning/alerting/context-guru.yml index 9e94138f..ee89a0b9 100644 --- a/deploy/grafana/provisioning/alerting/context-guru.yml +++ b/deploy/grafana/provisioning/alerting/context-guru.yml @@ -3,10 +3,11 @@ # cp -r deploy/grafana/provisioning/* /etc/grafana/provisioning/ # systemctl restart grafana-server # or: docker restart cg-grafana # -# The alerting directory shipped EMPTY, so nothing paged. Two rules, both chosen because +# The alerting directory shipped EMPTY, so nothing paged. Three rules, all chosen because # they are the failures a dashboard cannot catch: nobody is watching a screen when the -# service dies at 03:00, and a refusal rate is invisible on every panel that plots only -# the requests that got through. +# service dies at 03:00, a refusal rate is invisible on every panel that plots only the +# requests that got through, and a stale-but-served /metrics body (2026-09-02's dashboard +# outage) reads as a perfectly healthy scrape on every panel downstream of it. # # Rules fire into whatever notification policy the instance already has (Grafana's default # one if none was provisioned) — deliberately not overridden here, because a contact point @@ -121,3 +122,57 @@ groups: - evaluator: type: gt params: [0.1] + + # --- the exposition is stale, even though the scrape looks fine -------- + # + # 2026-09-02: promCacheTTL (10s) was shorter than the 15s scrape_interval, so the + # cached body was always already expired when a scrape arrived, every scrape paid a + # full render, and once a render exceeded scrape_timeout the target went down and + # STAYED down — every dashboard built on cg_* series read "No data" for a whole day + # before anyone noticed. Fixed in the render path (promCacheTTL now 60s, + # stale-while-revalidate instead of render-on-miss), but the failure mode this alert + # exists for is more general than that one bug: /metrics can start serving a body + # that stopped updating while `up{job="context-guru"}` still reads 1, because the + # scrape itself succeeds — cg-service-down cannot see this by construction. + # NoData is Alerting, same reasoning as cg-service-down: an absent series is at least + # as bad as a stale one. + - uid: cg-metrics-stale + title: context-guru's own metrics have stopped refreshing + condition: B + for: 5m + noDataState: Alerting + execErrState: Alerting + annotations: + summary: The /metrics exposition has not refreshed in over 2 minutes. + description: >- + cg_metrics_age_seconds is above 120s: whatever this dashboard shows for every + cg_* series is that old or older, even though Prometheus itself is scraping + fine. Check cg_metrics_render_seconds for whether a render is stuck (per-tenant + store queries are the usual suspect — see CachesplitHistoricalUSD's history) + and `journalctl -u context-guru` for what the render is stuck on. + labels: + severity: critical + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: cg-prom + model: + refId: A + editorMode: code + expr: max(cg_metrics_age_seconds) + instant: true + range: false + intervalMs: 15000 + maxDataPoints: 43200 + - refId: B + datasourceUid: __expr__ + model: + refId: B + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: [120] From 991f48cb2905238363081f70a2bf4478d531d970 Mon Sep 17 00:00:00 2001 From: Osher Elhadad Date: Thu, 3 Sep 2026 06:05:01 +0000 Subject: [PATCH 2/2] test(proxy): assert the dashboard-metric guard against the SERVED response, not renderMetrics() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestDashboardsOnlyQueryMetricsWeExport failed on this branch (both build-test and purego, same single test) reporting cg_metrics_age_seconds and cg_metrics_render_seconds as unexported. They are exported — #183 ships them — but not by renderMetrics(), which is what the test inspected. Those two are appended PER RESPONSE in writeMetrics rather than built into the cached body, and that is deliberate: the body is shared by every scrape served from one render, so its age differs for each of them. Baking an age into the cached body would make it report the same figure to every scrape, which is the exact lie those two series exist to expose. So the test was asserting against the wrong artifact. Its own doc comment says the property under test is "every cg_* name any dashboard queries must appear in the rendered exposition" — and what a scraper actually receives is the response. It now issues a real request through metricsHandler from loopback (metricsAllowed gates on that absent a METRICS_TOKEN) and asserts on that body, which covers both body-level and response-level series. Strictly stronger, not looser: verified by pointing a panel at a nonexistent cg_totally_bogus_metric_xyz, which the test still catches and names. Signed-off-by: Osher Elhadad --- proxy/dashboardmetrics_test.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/proxy/dashboardmetrics_test.go b/proxy/dashboardmetrics_test.go index c435f725..66c210ca 100644 --- a/proxy/dashboardmetrics_test.go +++ b/proxy/dashboardmetrics_test.go @@ -2,6 +2,8 @@ package proxy import ( "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" "regexp" @@ -27,9 +29,18 @@ func (stubTenantMetrics) TenantMetrics(int64) ([]TenantMetricRow, error) { // and the exporter is hand-written, so the two drift silently and the first symptom is an // operator reading a blank panel as "no problem". // -// So: every cg_* name any dashboard queries must appear in the rendered exposition. The -// per-tenant families need a tenant to have traffic, so the HELP line is what is checked -// for those — it is emitted with the header regardless of whether any series follows. +// So: every cg_* name any dashboard queries must appear in the exposition. The per-tenant +// families need a tenant to have traffic, so the HELP line is what is checked for those — it +// is emitted with the header regardless of whether any series follows. +// +// It checks the SERVED RESPONSE, not renderMetrics() alone, and the difference is load-bearing. +// Not every series lives in the cached body: cg_metrics_age_seconds and +// cg_metrics_render_seconds are appended per response, because the body is shared by every +// scrape served from one render and its age differs for each of them — baking an age into the +// cached body would make it report the same figure to every scrape, which is the exact lie +// those two series exist to expose. Asserting against renderMetrics() would therefore fail on +// series that a scraper does in fact receive. What Prometheus actually gets is the response, so +// that is what this asserts on, and it now covers both kinds. func TestDashboardsOnlyQueryMetricsWeExport(t *testing.T) { // A recorder and a tenant-metrics source, because whole families of series are // emitted only when those are wired — which is also why this test needs to exist: @@ -43,7 +54,15 @@ func TestDashboardsOnlyQueryMetricsWeExport(t *testing.T) { h := New(nil, nil, metrics.NewAggregator(), Options{ Dashboard: rec, TenantMetrics: stubTenantMetrics{}, }) - body := h.renderMetrics() + // Loopback, because metricsAllowed gates the endpoint on it when no METRICS_TOKEN is set. + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.RemoteAddr = "127.0.0.1:1" + w := httptest.NewRecorder() + h.metricsHandler(w, req) + if w.Code != http.StatusOK { + t.Fatalf("/metrics returned %d, want 200: %s", w.Code, w.Body.String()) + } + body := w.Body.String() files, err := filepath.Glob("../deploy/grafana/dashboards/*.json") if err != nil || len(files) == 0 {