From 6edee6e2ba7f28a8623f1c06764ef1d46f0c4afd Mon Sep 17 00:00:00 2001 From: Devadakene Date: Mon, 31 Aug 2026 08:30:44 +0100 Subject: [PATCH] feat: enhance metrics monitoring for keeper and rpc --- scripts/grafana-dashboard.json | 98 ++++++++++++++++++++++++++++++++++ scripts/keeper.ts | 87 ++++++++++++++++++++++++------ scripts/metrics-server.ts | 27 ++++++++++ scripts/rpc-client.ts | 2 + 4 files changed, 199 insertions(+), 15 deletions(-) diff --git a/scripts/grafana-dashboard.json b/scripts/grafana-dashboard.json index be6b53ea..b52a7d69 100644 --- a/scripts/grafana-dashboard.json +++ b/scripts/grafana-dashboard.json @@ -440,6 +440,104 @@ ], "title": "Avg Batch Size", "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 30 }, + "id": 9, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", + "expr": "increase(keeper_charge_results_total[$__rate_interval])", + "legendFormat": "{{result}}", + "range": true, + "refId": "A" + } + ], + "title": "Charge Results Breakdown", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 30 }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", + "expr": "rate(keeper_rpc_failovers_total[$__rate_interval])", + "refId": "A" + } + ], + "title": "RPC Failover Rate", + "type": "stat" } ], "refresh": "10s", diff --git a/scripts/keeper.ts b/scripts/keeper.ts index 4239d2f4..dff54132 100644 --- a/scripts/keeper.ts +++ b/scripts/keeper.ts @@ -44,6 +44,13 @@ import fs from "fs"; import path from "path"; import { Server, assembleTransaction } from "@stellar/stellar-sdk/rpc"; import { buildOptimizedBatches } from "./batch-optimizer"; +import { + startMetricsServer, + recordBatchCharge, + recordChargeResults, + incrementCycles, + setActiveSubscribers, +} from "./metrics-server"; import { Address, Contract, @@ -532,24 +539,47 @@ async function processPageDryRun(users: string[], pageOffset: number): Promise { // ── Post-cycle reporting ─────────────────────────────────────────────────── + setActiveSubscribers(report.totalChecked); + if (!isDryRun) { writeLatestLive(report); } else { @@ -1001,14 +1054,18 @@ async function main(): Promise { log(false, "Keeper started in LIVE mode"); } + startMetricsServer(); + if (once) { const report = await runCycle(); + incrementCycles(); process.exit(report.errors.length > 0 && report.totalCharged === 0 ? 1 : 0); } // Loop mode while (true) { const report = await runCycle(); + incrementCycles(); const nextRun = new Date(Date.now() + INTERVAL_SECONDS * 1000); log(DRY_RUN, `Next cycle at ${nextRun.toISOString()} (in ${INTERVAL_SECONDS}s)`); diff --git a/scripts/metrics-server.ts b/scripts/metrics-server.ts index db45e39f..e4b90ffa 100644 --- a/scripts/metrics-server.ts +++ b/scripts/metrics-server.ts @@ -109,6 +109,21 @@ const indexerDedupEvictionsTotal = new Counter({ registers: [registry], }); +/** Total RPC failovers across multiple endpoints. */ +const rpcFailoversTotal = new Counter({ + name: "keeper_rpc_failovers_total", + help: "Total number of RPC failovers triggered", + registers: [registry], +}); + +/** Granular outcomes for each subscriber checked. */ +const chargeResultsTotal = new Counter({ + name: "keeper_charge_results_total", + help: "Total number of charge outcomes labeled by specific contract result", + labelNames: ["result"] as const, + registers: [registry], +}); + // ── Public API for the keeper run loop ─────────────────────────────────────── /** @@ -143,11 +158,23 @@ export function recordBatchCharge(params: { } } +/** Record specific granular charge results. */ +export function recordChargeResults(results: Record): void { + for (const [result, count] of Object.entries(results)) { + chargeResultsTotal.inc({ result }, count); + } +} + /** Increment the RPC error counter. */ export function incrementRpcErrors(): void { rpcErrorsTotal.inc(1); } +/** Increment the RPC failovers counter. */ +export function incrementRpcFailovers(): void { + rpcFailoversTotal.inc(1); +} + /** Set the current active subscriber count gauge. */ export function setActiveSubscribers(count: number): void { activeSubscribers.set(count); diff --git a/scripts/rpc-client.ts b/scripts/rpc-client.ts index f2b7e27e..75a7c1c8 100644 --- a/scripts/rpc-client.ts +++ b/scripts/rpc-client.ts @@ -1,5 +1,6 @@ import { Account, Address, Contract, FeeBumpTransaction, Transaction, xdr, Networks } from "@stellar/stellar-sdk"; import { Server, Durability, Api } from "@stellar/stellar-sdk/rpc"; +import { incrementRpcFailovers } from "./metrics-server"; /** * Interface representing state of a single RPC endpoint. @@ -146,6 +147,7 @@ export class MultiEndpointServer { console.warn( `[RPC Failover] Endpoint ${failedUrl} failed: ${err?.message || err}. Retrying with ${nextUrl}...` ); + incrementRpcFailovers(); } }