Skip to content

Created local load gen setup and have a working baseline #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dj-wasabi merged 2 commits into from
May 29, 2026
Merged

Created local load gen setup and have a working baseline #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5119379
Created local load gen setup and have a working baseline
dj-wasabi May 29, 2026
b7c8a69
Fixed lint issue
dj-wasabi May 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,13 @@ labels-contract-check: ## Diff LABELS_CONTRACT.md against promforecast's copy.
dev-up: ## Start the minimum dev stack: VM, Prometheus, Redis, node-exporter, detector.
docker compose -f docker-compose.dev.yml up -d --build

.PHONY: dev-up-load
dev-up-load: ## Start the full demo stack: minimum + http-demo + loadgen + Grafana.
docker compose -f docker-compose.dev.yml --profile load up -d --build

.PHONY: dev-down
dev-down: ## Stop the local dev stack.
docker compose -f docker-compose.dev.yml down -v
dev-down: ## Stop the local dev stack (any profile).
docker compose -f docker-compose.dev.yml --profile load down -v

.PHONY: load-test
load-test: ## Synthetic high-cardinality run for safety-control validation.
Expand Down
4 changes: 1 addition & 3 deletions community/promanomaly-prophet/tests/test_detector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,7 @@ def test_fit_score_normalises_residual_by_band(stub_prophet: type) -> None:
assert bool(row["is_outside"]) is False


def test_params_forwarded_to_prophet(
monkeypatch: pytest.MonkeyPatch, stub_prophet: type
) -> None:
def test_params_forwarded_to_prophet(monkeypatch: pytest.MonkeyPatch, stub_prophet: type) -> None:
# Sanity check: custom params reach the Prophet constructor. We
# wrap the stub with a capturing subclass that records the kwargs
# it was called with — simpler than threading state through the
Expand Down
31 changes: 19 additions & 12 deletions dashboards/grafana/promanomaly-change-points.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
{
"annotations": {
"list": [
{
"name": "datasource",
"type": "datasource",
"label": "Datasource",
"query": "prometheus",
"current": {"text": "VictoriaMetrics", "value": "victoriametrics"}
},
{
"name": "change-points",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"enable": true,
"iconColor": "red",
"expr": "changes(anomaly_change_point_total{group=~\"$group\", id=~\"$id\"}[5m]) > 0",
Expand All @@ -21,7 +28,7 @@
{
"name": "group",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_change_point_total, group)",
"refresh": 2,
"multi": true,
Expand All @@ -30,7 +37,7 @@
{
"name": "id",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_change_point_total{group=~\"$group\"}, id)",
"refresh": 2,
"multi": true,
Expand All @@ -39,7 +46,7 @@
{
"name": "detector",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_change_point_total{group=~\"$group\", id=~\"$id\"}, detector)",
"refresh": 2,
"multi": true,
Expand All @@ -52,7 +59,7 @@
"id": 1,
"type": "stat",
"title": "Change-points in the last 24h",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "sum(increase(anomaly_change_point_total{group=~\"$group\", id=~\"$id\", detector=~\"$detector\"}[24h]))",
Expand All @@ -66,7 +73,7 @@
"id": 2,
"type": "stat",
"title": "Series with recent change-points",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "count(increase(anomaly_change_point_total{group=~\"$group\", id=~\"$id\", detector=~\"$detector\"}[1h]) > 0)",
Expand All @@ -80,7 +87,7 @@
"id": 3,
"type": "stat",
"title": "Active change-point detectors",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "count(count by (detector) (anomaly_change_point_total{group=~\"$group\", detector=~\"$detector\"}))",
Expand All @@ -93,7 +100,7 @@
"id": 4,
"type": "stat",
"title": "Change-points last 5m",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "sum(increase(anomaly_change_point_total{group=~\"$group\", id=~\"$id\", detector=~\"$detector\"}[5m]))",
Expand All @@ -108,7 +115,7 @@
"type": "timeseries",
"title": "Change-point rate by detector",
"description": "Rate of change-point firings per detector. Each spike is a regime shift the detector flagged.",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "sum by (detector) (rate(anomaly_change_point_total{group=~\"$group\", id=~\"$id\", detector=~\"$detector\"}[5m]))",
Expand All @@ -122,7 +129,7 @@
"type": "timeseries",
"title": "anomaly_score (change-point detectors)",
"description": "Underlying score driving change-point counters. For BOCPD the score is a posterior probability; for CUSUM it is a sigma multiplier.",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_score{group=~\"$group\", id=~\"$id\", detector=~\"BOCPD|CUSUM\"}",
Expand All @@ -136,7 +143,7 @@
"type": "table",
"title": "Top-N change-points (last 1h)",
"description": "Series with the most change-point firings in the last hour. Click through to drill down into the underlying signal.",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "topk(20, increase(anomaly_change_point_total{group=~\"$group\", id=~\"$id\", detector=~\"$detector\"}[1h]))",
Expand All @@ -151,7 +158,7 @@
"type": "timeseries",
"title": "Source signal with change-point overlay",
"description": "Underlying baseline overlaid for context. Annotations (top of dashboard) mark change-point firings.",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_baseline{group=~\"$group\", id=~\"$id\", detector=~\"$detector\"}",
Expand Down
25 changes: 16 additions & 9 deletions dashboards/grafana/promanomaly-cohort.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@
"tags": ["promanomaly", "anomaly-detection", "cohort"],
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"label": "Datasource",
"query": "prometheus",
"current": {"text": "VictoriaMetrics", "value": "victoriametrics"}
},
{
"name": "group",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_score{detector=\"Cohort\"}, group)",
"refresh": 2,
"multi": true,
Expand All @@ -18,7 +25,7 @@
{
"name": "id",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_score{detector=\"Cohort\", group=~\"$group\"}, id)",
"refresh": 2,
"multi": true,
Expand All @@ -31,7 +38,7 @@
"id": 1,
"type": "stat",
"title": "Cohort members currently diverging",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "count(anomaly_outside_threshold{detector=\"Cohort\", group=~\"$group\", id=~\"$id\"} == 1)",
Expand All @@ -45,7 +52,7 @@
"id": 2,
"type": "stat",
"title": "Active cohorts",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "count(count by (id, group) (anomaly_score{detector=\"Cohort\", group=~\"$group\", id=~\"$id\"}))",
Expand All @@ -59,7 +66,7 @@
"id": 3,
"type": "stat",
"title": "Max divergence score (60s)",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "max(max_over_time(anomaly_score{detector=\"Cohort\", group=~\"$group\", id=~\"$id\"}[1m]))",
Expand All @@ -73,7 +80,7 @@
"id": 4,
"type": "table",
"title": "Top-N currently diverging cohort members",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "topk(10, anomaly_score{detector=\"Cohort\", group=~\"$group\", id=~\"$id\"})",
Expand All @@ -88,7 +95,7 @@
"type": "timeseries",
"title": "Cohort divergence score over time",
"description": "Per-member divergence from the cohort baseline. A member rising above the rest is the 'one bad node in the fleet' case the Cohort detector targets.",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_score{detector=\"Cohort\", group=~\"$group\", id=~\"$id\"}",
Expand All @@ -102,7 +109,7 @@
"type": "timeseries",
"title": "Cohort baseline vs. individual members",
"description": "anomaly_baseline carries the cross-member median for the Cohort detector. Overlay the source signal to see where each member sits relative to the fleet.",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_baseline{detector=\"Cohort\", group=~\"$group\", id=~\"$id\"}",
Expand All @@ -116,7 +123,7 @@
"type": "timeseries",
"title": "Outside-threshold transitions (sustained divergence)",
"description": "anomaly_outside_threshold transitions for Cohort. Used by the AnomalyOutsideThreshold reference alert with for: 5m flap suppression.",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_outside_threshold{detector=\"Cohort\", group=~\"$group\", id=~\"$id\"}",
Expand Down
23 changes: 15 additions & 8 deletions dashboards/grafana/promanomaly-confidence.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@
"tags": ["promanomaly", "anomaly-detection", "confidence"],
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"label": "Datasource",
"query": "prometheus",
"current": {"text": "VictoriaMetrics", "value": "victoriametrics"}
},
{
"name": "group",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_confidence_score, group)",
"refresh": 2,
"multi": true,
Expand All @@ -18,7 +25,7 @@
{
"name": "id",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_confidence_score{group=~\"$group\"}, id)",
"refresh": 2,
"multi": true,
Expand All @@ -31,7 +38,7 @@
"id": 1,
"type": "timeseries",
"title": "Confidence score per detector",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_confidence_score{group=~\"$group\", id=~\"$id\"}",
Expand All @@ -58,7 +65,7 @@
"id": 2,
"type": "timeseries",
"title": "Baseline stability per detector",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_baseline_stability{group=~\"$group\", id=~\"$id\"}",
Expand All @@ -85,7 +92,7 @@
"id": 3,
"type": "table",
"title": "Low-confidence firing right now",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "topk(20, anomaly_confidence_score{group=~\"$group\"} < 0.5 and on (id, group, detector) anomaly_outside_threshold == 1)",
Expand All @@ -99,7 +106,7 @@
"id": 4,
"type": "stat",
"title": "High-confidence anomalies",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "count(anomaly_outside_threshold{group=~\"$group\"} == 1 and on (id, group, detector) anomaly_confidence_score >= 0.7)",
Expand All @@ -112,7 +119,7 @@
"id": 5,
"type": "stat",
"title": "Low-confidence anomalies",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "count(anomaly_outside_threshold{group=~\"$group\"} == 1 and on (id, group, detector) anomaly_confidence_score < 0.5)",
Expand All @@ -125,7 +132,7 @@
"id": 6,
"type": "stat",
"title": "Series with no calibration yet",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "count(anomaly_score{group=~\"$group\"}) - count(anomaly_confidence_score{group=~\"$group\"})",
Expand Down
23 changes: 15 additions & 8 deletions dashboards/grafana/promanomaly-detector-comparison.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@
"tags": ["promanomaly", "anomaly-detection", "detectors"],
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"label": "Datasource",
"query": "prometheus",
"current": {"text": "VictoriaMetrics", "value": "victoriametrics"}
},
{
"name": "group",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_score, group)",
"refresh": 2,
"multi": false,
Expand All @@ -18,7 +25,7 @@
{
"name": "id",
"type": "query",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"query": "label_values(anomaly_score{group=~\"$group\"}, id)",
"refresh": 2,
"multi": false,
Expand All @@ -31,7 +38,7 @@
"id": 1,
"type": "timeseries",
"title": "anomaly_score by detector",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_score{group=~\"$group\", id=~\"$id\"}",
Expand All @@ -44,7 +51,7 @@
"id": 2,
"type": "timeseries",
"title": "anomaly_outside_threshold by detector",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_outside_threshold{group=~\"$group\", id=~\"$id\"}",
Expand All @@ -60,7 +67,7 @@
"id": 3,
"type": "timeseries",
"title": "Composite (ensemble) verdict",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_composite_score{group=~\"$group\", id=~\"$id\"}",
Expand All @@ -77,7 +84,7 @@
"id": 4,
"type": "timeseries",
"title": "anomaly_duration_seconds by detector",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_duration_seconds{group=~\"$group\", id=~\"$id\"}",
Expand All @@ -90,7 +97,7 @@
"id": 5,
"type": "timeseries",
"title": "anomaly_confidence_score by detector",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_confidence_score{group=~\"$group\", id=~\"$id\"}",
Expand All @@ -106,7 +113,7 @@
"id": 6,
"type": "table",
"title": "Current per-detector verdict",
"datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"targets": [
{
"expr": "anomaly_score{group=~\"$group\", id=~\"$id\"}",
Expand Down
Loading
Loading