From 54de34c77ee2d2309cf3677f5a6ee5352e09b3eb Mon Sep 17 00:00:00 2001 From: Werner Dijkerman Date: Fri, 29 May 2026 21:14:41 +0200 Subject: [PATCH] Self-observability and production operations --- README.md | 9 +- .../promanomaly/ci/all-features-values.yaml | 11 +- charts/promanomaly/templates/configmap.yaml | 9 + .../promanomaly/templates/networkpolicy.yaml | 7 +- .../promanomaly/templates/prometheusrule.yaml | 50 +++ charts/promanomaly/values.schema.json | 11 +- charts/promanomaly/values.yaml | 45 ++ .../promanomaly-self-observability.json | 120 +++++ detector/src/promanomaly/cli/__init__.py | 295 +++++++++++- detector/src/promanomaly/cli/_cost.py | 312 +++++++++++++ detector/src/promanomaly/cli/_diagnose.py | 97 ++++ detector/src/promanomaly/cli/_inspect.py | 2 + detector/src/promanomaly/cli/_metadata.py | 80 ++++ detector/src/promanomaly/cli/_top.py | 12 + detector/src/promanomaly/config.py | 77 ++++ detector/src/promanomaly/diagnose.py | 350 +++++++++++++++ detector/src/promanomaly/exporter.py | 110 +++++ detector/src/promanomaly/ha.py | 22 +- detector/src/promanomaly/inspect.py | 14 + detector/src/promanomaly/main.py | 57 +++ detector/src/promanomaly/metadata_lint.py | 188 ++++++++ detector/src/promanomaly/runner.py | 95 +++- .../src/promanomaly/self_observability.py | 225 ++++++++++ detector/src/promanomaly/selftest.py | 185 ++++++++ detector/src/promanomaly/server.py | 62 ++- detector/src/promanomaly/source.py | 96 ++++ detector/src/promanomaly/state/__init__.py | 3 +- detector/src/promanomaly/state/discovery.py | 170 +++++++ detector/src/promanomaly/state/snapshot.py | 81 +++- detector/src/promanomaly/warmup.py | 420 ++++++++++++++++++ detector/tests/conftest.py | 6 + detector/tests/test_cli_estimate_cost.py | 159 +++++++ detector/tests/test_diagnose.py | 175 ++++++++ detector/tests/test_discovery_ha.py | 157 +++++++ detector/tests/test_exporter.py | 14 +- detector/tests/test_ha_integration.py | 10 +- detector/tests/test_metadata_lint.py | 252 +++++++++++ detector/tests/test_self_observability.py | 174 ++++++++ detector/tests/test_selftest.py | 133 ++++++ detector/tests/test_warmup.py | 281 ++++++++++++ docs/architecture/multi-cluster.md | 88 ++++ docs/cli.md | 74 ++- docs/operations.md | 70 ++- docs/operations/degraded-modes.md | 127 ++++++ docs/production-checklist.md | 72 +++ examples/alerts/promanomaly-rules.yaml | 60 +++ 46 files changed, 5005 insertions(+), 62 deletions(-) create mode 100644 dashboards/grafana/promanomaly-self-observability.json create mode 100644 detector/src/promanomaly/cli/_cost.py create mode 100644 detector/src/promanomaly/cli/_diagnose.py create mode 100644 detector/src/promanomaly/cli/_metadata.py create mode 100644 detector/src/promanomaly/diagnose.py create mode 100644 detector/src/promanomaly/metadata_lint.py create mode 100644 detector/src/promanomaly/self_observability.py create mode 100644 detector/src/promanomaly/selftest.py create mode 100644 detector/src/promanomaly/warmup.py create mode 100644 detector/tests/test_cli_estimate_cost.py create mode 100644 detector/tests/test_diagnose.py create mode 100644 detector/tests/test_discovery_ha.py create mode 100644 detector/tests/test_metadata_lint.py create mode 100644 detector/tests/test_self_observability.py create mode 100644 detector/tests/test_selftest.py create mode 100644 detector/tests/test_warmup.py create mode 100644 docs/architecture/multi-cluster.md create mode 100644 docs/operations/degraded-modes.md create mode 100644 docs/production-checklist.md diff --git a/README.md b/README.md index 14231a4..9f7b747 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,14 @@ monitoring, deploy-time silencing, cross-tool joins with promforecast). [`docs/when-to-use.md`](docs/when-to-use.md) walks through the "promforecast, promanomaly, or both?" decision tree, and [`docs/config-schema.md`](docs/config-schema.md) documents the v1 -schema stability commitments. +schema stability commitments. Before going to production, work through +the [`docs/production-checklist.md`](docs/production-checklist.md) +sizing guide and readiness checklist, and keep the +[degraded-mode playbook](docs/operations/degraded-modes.md) handy for +on-call. Running many clusters? The +[multi-cluster reference architectures](docs/architecture/multi-cluster.md) +spell out the three topologies and their cardinality / network / +blast-radius trade-offs. A self-monitoring example config is bundled at [`examples/configs/self-monitoring.yaml`](examples/configs/self-monitoring.yaml), diff --git a/charts/promanomaly/ci/all-features-values.yaml b/charts/promanomaly/ci/all-features-values.yaml index f747a62..ac19b04 100644 --- a/charts/promanomaly/ci/all-features-values.yaml +++ b/charts/promanomaly/ci/all-features-values.yaml @@ -6,6 +6,15 @@ serviceMonitor: enabled: true interval: 30s +# End-to-end detection self-test (dead-man's-switch): exercises the +# server.selftest configmap branch and keeps the AnomalyPipelineDead +# rule wired under the matrix render. +server: + selftest: + enabled: true + detector: MAD + threshold: 3.0 + prometheusRule: enabled: true labels: @@ -28,7 +37,7 @@ defaults: min_abs_delta: 0.0 min_relative_delta: 0.05 -# Output-label filtering: covers the v0.4 exporter.output_labels +# Output-label filtering: covers the exporter.output_labels # surface so the configmap.yaml conditional block renders under the # CI matrix. ``allow`` keeps an explicit set of instance/region labels # on the wire; ``drop`` strips a privacy-sensitive customer id even diff --git a/charts/promanomaly/templates/configmap.yaml b/charts/promanomaly/templates/configmap.yaml index de0239a..f5bc000 100644 --- a/charts/promanomaly/templates/configmap.yaml +++ b/charts/promanomaly/templates/configmap.yaml @@ -22,6 +22,15 @@ data: server: listen: {{ .Values.server.listen | quote }} refresh_interval: {{ .Values.server.refresh_interval | quote }} + {{- with .Values.server.expose_warmup_endpoint }} + expose_warmup_endpoint: {{ . }} + {{- end }} + {{- if .Values.server.selftest.enabled }} + selftest: + enabled: true + detector: {{ .Values.server.selftest.detector | quote }} + threshold: {{ .Values.server.selftest.threshold }} + {{- end }} reload: enabled: {{ .Values.server.reload.enabled }} watch_configmap: {{ .Values.server.reload.watch_configmap }} diff --git a/charts/promanomaly/templates/networkpolicy.yaml b/charts/promanomaly/templates/networkpolicy.yaml index ba915ae..f970c02 100644 --- a/charts/promanomaly/templates/networkpolicy.yaml +++ b/charts/promanomaly/templates/networkpolicy.yaml @@ -11,9 +11,10 @@ spec: - Ingress - Egress ingress: - # Allow /metrics scraping. The detector serves /-/reload and the - # debug endpoints from the same pod & port, so this rule - # deliberately allows the *namespace*, not the world. + # Allow /metrics scraping. The detector serves /-/reload, the debug + # endpoints, and the opt-in /warmup endpoint from the same pod & + # port, so this rule deliberately allows the *namespace*, not the + # world. - from: - podSelector: {} {{- range .Values.networkPolicy.allowedScrapeNamespaces }} diff --git a/charts/promanomaly/templates/prometheusrule.yaml b/charts/promanomaly/templates/prometheusrule.yaml index 5aec116..b02619a 100644 --- a/charts/promanomaly/templates/prometheusrule.yaml +++ b/charts/promanomaly/templates/prometheusrule.yaml @@ -211,4 +211,54 @@ spec: fail_ready policy is configured. {{- end }} {{- end }} + {{- with $rules.anomalyDetectorDegraded }} + {{- if .enabled }} + - alert: AnomalyDetectorDegraded + expr: anomaly_detect_success_ratio{window="{{ .window }}"} < {{ .threshold }} + for: {{ .for }} + labels: + severity: {{ .severity }} + annotations: + summary: "Detector {{ "{{" }} $labels.detector {{ "}}" }} degraded in {{ "{{" }} $labels.group {{ "}}" }}" + description: | + anomaly_detect_success_ratio for detector + {{ "{{" }} $labels.detector {{ "}}" }} in group {{ "{{" }} $labels.group {{ "}}" }} + has been below {{ .threshold }} over the {{ .window }} window — + the detector is failing or timing out on too many series and + its scores are missing. + {{- end }} + {{- end }} + {{- with $rules.anomalySnapshotStale }} + {{- if .enabled }} + - alert: AnomalySnapshotStale + expr: anomaly_snapshot_age_seconds > {{ .maxAgeSeconds }} + for: {{ .for }} + labels: + severity: {{ .severity }} + annotations: + summary: "promanomaly snapshot stale for {{ "{{" }} $labels.group {{ "}}" }}" + description: | + The most recent snapshot for group {{ "{{" }} $labels.group {{ "}}" }} + is older than {{ .maxAgeSeconds }}s. /metrics is serving stale + scores; cross-check AnomalySourceFailing and AnomalyStale. + {{- end }} + {{- end }} + {{- with $rules.anomalyPipelineDead }} + {{- if .enabled }} + - alert: AnomalyPipelineDead + expr: anomaly_selftest_ok == 0 + for: {{ .for }} + labels: + severity: {{ .severity }} + annotations: + summary: "promanomaly detection pipeline is dead (self-test failing)" + description: | + The end-to-end self-test for detector + {{ "{{" }} $labels.detector {{ "}}" }} has failed to catch its injected + synthetic anomaly for over {{ .for }}. The detect → threshold → export + pipeline is not surfacing anomalies even though the process is up — + check for a config/threshold mistake or an exporter regression. + See anomaly_selftest_failures_total. Requires server.selftest.enabled. + {{- end }} + {{- end }} {{- end }} diff --git a/charts/promanomaly/values.schema.json b/charts/promanomaly/values.schema.json index 6b5a4e8..3a000cf 100644 --- a/charts/promanomaly/values.schema.json +++ b/charts/promanomaly/values.schema.json @@ -53,7 +53,16 @@ "type": "object", "properties": { "listen": {"type": "string"}, - "refresh_interval": {"type": "string"} + "refresh_interval": {"type": "string"}, + "expose_warmup_endpoint": {"type": "boolean"}, + "selftest": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "detector": {"type": "string"}, + "threshold": {"type": "number", "exclusiveMinimum": 0} + } + } } }, "safety": { diff --git a/charts/promanomaly/values.yaml b/charts/promanomaly/values.yaml index c2fcd68..8ca9458 100644 --- a/charts/promanomaly/values.yaml +++ b/charts/promanomaly/values.yaml @@ -45,6 +45,20 @@ server: auth: type: none # none | bearer | mtls existingSecret: "" + # Opt-in GET /warmup endpoint (off by default). When true the + # detector reports per-(group, query) warm-up status; the shipped + # NetworkPolicy keeps it in-namespace alongside /debug/*. + expose_warmup_endpoint: false + # End-to-end detection self-test (dead-man's-switch). Off by default. + # When enabled, each run drives a synthetic series with a known + # injected anomaly through the real detect/threshold/export path and + # emits anomaly_selftest_ok (1/0); the AnomalyPipelineDead alert fires + # when it stops passing. Proves the detection path itself is live, not + # merely that the process is up. + selftest: + enabled: false + detector: MAD + threshold: 3.0 safety: max_series_per_query: 1000 @@ -296,3 +310,34 @@ prometheusRule: threshold: 0.1 for: 5m severity: warning + # Self-observability: the detector's own health. + # AnomalyDetectorDegraded fires when a detector's rolling compute + # success ratio drops — it is silently missing scores before any + # downstream anomaly alert can fire. + anomalyDetectorDegraded: + enabled: true + threshold: 0.9 + # Must be one of the emitted window labels: "1h" or "1d". The + # detector emits anomaly_detect_success_ratio only at these two + # windows, so any other value matches no series and the alert + # silently never fires. + window: 1h + for: 10m + severity: warning + # AnomalySnapshotStale fires when the served /metrics snapshot ages + # past maxAgeSeconds — distinct from AnomalyStale, which keys off the + # last successful-run timestamp. + anomalySnapshotStale: + enabled: true + maxAgeSeconds: 600 + for: 5m + severity: warning + # AnomalyPipelineDead (dead-man's-switch) fires when the opt-in + # end-to-end self-test stops catching its injected synthetic anomaly — + # the whole detect/threshold/export pipeline is silently dead. The + # underlying anomaly_selftest_ok series only exists when + # server.selftest.enabled, so this rule is dormant otherwise. + anomalyPipelineDead: + enabled: true + for: 5m + severity: critical diff --git a/dashboards/grafana/promanomaly-self-observability.json b/dashboards/grafana/promanomaly-self-observability.json new file mode 100644 index 0000000..d6c4f34 --- /dev/null +++ b/dashboards/grafana/promanomaly-self-observability.json @@ -0,0 +1,120 @@ +{ + "annotations": {"list": []}, + "editable": true, + "schemaVersion": 38, + "title": "promanomaly self-observability", + "description": "Is the detector itself healthy? Per-detector compute success ratio, per-group CPU/memory attribution, and per-series/snapshot freshness. Pairs with the AnomalyDetectorDegraded and AnomalySnapshotStale alerts.", + "tags": ["promanomaly", "anomaly-detection", "self-observability"], + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "label": "Datasource", + "query": "prometheus", + "current": {"text": "VictoriaMetrics", "value": "victoriametrics"} + }, + { + "name": "group", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "query": "label_values(anomaly_snapshot_age_seconds, group)", + "refresh": 2, + "multi": true, + "includeAll": true + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Detector compute success ratio (1h)", + "description": "Fraction of attempted detector computations that completed without raising, over the rolling 1h window. A drop below ~0.9 means the detector is timing out or erroring on a growing fraction of series — its scores are silently missing.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}, + "targets": [ + { + "expr": "anomaly_detect_success_ratio{group=~\"$group\", window=\"1h\"}", + "legendFormat": "{{group}}/{{detector}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0} + }, + { + "id": 2, + "type": "timeseries", + "title": "Detector compute success ratio (1d)", + "description": "The slower 24h trend of the same signal — distinguishes a transient blip from a sustained regression.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}, + "targets": [ + { + "expr": "anomaly_detect_success_ratio{group=~\"$group\", window=\"1d\"}", + "legendFormat": "{{group}}/{{detector}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0} + }, + { + "id": 3, + "type": "timeseries", + "title": "Snapshot age by group", + "description": "Age of each group's most recent successful snapshot. Climbs between runs and resets on each successful run; a monotonic climb past refresh_interval means the group has stopped producing fresh snapshots (drives AnomalySnapshotStale).", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + { + "expr": "anomaly_snapshot_age_seconds{group=~\"$group\"}", + "legendFormat": "{{group}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8} + }, + { + "id": 4, + "type": "timeseries", + "title": "Series staleness by group", + "description": "Age of the freshest per-series score in each group. Diverges from snapshot age when a group keeps writing snapshots that carry no scores (all series warming up, or scores dropped under drop_scores).", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + { + "expr": "anomaly_series_staleness_seconds{group=~\"$group\"}", + "legendFormat": "{{group}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8} + }, + { + "id": 5, + "type": "timeseries", + "title": "Per-group CPU rate", + "description": "Detector compute time attributed to each group, as a per-second rate of the cumulative counter. Surfaces which group dominates the detection compute budget.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + { + "expr": "rate(anomaly_group_cpu_seconds_total{group=~\"$group\"}[5m])", + "legendFormat": "{{group}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16} + }, + { + "id": 6, + "type": "timeseries", + "title": "Per-group snapshot memory", + "description": "Best-effort size of each group's output snapshot held by the exporter. Answers 'which group's snapshot is dominant' — not an exact process RSS breakdown.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + { + "expr": "anomaly_group_memory_bytes{group=~\"$group\"}", + "legendFormat": "{{group}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16} + } + ] +} diff --git a/detector/src/promanomaly/cli/__init__.py b/detector/src/promanomaly/cli/__init__.py index 9be6bcc..7c6882e 100644 --- a/detector/src/promanomaly/cli/__init__.py +++ b/detector/src/promanomaly/cli/__init__.py @@ -61,7 +61,11 @@ _round_to_dividing_bucket, ) from ._common import _coerce_param, _safe_float +from ._cost import run_estimate_cost as _run_estimate_cost +from ._diagnose import run_diagnose_target as _run_diagnose_target +from ._diagnose import run_diagnose_tsdb as _run_diagnose_tsdb from ._inspect import _print_inspect_text +from ._metadata import run_metadata_lint as _run_metadata_lint from ._probe import ( DiscoverProbeResult as _DiscoverProbeResult, ) @@ -127,7 +131,33 @@ def cli(ctx: click.Context, config_path: str | None, log_level: str) -> None: default=False, help=( "With --probe, exit non-zero when any query returns zero series " - "or errors. Suitable as a pre-deploy CI gate." + "or errors. With --estimate-cost, exit non-zero when the projected " + "series count exceeds safety.max_total_series. Suitable as a " + "pre-deploy CI gate." + ), +) +@click.option( + "--estimate-cost", + "estimate_cost", + is_flag=True, + default=False, + help=( + "Statically project cardinality, TSDB query load, and a coarse " + "CPU/memory estimate from the config — without touching the " + "datasource. Complements --probe (which checks the queries return " + "data)." + ), +) +@click.option( + "--lint-metadata", + "lint_metadata", + is_flag=True, + default=False, + help=( + "Query the datasource's /api/v1/metadata and warn when a query " + "feeds a raw counter to a detector without rate()/increase(). " + "Lint only — never rewrites the query. With --strict, any finding " + "exits non-zero. Composes with --probe and --estimate-cost." ), ) @click.option( @@ -142,36 +172,67 @@ def validate( config_path: str, probe: bool, strict: bool, + estimate_cost: bool, + lint_metadata: bool, datasource_url: str | None, ) -> None: - """Validate a config file (and optionally probe the datasource).""" + """Validate a config file (and optionally probe / estimate / lint it).""" try: cfg = load_config(config_path) except Exception as exc: click.echo(f"INVALID: {exc}", err=True) sys.exit(1) - if not probe: - if strict: - # --strict without --probe is a user error: schema validation - # is binary (pass / fail) and already exits non-zero on fail. - raise click.UsageError("--strict requires --probe") - click.echo(f"OK: apiVersion={cfg.apiVersion} groups={[g.name for g in cfg.groups]}") - return + runs_a_check = probe or estimate_cost or lint_metadata + if strict and not runs_a_check: + # --strict alone is a user error: schema validation is binary + # (pass / fail) and already exits non-zero on fail. It only gates + # something when paired with a check that has a soft verdict. + raise click.UsageError("--strict requires --probe, --estimate-cost, or --lint-metadata") + + # Each requested check runs unconditionally and prints its own report; + # the process exits non-zero if *any* of them failed. Collecting the + # codes (rather than short-circuiting with ``or``) means a failing + # earlier check never suppresses a later check's output — an operator + # passing several flags sees every report in one run. The static cost + # projection runs first so a CI gate sees the cardinality verdict even + # when the datasource is unreachable. + codes: list[int] = [] + if estimate_cost: + codes.append(_run_estimate_cost(cfg, strict=strict)) + + # --lint-metadata and --probe both touch the datasource. Pass the + # cli-module's ``PromQLSource`` symbol as the factory so tests that + # ``monkeypatch.setattr(cli, "PromQLSource", stub)`` win regardless of + # where the implementation lives. + if lint_metadata: + codes.append( + asyncio.run( + _run_metadata_lint( + cfg, + strict=strict, + datasource_override=datasource_url, + source_factory=PromQLSource, + ) + ) + ) - # Pass the cli-module's ``PromQLSource`` symbol as the factory so - # tests that ``monkeypatch.setattr(cli, "PromQLSource", stub)`` win - # regardless of where the probe implementation lives. Production - # callers see the real source class. - exit_code = asyncio.run( - _run_probe( - cfg, - strict=strict, - datasource_override=datasource_url, - source_factory=PromQLSource, + if probe: + codes.append( + asyncio.run( + _run_probe( + cfg, + strict=strict, + datasource_override=datasource_url, + source_factory=PromQLSource, + ) + ) ) - ) - sys.exit(exit_code) + + if not runs_a_check: + click.echo(f"OK: apiVersion={cfg.apiVersion} groups={[g.name for g in cfg.groups]}") + return + sys.exit(max(codes) if codes else 0) # The 450+ LOC of probe logic that used to live here moved to @@ -701,6 +762,17 @@ def inspect_cmd( type=int, help="Show at most this many series (ranked by severity).", ) +@click.option( + "--lint/--no-lint", + "lint", + default=True, + show_default=True, + help=( + "Ask the detector to include metadata-lint hints (counter fed to a " + "detector without rate()/increase()). Costs one extra metadata " + "query against the TSDB; pass --no-lint to skip it." + ), +) @click.option( "--output", type=click.Choice(["text", "json"]), @@ -719,6 +791,7 @@ def top_cmd( group: str | None, min_severity: float, limit: int, + lint: bool, output: str, timeout: float, ) -> None: @@ -731,7 +804,11 @@ def top_cmd( """ import httpx - params: dict[str, str] = {"min_severity": str(min_severity), "limit": str(limit)} + params: dict[str, str] = { + "min_severity": str(min_severity), + "limit": str(limit), + "lint": "true" if lint else "false", + } if group: params["group"] = group url = target_url.rstrip("/") + "/debug/anomalies" @@ -754,6 +831,176 @@ def top_cmd( _print_top_text(payload) +@cli.command(name="warmup") +@click.option( + "--target", + "target_url", + required=True, + help="Running detector base URL (e.g. http://localhost:9092).", +) +@click.option( + "--output", + type=click.Choice(["text", "json"]), + default="text", + show_default=True, + help="Output format. 'json' is the raw /warmup response.", +) +@click.option( + "--timeout", + default=30.0, + show_default=True, + help="HTTP timeout for the /warmup call, in seconds.", +) +def warmup_cmd(target_url: str, output: str, timeout: float) -> None: + """Report which series are still warming up and an ETA to readiness. + + Wraps a running detector's opt-in ``/warmup`` endpoint (enabled via + ``server.expose_warmup_endpoint``). Exit code is 0 when every query + in every group is ``ready`` (or there are no queries to warm up) and + 1 when at least one is still warming or errored — so a CI pre-flight + can gate "is the install done?" on it. Network failures exit 2. + Mirrors ``promforecast warmup`` for cross-tool muscle memory. + """ + import httpx + + from ..warmup import render_table, report_from_payload + + url = target_url.rstrip("/") + "/warmup" + try: + resp = httpx.get(url, timeout=timeout) + resp.raise_for_status() + payload = resp.json() + except httpx.HTTPError as exc: + click.echo(f"could not reach {target_url}: {exc}", err=True) + sys.exit(2) + except ValueError: + click.echo(f"non-JSON response from {url}", err=True) + sys.exit(2) + + if output == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + else: + click.echo(render_table(report_from_payload(payload)), nl=False) + + statuses = { + q.get("status", "") for group in payload.get("groups", []) for q in group.get("queries", []) + } + # Empty config (no queries) is "done", not "still warming", so a + # docs-recommended ``until promanomaly warmup`` loop terminates + # against a fresh install rather than spinning forever. + if statuses and statuses != {"ready"}: + sys.exit(1) + + +@cli.command(name="diagnose") +@click.option( + "--target", + "target_url", + default=None, + help=( + "Running detector base URL (e.g. http://localhost:9092). Reads one " + "/metrics scrape for a current-state read (firing now, warming now, " + "empty now). Cannot compute a firing rate over time — use " + "--datasource-url for that." + ), +) +@click.option( + "--datasource-url", + "datasource_url", + default=None, + help=( + "TSDB URL holding the detector's own scraped metrics. Aggregates " + "anomaly_* over --window for the full analysis (never-fires / " + "fires-often firing-rate bands, chronic warm-up, empty queries)." + ), +) +@click.option( + "--window", + default="24h", + show_default=True, + help="Lookback for the TSDB analysis (Prometheus duration: 24h, 7d, ...).", +) +@click.option( + "--config", + "config_path", + type=click.Path(exists=True, dir_okay=False), + default=None, + help=( + "Config file. With --datasource-url, cross-checks configured " + "detectors against emitted scores so a silent detector (dead config, " + "or a cohort below min_cohort_size) is flagged." + ), +) +@click.option( + "--fire-high", + default=0.5, + show_default=True, + type=float, + help="Firing-rate fraction at/above which a detector is flagged as too noisy.", +) +@click.option( + "--output", + type=click.Choice(["text", "json"]), + default="text", + show_default=True, + help="Output format. 'json' is suitable for CI consumption.", +) +@click.option("--timeout", default=30.0, show_default=True, help="HTTP timeout in seconds.") +def diagnose_cmd( + target_url: str | None, + datasource_url: str | None, + window: str, + config_path: str | None, + fire_high: float, + output: str, + timeout: float, +) -> None: + """Report mis-tuned detectors: never-fires, fires-often, stuck warming, empty. + + Pure analysis, no persisted state. Pairs with ``backtest`` and + ``calibrate-buckets`` to close the tuning loop. Parity with + ``promforecast diagnose``. + """ + from ..diagnose import render_report + + if bool(target_url) == bool(datasource_url): + raise click.UsageError("pass exactly one of --target or --datasource-url") + + if target_url is not None: + import httpx + + try: + report = _run_diagnose_target(target_url, timeout, fire_high=fire_high) + except httpx.HTTPError as exc: + click.echo(f"could not reach {target_url}: {exc}", err=True) + sys.exit(2) + else: + assert datasource_url is not None + cfg = load_config(config_path) if config_path else None + try: + report = asyncio.run( + _run_diagnose_tsdb( + datasource_url, + timeout, + window=window, + fire_high=fire_high, + never_eps=0.0, + source_factory=PromQLSource, + config=cfg, + ) + ) + except ValueError as exc: + raise click.UsageError(str(exc)) from exc + except SourceQueryError as exc: + click.echo(f"query failed: {exc}", err=True) + sys.exit(2) + + if output == "json": + click.echo(json.dumps(report.to_dict(), indent=2, sort_keys=True)) + else: + click.echo(render_report(report), nl=False) + + # Register the backtest command imported from promanomaly.backtest so # operators can invoke it as ``promanomaly backtest`` rather than the # longer ``python -m promanomaly.backtest`` (which still works because @@ -813,6 +1060,10 @@ def main() -> None: "_probe_discover_query", "_recommend_detector", "_round_to_dividing_bucket", + "_run_diagnose_target", + "_run_diagnose_tsdb", + "_run_estimate_cost", + "_run_metadata_lint", "_run_offline_change_points", "_run_probe", "_safe_float", diff --git a/detector/src/promanomaly/cli/_cost.py b/detector/src/promanomaly/cli/_cost.py new file mode 100644 index 0000000..daecf51 --- /dev/null +++ b/detector/src/promanomaly/cli/_cost.py @@ -0,0 +1,312 @@ +"""``validate --estimate-cost`` — static config cost projection. + +Unlike ``--probe`` (which executes every query against the live +datasource to confirm it returns data), ``--estimate-cost`` is a purely +static projection: it never touches the TSDB, so it runs in CI on a +config PR before the change ships. It answers three questions an +operator reviewing a config change wants answered at review time: + +* **Cardinality** — projected total output series per group and + globally, against ``safety.max_total_series``. Worst-case at + ``safety.max_series_per_query`` because, without a probe, the cap is + the only bound we can assert. +* **Query load** — projected TSDB queries per refresh interval: the + per-refresh short-window range queries, the discovery probe queries, + and the stratified-baseline refreshes (the long multi-week fetches, + amortised per day). +* **Compute** — a coarse CPU / memory estimate from the documented + detector resource model (``O(n log n)`` per series on the rolling + window; memory dominated by one float array per in-flight window). + Anchored to the guide's "~10k series at a 1-minute refresh on one + 1-CPU replica" reference point. + +``--strict`` turns a projected-series overshoot of ``max_total_series`` +into a non-zero exit so the check can gate a CI pipeline — the +cardinality complement to ``--probe --strict`` (which gates on whether +the queries actually return data). Flag-name parity with promforecast's +``validate --estimate-cost``. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass, field +from typing import Any + +import click + +from ..config import Config, parse_duration +from ..detectors import UnknownDetectorError +from ..detectors.registry import get as get_detector +from ..plan import merge_params +from ..stratified import baseline_refresh_seconds_for, query_window_seconds_for + +# Documented anchor from the project guide: a single 1-CPU replica +# handles ~10k series at a 1-minute (60s) refresh with the default +# detectors comfortably. The CPU projection scales linearly off this +# point — it is an order-of-magnitude sizing aid, not a benchmark. +_ANCHOR_SERIES = 10_000.0 +_ANCHOR_REFRESH_SECONDS = 60.0 +# One float array per in-flight rolling window dominates memory; 8 bytes +# per sample plus a small constant per series for the surrounding +# bookkeeping (timestamps, labels, the score frame). +_BYTES_PER_SAMPLE = 8.0 +_BYTES_PER_SERIES_OVERHEAD = 512.0 + + +@dataclass +class QueryCostEstimate: + """Projected cost for one configured query (worst case at the cap).""" + + group: str + query: str + is_discovery: bool + projected_series: int + detector_count: int + # Largest per-detector rolling window on this query, in samples + # (window_seconds / step_seconds) — drives the memory projection. + window_points: int + short_queries_per_refresh: int + discovery_probe_queries_per_refresh: int + stratified_baseline_queries_per_day: float + memory_bytes: float + cpu_cores: float + + +@dataclass +class GroupCostEstimate: + """Aggregated projection for one group.""" + + group: str + projected_series: int = 0 + short_queries_per_refresh: int = 0 + discovery_probe_queries_per_refresh: int = 0 + stratified_baseline_queries_per_day: float = 0.0 + memory_bytes: float = 0.0 + cpu_cores: float = 0.0 + queries: list[QueryCostEstimate] = field(default_factory=list) + + +def estimate_query_cost( + *, + cfg: Config, + group_name: str, + query: Any, +) -> QueryCostEstimate: + """Project one query's worst-case cost without touching the TSDB. + + Series are projected at ``max_series_per_query`` (the cardinality + control's per-query ceiling) because static analysis cannot know the + real fan-out. Discovery queries additionally count one probe per + declared variable per refresh; the rendered-variant range queries + are folded into the per-query short-query count as a single ``≥1`` + because the expansion size is only knowable via ``--probe``. + """ + series_cap = cfg.safety.max_series_per_query + step_seconds = max(cfg.defaults.step_seconds, 1e-9) + default_window = cfg.defaults.window_seconds + + detector_count = len(query.detectors) + # Each input series is scored by every configured detector, so the + # emitted ``anomaly_score`` family alone is series x detectors. The + # cardinality cap, however, is enforced on *input* series, so the + # projected-series figure that matters against ``max_total_series`` + # is the input series cap — detectors multiply the metric *families*, + # not the series the cap counts. + projected_series = series_cap + + # The memory / CPU model keys off the rolling *scoring* window (the + # detector's ``window`` param), not the stratified multi-week + # lookback: that lookback is fetched once per baseline-refresh and + # cached, so its cost is a TSDB query cost (counted in + # ``stratified_baseline_queries_per_day``) rather than a per-refresh + # dense float array held in memory every tick. + longest_window_seconds = default_window + stratified_per_day = 0.0 + for entry in query.detectors: + try: + cls = get_detector(entry.name) + except UnknownDetectorError: + continue + merged = merge_params(cls, entry.params, cfg.defaults) + scoring_window = _scoring_window_seconds(merged, default_window) + longest_window_seconds = max(longest_window_seconds, scoring_window) + if query_window_seconds_for(cls, merged) is not None: + refresh = baseline_refresh_seconds_for(cls, merged) + if refresh and refresh > 0: + stratified_per_day += 86400.0 / refresh + + window_points = int(longest_window_seconds / step_seconds) + + is_discovery = bool(getattr(query, "discover", None)) + discovery_probes = len(query.discover) if is_discovery else 0 + # Non-discovery queries issue exactly one short range query per + # refresh. Discovery queries issue at least one rendered-variant + # query per refresh (the true count needs --probe). + short_queries = 1 + + # Memory: one float array per in-flight rolling window per series. + memory_bytes = projected_series * ( + window_points * _BYTES_PER_SAMPLE + _BYTES_PER_SERIES_OVERHEAD + ) + # CPU: O(n log n) per series per detector, anchored to the documented + # 10k-series / 60s-refresh / 1-CPU reference. The window-size factor + # scales the anchor (which assumes the 1h default window) by how much + # more (or less) work this query's longest window implies. + anchor_points = max(default_window / step_seconds, 1.0) + work_factor = _nlogn(window_points) / _nlogn(int(anchor_points)) + refresh_seconds = max(cfg.server.refresh_interval_seconds, 1e-9) + cpu_cores = ( + (projected_series * detector_count / _ANCHOR_SERIES) + * (_ANCHOR_REFRESH_SECONDS / refresh_seconds) + * work_factor + ) + + return QueryCostEstimate( + group=group_name, + query=query.id, + is_discovery=is_discovery, + projected_series=projected_series, + detector_count=detector_count, + window_points=window_points, + short_queries_per_refresh=short_queries, + discovery_probe_queries_per_refresh=discovery_probes, + stratified_baseline_queries_per_day=round(stratified_per_day, 2), + memory_bytes=memory_bytes, + cpu_cores=cpu_cores, + ) + + +def estimate_cost(cfg: Config) -> list[GroupCostEstimate]: + """Project per-group cost across the whole config.""" + groups: list[GroupCostEstimate] = [] + for group in cfg.groups: + gest = GroupCostEstimate(group=group.name) + for query in group.queries: + qest = estimate_query_cost(cfg=cfg, group_name=group.name, query=query) + gest.queries.append(qest) + gest.projected_series += qest.projected_series + gest.short_queries_per_refresh += qest.short_queries_per_refresh + gest.discovery_probe_queries_per_refresh += qest.discovery_probe_queries_per_refresh + gest.stratified_baseline_queries_per_day += qest.stratified_baseline_queries_per_day + gest.memory_bytes += qest.memory_bytes + gest.cpu_cores += qest.cpu_cores + groups.append(gest) + return groups + + +def run_estimate_cost(cfg: Config, *, strict: bool) -> int: + """Print the JSON-lines cost projection; return a CLI exit code. + + One ``{"group": ..., "status": "estimate", ...}`` line per group, + then a ``{"summary": {...}}`` line. ``strict`` makes a projected + total over ``safety.max_total_series`` exit non-zero. + """ + groups = estimate_cost(cfg) + projected_total = 0 + total_memory = 0.0 + total_cpu = 0.0 + short_total = 0 + probe_total = 0 + stratified_total = 0.0 + has_discovery = False + for gest in groups: + projected_total += gest.projected_series + total_memory += gest.memory_bytes + total_cpu += gest.cpu_cores + short_total += gest.short_queries_per_refresh + probe_total += gest.discovery_probe_queries_per_refresh + stratified_total += gest.stratified_baseline_queries_per_day + group_has_discovery = any(q.is_discovery for q in gest.queries) + has_discovery = has_discovery or group_has_discovery + entry: dict[str, Any] = { + "group": gest.group, + "status": "estimate", + "projected_series": gest.projected_series, + "short_queries_per_refresh": gest.short_queries_per_refresh, + "discovery_probe_queries_per_refresh": (gest.discovery_probe_queries_per_refresh), + "stratified_baseline_queries_per_day": round( + gest.stratified_baseline_queries_per_day, 2 + ), + "estimated_memory_bytes": int(gest.memory_bytes), + "estimated_cpu_cores": round(gest.cpu_cores, 3), + } + if group_has_discovery: + # A discover: query fans out to N rendered variants at run + # time, each up to max_series_per_query. Static analysis can't + # know N, so projected_series counts one variant's worth — a + # lower bound. Flag it so an operator never reads a discovery + # group's "under budget" as a guarantee; run --probe for the + # real fan-out against the live TSDB. + entry["projected_series_is_lower_bound"] = True + click.echo(json.dumps(entry, sort_keys=True)) + over_budget = projected_total > cfg.safety.max_total_series + summary: dict[str, Any] = { + "projected_total_series": projected_total, + "max_total_series": cfg.safety.max_total_series, + "over_budget": over_budget, + "short_queries_per_refresh": short_total, + "discovery_probe_queries_per_refresh": probe_total, + "stratified_baseline_queries_per_day": round(stratified_total, 2), + "estimated_memory_bytes": int(total_memory), + "estimated_cpu_cores": round(total_cpu, 3), + "refresh_interval_seconds": cfg.server.refresh_interval_seconds, + } + if has_discovery: + # The total excludes discovery fan-out (see per-group flag), so it + # is a lower bound — don't let a green "under budget" mislead. + summary["projected_series_is_lower_bound"] = True + summary["discovery_note"] = ( + "projected_total_series excludes discover: fan-out; " + "run --probe for the real expansion against max_total_series" + ) + click.echo(json.dumps({"summary": summary}, sort_keys=True)) + if over_budget: + # Always surface the overshoot; only fail the process under + # --strict so the advisory mode stays usable interactively. + click.echo( + json.dumps( + { + "status": "cardinality_exceeded", + "projected_total_series": projected_total, + "max_total_series": cfg.safety.max_total_series, + }, + sort_keys=True, + ), + err=True, + ) + if strict: + return 1 + return 0 + + +def _scoring_window_seconds(merged: dict[str, Any], default_window: float) -> float: + """Resolve a detector's rolling scoring window from its merged params. + + ``merge_params`` always seeds ``window`` from the group default, so + the key is present; a malformed value falls back to the default + rather than crashing the static estimate. + """ + raw = merged.get("window") + if raw is None: + return default_window + try: + return parse_duration(raw) + except (ValueError, TypeError): + return default_window + + +def _nlogn(points: int) -> float: + """``n log2 n`` with a floor so tiny / zero windows don't blow up.""" + n = max(points, 1) + return n * math.log2(n + 1) + + +__all__ = [ + "GroupCostEstimate", + "QueryCostEstimate", + "estimate_cost", + "estimate_query_cost", + "run_estimate_cost", +] diff --git a/detector/src/promanomaly/cli/_diagnose.py b/detector/src/promanomaly/cli/_diagnose.py new file mode 100644 index 0000000..8e05dbb --- /dev/null +++ b/detector/src/promanomaly/cli/_diagnose.py @@ -0,0 +1,97 @@ +"""Orchestration for the ``promanomaly diagnose`` subcommand. + +Pure analysis lives in :mod:`promanomaly.diagnose`; this module does the +I/O — issuing the instant queries against the TSDB (lookback mode) or +scraping the detector's ``/metrics`` (snapshot mode) — and assembles a +:class:`promanomaly.diagnose.DiagnoseReport`. +""" + +from __future__ import annotations + +import re +from typing import Any + +from ..config import Config +from ..diagnose import ( + DiagnoseReport, + diagnose_from_metrics_text, + evaluate_empty, + evaluate_firing_rates, + evaluate_silent_detectors, + evaluate_warming, +) + +# Restrict the lookback to a Prometheus range-duration literal so it can +# be interpolated into a ``[...]`` selector without opening a PromQL +# injection. Matches the duration grammar the config schema accepts. +_RANGE_RE = re.compile(r"^\d+(?:ms|s|m|h|d|w)$") + + +def validate_window(window: str) -> str: + """Return ``window`` if it is a safe range literal, else raise ``ValueError``.""" + if not _RANGE_RE.match(window.strip()): + raise ValueError( + f"invalid --window {window!r}; expected a Prometheus duration like '24h', '7d'" + ) + return window.strip() + + +async def run_diagnose_tsdb( + cfg_datasource_url: str, + timeout_seconds: float, + *, + window: str, + fire_high: float, + never_eps: float, + source_factory: Any, + config: Config | None, +) -> DiagnoseReport: + """Diagnose against the TSDB holding the detector's own metrics.""" + w = validate_window(window) + source = source_factory(cfg_datasource_url, timeout_seconds) + findings = [] + try: + await source.start() + firing_rows = await source.instant_query( + f"avg by (id, group, detector, detector_instance) " + f"(avg_over_time(anomaly_outside_threshold[{w}]))" + ) + findings.extend( + evaluate_firing_rates(firing_rows, fire_high=fire_high, never_eps=never_eps) + ) + warming_rows = await source.instant_query( + f"count by (group, id) (min_over_time(anomaly_warming_up[{w}]) >= 1)" + ) + findings.extend(evaluate_warming(warming_rows)) + empty_rows = await source.instant_query(f"max_over_time(anomaly_series_count[{w}])") + findings.extend(evaluate_empty(empty_rows)) + if config is not None: + score_rows = await source.instant_query( + f"count by (group, detector, detector_instance) " + f"(count_over_time(anomaly_score[{w}]))" + ) + emitting = { + (labels.get("group", ""), labels.get("detector", "")) for labels, _ in score_rows + } + findings.extend(evaluate_silent_detectors(config, emitting)) + finally: + await source.close() + return DiagnoseReport(mode="tsdb", window=w, findings=findings) + + +def run_diagnose_target( + target_url: str, + timeout_seconds: float, + *, + fire_high: float, +) -> DiagnoseReport: + """Diagnose from one ``/metrics`` scrape of a running detector.""" + import httpx + + url = target_url.rstrip("/") + "/metrics" + resp = httpx.get(url, timeout=timeout_seconds) + resp.raise_for_status() + return diagnose_from_metrics_text(resp.text, fire_high=fire_high) + + +__all__ = ["run_diagnose_target", "run_diagnose_tsdb", "validate_window"] diff --git a/detector/src/promanomaly/cli/_inspect.py b/detector/src/promanomaly/cli/_inspect.py index ec90ac1..e0d1ae9 100644 --- a/detector/src/promanomaly/cli/_inspect.py +++ b/detector/src/promanomaly/cli/_inspect.py @@ -18,6 +18,8 @@ def _print_inspect_text(payload: dict[str, Any]) -> None: sel_repr = ",".join(f"{k}={v}" for k, v in sorted(sel.items())) if sel else "(none)" click.echo(f"labels : {sel_repr}") click.echo(f"matched: {payload['matched_series']} / {payload['total_series']} series") + for hint in payload.get("metadata_lint", []): + click.echo(f"lint : {hint.get('message', '')}") for series in payload.get("series", []): labels_repr = ",".join(f"{k}={v}" for k, v in sorted(series["labels"].items())) or "-" click.echo("") diff --git a/detector/src/promanomaly/cli/_metadata.py b/detector/src/promanomaly/cli/_metadata.py new file mode 100644 index 0000000..a2db19b --- /dev/null +++ b/detector/src/promanomaly/cli/_metadata.py @@ -0,0 +1,80 @@ +"""``validate --lint-metadata`` — metric-type mismatch lint. + +Queries the datasource's ``/api/v1/metadata`` and warns when a query +feeds a raw counter to a detector without a ``rate()`` / ``increase()`` +wrapper. Lint only: ``--strict`` turns findings into a non-zero exit so +it can gate CI, but it never rewrites the query. Composes with +``--probe`` and ``--estimate-cost``. +""" + +from __future__ import annotations + +import json +from typing import Any + +import click + +from ..config import Config +from ..metadata_lint import lint_config_metadata +from ..source import SourceQueryError + + +async def run_metadata_lint( + cfg: Config, + *, + strict: bool, + datasource_override: str | None, + source_factory: Any, +) -> int: + """Fetch metadata, lint the config, print findings; return an exit code. + + One ``{"status": "lint", ...}`` line per finding, then a + ``{"summary": {...}}`` line. ``strict`` makes any finding exit + non-zero. A datasource that can't be reached is reported and, under + ``strict``, exits non-zero (so a broken pre-deploy check fails loud + rather than silently passing). + """ + url = datasource_override or cfg.datasource.url + source = source_factory( + url, + cfg.datasource.timeout_seconds, + auth=cfg.datasource.auth, + ) + try: + await source.start() + try: + metadata = await source.metric_metadata() + except SourceQueryError as exc: + click.echo( + json.dumps( + {"status": "metadata_unavailable", "reason": exc.reason, "detail": exc.message}, + sort_keys=True, + ), + err=True, + ) + # Lint is advisory; an unreachable datasource is only fatal + # under --strict, where the operator asked for a hard gate. + return 1 if strict else 0 + finally: + await source.close() + + findings = lint_config_metadata(cfg, metadata) + for finding in findings: + click.echo(json.dumps({"status": "lint", **finding.to_dict()}, sort_keys=True)) + click.echo( + json.dumps( + { + "summary": { + "metrics_with_metadata": len(metadata), + "findings": len(findings), + } + }, + sort_keys=True, + ) + ) + if findings and strict: + return 1 + return 0 + + +__all__ = ["run_metadata_lint"] diff --git a/detector/src/promanomaly/cli/_top.py b/detector/src/promanomaly/cli/_top.py index a5f5bb0..aa38148 100644 --- a/detector/src/promanomaly/cli/_top.py +++ b/detector/src/promanomaly/cli/_top.py @@ -24,7 +24,19 @@ def _format_duration(seconds: float) -> str: return "".join(parts) or "0s" +def _print_lint_hints(payload: dict[str, Any]) -> None: + hints = payload.get("metadata_lint") or [] + if not hints: + return + click.echo("metadata lint:") + for hint in hints: + target = f"{hint.get('group', '')}/{hint.get('query', '')}" + click.echo(f" ! {target}: {hint.get('message', '')}") + click.echo("") + + def _print_top_text(payload: dict[str, Any]) -> None: + _print_lint_hints(payload) anomalies = payload.get("anomalies", []) if not anomalies: click.echo("no firing anomalies") diff --git a/detector/src/promanomaly/config.py b/detector/src/promanomaly/config.py index e4a48fc..0b9e64a 100644 --- a/detector/src/promanomaly/config.py +++ b/detector/src/promanomaly/config.py @@ -164,11 +164,50 @@ class ReloadConfig(_ModelBase): auth: ReloadAuthConfig = Field(default_factory=ReloadAuthConfig) +class SelfTestConfig(_ModelBase): + """End-to-end detection self-test (dead-man's-switch). + + Off by default. When ``enabled``, each scheduled run drives one + synthetic series carrying a known injected anomaly through the real + detect → threshold → export path and asserts it is caught. It proves + the *detection pipeline itself* is live — a global misconfiguration + (an exporter regression, an absurd threshold) can leave the process + up and emitting nothing while silently catching no anomalies, which + the per-detector success ratios won't reveal. + + Emits ``anomaly_selftest_ok`` (1/0) and + ``anomaly_selftest_failures_total``; the ``AnomalyPipelineDead`` + reference alert fires when it stops passing. Stateless and bounded to + one synthetic series. Distinct from the self-monitoring example + config, which watches the detector's *operational* metrics rather + than re-driving the detection path. + """ + + enabled: bool = False + # Detector run through the synthetic path. Defaults to MAD — a robust + # baseline detector that reliably flags the injected point spike. + detector: str = "MAD" + # Alert threshold the synthetic score is checked against. Mirrors the + # default ``alert_thresholds.score`` so the self-test exercises the + # same threshold semantics the real pipeline uses. + threshold: float = Field(default=3.0, gt=0) + + class ServerConfig(_ModelBase): listen: str = ":9092" refresh_interval: Duration = "1m" reload: ReloadConfig = Field(default_factory=ReloadConfig) ready_endpoint: bool = True + # Opt-in ``GET /warmup`` endpoint reporting per-(group, query) "what + # is still loading?" status on a fresh install or after a restart. + # Off by default because even its one cheap probe per query is wasted + # work on the steady-state path — the operator workflow is "hit it a + # few times during the first refresh, then never again". The shipped + # NetworkPolicy denies external ingress to it alongside the /debug + # endpoints. See docs/operations.md. + expose_warmup_endpoint: bool = False + # End-to-end detection self-test (dead-man's-switch). Off by default. + selftest: SelfTestConfig = Field(default_factory=SelfTestConfig) @property def refresh_interval_seconds(self) -> float: @@ -754,9 +793,46 @@ def load_config(path: str | Path) -> Config: raise ValueError(f"config {path}: top level must be a mapping") cfg = Config.model_validate(raw) _validate_detector_params(cfg) + _validate_selftest_detector(cfg) return cfg +def _validate_selftest_detector(cfg: Config) -> None: + """Reject a self-test detector that can't work on one short window. + + The self-test drives a single synthetic rolling-window series. A + stratified detector needs a multi-week lookback and a cohort detector + needs cross-series context — neither is available, so they would + silently always report ``anomaly_selftest_ok=0`` and fire a permanent + false ``AnomalyPipelineDead``. Fail fast at load rather than at 2 a.m. + Unknown / custom detectors are left alone ("no opinion"). Only + consulted when the self-test is enabled. + """ + if not cfg.server.selftest.enabled: + return + from .detectors import UnknownDetectorError + from .detectors.cohort import is_cohort_aware + from .detectors.registry import get as get_detector + from .stratified import query_window_seconds_for + + name = cfg.server.selftest.detector + try: + cls = get_detector(name) + except UnknownDetectorError: + return + hint = "use a rolling-window detector (e.g. MAD, Hampel, ZScoreEWMA, IQR)" + if is_cohort_aware(cls): + raise ValueError( + f"server.selftest.detector={name!r}: cohort detectors need cross-series " + f"context the single synthetic self-test series can't provide; {hint}" + ) + if query_window_seconds_for(cls, dict(getattr(cls, "defaults", {}) or {})) is not None: + raise ValueError( + f"server.selftest.detector={name!r}: stratified detectors need a multi-week " + f"lookback the synthetic self-test window can't fill; {hint}" + ) + + def _validate_detector_params(cfg: Config) -> None: """Type-check user-supplied detector params against each detector's ParamSpec. @@ -819,6 +895,7 @@ def _validate_detector_params(cfg: Config) -> None: "ReloadAuthConfig", "ReloadConfig", "SafetyConfig", + "SelfTestConfig", "ServerConfig", "TelemetryConfig", "load_config", diff --git a/detector/src/promanomaly/diagnose.py b/detector/src/promanomaly/diagnose.py new file mode 100644 index 0000000..8d14107 --- /dev/null +++ b/detector/src/promanomaly/diagnose.py @@ -0,0 +1,350 @@ +"""Config health and tuning diagnostics (``promanomaly diagnose``). + +Tells an operator which of their detectors are mis-tuned, from evidence +rather than guesswork: + +* **never fires** — dead config: a detector that hasn't crossed threshold + over the whole lookback (wrong detector, threshold too high, or a query + that doesn't carry the signal it's meant to). +* **fires often** — threshold too low or the wrong detector: a detector + firing more than a configurable fraction of the time is alert noise. +* **chronically warming** — series stuck below ``min_points`` for the + whole window (a too-sparse query or a ``min_points`` set too high). +* **empty query** — a query that returned no series over the window. +* **silent detector** — (with ``--config``) a configured detector that + emitted no score at all, which includes cohort detectors silently + dropping cohorts below ``min_cohort_size``. + +Two data sources, sharing one report shape: + +* **TSDB lookback** (``--datasource-url``) — instant queries that + aggregate the detector's own metrics over ``--window``. This is the + mode that answers "firing rate over a lookback"; ``never_fires`` / + ``fires_often`` are only meaningful here. +* **live snapshot** (``--target``) — one scrape of the detector's + ``/metrics``, for a quick current-state read (firing now, warming now, + empty now). It cannot compute a rate, so it never reports + ``never_fires``. + +Pure analysis, no persisted state — parity with promforecast's +``diagnose``. Pairs with ``backtest`` and ``calibrate-buckets`` to close +the tuning loop. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import Any + +from .config import Config + +# Default bands. A detector firing more than half the lookback is almost +# certainly mis-tuned; one that never fires is dead config. Both are +# advisory and overridable on the CLI. +DEFAULT_FIRE_HIGH = 0.5 +# A firing rate at or below this counts as "never fires" — a tiny epsilon +# rather than exactly 0 so a single stray sample in a long window doesn't +# rescue a detector that is effectively dead. +DEFAULT_NEVER_EPS = 0.0 + +InstantRows = list[tuple[dict[str, str], float]] + + +@dataclass(frozen=True) +class DiagnoseFinding: + """One tuning issue found for a group / query / detector.""" + + category: str + group: str + query: str + detector: str + detail: str + value: float + + def to_dict(self) -> dict[str, Any]: + return { + "category": self.category, + "group": self.group, + "query": self.query, + "detector": self.detector, + "detail": self.detail, + "value": self.value, + } + + +@dataclass(frozen=True) +class DiagnoseReport: + """The full diagnose result.""" + + mode: str + window: str | None + findings: list[DiagnoseFinding] + + def to_dict(self) -> dict[str, Any]: + return { + "mode": self.mode, + "window": self.window, + "findings": [f.to_dict() for f in self.findings], + } + + +def _detector_label(labels: dict[str, str]) -> str: + detector = labels.get("detector", "") + instance = labels.get("detector_instance") + return f"{detector}[{instance}]" if instance else detector + + +def evaluate_firing_rates( + rows: InstantRows, + *, + fire_high: float = DEFAULT_FIRE_HIGH, + never_eps: float = DEFAULT_NEVER_EPS, +) -> list[DiagnoseFinding]: + """Flag per-detector firing rates outside the never / often bands. + + ``rows`` come from ``avg by (id, group, detector, detector_instance) + (avg_over_time(anomaly_outside_threshold[W]))`` — one row per detector + series, value in ``[0, 1]``. + """ + findings: list[DiagnoseFinding] = [] + for labels, rate in rows: + group = labels.get("group", "") + query = labels.get("id", "") + det = _detector_label(labels) + if rate <= never_eps: + findings.append( + DiagnoseFinding( + category="never_fires", + group=group, + query=query, + detector=det, + detail=( + "detector never crossed threshold over the lookback — " + "dead config: wrong detector, threshold too high, or the " + "query doesn't carry the signal" + ), + value=rate, + ) + ) + elif rate >= fire_high: + findings.append( + DiagnoseFinding( + category="fires_often", + group=group, + query=query, + detector=det, + detail=( + f"detector fired {rate:.0%} of the lookback — threshold " + "likely too low or the wrong detector for this signal" + ), + value=rate, + ) + ) + return findings + + +def evaluate_warming(rows: InstantRows) -> list[DiagnoseFinding]: + """Flag series stuck warming up for the whole window. + + ``rows`` come from ``count by (group, id) + (min_over_time(anomaly_warming_up[W]) >= 1)`` — value is the number of + series still warming across the entire lookback. + """ + findings: list[DiagnoseFinding] = [] + for labels, count in rows: + if count <= 0: + continue + findings.append( + DiagnoseFinding( + category="chronically_warming", + group=labels.get("group", ""), + query=labels.get("id", ""), + detector="", + detail=( + f"{int(count)} series have been warming up the whole lookback — " + "the query is too sparse or min_points is set too high" + ), + value=float(count), + ) + ) + return findings + + +def evaluate_empty(rows: InstantRows) -> list[DiagnoseFinding]: + """Flag groups whose series count was zero over the window. + + ``rows`` come from ``max_over_time(anomaly_series_count[W])`` — one row + per group, value the peak series count. + """ + findings: list[DiagnoseFinding] = [] + for labels, peak in rows: + if peak > 0: + continue + findings.append( + DiagnoseFinding( + category="empty_query", + group=labels.get("group", ""), + query="", + detector="", + detail="group produced no series over the lookback — check the PromQL", + value=float(peak), + ) + ) + return findings + + +def evaluate_silent_detectors( + config: Config, emitting: set[tuple[str, str]] +) -> list[DiagnoseFinding]: + """Flag configured detectors that emitted no score over the window. + + ``emitting`` is the set of ``(group, detector)`` pairs that produced + at least one ``anomaly_score`` series. A configured detector missing + from it is silent — dead config, or a cohort detector dropping every + cohort below ``min_cohort_size``. + """ + findings: list[DiagnoseFinding] = [] + seen: set[tuple[str, str, str]] = set() + for group in config.groups: + for query in group.queries: + for det in query.detectors: + key = (group.name, det.name) + dedup = (group.name, query.id, det.name) + if dedup in seen: + continue + seen.add(dedup) + if key in emitting: + continue + cohort_note = ( + " (cohort detectors emit nothing when every cohort is below min_cohort_size)" + if det.name == "Cohort" + else "" + ) + findings.append( + DiagnoseFinding( + category="silent_detector", + group=group.name, + query=query.id, + detector=det.name, + detail=( + "configured detector emitted no anomaly_score over the " + f"lookback{cohort_note}" + ), + value=0.0, + ) + ) + return findings + + +def diagnose_from_metrics_text( + text: str, *, fire_high: float = DEFAULT_FIRE_HIGH +) -> DiagnoseReport: + """Current-state diagnose from one ``/metrics`` scrape (``--target``). + + Reports empty queries, currently-warming series, and detectors firing + on a large fraction of their series *right now*. A single snapshot + cannot establish a rate over time, so ``never_fires`` is deliberately + not reported here — use the TSDB lookback mode for that. + """ + from prometheus_client.parser import text_string_to_metric_families + + series_count_by_group: dict[str, float] = {} + warming_by_query: dict[tuple[str, str], int] = defaultdict(int) + firing_by_detector: dict[tuple[str, str, str], list[int]] = defaultdict(lambda: [0, 0]) + + for family in text_string_to_metric_families(text): + for sample in family.samples: + name = sample.name + labels = sample.labels + value = sample.value + group = labels.get("group", "") + if name == "anomaly_series_count": + series_count_by_group[group] = value + elif name == "anomaly_warming_up" and value >= 1: + warming_by_query[(group, labels.get("id", ""))] += 1 + elif name == "anomaly_outside_threshold": + key = (group, labels.get("id", ""), _detector_label(labels)) + bucket = firing_by_detector[key] + bucket[1] += 1 + if value >= 1: + bucket[0] += 1 + + findings: list[DiagnoseFinding] = [] + for group, count in sorted(series_count_by_group.items()): + if count <= 0: + findings.append( + DiagnoseFinding( + category="empty_query", + group=group, + query="", + detector="", + detail="group currently has zero series — check the PromQL", + value=count, + ) + ) + for (group, query_id), n in sorted(warming_by_query.items()): + findings.append( + DiagnoseFinding( + category="warming", + group=group, + query=query_id, + detector="", + detail=f"{n} series are warming up right now", + value=float(n), + ) + ) + for (group, query_id, det), (active, total) in sorted(firing_by_detector.items()): + if total == 0: + continue + frac = active / total + if frac >= fire_high: + findings.append( + DiagnoseFinding( + category="fires_often", + group=group, + query=query_id, + detector=det, + detail=( + f"{active}/{total} series firing right now ({frac:.0%}) — " + "threshold likely too low or the wrong detector" + ), + value=frac, + ) + ) + return DiagnoseReport(mode="snapshot", window=None, findings=findings) + + +def render_report(report: DiagnoseReport) -> str: + """Render a diagnose report as human-readable text.""" + lines: list[str] = [] + scope = f"mode={report.mode}" + if report.window: + scope += f" window={report.window}" + lines.append(f"promanomaly diagnose ({scope})") + if not report.findings: + lines.append(" no tuning issues found") + return "\n".join(lines) + "\n" + by_category: dict[str, list[DiagnoseFinding]] = defaultdict(list) + for finding in report.findings: + by_category[finding.category].append(finding) + for category in sorted(by_category): + lines.append(f"\n[{category}] ({len(by_category[category])})") + for f in by_category[category]: + target = "/".join(p for p in (f.group, f.query, f.detector) if p) + lines.append(f" - {target}: {f.detail}") + return "\n".join(lines) + "\n" + + +__all__ = [ + "DEFAULT_FIRE_HIGH", + "DEFAULT_NEVER_EPS", + "DiagnoseFinding", + "DiagnoseReport", + "diagnose_from_metrics_text", + "evaluate_empty", + "evaluate_firing_rates", + "evaluate_silent_detectors", + "evaluate_warming", + "render_report", +] diff --git a/detector/src/promanomaly/exporter.py b/detector/src/promanomaly/exporter.py index b2e5baf..953a38d 100644 --- a/detector/src/promanomaly/exporter.py +++ b/detector/src/promanomaly/exporter.py @@ -16,6 +16,7 @@ from __future__ import annotations import re +import time from collections import defaultdict from collections.abc import Iterable @@ -27,6 +28,7 @@ generate_latest, ) +from .self_observability import DetectOutcomeTracker from .source import PromQLSource from .state import Sample, SnapshotStore @@ -258,10 +260,17 @@ class OperationalMetrics: def __init__(self) -> None: self.registry = CollectorRegistry() + # The rolling detector-success ledger is owned here so both the + # runner (which records per-run outcomes) and the exporter (which + # renders the derived ratio gauge at scrape time) reach it through + # the one shared ``OperationalMetrics`` instance. + self.detect_outcomes = DetectOutcomeTracker() self._declare_runner_metrics() self._declare_cache_metrics() self._declare_leader_metrics() self._declare_discovery_metrics() + self._declare_self_observability_metrics() + self._declare_selftest_metrics() # ------------------------------------------------------------------ # Runner-owned metrics: per-group timing, readiness, source-failure @@ -415,6 +424,76 @@ def _declare_discovery_metrics(self) -> None: registry=self.registry, ) + # ------------------------------------------------------------------ + # Self-observability metrics. Telemetry about the detector's + # own internals: per-detector compute success rate, per-group CPU / + # memory attribution, and per-series freshness. All bounded by the + # configured group / detector cardinality. ``detect_success_ratio`` + # and the resource gauges are repopulated from the trackers on every + # render; the staleness / snapshot-age gauges are time-relative and + # set at render time straight from the snapshot store. + # ------------------------------------------------------------------ + def _declare_self_observability_metrics(self) -> None: + self.detect_success_ratio = Gauge( + "anomaly_detect_success_ratio", + ( + "Fraction of attempted detector computations that completed " + "without raising, over a rolling window. A degrading detector " + "surfaces here before its missed scores reach downstream alerts." + ), + ["group", "detector", "window"], + registry=self.registry, + ) + self.group_cpu_seconds_total = Counter( + "anomaly_group_cpu_seconds", + "Cumulative detector compute time attributed to a group, in seconds.", + ["group"], + registry=self.registry, + ) + self.group_memory_bytes = Gauge( + "anomaly_group_memory_bytes", + "Best-effort size of the per-group output snapshot held by the exporter.", + ["group"], + registry=self.registry, + ) + self.series_staleness_seconds = Gauge( + "anomaly_series_staleness_seconds", + "Age in seconds of the freshest per-series score in a group.", + ["group"], + registry=self.registry, + ) + self.snapshot_age_seconds = Gauge( + "anomaly_snapshot_age_seconds", + "Age in seconds of the group's most recent successful snapshot.", + ["group"], + registry=self.registry, + ) + + # ------------------------------------------------------------------ + # Self-test (dead-man's-switch) metrics. Carry a ``detector`` + # label so the series only appears once the self-test has actually run + # — a disabled self-test emits nothing and ``AnomalyPipelineDead`` + # ( anomaly_selftest_ok == 0 ) never fires on an opted-out deployment. + # ------------------------------------------------------------------ + def _declare_selftest_metrics(self) -> None: + self.selftest_ok = Gauge( + "anomaly_selftest_ok", + ( + "1 when the end-to-end detection self-test caught its injected " + "synthetic anomaly on the last run, 0 when the detect/threshold/" + "export pipeline failed to surface it. Emitted only when " + "server.selftest.enabled." + ), + ["detector"], + registry=self.registry, + ) + self.selftest_failures_total = Counter( + "anomaly_selftest_failures_total", + "Total self-test runs that failed to catch the injected anomaly.", + ["detector"], + registry=self.registry, + ) + def render(self) -> bytes: return generate_latest(self.registry) @@ -460,6 +539,7 @@ def __init__( def render(self) -> tuple[bytes, str]: self._refresh_cache_metrics() + self._refresh_self_observability_metrics() op_bytes = self._operational.render() snapshots = self._store.all_snapshots() key = tuple(sorted((snap.group, snap.timestamp) for snap in snapshots)) @@ -470,6 +550,36 @@ def render(self) -> tuple[bytes, str]: body = op_bytes + self._snapshot_render_bytes return body, CONTENT_TYPE_LATEST + def _refresh_self_observability_metrics(self) -> None: + """Repopulate the render-time self-observability gauges. + + ``anomaly_detect_success_ratio`` is derived from the rolling + outcome ledger; ``anomaly_series_staleness_seconds`` and + ``anomaly_snapshot_age_seconds`` are time-relative and must be + recomputed on every scrape so they keep climbing between runs + (an operator alerting on ``... > 600`` needs the value to grow, + not freeze at the last run's timestamp). The per-group CPU + counter and memory gauge are written directly by the runner and + need no render-time work. + """ + ops = self._operational + for point in ops.detect_outcomes.render(): + ops.detect_success_ratio.labels( + group=point.group, + detector=point.detector, + window=point.window_label, + ).set(point.ratio) + now = time.time() + for snapshot in self._store.all_snapshots(): + ops.snapshot_age_seconds.labels(group=snapshot.group).set( + max(0.0, now - snapshot.timestamp) + ) + scored_at = self._store.last_scored_at(snapshot.group) + if scored_at is not None: + ops.series_staleness_seconds.labels(group=snapshot.group).set( + max(0.0, now - scored_at) + ) + def _refresh_cache_metrics(self) -> None: if self._source is None or self._source.cache is None: return diff --git a/detector/src/promanomaly/ha.py b/detector/src/promanomaly/ha.py index dc0b27d..a23438f 100644 --- a/detector/src/promanomaly/ha.py +++ b/detector/src/promanomaly/ha.py @@ -27,7 +27,12 @@ from .config import HighAvailabilityConfig, RedisConfig from .leader import LeaderElector, LeaseClient from .logging import get_logger -from .state import FollowerSyncer, SharedSnapshotCache, SnapshotStore +from .state import ( + FollowerSyncer, + RedisDiscoveryMissTracker, + SharedSnapshotCache, + SnapshotStore, +) logger = get_logger(__name__) @@ -39,6 +44,7 @@ class HAComponents: elector: LeaderElector snapshot_cache: SharedSnapshotCache follower_syncer: FollowerSyncer + discovery_tracker: RedisDiscoveryMissTracker follower_task: asyncio.Task[None] | None = field(default=None) @@ -49,6 +55,7 @@ def build_ha_components( redis_client: Any, store: SnapshotStore, identity: str, + refresh_interval_seconds: float, on_started_leading: Callable[[], Awaitable[None]], on_stopped_leading: Callable[[], Awaitable[None]], ) -> HAComponents: @@ -66,6 +73,18 @@ def build_ha_components( key_prefix=f"{redis_config.key_prefix}:snapshot", ) follower_syncer = FollowerSyncer(store=store, shared=snapshot_cache) + # Discovery-absence miss counts live in Redis so they survive a Lease + # failover (the in-memory tracker would reset the grace window on the + # new leader). The TTL is refreshed on every write, so while a leader + # writes each refresh the key never expires; it only matters for the + # failover gap and orphan cleanup, so a few refresh intervals (floored + # at the snapshot TTL) is ample. + discovery_ttl = max(config.snapshot_ttl_seconds, refresh_interval_seconds * 4.0) + discovery_tracker = RedisDiscoveryMissTracker( + client=redis_client, + ttl_seconds=discovery_ttl, + key_prefix=f"{redis_config.key_prefix}:discovery", + ) elector = LeaderElector( ha=config, identity=identity, @@ -81,6 +100,7 @@ def build_ha_components( elector=elector, snapshot_cache=snapshot_cache, follower_syncer=follower_syncer, + discovery_tracker=discovery_tracker, ) diff --git a/detector/src/promanomaly/inspect.py b/detector/src/promanomaly/inspect.py index c07c921..a2e24c9 100644 --- a/detector/src/promanomaly/inspect.py +++ b/detector/src/promanomaly/inspect.py @@ -37,6 +37,7 @@ cohort_key_for, is_cohort_aware, ) +from .metadata_lint import lint_query from .plan import build_detector as _build_detector from .plan import build_detector_plans, effective_threshold from .source import PromQLSource, QuerySeries, SourceQueryError @@ -203,6 +204,18 @@ async def inspect_series( "label_selector": selector, } + # Metadata-aware lint hint: surface the same counter-not-rated warning + # the validate lint reports, computed live for this query. Best-effort + # — a TSDB without a metadata endpoint just yields no hint rather than + # failing the inspect. + metadata_lint: list[dict[str, Any]] = [] + try: + metadata = await source.metric_metadata() + except SourceQueryError: + metadata = {} + for finding in lint_query(group.name, query, metadata): + metadata_lint.append(finding.to_dict()) + matching = [s for s in qresult.series if _matches(s, selector)] # Pre-compute cohort baselines from the *unfiltered* result set so a # cohort-aware detector inspecting one member still sees the full @@ -321,6 +334,7 @@ async def inspect_series( "warmup_policy": defaults.warmup_policy, "matched_series": len(series_payload), "total_series": len(qresult.series), + "metadata_lint": metadata_lint, "series": series_payload, } diff --git a/detector/src/promanomaly/main.py b/detector/src/promanomaly/main.py index 7798418..43cac6a 100644 --- a/detector/src/promanomaly/main.py +++ b/detector/src/promanomaly/main.py @@ -39,6 +39,7 @@ from .leader import resolve_identity from .logging import configure_logging, get_logger from .runner import Runner +from .selftest import run_selftest from .server import build_app as _build_app from .source import PromQLSource, QueryResult from .state import SnapshotStore @@ -137,6 +138,8 @@ async def startup(self) -> None: # /metrics has data before the first scheduler tick. for group in self._config.groups: self._spawn(self._safe_group_run(group.name)) + if self._config.server.selftest.enabled: + self._spawn(self._selftest_tick()) async def shutdown(self) -> None: await stop_ha(self._ha) @@ -233,6 +236,19 @@ def _reschedule_jobs(self, scheduler: AsyncIOScheduler) -> None: max_instances=1, coalesce=True, ) + # End-to-end detection self-test, on the same cadence as the + # detector runs. Gated inside ``_selftest_tick`` by the leader flag + # like ``_tick``, so it fires on every replica but only the leader + # runs it in HA mode. + if self._config.server.selftest.enabled: + scheduler.add_job( + self._selftest_tick, + trigger=IntervalTrigger(seconds=interval), + id="selftest", + replace_existing=True, + max_instances=1, + coalesce=True, + ) async def _tick(self, group_name: str) -> None: # In HA mode followers must never run the detector pipeline — @@ -286,6 +302,38 @@ async def _safe_group_run(self, group_name: str) -> None: if not tasks: self._running_group_tasks.pop(group_name, None) + async def _selftest_tick(self) -> None: + # Leader-gated like ``_tick``: the self-test fires on every replica + # but only the elected leader actually runs it in HA mode, so the + # ``anomaly_selftest_ok`` gauge is published once per cluster. + if self._ha_enabled and not self._is_leader: + return + self._run_selftest() + + def _run_selftest(self) -> None: + """Run one end-to-end self-test and publish the result metrics. + + Synchronous and tiny (one detector over a ~60-sample synthetic + window); runs inline. Exceptions inside ``run_selftest`` are caught + there and returned as a failed result, so this never raises. + """ + cfg = self._config.server.selftest + result = run_selftest( + detector_name=cfg.detector, + threshold=cfg.threshold, + min_points=self._config.defaults.min_points, + ) + self._ops.selftest_ok.labels(detector=cfg.detector).set(1.0 if result.ok else 0.0) + if not result.ok: + self._ops.selftest_failures_total.labels(detector=cfg.detector).inc() + logger.warning( + "selftest_failed", + detector=cfg.detector, + score=result.score, + threshold=result.threshold, + detail=result.detail, + ) + def _spawn(self, coro: Any) -> None: """Track background tasks so the GC keeps them alive (RUF006).""" task = asyncio.create_task(coro) @@ -313,9 +361,16 @@ async def _start_ha(self) -> None: redis_client=self._redis, store=self._store, identity=self._identity, + refresh_interval_seconds=self._config.server.refresh_interval_seconds, on_started_leading=self._on_started_leading, on_stopped_leading=self._on_stopped_leading, ) + # Point the snapshot store at the Redis-backed discovery tracker so + # per-series miss counts are shared across replicas and survive a + # Lease failover. Both leader and followers set it (followers never + # write — they don't run the pipeline — but a demoted-then-promoted + # replica then has it ready). + self._store.set_discovery_tracker(self._ha.discovery_tracker) self._ha.elector.start() self._ha.follower_task = asyncio.create_task( run_follower_loop( @@ -345,6 +400,8 @@ async def _on_started_leading(self) -> None: self._runner.surface_misconfigurations() for group in self._config.groups: self._spawn(self._safe_group_run(group.name)) + if self._config.server.selftest.enabled: + self._spawn(self._selftest_tick()) async def _on_stopped_leading(self) -> None: """Stop being leader: cancel in-flight runs and cool down. diff --git a/detector/src/promanomaly/metadata_lint.py b/detector/src/promanomaly/metadata_lint.py new file mode 100644 index 0000000..67a6f02 --- /dev/null +++ b/detector/src/promanomaly/metadata_lint.py @@ -0,0 +1,188 @@ +"""Metadata-aware validation lint. + +The TSDB already knows each metric's type via ``/api/v1/metadata``. This +module uses that to catch a common config mistake at validate time +rather than in production: a **counter fed to a detector without being +wrapped in ``rate()`` / ``increase()``**. A raw counter monotonically +climbs, so every statistical baseline detector (MAD, Hampel, Z-score, +IQR, the stratified detectors) sees an ever-rising ramp and scores +nonsense; the operator usually meant ``rate(http_requests_total[5m])``. + +It is **lint only**. It reports; under ``--strict`` it can fail CI; it +never rewrites the query or auto-applies ``rate()`` — that would violate +promanomaly's explicit, config-driven principle. Absent metadata is +treated as "no opinion" (no finding), so a metric the TSDB doesn't carry +metadata for, or a metadata endpoint that returns nothing, never +produces a false positive. The same findings are surfaced live in +``inspect`` and ``promanomaly top``. + +The PromQL analysis is deliberately a lightweight regex pass, not a full +parser: it only needs to answer "does this known counter appear without +a rate-like wrapper?" — and it only ever looks at metric names the TSDB +metadata confirms are counters, so an over-broad match cannot +manufacture a finding for a metric it knows nothing about. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from .config import Config, QueryConfig + +# Functions that turn a counter into a rate / delta, making a counter +# input legitimate. A counter appearing only inside one of these is +# "rate-wrapped" and never flagged. +_COUNTER_FUNCTIONS: tuple[str, ...] = ( + "rate", + "irate", + "increase", + "delta", + "idelta", + "deriv", + "resets", +) + +# Series-identity aggregations that use only the *presence* of a series, +# not its value — ``count(node_cpu_seconds_total)`` counts cores, it does +# not score the raw counter. Treating the counter as safe inside these +# avoids the most common false positive (a counter used purely as a +# denominator's series count) without a full PromQL parser. +_IDENTITY_FUNCTIONS: tuple[str, ...] = ( + "count", + "count_values", + "group", + "absent", + "absent_over_time", + "present_over_time", +) + +# PromQL identifier (metric or function name). Colons are allowed because +# recording-rule names carry them; our own output never does. +_IDENT_RE = re.compile(r"[a-zA-Z_:][a-zA-Z0-9_:]*") + +# A function call wrapping a metric: capture the first identifier inside +# the call. Handles the nested case (``sum(rate(foo_total[5m]))``) because +# the wrapper directly precedes the metric regardless of any outer +# aggregation. Both rate-like and series-identity functions count as a +# legitimate wrapper for a counter. +_WRAP_RE = re.compile( + r"\b(?:" + + "|".join((*_COUNTER_FUNCTIONS, *_IDENTITY_FUNCTIONS)) + + r")\s*" + # Optional aggregation modifier: ``count by (instance) (metric)`` / + # ``sum without (cpu) (metric)``. Rate-like functions never carry one, + # so making it optional is harmless for them. + + r"(?:(?:by|without)\s*\([^)]*\)\s*)?" + + r"\(\s*([a-zA-Z_:][a-zA-Z0-9_:]*)" +) + + +@dataclass(frozen=True) +class MetadataFinding: + """One metadata/detector mismatch for a configured query.""" + + group: str + query_id: str + metric: str + metric_type: str + issue: str + detectors: list[str] + message: str + + def to_dict(self) -> dict[str, Any]: + return { + "group": self.group, + "query": self.query_id, + "metric": self.metric, + "metric_type": self.metric_type, + "issue": self.issue, + "detectors": list(self.detectors), + "message": self.message, + } + + +def normalise_metadata(payload: Any) -> dict[str, str]: + """Reduce a ``/api/v1/metadata`` ``data`` map to ``{metric: type}``. + + Accepts the already-reduced ``{metric: type}`` dict (what + :meth:`PromQLSource.metric_metadata` returns) unchanged, or the raw + ``{metric: [{"type": ...}, ...]}`` shape, so callers can pass either. + """ + if not isinstance(payload, dict): + return {} + out: dict[str, str] = {} + for metric, value in payload.items(): + if not isinstance(metric, str): + continue + if isinstance(value, str): + out[metric] = value + elif isinstance(value, list) and value and isinstance(value[0], dict): + metric_type = value[0].get("type") + if isinstance(metric_type, str) and metric_type: + out[metric] = metric_type + return out + + +def unrated_counters(promql: str, metadata: dict[str, str]) -> list[str]: + """Counter metrics in ``promql`` used without a rate-like wrapper. + + Only metrics the metadata confirms are counters are considered, so a + metric the TSDB has no opinion on is never flagged. A counter that + appears anywhere inside ``rate()`` / ``increase()`` / … is treated as + correctly wrapped. + """ + idents = set(_IDENT_RE.findall(promql)) + counters_used = {m for m in idents if metadata.get(m) == "counter"} + if not counters_used: + return [] + wrapped = set(_WRAP_RE.findall(promql)) + return sorted(counters_used - wrapped) + + +def lint_query( + group_name: str, + query: QueryConfig, + metadata: dict[str, str], +) -> list[MetadataFinding]: + """Lint one query against the TSDB metric metadata.""" + findings: list[MetadataFinding] = [] + detector_names = [d.name for d in query.detectors] + for metric in unrated_counters(query.promql, metadata): + det_repr = ", ".join(detector_names) or "(none)" + findings.append( + MetadataFinding( + group=group_name, + query_id=query.id, + metric=metric, + metric_type="counter", + issue="counter_not_rated", + detectors=detector_names, + message=( + f"metric {metric!r} is a counter used without " + f"rate()/increase(); detector(s) [{det_repr}] would score " + "the raw monotonically-increasing value. Wrap it in " + f"rate({metric}[...]) or increase({metric}[...])." + ), + ) + ) + return findings + + +def lint_config_metadata(config: Config, metadata: dict[str, str]) -> list[MetadataFinding]: + """Lint every query in ``config`` against the metric metadata map.""" + findings: list[MetadataFinding] = [] + for group in config.groups: + for query in group.queries: + findings.extend(lint_query(group.name, query, metadata)) + return findings + + +__all__ = [ + "MetadataFinding", + "lint_config_metadata", + "lint_query", + "normalise_metadata", + "unrated_counters", +] diff --git a/detector/src/promanomaly/runner.py b/detector/src/promanomaly/runner.py index 875f660..1a9b5bb 100644 --- a/detector/src/promanomaly/runner.py +++ b/detector/src/promanomaly/runner.py @@ -64,6 +64,7 @@ merge_params, ) from .selector import select_winner +from .self_observability import estimate_samples_bytes from .severity import compute_severity from .source import PromQLSource, QueryResult, QuerySeries, SourceQueryError from .state import GroupSnapshot, Sample, SnapshotStore @@ -174,6 +175,34 @@ def _current_detect_durations() -> dict[str, float]: return durations +# Per-detector (attempts, successes) accumulator for one group run, +# flushed onto the rolling ``anomaly_detect_success_ratio`` ledger at the +# end of that run. Task-local for the same reason as ``_DETECT_DURATIONS`` +# — concurrent group runs share one Runner and an instance dict would +# cross-attribute outcomes between groups. Recorded once per run rather +# than per series so the rolling ledger stays bounded at runs-per-window. +_DETECT_OUTCOMES: contextvars.ContextVar[dict[str, list[int]]] = contextvars.ContextVar( + "promanomaly_detect_outcomes" +) + + +def _record_detector_attempt(detector_name: str, *, success: bool) -> None: + """Tally one detector attempt for the current group run. + + No-op outside a seeded run context (the isolated-``_run_detector`` + tests) so unit tests of a single detector don't need to set up the + ContextVar. ``[attempts, successes]`` per detector name. + """ + try: + outcomes = _DETECT_OUTCOMES.get() + except LookupError: + return + tally = outcomes.setdefault(detector_name, [0, 0]) + tally[0] += 1 + if success: + tally[1] += 1 + + class Runner: """Owns the detection pipeline shared across all groups. @@ -279,6 +308,11 @@ def replace_config(self, new_config: Config) -> None: self._group_locks.pop(removed, None) for added in new_names - old_names: self._group_locks[added] = asyncio.Lock() + # Bound the rolling detector-success ledger to live groups so a + # group removed by a reload stops accumulating in-memory history + # (its gauge row lingers until restart, consistent with the other + # per-group operational gauges). + self._ops.detect_outcomes.retain_groups(new_names) self._store.clear_calibrations() # StratifiedFetcher owns its cache + discovery-churn tracker; # ``replace_config`` here both updates its config snapshot and @@ -319,6 +353,7 @@ async def run_group(self, group_name: str) -> GroupRunResult: # Isolated per group because each group runs in its own # concurrent scheduler task (see ``_DETECT_DURATIONS`` note). _DETECT_DURATIONS.set({}) + _DETECT_OUTCOMES.set({}) result = GroupRunResult(group=group_name) source_failed = False try: @@ -558,18 +593,37 @@ async def _process_discovered_query( # (transient pods, replicas) can't grow the dict without end. grace = query.expect_grace_runs forget_after = self._config.safety.discovery.forget_after_runs - transitioned = self._store.record_discovery_observation( - group.name, - query.id, - seen_keys, - grace, - forget_after_runs=forget_after, - ) + # In HA mode the tracker is Redis-backed (blocking socket I/O), so + # offload the read-modify-write off the event loop. Single-replica + # uses the in-memory tracker and runs inline. + if self._store.discovery_blocking_io: + transitioned = await asyncio.to_thread( + self._store.record_discovery_observation, + group.name, + query.id, + seen_keys, + grace, + forget_after_runs=forget_after, + ) + else: + transitioned = self._store.record_discovery_observation( + group.name, + query.id, + seen_keys, + grace, + forget_after_runs=forget_after, + ) for _series_key in transitioned: self._ops.signal_missing_total.labels( group=group.name, reason="missing_grace_exhausted" ).inc() - for series_key in self._store.discovery_known_missing(group.name, query.id, grace): + if self._store.discovery_blocking_io: + known_missing = await asyncio.to_thread( + self._store.discovery_known_missing, group.name, query.id, grace + ) + else: + known_missing = self._store.discovery_known_missing(group.name, query.id, grace) + for series_key in known_missing: # Render the id using the discovered labels so a missing # series carries the same ``id`` label value it carried # when it was present. Without this, an operator alerting @@ -1127,6 +1181,7 @@ async def _score_series( self._ops.failures_total.labels( group=group.name, detector=plan.name, reason=REASON_TIMEOUT ).inc() + _record_detector_attempt(plan.name, success=False) continue except Exception as exc: logger.exception( @@ -1142,7 +1197,9 @@ async def _score_series( detector=plan.name, reason=REASON_DETECTOR_EXCEPTION, ).inc() + _record_detector_attempt(plan.name, success=False) continue + _record_detector_attempt(plan.name, success=True) score, outside, samples = self._format_detector_samples( group=group, @@ -1840,10 +1897,30 @@ def _update_operational_metrics(self, result: GroupRunResult) -> None: # Read the task-local accumulator seeded at the top of this run; # ``_update_operational_metrics`` runs in the same task as # ``run_group`` so the ContextVar resolves to this group's dict. - for detector_name, elapsed in _DETECT_DURATIONS.get({}).items(): + durations = _DETECT_DURATIONS.get({}) + for detector_name, elapsed in durations.items(): self._ops.detect_duration_seconds.labels( group=result.group, detector=detector_name ).set(elapsed) + # Self-observability: attribute this run's detector compute + # time to the group's cumulative CPU counter, record the + # per-detector success ledger, and snapshot the group's output + # memory footprint. CPU is summed across detectors so the counter + # is a true per-group total regardless of detector count. + cpu_seconds = sum(durations.values()) + if cpu_seconds > 0: + self._ops.group_cpu_seconds_total.labels(group=result.group).inc(cpu_seconds) + for detector_name, (attempts, successes) in _DETECT_OUTCOMES.get({}).items(): + self._ops.detect_outcomes.record( + group=result.group, + detector=detector_name, + attempts=attempts, + successes=successes, + ) + if result.succeeded: + self._ops.group_memory_bytes.labels(group=result.group).set( + estimate_samples_bytes(result.samples) + ) # Per-kind log emitters for stratified misconfigurations. Each diff --git a/detector/src/promanomaly/self_observability.py b/detector/src/promanomaly/self_observability.py new file mode 100644 index 0000000..4816b39 --- /dev/null +++ b/detector/src/promanomaly/self_observability.py @@ -0,0 +1,225 @@ +"""Self-observability metrics for the detector itself. + +The detector emits *correct* anomaly scores; this module gives operators +telemetry about the detector's own health so a silently degrading +component is caught before it stops catching anomalies. Four families, +all bounded by configuration size (group / detector) — none touch the +per-series cardinality budget: + +* ``anomaly_detect_success_ratio{group, detector, window}`` — the + fraction of attempted detector computations that completed without + raising (timeout / exception), over a rolling window. Two windows are + emitted (``1h`` and ``24h`` by default): the 1h flavour spots a + detector that just started failing, the 24h flavour is the slower + trend. A degrading detector surfaces here before its missed scores + pollute downstream alerts. +The remaining three families are owned by the exporter / runner +directly because their natural representation is a plain +``prometheus_client`` Counter / Gauge: + +* ``anomaly_group_cpu_seconds_total{group}`` — cumulative detector + compute time attributed to a group (the sum of the per-detector + wall-clock the runner already tracks in ``_DETECT_DURATIONS``); the + runner ``inc()``s the counter at the end of each run. +* ``anomaly_group_memory_bytes{group}`` — a best-effort + ``sys.getsizeof`` snapshot of the per-group output samples; the runner + sets the gauge after each successful run. +* ``anomaly_series_staleness_seconds{group}`` and + ``anomaly_snapshot_age_seconds{group}`` — render-time freshness gauges + computed by the exporter directly from the snapshot store timestamps + (see :mod:`promanomaly.exporter`). + +Mirrors promforecast's ``self_observability`` module so the two tools +report their health the same way (``forecast_model_fit_success_ratio`` +is the sibling of ``anomaly_detect_success_ratio``). +""" + +from __future__ import annotations + +import threading +import time +from collections import deque +from collections.abc import Iterable +from dataclasses import dataclass + +# Default rolling windows for the success-ratio gauge. The 1h window +# spots a detector that just started failing; the 24h window is the +# slower trend. Both work against the chart's alert language +# ("ratio < 0.9 for 5m"). +DEFAULT_SUCCESS_WINDOWS_SECONDS: tuple[int, ...] = (3600, 86400) + + +@dataclass(frozen=True) +class DetectOutcomeSample: + """One aggregated (attempts, successes, timestamp) observation. + + One sample is recorded per group-run per detector — aggregating the + per-series attempts inside that run into a single ``(attempts, + successes)`` pair. This bounds the deque at *runs-per-window* rather + than *series x runs-per-window*: a 10k-series group at a 1m refresh + over 24h keeps 1440 samples per detector, not 14.4M. + """ + + attempts: int + successes: int + timestamp: float + + +@dataclass(frozen=True) +class SuccessRatioPoint: + """Rendered point for ``anomaly_detect_success_ratio``.""" + + group: str + detector: str + window_label: str + ratio: float + + +class DetectOutcomeTracker: + """Per-(group, detector) rolling success/attempt ledger. + + A ``deque`` per (group, detector) keeps every aggregated observation + inside the longest configured window; :meth:`render` returns the + fraction of successful attempts that fall inside each asked-for + window. Observations older than the longest window are pruned at + record time so the deque stays bounded at the operational + cardinality the runner already imposes. + """ + + def __init__(self, *, windows_seconds: Iterable[int] = DEFAULT_SUCCESS_WINDOWS_SECONDS) -> None: + self._windows_seconds: tuple[int, ...] = tuple(sorted({int(w) for w in windows_seconds})) + if not self._windows_seconds: + raise ValueError("DetectOutcomeTracker requires at least one window") + self._max_window = max(self._windows_seconds) + self._lock = threading.Lock() + self._samples: dict[tuple[str, str], deque[DetectOutcomeSample]] = {} + + @property + def windows_seconds(self) -> tuple[int, ...]: + return self._windows_seconds + + def record( + self, + *, + group: str, + detector: str, + attempts: int, + successes: int, + now: float | None = None, + ) -> None: + """Record one group-run's aggregated outcome for ``detector``. + + ``now`` defaults to ``time.time()`` and is overridable in tests + so rolling-window expiry is deterministic without sleeps. A run + in which the detector was never attempted (``attempts == 0``) is + ignored so an idle detector doesn't dilute the ratio. + """ + if attempts <= 0: + return + ts = time.time() if now is None else now + with self._lock: + bucket = self._samples.setdefault((group, detector), deque()) + bucket.append(DetectOutcomeSample(attempts=attempts, successes=successes, timestamp=ts)) + self._prune_locked(bucket, ts) + + def render(self, *, now: float | None = None) -> list[SuccessRatioPoint]: + """Snapshot the success ratio at every configured window. + + Returns one point per (group, detector, window) that has at + least one attempt inside the window. Empty windows are skipped — + a 0/0 ratio is not a "100% healthy" signal, and emitting NaN + would just confuse the alert path. + """ + ts = time.time() if now is None else now + out: list[SuccessRatioPoint] = [] + with self._lock: + for (group, detector), bucket in self._samples.items(): + self._prune_locked(bucket, ts) + for window in self._windows_seconds: + ratio = _ratio_in_window(bucket, ts, window) + if ratio is None: + continue + out.append( + SuccessRatioPoint( + group=group, + detector=detector, + window_label=_format_window(window), + ratio=ratio, + ) + ) + return out + + def retain_groups(self, names: set[str]) -> None: + """Drop per-(group, detector) buckets for groups removed by a reload.""" + with self._lock: + self._samples = { + key: bucket for key, bucket in self._samples.items() if key[0] in names + } + + def _prune_locked(self, bucket: deque[DetectOutcomeSample], now: float) -> None: + cutoff = now - self._max_window + while bucket and bucket[0].timestamp < cutoff: + bucket.popleft() + + +def _ratio_in_window( + bucket: deque[DetectOutcomeSample], now: float, window_seconds: int +) -> float | None: + cutoff = now - window_seconds + attempts = 0 + successes = 0 + for sample in bucket: + if sample.timestamp < cutoff: + continue + attempts += sample.attempts + successes += sample.successes + if attempts == 0: + return None + return successes / attempts + + +def estimate_samples_bytes(samples: Iterable[object]) -> float: + """Best-effort ``sys.getsizeof`` sum of a group's output samples. + + Walks the dataclass ``Sample`` objects and their label tuples rather + than guessing — cheap relative to the detection run and accurate + enough to answer "which group's snapshot is dominant". Not an exact + process-level breakdown: shared interned strings are double-counted, + which is acceptable for a relative gauge. + """ + import sys + + total = 0 + for sample in samples: + total += sys.getsizeof(sample) + labels = getattr(sample, "labels", None) + if labels is None: + continue + total += sys.getsizeof(labels) + for pair in labels: + total += sys.getsizeof(pair) + for item in pair: + total += sys.getsizeof(item) + return float(total) + + +def _format_window(seconds: int) -> str: + """Render a duration as the Prometheus-style ```` label.""" + if seconds <= 0: + return "0s" + if seconds % 86_400 == 0: + return f"{seconds // 86_400}d" + if seconds % 3_600 == 0: + return f"{seconds // 3_600}h" + if seconds % 60 == 0: + return f"{seconds // 60}m" + return f"{seconds}s" + + +__all__ = [ + "DEFAULT_SUCCESS_WINDOWS_SECONDS", + "DetectOutcomeSample", + "DetectOutcomeTracker", + "SuccessRatioPoint", + "estimate_samples_bytes", +] diff --git a/detector/src/promanomaly/selftest.py b/detector/src/promanomaly/selftest.py new file mode 100644 index 0000000..8f53cba --- /dev/null +++ b/detector/src/promanomaly/selftest.py @@ -0,0 +1,185 @@ +"""End-to-end detection self-test (dead-man's-switch). + +Drives one synthetic series carrying a known injected point-spike anomaly +through the *real* detect → threshold → export path each run and asserts +it is caught. The point is to prove the whole pipeline is live, not just +that the process is up: a global misconfiguration — an exporter +regression, an absurd ``alert_thresholds.score``, a detector that silently +returns nothing — can leave promanomaly running and emitting no anomalies +while the per-detector success ratios still look healthy. + +Built on the same synthetic-injection primitive the calibration cycle +uses (:data:`promanomaly.calibration.INJECTION_PATTERNS`), stateless, and +bounded to one synthetic series. The result drives ``anomaly_selftest_ok`` +(1/0) and ``anomaly_selftest_failures_total``; the ``AnomalyPipelineDead`` +reference alert fires when it stops passing. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import structlog + +from .calibration import INJECTION_PATTERNS +from .detectors import UnknownDetectorError +from .detectors.registry import get as get_detector +from .exporter import InvalidMetricNameError, render_samples +from .plan import build_detector +from .state import Sample + +logger = structlog.get_logger(__name__) + +# Deterministic synthetic window: a clean baseline with a small amount of +# fixed-seed noise (so the baseline MAD/std is non-zero and the detector +# isn't dividing by zero) plus a single injected point spike on the last +# sample. The seed is fixed so the self-test is reproducible run-to-run — +# a flapping self-test would be worse than none. +_SEED = 1729 +_BASELINE = 100.0 +_NOISE_SD = 1.0 +# Floor on the synthetic window length so even a tiny ``min_points`` still +# gives the detector enough context to compute a stable baseline. +_MIN_WINDOW = 60 + + +@dataclass(frozen=True) +class SelfTestResult: + """Outcome of one self-test run.""" + + ok: bool + detector: str + score: float + threshold: float + outside: bool + detail: str + + def to_dict(self) -> dict[str, object]: + return { + "ok": self.ok, + "detector": self.detector, + "score": self.score, + "threshold": self.threshold, + "outside": self.outside, + "detail": self.detail, + } + + +def _synthetic_window(min_points: int) -> pd.DataFrame: + n = max(min_points, _MIN_WINDOW) + rng = np.random.default_rng(_SEED) + clean = _BASELINE + rng.normal(0.0, _NOISE_SD, size=n) + spiked = INJECTION_PATTERNS["point_spike"](clean) + # 15s spacing mirrors the default scoring step; the exact cadence is + # irrelevant to a point-spike detector but keeps the frame realistic. + timestamps = np.arange(n, dtype=float) * 15.0 + return pd.DataFrame({"timestamp": timestamps, "y": spiked}) + + +def run_selftest(*, detector_name: str, threshold: float, min_points: int) -> SelfTestResult: + """Run one synthetic detect → threshold → export cycle. + + Returns ``ok=True`` only when the injected anomaly is both flagged + outside threshold by the detector *and* survives rendering through the + exporter (so an exporter regression that rejects the sample also trips + the self-test). Any exception along the way is caught and reported as + a failure rather than raised — the dead-man's-switch must itself never + take the process down. + """ + try: + cls = get_detector(detector_name) + except UnknownDetectorError as exc: + return SelfTestResult( + ok=False, + detector=detector_name, + score=0.0, + threshold=threshold, + outside=False, + detail=f"unknown detector: {exc}", + ) + + params = dict(getattr(cls, "defaults", {}) or {}) + window = _synthetic_window(min_points) + try: + detector = build_detector(cls, params) + row = detector.fit_score(window, "", params).iloc[-1] + except Exception as exc: + return SelfTestResult( + ok=False, + detector=detector_name, + score=0.0, + threshold=threshold, + outside=False, + detail=f"detector raised: {exc}", + ) + + score = float(row.get("score", 0.0)) + # Mirror the runner's verdict: a detector that owns its threshold + # semantics flags ``is_outside`` directly; otherwise compare the raw + # score. No practical-significance floors apply — the synthetic spike + # is a pure statistical signal. + outside = bool(row.get("is_outside", False)) or score >= threshold + + # Exercise the export path with the same Sample shape the runner emits, + # so an exporter regression (e.g. a name that fails validation) is + # caught here rather than silently blanking /metrics. + labels: tuple[tuple[str, str], ...] = ( + ("detector", detector_name), + ("group", "_selftest"), + ("id", "_selftest"), + ) + samples = [ + Sample(metric="anomaly_score", labels=labels, value=score), + Sample( + metric="anomaly_outside_threshold", + labels=labels, + value=1.0 if outside else 0.0, + ), + ] + try: + rendered = render_samples(samples) + except InvalidMetricNameError as exc: + return SelfTestResult( + ok=False, + detector=detector_name, + score=score, + threshold=threshold, + outside=outside, + detail=f"export path rejected the sample: {exc}", + ) + + exported_fired = "anomaly_outside_threshold{" in rendered and "} 1.0" in rendered + if not outside: + return SelfTestResult( + ok=False, + detector=detector_name, + score=score, + threshold=threshold, + outside=False, + detail=( + "injected anomaly not caught: score " + f"{score:.3f} did not cross threshold {threshold}" + ), + ) + if not exported_fired: + return SelfTestResult( + ok=False, + detector=detector_name, + score=score, + threshold=threshold, + outside=outside, + detail="detector fired but the export path did not surface it", + ) + return SelfTestResult( + ok=True, + detector=detector_name, + score=score, + threshold=threshold, + outside=True, + detail="ok", + ) + + +__all__ = ["SelfTestResult", "run_selftest"] diff --git a/detector/src/promanomaly/server.py b/detector/src/promanomaly/server.py index 1177430..4e84451 100644 --- a/detector/src/promanomaly/server.py +++ b/detector/src/promanomaly/server.py @@ -19,6 +19,9 @@ from .config import Config, ReloadAuthConfig from .detectors import list_meta from .inspect import InspectError, inspect_series +from .metadata_lint import lint_config_metadata +from .source import SourceQueryError +from .warmup import compute_warmup if TYPE_CHECKING: from .main import Application @@ -88,6 +91,7 @@ async def debug_anomalies( group: str | None = None, min_severity: float = 0.0, limit: int = 50, + lint: bool = False, ) -> JSONResponse: # Live triage: the currently-firing series across all groups, # ranked by severity, read straight from the snapshot store (no @@ -100,14 +104,40 @@ async def debug_anomalies( min_severity=min_severity, limit=max(0, limit), ) - return JSONResponse( - { - "count": len(firing), - "min_severity": min_severity, - "group": group, - "anomalies": [f.as_dict() for f in firing], - } - ) + body: dict[str, Any] = { + "count": len(firing), + "min_severity": min_severity, + "group": group, + "anomalies": [f.as_dict() for f in firing], + } + # Opt-in metadata lint (off by default so the no-extra-TSDB-query + # contract holds for the common scrape path). ``promanomaly top`` + # requests it so an operator triaging an anomaly is reminded when + # the offending query is feeding a raw counter to a detector. + if lint: + body["metadata_lint"] = await _metadata_lint_hints(application) + return JSONResponse(body) + + @app.get("/warmup") + async def warmup_endpoint() -> Response: + # Opt-in per server.expose_warmup_endpoint. Reports per-(group, + # query) warm-up status by issuing one cheap probe per query — + # the operator's answer to "the dashboard is empty, do I wait or + # debug?" on a fresh install. The shipped NetworkPolicy denies + # external ingress here alongside /debug/*. + if not application._config.server.expose_warmup_endpoint: + return PlainTextResponse("warmup endpoint disabled", status_code=404) + try: + report = await compute_warmup( + config=application._config, + source=application._source, + ) + except Exception as exc: # pragma: no cover - defensive + return JSONResponse( + {"error": "internal", "message": str(exc)}, + status_code=500, + ) + return JSONResponse(report.to_dict()) @app.get("/debug/inspect") async def debug_inspect( @@ -137,6 +167,22 @@ async def debug_inspect( return app +async def _metadata_lint_hints(application: Application) -> list[dict[str, Any]]: + """Best-effort metadata-lint findings for the live config. + + Used by ``/debug/anomalies?lint=true`` (and thus ``promanomaly top + --lint``). A TSDB without a metadata endpoint, or a transient query + failure, yields no hints rather than failing triage. + """ + try: + metadata = await application._source.metric_metadata() + except SourceQueryError: + return [] + except Exception: # pragma: no cover - defensive + return [] + return [f.to_dict() for f in lint_config_metadata(application._config, metadata)] + + def authorise_reload(auth: ReloadAuthConfig, request: Request) -> None: """Reject the request unless it satisfies the configured reload auth. diff --git a/detector/src/promanomaly/source.py b/detector/src/promanomaly/source.py index 5c7c99f..7ec6a26 100644 --- a/detector/src/promanomaly/source.py +++ b/detector/src/promanomaly/source.py @@ -261,6 +261,102 @@ async def range_query( await self._cache_put(cache_key, query_result) return query_result + async def instant_query(self, promql: str) -> list[tuple[dict[str, str], float]]: + """Run an instant PromQL query and return ``(labels, value)`` rows. + + Used by ``promanomaly diagnose`` to aggregate the detector's own + metrics over a lookback (e.g. ``avg_over_time(...[24h])``). Returns + one row per result-vector series; scalar / non-finite values are + dropped. Raises :class:`SourceQueryError` on a TSDB-side failure so + the caller can report it. + """ + if self._client is None: + await self.start() + assert self._client is not None + try: + response = await self._client.get("/api/v1/query", params={"query": promql}) + except httpx.TimeoutException as exc: + raise SourceQueryError("timeout", _exc_detail(exc)) from exc + except httpx.HTTPError as exc: + raise SourceQueryError("transport_error", _exc_detail(exc)) from exc + if response.status_code >= 500: + raise SourceQueryError("server_error", f"HTTP {response.status_code}") + if response.status_code >= 400: + raise SourceQueryError( + "client_error", f"HTTP {response.status_code}: {response.text[:200]}" + ) + try: + payload = response.json() + except ValueError as exc: + raise SourceQueryError("invalid_response", str(exc)) from exc + if payload.get("status") != "success": + raise SourceQueryError("query_error", str(payload.get("error") or "unknown")) + data = payload.get("data") or {} + result = data.get("result") or [] + rows: list[tuple[dict[str, str], float]] = [] + for entry in result: + labels = dict(entry.get("metric") or {}) + value_pair = entry.get("value") + if not isinstance(value_pair, list) or len(value_pair) != 2: + continue + try: + value = float(value_pair[1]) + except (TypeError, ValueError): + continue + if not np.isfinite(value): + continue + rows.append((labels, value)) + return rows + + async def metric_metadata(self) -> dict[str, str]: + """Fetch ``/api/v1/metadata`` and return ``{metric_name: type}``. + + Powers the metadata-aware validation lint: the TSDB already knows + each metric's type (``counter`` / ``gauge`` / ``histogram`` / + ``summary``), so the lint can warn when a query feeds a raw + counter to a detector without wrapping it in ``rate()`` / + ``increase()``. + + Returns one type per metric (the first entry when a metric reports + several). Raises :class:`SourceQueryError` on a transport / HTTP + failure so the caller can report "metadata unavailable"; an + endpoint that simply has no metadata returns ``{}`` ("no opinion"), + not an error. Metadata-endpoint quirks across VictoriaMetrics / + Mimir / Thanos are absorbed: a malformed envelope yields ``{}`` + rather than crashing the lint. + """ + if self._client is None: + await self.start() + assert self._client is not None + try: + response = await self._client.get("/api/v1/metadata") + except httpx.TimeoutException as exc: + raise SourceQueryError("timeout", _exc_detail(exc)) from exc + except httpx.HTTPError as exc: + raise SourceQueryError("transport_error", _exc_detail(exc)) from exc + if response.status_code >= 500: + raise SourceQueryError("server_error", f"HTTP {response.status_code}") + if response.status_code >= 400: + raise SourceQueryError( + "client_error", f"HTTP {response.status_code}: {response.text[:200]}" + ) + try: + payload = response.json() + except ValueError as exc: + raise SourceQueryError("invalid_response", str(exc)) from exc + data = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(data, dict): + return {} + out: dict[str, str] = {} + for metric, entries in data.items(): + if not isinstance(metric, str) or not isinstance(entries, list) or not entries: + continue + first = entries[0] + metric_type = first.get("type") if isinstance(first, dict) else None + if isinstance(metric_type, str) and metric_type: + out[metric] = metric_type + return out + async def _cache_get(self, cache_key: tuple[str, float, float, int, int]) -> QueryResult | None: """Read from the cache without blocking the event loop. diff --git a/detector/src/promanomaly/state/__init__.py b/detector/src/promanomaly/state/__init__.py index 43d5e2b..3a8ef81 100644 --- a/detector/src/promanomaly/state/__init__.py +++ b/detector/src/promanomaly/state/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from .discovery import DiscoveryMissTracker +from .discovery import DiscoveryMissTracker, RedisDiscoveryMissTracker from .shared_snapshot import FollowerSyncer, SharedSnapshotCache from .snapshot import GroupSnapshot, Sample, SnapshotStore @@ -10,6 +10,7 @@ "DiscoveryMissTracker", "FollowerSyncer", "GroupSnapshot", + "RedisDiscoveryMissTracker", "Sample", "SharedSnapshotCache", "SnapshotStore", diff --git a/detector/src/promanomaly/state/discovery.py b/detector/src/promanomaly/state/discovery.py index af74ed3..e48b4d9 100644 --- a/detector/src/promanomaly/state/discovery.py +++ b/detector/src/promanomaly/state/discovery.py @@ -14,15 +14,54 @@ from __future__ import annotations +import json import threading from collections.abc import Iterable +from typing import Any + +import structlog + +logger = structlog.get_logger(__name__) SeriesKey = tuple[tuple[str, str], ...] +def _encode_series_key(series_key: SeriesKey) -> str: + """Serialise a series key to a stable string for a Redis hash field. + + The key is already a sorted tuple of ``(label, value)`` pairs, so the + JSON encoding is deterministic and round-trips back to the same tuple. + """ + return json.dumps([list(pair) for pair in series_key], separators=(",", ":")) + + +def _decode_series_key(field: str) -> SeriesKey | None: + """Inverse of :func:`_encode_series_key`; ``None`` on a corrupt field.""" + try: + raw = json.loads(field) + except (TypeError, ValueError): + return None + if not isinstance(raw, list): + return None + pairs: list[tuple[str, str]] = [] + for item in raw: + if not isinstance(item, list) or len(item) != 2: + return None + k, v = item + if not isinstance(k, str) or not isinstance(v, str): + return None + pairs.append((k, v)) + return tuple(pairs) + + class DiscoveryMissTracker: """Thread-safe miss-counter for discover-based absence detection.""" + # In-memory tracker: lookups are instant, so callers run it inline on + # the event loop. The Redis variant flips this so the runner offloads + # its blocking socket I/O to a worker thread. + is_blocking_io: bool = False + def __init__(self) -> None: self._lock = threading.RLock() # (group, query_id) -> series_key -> consecutive miss count. @@ -92,3 +131,134 @@ def drop_group(self, group: str) -> None: for key in list(self._series): if key[0] == group: del self._series[key] + + +class RedisDiscoveryMissTracker: + """Redis-backed miss-counter for discover-based absence detection (HA). + + Drop-in replacement for :class:`DiscoveryMissTracker` whose state lives + in Redis instead of leader-local memory, so a Lease failover no longer + resets the per-series miss counts: a newly-elected leader inherits them + and a series that disappeared right before the handover still trips + ``anomaly_signal_missing`` on schedule rather than restarting its grace + window from zero. Resolves the HA caveat documented in + ``docs/operations.md`` for signal-absence detection. + + Only the elected leader runs the detector pipeline, so there is exactly + one writer; a plain read-modify-write per ``(group, query_id)`` is safe + without Lua/WATCH. Each ``(group, query_id)`` is one Redis hash whose + fields are JSON-encoded series keys and whose values are the + consecutive-miss counts. A generous TTL is refreshed on every write so + the state survives a failover gap yet a fully-removed deployment's keys + eventually expire rather than leaking; while a leader is actively + writing every refresh the key never expires. + + Best-effort: every Redis op is wrapped so a transient Redis outage + degrades absence detection (no missing-signal emitted that run) rather + than crashing the detection run — consistent with the shared snapshot + cache's philosophy. ``forget_after_runs`` bounds the hash size exactly + as the in-memory tracker does. + """ + + is_blocking_io: bool = True + + def __init__( + self, + *, + client: Any, + ttl_seconds: float, + key_prefix: str = "promanomaly:discovery", + ) -> None: + if ttl_seconds <= 0: + raise ValueError("ttl_seconds must be > 0") + self._client = client + self._ttl = int(max(1, ttl_seconds)) + self._prefix = key_prefix + + def _key_for(self, group: str, query_id: str) -> str: + # group / query_id are validated Prometheus names (no colons), so + # ':' is an unambiguous separator and the drop_group SCAN glob is + # safe. + return f"{self._prefix}:{group}:{query_id}" + + def _load(self, group: str, query_id: str) -> dict[SeriesKey, int]: + try: + raw = self._client.hgetall(self._key_for(group, query_id)) + except Exception as exc: + logger.warning( + "discovery_redis_load_failed", group=group, query=query_id, error=str(exc) + ) + return {} + known: dict[SeriesKey, int] = {} + for field, value in (raw or {}).items(): + field_str = field.decode("utf-8") if isinstance(field, bytes) else str(field) + series_key = _decode_series_key(field_str) + if series_key is None: + continue + try: + known[series_key] = int(value) + except (TypeError, ValueError): + continue + return known + + def record_observation( + self, + group: str, + query_id: str, + seen: Iterable[SeriesKey], + grace_runs: int, + *, + forget_after_runs: int | None = None, + ) -> list[SeriesKey]: + seen_set = {tuple(s) for s in seen} + known = self._load(group, query_id) + new_state: dict[SeriesKey, int] = {} + transitioned: list[SeriesKey] = [] + for series_key in seen_set: + new_state[series_key] = 0 + for series_key, miss_count in known.items(): + if series_key in seen_set: + continue + new_count = miss_count + 1 + if forget_after_runs is not None and new_count > forget_after_runs: + continue + new_state[series_key] = new_count + if new_count == grace_runs: + transitioned.append(series_key) + self._persist(group, query_id, new_state) + return transitioned + + def _persist(self, group: str, query_id: str, state: dict[SeriesKey, int]) -> None: + key = self._key_for(group, query_id) + mapping = {_encode_series_key(k): str(v) for k, v in state.items()} + try: + pipe = self._client.pipeline() + pipe.delete(key) + if mapping: + pipe.hset(key, mapping=mapping) + pipe.expire(key, self._ttl) + pipe.execute() + except Exception as exc: + logger.warning( + "discovery_redis_persist_failed", group=group, query=query_id, error=str(exc) + ) + + def known_missing( + self, + group: str, + query_id: str, + grace_runs: int, + ) -> list[SeriesKey]: + known = self._load(group, query_id) + return [s for s, miss in known.items() if miss >= grace_runs] + + def known_series(self, group: str, query_id: str) -> set[SeriesKey]: + return set(self._load(group, query_id).keys()) + + def drop_group(self, group: str) -> None: + try: + keys = list(self._client.scan_iter(match=f"{self._prefix}:{group}:*", count=200)) + if keys: + self._client.delete(*keys) + except Exception as exc: + logger.warning("discovery_redis_drop_failed", group=group, error=str(exc)) diff --git a/detector/src/promanomaly/state/snapshot.py b/detector/src/promanomaly/state/snapshot.py index 0f08efe..92a194c 100644 --- a/detector/src/promanomaly/state/snapshot.py +++ b/detector/src/promanomaly/state/snapshot.py @@ -13,11 +13,41 @@ import threading from collections.abc import Iterable from dataclasses import dataclass, field +from typing import Protocol from ..calibration import DetectorCalibration from ..duration import DurationTracker from .discovery import DiscoveryMissTracker +SeriesKey = tuple[tuple[str, str], ...] + + +class _DiscoveryTracker(Protocol): + """Structural type the snapshot store accepts for its discovery tracker. + + Both the in-memory :class:`DiscoveryMissTracker` and the Redis-backed + :class:`RedisDiscoveryMissTracker` satisfy it, so HA mode can swap one + for the other without the store knowing which it holds. + """ + + is_blocking_io: bool + + def record_observation( + self, + group: str, + query_id: str, + seen: Iterable[SeriesKey], + grace_runs: int, + *, + forget_after_runs: int | None = ..., + ) -> list[SeriesKey]: ... + + def known_missing(self, group: str, query_id: str, grace_runs: int) -> list[SeriesKey]: ... + + def known_series(self, group: str, query_id: str) -> set[SeriesKey]: ... + + def drop_group(self, group: str) -> None: ... + @dataclass(frozen=True) class Sample: @@ -58,6 +88,14 @@ def __init__(self) -> None: self._lock = threading.RLock() self._snapshots: dict[str, GroupSnapshot] = {} self._group_ready: dict[str, bool] = {} + # group -> timestamp of the most recent snapshot that carried at + # least one ``anomaly_score`` sample. Distinct from the snapshot + # timestamp because a group can write a fresh snapshot that only + # contains warm-up flags (no scores yet) or, under drop_scores, + # only operational rollups; ``anomaly_series_staleness_seconds`` + # must keep climbing in that case while ``anomaly_snapshot_age`` + # resets. Rebuilt from zero on restart like every other counter. + self._last_scored_at: dict[str, float] = {} # (group, query_id, series_key) -> consecutive observation count. # Gates the warm-up window. self._observed_runs: dict[tuple[str, str, tuple[tuple[str, str], ...]], int] = {} @@ -85,7 +123,31 @@ def __init__(self) -> None: tuple[str, str, tuple[tuple[str, str], ...], str, str | None], int, ] = {} - self._discovery = DiscoveryMissTracker() + # Discovery-absence miss tracker. In-memory by default; HA mode + # swaps in a Redis-backed tracker via ``set_discovery_tracker`` so + # the miss counts survive leader failover. + self._discovery: _DiscoveryTracker = DiscoveryMissTracker() + + def set_discovery_tracker(self, tracker: _DiscoveryTracker) -> None: + """Replace the discovery-absence tracker (HA mode wiring). + + Called once at HA startup to point the store at the Redis-backed + tracker so per-series miss counts are shared across replicas and + survive a Lease failover. Single-replica deployments keep the + in-memory default. + """ + with self._lock: + self._discovery = tracker + + @property + def discovery_blocking_io(self) -> bool: + """Whether the discovery tracker does blocking I/O (Redis backend). + + The runner reads this to decide whether to offload the per-run + discovery-observation calls to a worker thread instead of running + them inline on the event loop. + """ + return bool(getattr(self._discovery, "is_blocking_io", False)) # ------------------------------------------------------------------ # Snapshots @@ -109,6 +171,8 @@ def write(self, snapshot: GroupSnapshot, *, mark_ready: bool = True) -> None: with self._lock: self._snapshots[snapshot.group] = snapshot self._group_ready[snapshot.group] = mark_ready + if any(sample.metric == "anomaly_score" for sample in snapshot.samples): + self._last_scored_at[snapshot.group] = snapshot.timestamp def get(self, group: str) -> GroupSnapshot | None: with self._lock: @@ -118,6 +182,18 @@ def all_snapshots(self) -> list[GroupSnapshot]: with self._lock: return list(self._snapshots.values()) + def last_scored_at(self, group: str) -> float | None: + """Timestamp of the freshest scoring snapshot for ``group``. + + ``None`` until the group has produced at least one snapshot + carrying an ``anomaly_score`` sample — feeds + ``anomaly_series_staleness_seconds`` (the exporter omits the row + while this is ``None`` so a still-warming group doesn't report a + misleading staleness number). + """ + with self._lock: + return self._last_scored_at.get(group) + def all_samples(self) -> Iterable[Sample]: for snap in self.all_snapshots(): yield from snap.samples @@ -127,6 +203,7 @@ def remove_group(self, group: str) -> None: with self._lock: self._snapshots.pop(group, None) self._group_ready.pop(group, None) + self._last_scored_at.pop(group, None) for obs_key in list(self._observed_runs): if obs_key[0] == group: del self._observed_runs[obs_key] @@ -317,7 +394,7 @@ def clear_calibrations(self) -> None: self._calibrations.clear() # ------------------------------------------------------------------ - # Discovery-based absence tracking (v0.4) — delegated to + # Discovery-based absence tracking — delegated to # :class:`DiscoveryMissTracker`. # ------------------------------------------------------------------ def record_discovery_observation( diff --git a/detector/src/promanomaly/warmup.py b/detector/src/promanomaly/warmup.py new file mode 100644 index 0000000..01277ee --- /dev/null +++ b/detector/src/promanomaly/warmup.py @@ -0,0 +1,420 @@ +"""Detector warm-up status. + +Reports per-(group, query) "what is still loading?" status for a fresh +install or a just-restarted pod. It answers the questions an operator +asks during the first cycle after ``helm install`` when the dashboard is +empty: + +* How many rolling-window samples does this query already have? +* How many does it need before the detector will score (``min_points``)? +* What is currently blocking it (no data yet, not enough points, a + stratified baseline still filling its multi-week lookback)? +* Roughly how long until the first usable score? + +The status is computed on demand — there is no background poll. Each +call issues one cheap range query per configured query (a small +``range_query`` over the rolling window, returning at most +``window / step`` samples). Probes fan out concurrently across queries +and across groups, so total wall time is bounded by the slowest probe +rather than the sum. + +The endpoint is opt-in via ``server.expose_warmup_endpoint`` because +even the one-probe-per-query cost is wasted on the steady-state path: +the operator workflow is "hit it a few times during the first refresh, +then never again". For the same reason there is no caching layer — a +stale cached answer is more confusing than a fresh probe. + +Mirrors promforecast's warmup surface so the cross-tool operator +experience is identical (``promanomaly warmup`` ↔ ``promforecast +warmup``, same JSON shape and table columns). +""" + +from __future__ import annotations + +import asyncio +import math +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +import structlog + +from .config import Config, DefaultsConfig, GroupConfig, QueryConfig +from .detectors import UnknownDetectorError +from .detectors.registry import get as get_detector +from .plan import merge_params +from .source import SourceQueryError +from .stratified import query_window_seconds_for + +if TYPE_CHECKING: + from .source import PromQLSource + +logger = structlog.get_logger(__name__) + + +# Bounded set of ``blocked_by`` reasons. ``lookback`` = the TSDB returned +# no samples at all; ``min_points`` = some samples but below the +# threshold; ``baseline`` = a stratified detector still filling its +# multi-week lookback; ``discovery`` = a templated query we can't probe +# until discovery resolves; ``none`` = ready. +BlockedBy = Literal["none", "min_points", "lookback", "baseline", "discovery"] + +# Upper bound on samples a single warm-up probe pulls per series. Keeps +# the probe cheap even for a multi-week stratified lookback: the step is +# coarsened so the range query returns at most this many points. Short +# rolling windows never reach the cap and probe at their native step. +_MAX_PROBE_POINTS = 1500 + + +@dataclass(frozen=True) +class QueryWarmupStatus: + """Per-query warm-up status row.""" + + id: str + status: Literal["ready", "warming", "error"] + points_available: int + points_needed: int + blocked_by: str + eta_seconds: float + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "status": self.status, + "points_available": self.points_available, + "points_needed": self.points_needed, + "blocked_by": self.blocked_by, + "eta_seconds": self.eta_seconds, + } + + +@dataclass(frozen=True) +class GroupWarmupStatus: + """Per-group warm-up status.""" + + name: str + queries: list[QueryWarmupStatus] + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "queries": [q.to_dict() for q in self.queries]} + + +@dataclass(frozen=True) +class WarmupReport: + """Full warm-up report.""" + + groups: list[GroupWarmupStatus] + + def to_dict(self) -> dict[str, Any]: + return {"groups": [g.to_dict() for g in self.groups]} + + +async def compute_warmup( + *, + config: Config, + source: PromQLSource, + query_timeout_seconds: float | None = None, + now: float | None = None, +) -> WarmupReport: + """Probe every configured query and return its warm-up status. + + Fan-out is concurrent across queries and groups. ``query_timeout_seconds`` + overrides the per-probe timeout (defaults to ``safety.query_timeout`` + if present, else the datasource timeout). Probe failures degrade the + row to ``status: "error"`` so one bad PromQL never sinks the report. + """ + timeout = ( + query_timeout_seconds + if query_timeout_seconds is not None + else config.datasource.timeout_seconds + ) + ts = time.time() if now is None else now + tasks = [ + _probe_group( + config=config, + group=group, + source=source, + query_timeout_seconds=timeout, + now=ts, + ) + for group in config.groups + ] + groups = await asyncio.gather(*tasks) if tasks else [] + return WarmupReport(groups=list(groups)) + + +async def _probe_group( + *, + config: Config, + group: GroupConfig, + source: PromQLSource, + query_timeout_seconds: float, + now: float, +) -> GroupWarmupStatus: + tasks = [ + _probe_query( + config=config, + query=query, + source=source, + query_timeout_seconds=query_timeout_seconds, + now=now, + ) + for query in group.queries + ] + queries = await asyncio.gather(*tasks) + return GroupWarmupStatus(name=group.name, queries=list(queries)) + + +async def _probe_query( + *, + config: Config, + query: QueryConfig, + source: PromQLSource, + query_timeout_seconds: float, + now: float, +) -> QueryWarmupStatus: + """Probe one query and translate the result into a status row.""" + defaults = config.defaults + step_seconds = max(defaults.step_seconds, 1.0) + min_points = defaults.min_points + + if query.discover: + # Templated queries can't be probed until discovery resolves the + # rendered string — probing the raw template would just surface a + # parse error and confuse the operator. Reported ``ready`` so the + # CLI's exit-code loop treats it as non-blocking, but ``blocked_by`` + # makes the dependency explicit ("check the discovery counters"). + return QueryWarmupStatus( + id=query.id, + status="ready", + points_available=0, + points_needed=0, + blocked_by="discovery", + eta_seconds=0.0, + ) + + # A stratified detector wants a multi-week lookback before its bucket + # baseline is meaningful. Probe over the longest window any detector + # on the query asks for so the report reflects what the heaviest + # detector actually needs, not just the short rolling window. + window_seconds, needs_baseline = _probe_window_seconds(query, defaults) + # Keep the probe cheap regardless of window length: a multi-week + # stratified lookback at the 15s scoring step would pull >150k points + # per series, defeating the "one cheap probe per query" promise and + # hammering the TSDB. Coarsen the step so the response is bounded to + # ~``_MAX_PROBE_POINTS`` samples; for the common short-window case the + # cap never trips and the step is unchanged. ``points_needed`` is + # measured at the same coarsened step so the available/needed fraction + # stays a faithful "how full is the lookback" ratio. + probe_step = step_seconds + if window_seconds / probe_step > _MAX_PROBE_POINTS: + probe_step = window_seconds / _MAX_PROBE_POINTS + + try: + result = await asyncio.wait_for( + source.range_query( + promql=query.promql, + end=now, + window_seconds=window_seconds, + step_seconds=probe_step, + max_series=config.safety.max_series_per_query, + ), + timeout=query_timeout_seconds, + ) + except TimeoutError: + logger.warning("warmup_probe_timeout", query=query.id) + return _error_status(query.id, min_points) + except SourceQueryError as exc: + logger.warning("warmup_probe_failed", query=query.id, reason=exc.reason) + return _error_status(query.id, min_points) + except Exception as exc: # pragma: no cover - defensive + logger.warning("warmup_probe_failed", query=query.id, error=str(exc)) + return _error_status(query.id, min_points) + + points_available = _points_in_longest_series(result.series) + + # For a stratified detector the "enough" bar is a full lookback, not + # ``min_points`` — surface ``baseline`` as the blocker so an operator + # understands the wait is the multi-week fill, not a thin signal. The + # needed-points target is computed at the (possibly coarsened) probe + # step so it matches what ``points_available`` was measured against. + if needs_baseline: + needed = max(min_points, int(window_seconds / probe_step)) + if points_available < needed: + return QueryWarmupStatus( + id=query.id, + status="warming", + points_available=points_available, + points_needed=needed, + blocked_by="baseline" if points_available > 0 else "lookback", + eta_seconds=_estimate_eta(points_available, needed, probe_step), + ) + return _ready_status(query.id, points_available, needed) + + if points_available < min_points: + return QueryWarmupStatus( + id=query.id, + status="warming", + points_available=points_available, + points_needed=min_points, + blocked_by="min_points" if points_available > 0 else "lookback", + eta_seconds=_estimate_eta(points_available, min_points, step_seconds), + ) + return _ready_status(query.id, points_available, min_points) + + +def _probe_window_seconds(query: QueryConfig, defaults: DefaultsConfig) -> tuple[float, bool]: + """Resolve the probe window and whether a stratified baseline applies. + + Returns ``(window_seconds, needs_baseline)``. ``needs_baseline`` is + True when at least one detector on the query opts into the + sliding-window stratified strategy (its lookback is what gates + readiness, not ``min_points``). + """ + window = defaults.window_seconds + needs_baseline = False + for entry in query.detectors: + try: + cls = get_detector(entry.name) + except UnknownDetectorError: + continue + merged = merge_params(cls, entry.params, defaults) + lookback = query_window_seconds_for(cls, merged) + if lookback is not None and lookback > window: + window = lookback + needs_baseline = True + return window, needs_baseline + + +def _ready_status(query_id: str, points_available: int, points_needed: int) -> QueryWarmupStatus: + return QueryWarmupStatus( + id=query_id, + status="ready", + points_available=points_available, + points_needed=points_needed, + blocked_by="none", + eta_seconds=0.0, + ) + + +def _error_status(query_id: str, min_points: int) -> QueryWarmupStatus: + return QueryWarmupStatus( + id=query_id, + status="error", + points_available=0, + points_needed=min_points, + blocked_by="lookback", + eta_seconds=math.inf, + ) + + +def _points_in_longest_series(series: list[Any]) -> int: + """Length of the series with the most samples (0 when empty). + + The runner scores each series independently, so the "best off" series + is what unblocks the query first; the mean would smear over a single + slow-to-warm peer. + """ + if not series: + return 0 + return max(len(s.samples) for s in series) + + +def _estimate_eta(points_available: int, points_needed: int, step_seconds: float) -> float: + """Rough seconds-to-ready estimate ("each step adds one sample"). + + Underestimates for sparse series and overestimates slightly when + backfill inserts points faster than the scrape cadence; meant to + answer "minutes, hours, or days?", not to be a hard deadline. + """ + if points_available >= points_needed: + return 0.0 + return float((points_needed - points_available) * step_seconds) + + +def render_table(report: WarmupReport) -> str: + """Render the report as a plain-text aligned table for the CLI.""" + if not report.groups: + return "no groups configured\n" + rows: list[tuple[str, ...]] = [ + ("group", "query", "status", "points", "needed", "eta", "blocked_by") + ] + for group in report.groups: + for query in group.queries: + rows.append( + ( + group.name, + query.id, + query.status, + str(query.points_available), + str(query.points_needed), + _format_eta(query.eta_seconds), + query.blocked_by, + ) + ) + widths = [max(len(row[i]) for row in rows) for i in range(7)] + lines: list[str] = [] + for i, row in enumerate(rows): + formatted = " ".join( + cell.ljust(widths[j]) if j in {0, 1, 2, 6} else cell.rjust(widths[j]) + for j, cell in enumerate(row) + ) + lines.append(formatted) + if i == 0: + lines.append(" ".join("-" * w for w in widths)) + return "\n".join(lines) + "\n" + + +_SECONDS_PER_MINUTE = 60 +_SECONDS_PER_HOUR = 3_600 +_SECONDS_PER_DAY = 86_400 + + +def _format_eta(eta_seconds: float) -> str: + """Compact Prometheus-style duration for the table.""" + if not math.isfinite(eta_seconds): + return "inf" + if eta_seconds <= 0: + return "-" + seconds = round(eta_seconds) + if seconds >= _SECONDS_PER_DAY: + return f"{seconds // _SECONDS_PER_DAY}d" + if seconds >= _SECONDS_PER_HOUR: + return f"{seconds // _SECONDS_PER_HOUR}h" + if seconds >= _SECONDS_PER_MINUTE: + return f"{seconds // _SECONDS_PER_MINUTE}m" + return f"{seconds}s" + + +def report_from_payload(payload: dict[str, Any]) -> WarmupReport: + """Reconstruct a :class:`WarmupReport` from a JSON envelope. + + Lets the CLI reuse :func:`render_table` on the HTTP response without + duplicating the table formatting. + """ + groups: list[GroupWarmupStatus] = [] + for group in payload.get("groups", []): + queries = [ + QueryWarmupStatus( + id=str(q.get("id", "")), + status=q.get("status", "ready"), + points_available=int(q.get("points_available", 0)), + points_needed=int(q.get("points_needed", 0)), + blocked_by=str(q.get("blocked_by", "none")), + eta_seconds=float(q.get("eta_seconds", 0.0)), + ) + for q in group.get("queries", []) + ] + groups.append(GroupWarmupStatus(name=str(group.get("name", "")), queries=queries)) + return WarmupReport(groups=groups) + + +__all__ = [ + "BlockedBy", + "GroupWarmupStatus", + "QueryWarmupStatus", + "WarmupReport", + "compute_warmup", + "render_table", + "report_from_payload", +] diff --git a/detector/tests/conftest.py b/detector/tests/conftest.py index 6506910..3bd24e6 100644 --- a/detector/tests/conftest.py +++ b/detector/tests/conftest.py @@ -25,6 +25,9 @@ def __init__(self) -> None: super().__init__(base_url="http://stub", timeout=1.0) self.queries: list[dict[str, Any]] = [] self._responder: Callable[[str], StubResponse] = lambda _: StubResponse() + # {metric_name: type} served by ``metric_metadata`` (metadata-lint). + # Empty by default so the lint reports nothing unless a test opts in. + self.metadata: dict[str, str] = {} def respond(self, fn: Callable[[str], StubResponse]) -> None: self._responder = fn @@ -35,6 +38,9 @@ async def start(self) -> None: async def close(self) -> None: return None + async def metric_metadata(self) -> dict[str, str]: # type: ignore[override] + return dict(self.metadata) + async def range_query( # type: ignore[override] self, promql: str, diff --git a/detector/tests/test_cli_estimate_cost.py b/detector/tests/test_cli_estimate_cost.py new file mode 100644 index 0000000..b7442e4 --- /dev/null +++ b/detector/tests/test_cli_estimate_cost.py @@ -0,0 +1,159 @@ +"""``validate --estimate-cost`` — static cardinality / cost projection.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import yaml +from click.testing import CliRunner + +from promanomaly.cli import cli +from promanomaly.cli._cost import estimate_cost +from promanomaly.config import CURRENT_API_VERSION, Config + + +def _write_config(tmp_path: Path, **extras: Any) -> Path: + cfg: dict[str, Any] = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "groups": [ + { + "name": "g", + "queries": [{"id": "m", "promql": "up", "detectors": [{"name": "MAD"}]}], + } + ], + } + cfg.update(extras) + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(cfg)) + return path + + +def _summary_line(output: str) -> dict[str, Any]: + for line in output.splitlines(): + obj = json.loads(line) + if "summary" in obj: + return obj["summary"] + raise AssertionError("no summary line in output") + + +def test_estimate_cost_emits_per_group_and_summary(tmp_path: Path) -> None: + path = _write_config(tmp_path) + result = CliRunner().invoke(cli, ["validate", "--config", str(path), "--estimate-cost"]) + assert result.exit_code == 0 + summary = _summary_line(result.output) + # One group, one query, projected at the default per-query cap (1000). + assert summary["projected_total_series"] == 1000 + assert summary["max_total_series"] == 20000 + assert summary["over_budget"] is False + assert summary["short_queries_per_refresh"] == 1 + assert summary["estimated_cpu_cores"] >= 0.0 + assert summary["estimated_memory_bytes"] > 0 + + +def test_estimate_cost_does_not_touch_datasource(tmp_path: Path) -> None: + # The datasource URL is unreachable; --estimate-cost must still + # succeed because it is a purely static projection. + path = _write_config(tmp_path, datasource={"url": "http://127.0.0.1:1/unreachable"}) + result = CliRunner().invoke(cli, ["validate", "--config", str(path), "--estimate-cost"]) + assert result.exit_code == 0 + + +def test_estimate_cost_strict_exits_nonzero_over_budget(tmp_path: Path) -> None: + # Tighten the global cap below the per-query worst case so the + # projection overshoots. + path = _write_config( + tmp_path, + safety={"max_series_per_query": 1000, "max_total_series": 500}, + ) + result = CliRunner().invoke( + cli, ["validate", "--config", str(path), "--estimate-cost", "--strict"] + ) + assert result.exit_code == 1 + summary = _summary_line(result.output) + assert summary["over_budget"] is True + + +def test_estimate_cost_over_budget_without_strict_is_advisory(tmp_path: Path) -> None: + path = _write_config( + tmp_path, + safety={"max_series_per_query": 1000, "max_total_series": 500}, + ) + result = CliRunner().invoke(cli, ["validate", "--config", str(path), "--estimate-cost"]) + # Advisory only: over-budget is reported but the process exits 0. + assert result.exit_code == 0 + assert _summary_line(result.output)["over_budget"] is True + + +def test_strict_alone_still_requires_probe_or_estimate(tmp_path: Path) -> None: + path = _write_config(tmp_path) + result = CliRunner().invoke(cli, ["validate", "--config", str(path), "--strict"]) + assert result.exit_code != 0 + assert "--strict requires" in result.output + + +def test_estimate_cost_counts_discovery_probe_queries() -> None: + raw = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "groups": [ + { + "name": "g", + "queries": [ + { + "id": "m_{{ node }}", + "promql": 'rate(http_requests_total{node="{{ node }}"}[5m])', + "detectors": [{"name": "MAD"}], + "discover": [ + { + "variable": "node", + "probe": "count by (node) (up)", + "label": "node", + } + ], + } + ], + } + ], + } + cfg = Config.model_validate(raw) + groups = estimate_cost(cfg) + assert groups[0].discovery_probe_queries_per_refresh == 1 + assert groups[0].queries[0].is_discovery is True + + +def test_estimate_cost_flags_discovery_fanout_as_lower_bound(tmp_path: Path) -> None: + raw = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "groups": [ + { + "name": "g", + "queries": [ + { + "id": "m_{{ node }}", + "promql": 'rate(http_requests_total{node="{{ node }}"}[5m])', + "detectors": [{"name": "MAD"}], + "discover": [ + { + "variable": "node", + "probe": "count by (node) (up)", + "label": "node", + } + ], + } + ], + } + ], + } + path = tmp_path / "disc.yaml" + path.write_text(yaml.safe_dump(raw)) + result = CliRunner().invoke(cli, ["validate", "--config", str(path), "--estimate-cost"]) + assert result.exit_code == 0 + summary = _summary_line(result.output) + # Discovery fan-out is unbounded statically, so the total is a lower + # bound — the operator must not read "under budget" as a guarantee. + assert summary["projected_series_is_lower_bound"] is True + assert "discovery_note" in summary diff --git a/detector/tests/test_diagnose.py b/detector/tests/test_diagnose.py new file mode 100644 index 0000000..7047f2c --- /dev/null +++ b/detector/tests/test_diagnose.py @@ -0,0 +1,175 @@ +"""Config health and tuning diagnostics (``promanomaly diagnose``).""" + +from __future__ import annotations + +import json +from typing import Any + +from click.testing import CliRunner + +from promanomaly import cli as cli_module +from promanomaly.cli import cli +from promanomaly.config import CURRENT_API_VERSION, Config +from promanomaly.diagnose import ( + diagnose_from_metrics_text, + evaluate_empty, + evaluate_firing_rates, + evaluate_silent_detectors, + evaluate_warming, +) + + +# ---------------------------------------------------------------------- +# Pure evaluators +# ---------------------------------------------------------------------- +def test_firing_rate_bands() -> None: + rows = [ + ({"id": "a", "group": "g", "detector": "MAD"}, 0.0), + ({"id": "b", "group": "g", "detector": "Hampel"}, 0.8), + ({"id": "c", "group": "g", "detector": "IQR"}, 0.2), + ] + findings = evaluate_firing_rates(rows, fire_high=0.5) + cats = {(f.query, f.category) for f in findings} + assert ("a", "never_fires") in cats + assert ("b", "fires_often") in cats + # 0.2 is in the healthy band — no finding. + assert not any(f.query == "c" for f in findings) + + +def test_warming_findings() -> None: + rows = [({"group": "g", "id": "a"}, 3.0), ({"group": "g", "id": "b"}, 0.0)] + findings = evaluate_warming(rows) + assert len(findings) == 1 + assert findings[0].query == "a" + assert findings[0].value == 3.0 + + +def test_empty_findings() -> None: + rows = [({"group": "g1"}, 0.0), ({"group": "g2"}, 12.0)] + findings = evaluate_empty(rows) + assert [f.group for f in findings] == ["g1"] + + +def _config_with_cohort() -> Config: + return Config.model_validate( + { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "groups": [ + { + "name": "g", + "queries": [ + { + "id": "m1", + "promql": "up", + "detectors": [{"name": "MAD"}, {"name": "Cohort"}], + } + ], + } + ], + } + ) + + +def test_silent_detectors_flags_unemitting() -> None: + cfg = _config_with_cohort() + # MAD emitted; Cohort did not (e.g. cohorts below min_cohort_size). + findings = evaluate_silent_detectors(cfg, emitting={("g", "MAD")}) + assert len(findings) == 1 + assert findings[0].detector == "Cohort" + assert "min_cohort_size" in findings[0].detail + + +# ---------------------------------------------------------------------- +# Snapshot mode (parse /metrics text) +# ---------------------------------------------------------------------- +_METRICS = """ +# HELP anomaly_series_count x +# TYPE anomaly_series_count gauge +anomaly_series_count{group="g1"} 5 +anomaly_series_count{group="g2"} 0 +# TYPE anomaly_warming_up gauge +anomaly_warming_up{group="g1",id="m1",detector="MAD"} 1 +# TYPE anomaly_outside_threshold gauge +anomaly_outside_threshold{group="g1",id="m1",detector="MAD"} 1 +anomaly_outside_threshold{group="g1",id="m1",detector="MAD",instance="b"} 1 +""" + + +def test_diagnose_from_metrics_text() -> None: + report = diagnose_from_metrics_text(_METRICS, fire_high=0.5) + cats = {f.category for f in report.findings} + assert report.mode == "snapshot" + assert "empty_query" in cats # g2 has 0 series + assert "warming" in cats + assert "fires_often" in cats # both MAD series firing -> 100% + # Snapshot mode never reports never_fires (no rate over time). + assert "never_fires" not in cats + + +# ---------------------------------------------------------------------- +# CLI +# ---------------------------------------------------------------------- +def test_diagnose_requires_exactly_one_source() -> None: + result = CliRunner().invoke(cli, ["diagnose"]) + assert result.exit_code != 0 + assert "exactly one" in result.output + + +def test_diagnose_target_mode(monkeypatch: Any) -> None: + class _Resp: + text = _METRICS + + def raise_for_status(self) -> None: + return None + + monkeypatch.setattr("httpx.get", lambda *a, **k: _Resp()) + result = CliRunner().invoke(cli, ["diagnose", "--target", "http://x", "--output", "json"]) + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["mode"] == "snapshot" + assert any(f["category"] == "empty_query" for f in payload["findings"]) + + +def test_diagnose_tsdb_mode(monkeypatch: Any) -> None: + class _StubSource: + def __init__(self, *_a: Any, **_k: Any) -> None: + pass + + async def start(self) -> None: + return None + + async def close(self) -> None: + return None + + async def instant_query(self, promql: str) -> list[tuple[dict[str, str], float]]: + if "anomaly_outside_threshold" in promql: + return [({"id": "m1", "group": "g", "detector": "MAD"}, 0.0)] + if "anomaly_warming_up" in promql: + return [] + if "anomaly_series_count" in promql: + return [({"group": "g"}, 5.0)] + return [] + + monkeypatch.setattr(cli_module, "PromQLSource", _StubSource) + result = CliRunner().invoke( + cli, ["diagnose", "--datasource-url", "http://tsdb", "--window", "24h", "--output", "json"] + ) + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["mode"] == "tsdb" + assert payload["window"] == "24h" + assert any(f["category"] == "never_fires" for f in payload["findings"]) + + +def test_diagnose_rejects_bad_window(monkeypatch: Any) -> None: + class _StubSource: + def __init__(self, *_a: Any, **_k: Any) -> None: + pass + + monkeypatch.setattr(cli_module, "PromQLSource", _StubSource) + result = CliRunner().invoke( + cli, ["diagnose", "--datasource-url", "http://tsdb", "--window", "24 hours; drop"] + ) + assert result.exit_code != 0 + assert "invalid --window" in result.output diff --git a/detector/tests/test_discovery_ha.py b/detector/tests/test_discovery_ha.py new file mode 100644 index 0000000..70e82e7 --- /dev/null +++ b/detector/tests/test_discovery_ha.py @@ -0,0 +1,157 @@ +"""HA-correct discovery-absence tracking. + +The in-memory :class:`DiscoveryMissTracker` resets on Lease failover, so +a series that went missing right before a handover would restart its +``expect_grace_runs`` window on the new leader. The Redis-backed +:class:`RedisDiscoveryMissTracker` keeps the per-series miss counts in +the shared cache so a newly-elected leader inherits them. + +These tests exercise the Redis tracker against ``fakeredis`` and assert +parity with the in-memory tracker's semantics plus the failover-survival +property that is the whole point of the feature. +""" + +from __future__ import annotations + +import fakeredis +import pytest + +from promanomaly.state import SnapshotStore +from promanomaly.state.discovery import ( + DiscoveryMissTracker, + RedisDiscoveryMissTracker, + _decode_series_key, + _encode_series_key, +) + +A = (("instance", "a"),) +B = (("instance", "b"),) + + +@pytest.fixture +def redis_client() -> fakeredis.FakeStrictRedis: + return fakeredis.FakeStrictRedis() + + +def test_series_key_round_trips() -> None: + key = (("instance", "node-17"), ("job", "node")) + assert _decode_series_key(_encode_series_key(key)) == key + + +def test_decode_rejects_corrupt_field() -> None: + assert _decode_series_key("not json") is None + assert _decode_series_key('{"not": "a list"}') is None + assert _decode_series_key('[["k"]]') is None # wrong pair arity + + +def test_redis_tracker_marks_blocking_io(redis_client: fakeredis.FakeStrictRedis) -> None: + tracker = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + assert tracker.is_blocking_io is True + assert DiscoveryMissTracker().is_blocking_io is False + + +def test_redis_tracker_transitions_at_grace(redis_client: fakeredis.FakeStrictRedis) -> None: + tracker = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + # Run 1: A and B both present — no transitions, both known. + assert tracker.record_observation("g", "q", [A, B], grace_runs=2) == [] + assert tracker.known_series("g", "q") == {A, B} + # Run 2: B missing once (below grace). + assert tracker.record_observation("g", "q", [A], grace_runs=2) == [] + assert tracker.known_missing("g", "q", 2) == [] + # Run 3: B missing twice — crosses grace_runs and transitions. + assert tracker.record_observation("g", "q", [A], grace_runs=2) == [B] + assert tracker.known_missing("g", "q", 2) == [B] + + +def test_redis_tracker_reset_on_reappearance(redis_client: fakeredis.FakeStrictRedis) -> None: + tracker = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + tracker.record_observation("g", "q", [A], grace_runs=1) + tracker.record_observation("g", "q", [], grace_runs=1) + assert tracker.known_missing("g", "q", 1) == [A] + # A reappears — its miss count resets, so it is no longer missing. + tracker.record_observation("g", "q", [A], grace_runs=1) + assert tracker.known_missing("g", "q", 1) == [] + + +def test_redis_tracker_forget_after_runs_bounds_state( + redis_client: fakeredis.FakeStrictRedis, +) -> None: + tracker = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + tracker.record_observation("g", "q", [A], grace_runs=1) + # Miss A three times with forget_after_runs=2 — after the 3rd it is + # dropped entirely and treated as a fresh discovery if it returns. + tracker.record_observation("g", "q", [], grace_runs=1, forget_after_runs=2) # miss 1 + tracker.record_observation("g", "q", [], grace_runs=1, forget_after_runs=2) # miss 2 + assert A in tracker.known_series("g", "q") + tracker.record_observation( + "g", "q", [], grace_runs=1, forget_after_runs=2 + ) # miss 3 -> forgotten + assert tracker.known_series("g", "q") == set() + + +def test_redis_tracker_survives_failover(redis_client: fakeredis.FakeStrictRedis) -> None: + """The failover property: a second tracker pointed at the same Redis + inherits the first leader's miss counts rather than starting at zero.""" + leader1 = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + leader1.record_observation("g", "q", [A], grace_runs=3) + leader1.record_observation("g", "q", [], grace_runs=3) # miss 1 + leader1.record_observation("g", "q", [], grace_runs=3) # miss 2 + + # Lease failover: a brand-new tracker (new leader process) reads the + # same Redis. One more missing run crosses grace=3 — the count was + # inherited, not reset. + leader2 = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + assert leader2.record_observation("g", "q", [], grace_runs=3) == [A] + + +def test_redis_tracker_drop_group(redis_client: fakeredis.FakeStrictRedis) -> None: + tracker = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + tracker.record_observation("g1", "q", [A], grace_runs=1) + tracker.record_observation("g2", "q", [B], grace_runs=1) + tracker.drop_group("g1") + assert tracker.known_series("g1", "q") == set() + # g2 is untouched. + assert tracker.known_series("g2", "q") == {B} + + +def test_redis_tracker_sets_ttl(redis_client: fakeredis.FakeStrictRedis) -> None: + tracker = RedisDiscoveryMissTracker( + client=redis_client, ttl_seconds=120.0, key_prefix="pre:discovery" + ) + tracker.record_observation("g", "q", [A], grace_runs=1) + ttl = redis_client.ttl("pre:discovery:g:q") + assert 0 < ttl <= 120 + + +def test_store_swaps_tracker_and_reports_blocking( + redis_client: fakeredis.FakeStrictRedis, +) -> None: + store = SnapshotStore() + assert store.discovery_blocking_io is False + tracker = RedisDiscoveryMissTracker(client=redis_client, ttl_seconds=60.0) + store.set_discovery_tracker(tracker) + assert store.discovery_blocking_io is True + # And the store delegates to the swapped tracker. + store.record_discovery_observation("g", "q", [A], 1) + assert store.discovery_known_series("g", "q") == {A} + + +def test_redis_tracker_degrades_on_redis_error() -> None: + """A Redis outage must not crash the detection run — the tracker + degrades to "no observation recorded / nothing missing" instead.""" + + class _BrokenRedis: + def hgetall(self, *_a: object, **_k: object) -> dict[str, str]: + raise ConnectionError("redis down") + + def pipeline(self) -> object: + raise ConnectionError("redis down") + + def scan_iter(self, *_a: object, **_k: object) -> list[str]: + raise ConnectionError("redis down") + + tracker = RedisDiscoveryMissTracker(client=_BrokenRedis(), ttl_seconds=60.0) + # No exception bubbles out; the run continues with degraded tracking. + assert tracker.record_observation("g", "q", [A], grace_runs=1) == [] + assert tracker.known_missing("g", "q", 1) == [] + tracker.drop_group("g") diff --git a/detector/tests/test_exporter.py b/detector/tests/test_exporter.py index 739d59a..7ee8fd7 100644 --- a/detector/tests/test_exporter.py +++ b/detector/tests/test_exporter.py @@ -74,15 +74,15 @@ def test_exporter_caches_rendered_snapshot_bytes() -> None: ] store.write(GroupSnapshot(group="g", timestamp=100.0, samples=samples)) - body1, _ = exporter.render() + exporter.render() cached_bytes = exporter._snapshot_render_bytes # type: ignore[reportPrivateUsage] - body2, _ = exporter.render() - # Second render reuses the cached bytes (no re-render, no new alloc). + exporter.render() + # Second render reuses the cached *snapshot* bytes (no re-render, no + # new alloc). The cache-bytes identity is the real assertion — the + # full body legitimately differs scrape-to-scrape now that the + # time-relative self-observability gauges (snapshot_age) re-render + # per scrape. assert exporter._snapshot_render_bytes is cached_bytes # type: ignore[reportPrivateUsage] - # Operational metrics still re-render per scrape, so the full body - # is not guaranteed to be identical (it is, but only because nothing - # changed). The cache-bytes identity is the real assertion. - assert body1 == body2 # New snapshot at a new timestamp busts the cache and the bytes # object differs. diff --git a/detector/tests/test_ha_integration.py b/detector/tests/test_ha_integration.py index eb2c7b5..42b77d4 100644 --- a/detector/tests/test_ha_integration.py +++ b/detector/tests/test_ha_integration.py @@ -102,8 +102,11 @@ def ha_app(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Application: from unittest.mock import MagicMock from promanomaly.ha import HAComponents - from promanomaly.state import SharedSnapshotCache + from promanomaly.state import RedisDiscoveryMissTracker, SharedSnapshotCache + discovery_tracker = RedisDiscoveryMissTracker( + client=fake, ttl_seconds=60.0, key_prefix="promanomaly:discovery" + ) app._ha = HAComponents( elector=MagicMock(), snapshot_cache=SharedSnapshotCache( @@ -112,7 +115,12 @@ def ha_app(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Application: key_prefix="promanomaly:snapshot", ), follower_syncer=MagicMock(), + discovery_tracker=discovery_tracker, ) + # The store swap normally happens in _start_ha(); the fixture wires + # _ha by hand (without launching the elector), so mirror it here so + # the discovery-absence path uses the Redis-backed tracker. + app._store.set_discovery_tracker(discovery_tracker) return app diff --git a/detector/tests/test_metadata_lint.py b/detector/tests/test_metadata_lint.py new file mode 100644 index 0000000..4dba174 --- /dev/null +++ b/detector/tests/test_metadata_lint.py @@ -0,0 +1,252 @@ +"""Metadata-aware validation lint. + +Covers the pure PromQL/metadata analysis, the ``validate --lint-metadata`` +CLI surface, and the live surfacing in ``inspect`` / ``top``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml +from click.testing import CliRunner +from fastapi.testclient import TestClient + +from promanomaly import cli as cli_module +from promanomaly.cli import cli +from promanomaly.config import CURRENT_API_VERSION, Config +from promanomaly.main import Application +from promanomaly.metadata_lint import ( + lint_config_metadata, + normalise_metadata, + unrated_counters, +) +from tests.conftest import StubResponse, StubSource, make_series +from tests.fixtures import clean_baseline + +COUNTER_META = {"http_requests_total": "counter"} + + +# ---------------------------------------------------------------------- +# Pure analysis +# ---------------------------------------------------------------------- +def test_raw_counter_is_flagged() -> None: + assert unrated_counters("http_requests_total", COUNTER_META) == ["http_requests_total"] + + +def test_rate_wrapped_counter_is_not_flagged() -> None: + assert unrated_counters("rate(http_requests_total[5m])", COUNTER_META) == [] + + +def test_nested_rate_is_not_flagged() -> None: + assert unrated_counters("sum by (job)(rate(http_requests_total[5m]))", COUNTER_META) == [] + + +def test_increase_and_other_counter_funcs_not_flagged() -> None: + assert unrated_counters("increase(http_requests_total[1h])", COUNTER_META) == [] + assert unrated_counters("irate(http_requests_total[5m])", COUNTER_META) == [] + + +def test_counter_inside_count_is_not_flagged() -> None: + # `node_load1 / count(node_cpu_seconds_total)` uses the counter only to + # count cores — the raw value never reaches the detector, so it's safe. + meta = {"node_cpu_seconds_total": "counter"} + assert unrated_counters("node_load1 / count(node_cpu_seconds_total)", meta) == [] + + +def test_counter_inside_count_by_is_not_flagged() -> None: + # The aggregation-modifier form `count by (instance) (metric)` is the + # one node-exporter's node_load1_per_core uses in the wild. + meta = {"node_cpu_seconds_total": "counter"} + promql = 'node_load1 / on (instance) count by (instance) (node_cpu_seconds_total{mode="idle"})' + assert unrated_counters(promql, meta) == [] + + +def test_gauge_is_never_flagged() -> None: + assert unrated_counters("node_memory_free_bytes", {"node_memory_free_bytes": "gauge"}) == [] + + +def test_unknown_metric_is_no_opinion() -> None: + # Absent metadata -> no finding, even for a bare metric. + assert unrated_counters("some_metric_total", {}) == [] + + +def test_normalise_metadata_accepts_raw_and_reduced() -> None: + raw = {"m": [{"type": "counter", "help": "x"}]} + assert normalise_metadata(raw) == {"m": "counter"} + assert normalise_metadata({"m": "gauge"}) == {"m": "gauge"} + + +def _config(promql: str) -> Config: + return Config.model_validate( + { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "groups": [ + { + "name": "g1", + "queries": [{"id": "m1", "promql": promql, "detectors": [{"name": "MAD"}]}], + } + ], + } + ) + + +def test_lint_config_reports_counter_finding() -> None: + findings = lint_config_metadata(_config("http_requests_total"), COUNTER_META) + assert len(findings) == 1 + assert findings[0].metric == "http_requests_total" + assert findings[0].issue == "counter_not_rated" + assert findings[0].detectors == ["MAD"] + + +def test_lint_config_clean_when_rated() -> None: + findings = lint_config_metadata(_config("rate(http_requests_total[5m])"), COUNTER_META) + assert findings == [] + + +# ---------------------------------------------------------------------- +# validate --lint-metadata CLI +# ---------------------------------------------------------------------- +def _write_config(tmp_path: Path, promql: str) -> Path: + cfg = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "groups": [ + { + "name": "g1", + "queries": [{"id": "m1", "promql": promql, "detectors": [{"name": "MAD"}]}], + } + ], + } + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(cfg)) + return path + + +def _stub_factory(metadata: dict[str, str]) -> Any: + def factory(*_a: Any, **_k: Any) -> StubSource: + stub = StubSource() + stub.metadata = dict(metadata) + return stub + + return factory + + +def test_validate_lint_metadata_flags_counter(tmp_path: Path, monkeypatch: Any) -> None: + monkeypatch.setattr(cli_module, "PromQLSource", _stub_factory(COUNTER_META)) + path = _write_config(tmp_path, "http_requests_total") + result = CliRunner().invoke(cli, ["validate", "--config", str(path), "--lint-metadata"]) + assert result.exit_code == 0 # advisory without --strict + assert "counter_not_rated" in result.output + assert "http_requests_total" in result.output + + +def test_validate_lint_metadata_strict_fails(tmp_path: Path, monkeypatch: Any) -> None: + monkeypatch.setattr(cli_module, "PromQLSource", _stub_factory(COUNTER_META)) + path = _write_config(tmp_path, "http_requests_total") + result = CliRunner().invoke( + cli, ["validate", "--config", str(path), "--lint-metadata", "--strict"] + ) + assert result.exit_code == 1 + + +def test_validate_lint_metadata_clean_passes(tmp_path: Path, monkeypatch: Any) -> None: + monkeypatch.setattr(cli_module, "PromQLSource", _stub_factory(COUNTER_META)) + path = _write_config(tmp_path, "rate(http_requests_total[5m])") + result = CliRunner().invoke( + cli, ["validate", "--config", str(path), "--lint-metadata", "--strict"] + ) + assert result.exit_code == 0 + assert '"findings": 0' in result.output + + +def test_validate_runs_all_checks_even_when_estimate_cost_fails( + tmp_path: Path, monkeypatch: Any +) -> None: + # --estimate-cost --strict over budget exits non-zero; the metadata + # lint must still run and print its report (no short-circuit), and the + # process must still exit non-zero overall. + monkeypatch.setattr(cli_module, "PromQLSource", _stub_factory(COUNTER_META)) + cfg = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "safety": {"max_total_series": 1, "max_series_per_query": 1000}, + "groups": [ + { + "name": "g1", + "queries": [ + {"id": "m1", "promql": "http_requests_total", "detectors": [{"name": "MAD"}]} + ], + } + ], + } + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(cfg)) + result = CliRunner().invoke( + cli, + ["validate", "--config", str(path), "--estimate-cost", "--lint-metadata", "--strict"], + ) + assert result.exit_code == 1 + # Both reports present: the cardinality overshoot AND the lint finding. + assert "cardinality_exceeded" in result.output + assert "counter_not_rated" in result.output + + +def test_validate_strict_alone_mentions_lint_metadata(tmp_path: Path) -> None: + path = _write_config(tmp_path, "up") + result = CliRunner().invoke(cli, ["validate", "--config", str(path), "--strict"]) + assert result.exit_code != 0 + assert "--lint-metadata" in result.output + + +# ---------------------------------------------------------------------- +# inspect / top surfacing +# ---------------------------------------------------------------------- +def _yaml_config(tmp_path: Path, promql: str) -> Path: + cfg = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "server": {"listen": ":0"}, + "defaults": {"window": "5m", "step": "15s", "min_points": 5}, + "groups": [ + { + "name": "g1", + "queries": [{"id": "m1", "promql": promql, "detectors": [{"name": "MAD"}]}], + } + ], + } + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(cfg)) + return path + + +def test_inspect_endpoint_includes_metadata_lint(tmp_path: Path, monkeypatch: Any) -> None: + stub = StubSource() + stub.metadata = COUNTER_META + stub.respond(lambda _: StubResponse(series=[make_series({"i": "h"}, clean_baseline(n=120))])) + monkeypatch.setattr("promanomaly.main.PromQLSource", lambda *a, **k: stub) + path = _yaml_config(tmp_path, "http_requests_total") + app = Application(Config(**yaml.safe_load(path.read_text())), tmp_path) + with TestClient(app.build_app()) as client: + resp = client.get("/debug/inspect", params={"id": "m1"}) + assert resp.status_code == 200 + body = resp.json() + assert body["metadata_lint"] + assert body["metadata_lint"][0]["metric"] == "http_requests_total" + + +def test_top_endpoint_lint_param_returns_hints(tmp_path: Path, monkeypatch: Any) -> None: + stub = StubSource() + stub.metadata = COUNTER_META + stub.respond(lambda _: StubResponse(series=[make_series({"i": "h"}, clean_baseline(n=120))])) + monkeypatch.setattr("promanomaly.main.PromQLSource", lambda *a, **k: stub) + path = _yaml_config(tmp_path, "http_requests_total") + app = Application(Config(**yaml.safe_load(path.read_text())), tmp_path) + with TestClient(app.build_app()) as client: + with_lint = client.get("/debug/anomalies", params={"lint": "true"}) + without_lint = client.get("/debug/anomalies", params={"lint": "false"}) + assert with_lint.json()["metadata_lint"][0]["metric"] == "http_requests_total" + # Default (no extra TSDB query) omits the field entirely. + assert "metadata_lint" not in without_lint.json() diff --git a/detector/tests/test_self_observability.py b/detector/tests/test_self_observability.py new file mode 100644 index 0000000..47176b4 --- /dev/null +++ b/detector/tests/test_self_observability.py @@ -0,0 +1,174 @@ +"""Self-observability metrics: detector-health telemetry. + +Pins the four families this surface ships: + +* ``anomaly_detect_success_ratio{group, detector, window}`` +* ``anomaly_group_cpu_seconds_total{group}`` +* ``anomaly_group_memory_bytes{group}`` +* ``anomaly_series_staleness_seconds{group}`` / + ``anomaly_snapshot_age_seconds{group}`` +""" + +from __future__ import annotations + +from typing import Any, ClassVar + +import pandas as pd +import pytest + +from promanomaly.detectors import registry +from promanomaly.exporter import Exporter, OperationalMetrics +from promanomaly.self_observability import ( + DetectOutcomeTracker, + estimate_samples_bytes, +) +from promanomaly.state import GroupSnapshot, Sample, SnapshotStore +from tests.conftest import StubResponse, StubSource, make_series +from tests.fixtures import clean_baseline, point_spike +from tests.test_runner import _config, _runner + + +# ---------------------------------------------------------------------- +# DetectOutcomeTracker — rolling success ratio +# ---------------------------------------------------------------------- +def test_outcome_tracker_aggregates_attempts_within_window() -> None: + tracker = DetectOutcomeTracker(windows_seconds=(3600, 86400)) + # Two runs inside the 1h window: 9/10 then 10/10 -> 19/20 = 0.95. + tracker.record(group="g", detector="MAD", attempts=10, successes=9, now=1000.0) + tracker.record(group="g", detector="MAD", attempts=10, successes=10, now=1010.0) + points = {(p.window_label): p.ratio for p in tracker.render(now=1020.0)} + assert points["1h"] == pytest.approx(0.95) + assert points["1d"] == pytest.approx(0.95) + + +def test_outcome_tracker_expires_old_samples() -> None: + tracker = DetectOutcomeTracker(windows_seconds=(3600,)) + tracker.record(group="g", detector="MAD", attempts=4, successes=0, now=0.0) + # 2h later, the failing run has aged out of the 1h window; a fresh + # clean run leaves the ratio at 1.0 rather than dragged down by history. + tracker.record(group="g", detector="MAD", attempts=4, successes=4, now=7200.0) + points = {p.window_label: p.ratio for p in tracker.render(now=7200.0)} + assert points["1h"] == pytest.approx(1.0) + + +def test_outcome_tracker_skips_empty_windows_and_zero_attempts() -> None: + tracker = DetectOutcomeTracker(windows_seconds=(3600,)) + # Zero-attempt records are ignored entirely (idle detector). + tracker.record(group="g", detector="MAD", attempts=0, successes=0, now=0.0) + assert tracker.render(now=0.0) == [] + # A real record more than a window ago renders nothing (no points in window). + tracker.record(group="g", detector="MAD", attempts=2, successes=2, now=0.0) + assert tracker.render(now=100_000.0) == [] + + +def test_outcome_tracker_retain_groups_drops_removed() -> None: + tracker = DetectOutcomeTracker(windows_seconds=(3600,)) + tracker.record(group="keep", detector="MAD", attempts=1, successes=1, now=0.0) + tracker.record(group="drop", detector="MAD", attempts=1, successes=1, now=0.0) + tracker.retain_groups({"keep"}) + groups = {p.group for p in tracker.render(now=0.0)} + assert groups == {"keep"} + + +# ---------------------------------------------------------------------- +# estimate_samples_bytes +# ---------------------------------------------------------------------- +def test_estimate_samples_bytes_grows_with_more_samples() -> None: + one = [Sample(metric="anomaly_score", labels=(("id", "a"),), value=1.0)] + many = [Sample(metric="anomaly_score", labels=(("id", f"a{i}"),), value=1.0) for i in range(10)] + assert estimate_samples_bytes(many) > estimate_samples_bytes(one) > 0 + assert estimate_samples_bytes([]) == 0.0 + + +# ---------------------------------------------------------------------- +# Exporter render-time gauges +# ---------------------------------------------------------------------- +def test_snapshot_age_and_staleness_climb_between_renders() -> None: + store = SnapshotStore() + ops = OperationalMetrics() + exporter = Exporter(store, ops, source=None) + store.write( + GroupSnapshot( + group="g", + timestamp=100.0, + samples=[Sample(metric="anomaly_score", labels=(("group", "g"),), value=1.0)], + ) + ) + body, _ = exporter.render() + text = body.decode("utf-8") + # Both families present and, because the snapshot timestamp is far in + # the past, the age is large and positive. + assert 'anomaly_snapshot_age_seconds{group="g"}' in text + assert 'anomaly_series_staleness_seconds{group="g"}' in text + + +def test_staleness_absent_until_a_score_is_emitted() -> None: + store = SnapshotStore() + ops = OperationalMetrics() + exporter = Exporter(store, ops, source=None) + # A warming-up-only snapshot carries no anomaly_score sample. + store.write( + GroupSnapshot( + group="g", + timestamp=100.0, + samples=[Sample(metric="anomaly_warming_up", labels=(("group", "g"),), value=1.0)], + ) + ) + body, _ = exporter.render() + text = body.decode("utf-8") + # snapshot_age is reported (the group ran) but staleness is not (no + # score has ever been produced, so a staleness number would mislead). + assert 'anomaly_snapshot_age_seconds{group="g"}' in text + assert 'anomaly_series_staleness_seconds{group="g"}' not in text + + +# ---------------------------------------------------------------------- +# Runner integration +# ---------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_run_records_success_ratio_cpu_and_memory(stub_source: StubSource) -> None: + df = point_spike(n=120) + stub_source.respond(lambda _: StubResponse(series=[make_series({"instance": "h"}, df)])) + cfg = _config(min_points=30) + runner, _store, ops = _runner(cfg, stub_source) + + result = await runner.run_group("g1") + assert result.succeeded + + # Success ratio recorded for the detector that ran. + points = ops.detect_outcomes.render() + assert any(p.group == "g1" and p.detector == "MAD" and p.ratio == 1.0 for p in points) + # CPU counter advanced and memory gauge set. + cpu = ops.group_cpu_seconds_total.labels(group="g1")._value.get() + assert cpu > 0.0 + mem = ops.group_memory_bytes.labels(group="g1")._value.get() + assert mem > 0.0 + + +@pytest.mark.asyncio +async def test_detector_failure_drags_success_ratio_below_one( + stub_source: StubSource, +) -> None: + df = clean_baseline(n=120) + stub_source.respond(lambda _: StubResponse(series=[make_series({"instance": "h"}, df)])) + + class BoomDetector: + name: ClassVar[str] = "BoomSO" + description: ClassVar[str] = "always raises" + + def fit_score(self, *_: Any, **__: Any) -> pd.DataFrame: + raise RuntimeError("intentional") + + registry.register(BoomDetector) # type: ignore[arg-type] + try: + cfg = _config(min_points=30, detectors=[{"name": "MAD"}, {"name": "BoomSO"}]) + runner, _store, ops = _runner(cfg, stub_source) + await runner.run_group("g1") + points = { + p.detector: p.ratio for p in ops.detect_outcomes.render() if p.window_label == "1h" + } + # MAD succeeded (1.0); the boom detector failed on its series (0.0). + assert points.get("MAD") == pytest.approx(1.0) + assert points.get("BoomSO") == pytest.approx(0.0) + finally: + registry.unregister("BoomSO") diff --git a/detector/tests/test_selftest.py b/detector/tests/test_selftest.py new file mode 100644 index 0000000..075fc95 --- /dev/null +++ b/detector/tests/test_selftest.py @@ -0,0 +1,133 @@ +"""End-to-end detection self-test / dead-man's-switch.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +import yaml +from fastapi.testclient import TestClient + +from promanomaly.config import CURRENT_API_VERSION, Config +from promanomaly.main import Application +from promanomaly.selftest import run_selftest +from tests.conftest import StubResponse, StubSource, make_series +from tests.fixtures import clean_baseline + + +def test_selftest_passes_with_default_detector() -> None: + result = run_selftest(detector_name="MAD", threshold=3.0, min_points=60) + assert result.ok is True + assert result.outside is True + assert result.score >= 3.0 + assert result.detail == "ok" + + +def test_selftest_fails_with_absurd_threshold() -> None: + # A threshold no synthetic spike can cross simulates a global + # mis-tuning — the dead-man's-switch must report it, not pass. + result = run_selftest(detector_name="MAD", threshold=1e9, min_points=60) + assert result.ok is False + assert result.outside is False + assert "not caught" in result.detail + + +def test_selftest_unknown_detector_is_failure() -> None: + result = run_selftest(detector_name="DoesNotExist", threshold=3.0, min_points=60) + assert result.ok is False + assert "unknown detector" in result.detail + + +def _selftest_config(detector: str) -> dict[str, Any]: + return { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "server": {"selftest": {"enabled": True, "detector": detector}}, + "groups": [ + { + "name": "g1", + "queries": [{"id": "m1", "promql": "up", "detectors": [{"name": "MAD"}]}], + } + ], + } + + +def test_config_rejects_stratified_selftest_detector(tmp_path: Path) -> None: + from promanomaly.config import load_config + + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(_selftest_config("DayOfWeekMAD"))) + with pytest.raises(ValueError, match="stratified"): + load_config(path) + + +def test_config_rejects_cohort_selftest_detector(tmp_path: Path) -> None: + from promanomaly.config import load_config + + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(_selftest_config("Cohort"))) + with pytest.raises(ValueError, match="cohort"): + load_config(path) + + +def test_config_allows_rolling_window_selftest_detector(tmp_path: Path) -> None: + from promanomaly.config import load_config + + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(_selftest_config("Hampel"))) + cfg = load_config(path) + assert cfg.server.selftest.detector == "Hampel" + + +def _yaml_config(tmp_path: Path, *, enabled: bool, threshold: float = 3.0) -> Path: + cfg = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "server": { + "listen": ":0", + "selftest": {"enabled": enabled, "detector": "MAD", "threshold": threshold}, + }, + "defaults": {"window": "5m", "step": "15s", "min_points": 5}, + "groups": [ + { + "name": "g1", + "queries": [{"id": "m1", "promql": "up", "detectors": [{"name": "MAD"}]}], + } + ], + } + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(cfg)) + return path + + +def _app(tmp_path: Path, monkeypatch: Any, *, enabled: bool, threshold: float = 3.0) -> Application: + stub = StubSource() + stub.respond(lambda _: StubResponse(series=[make_series({"i": "h"}, clean_baseline(n=120))])) + monkeypatch.setattr("promanomaly.main.PromQLSource", lambda *a, **k: stub) + path = _yaml_config(tmp_path, enabled=enabled, threshold=threshold) + return Application(Config(**yaml.safe_load(path.read_text())), tmp_path) + + +def test_selftest_metric_absent_when_disabled(tmp_path: Path, monkeypatch: Any) -> None: + app = _app(tmp_path, monkeypatch, enabled=False) + with TestClient(app.build_app()) as client: + body = client.get("/metrics").text + # Disabled: the labelled series never appears, so AnomalyPipelineDead + # ( anomaly_selftest_ok == 0 ) can't fire on an opted-out deployment. + assert "anomaly_selftest_ok{" not in body + + +def test_selftest_emits_ok_when_enabled_and_passing(tmp_path: Path, monkeypatch: Any) -> None: + app = _app(tmp_path, monkeypatch, enabled=True) + app._run_selftest() + body = app._exporter.render()[0].decode("utf-8") + assert 'anomaly_selftest_ok{detector="MAD"} 1.0' in body + + +def test_selftest_emits_zero_and_bumps_counter_on_failure(tmp_path: Path, monkeypatch: Any) -> None: + app = _app(tmp_path, monkeypatch, enabled=True, threshold=1e9) + app._run_selftest() + body = app._exporter.render()[0].decode("utf-8") + assert 'anomaly_selftest_ok{detector="MAD"} 0.0' in body + assert 'anomaly_selftest_failures_total{detector="MAD"} 1.0' in body diff --git a/detector/tests/test_warmup.py b/detector/tests/test_warmup.py new file mode 100644 index 0000000..d623c8c --- /dev/null +++ b/detector/tests/test_warmup.py @@ -0,0 +1,281 @@ +"""Warm-up status endpoint + CLI.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +import yaml +from click.testing import CliRunner +from fastapi.testclient import TestClient + +from promanomaly.cli import cli +from promanomaly.config import CURRENT_API_VERSION, Config +from promanomaly.main import Application +from promanomaly.warmup import compute_warmup, render_table, report_from_payload +from tests.conftest import StubResponse, StubSource, make_series +from tests.fixtures import clean_baseline + + +def _config(min_points: int = 30, detectors: list[dict[str, Any]] | None = None) -> Config: + raw = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "defaults": { + "window": "15m", + "step": "15s", + "min_points": min_points, + }, + "groups": [ + { + "name": "g1", + "queries": [ + { + "id": "m1", + "promql": "up", + "detectors": detectors or [{"name": "MAD"}], + } + ], + } + ], + } + return Config.model_validate(raw) + + +@pytest.mark.asyncio +async def test_warmup_reports_ready_when_enough_points(stub_source: StubSource) -> None: + df = clean_baseline(n=120) + stub_source.respond(lambda _: StubResponse(series=[make_series({"instance": "h"}, df)])) + report = await compute_warmup(config=_config(min_points=30), source=stub_source, now=1000.0) + q = report.groups[0].queries[0] + assert q.status == "ready" + assert q.blocked_by == "none" + assert q.points_available >= 30 + + +@pytest.mark.asyncio +async def test_warmup_reports_warming_with_eta(stub_source: StubSource) -> None: + df = clean_baseline(n=10) + stub_source.respond(lambda _: StubResponse(series=[make_series({"instance": "h"}, df)])) + report = await compute_warmup(config=_config(min_points=30), source=stub_source, now=1000.0) + q = report.groups[0].queries[0] + assert q.status == "warming" + assert q.blocked_by == "min_points" + assert q.points_available == 10 + assert q.points_needed == 30 + # 20 missing points x 15s step. + assert q.eta_seconds == pytest.approx(300.0) + + +@pytest.mark.asyncio +async def test_warmup_blocked_by_lookback_when_no_data(stub_source: StubSource) -> None: + stub_source.respond(lambda _: StubResponse(series=[])) + report = await compute_warmup(config=_config(), source=stub_source, now=1000.0) + q = report.groups[0].queries[0] + assert q.status == "warming" + assert q.blocked_by == "lookback" + assert q.points_available == 0 + + +@pytest.mark.asyncio +async def test_warmup_probe_error_degrades_single_row(stub_source: StubSource) -> None: + from promanomaly.source import SourceQueryError + + stub_source.respond(lambda _: StubResponse(error=SourceQueryError("boom", reason="timeout"))) + report = await compute_warmup(config=_config(), source=stub_source, now=1000.0) + q = report.groups[0].queries[0] + assert q.status == "error" + + +@pytest.mark.asyncio +async def test_warmup_discovery_query_reports_discovery_blocker( + stub_source: StubSource, +) -> None: + raw = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "defaults": {"window": "15m", "step": "15s", "min_points": 30}, + "groups": [ + { + "name": "g1", + "queries": [ + { + "id": "m_{{ node }}", + "promql": 'rate(x{node="{{ node }}"}[5m])', + "detectors": [{"name": "MAD"}], + "discover": [ + {"variable": "node", "probe": "count by (node)(up)", "label": "node"} + ], + } + ], + } + ], + } + cfg = Config.model_validate(raw) + report = await compute_warmup(config=cfg, source=stub_source, now=1000.0) + q = report.groups[0].queries[0] + # Templated query isn't probed; reported ready-but-blocked-on-discovery. + assert q.status == "ready" + assert q.blocked_by == "discovery" + + +@pytest.mark.asyncio +async def test_warmup_coarsens_probe_step_for_long_stratified_window( + stub_source: StubSource, +) -> None: + # A stratified detector (DayOfWeekMAD, multi-week lookback) must not + # make the warm-up probe pull >150k points at the 15s scoring step — + # the probe step is coarsened so the response stays bounded. + df = clean_baseline(n=120) + stub_source.respond(lambda _: StubResponse(series=[make_series({"i": "h"}, df)])) + cfg = _config(detectors=[{"name": "DayOfWeekMAD"}]) + await compute_warmup(config=cfg, source=stub_source, now=1000.0) + probe = stub_source.queries[-1] + # The multi-week window at the coarsened step yields <= the cap, not + # the ~160k points a 15s step over 4 weeks would have produced. + assert probe["window"] / probe["step"] <= 1500 + 1 + # And the step was genuinely coarsened above the 15s scoring step. + assert probe["step"] > 15.0 + + +def test_render_table_round_trips_payload() -> None: + raw = { + "groups": [ + { + "name": "g1", + "queries": [ + { + "id": "m1", + "status": "warming", + "points_available": 5, + "points_needed": 30, + "blocked_by": "min_points", + "eta_seconds": 375.0, + } + ], + } + ] + } + table = render_table(report_from_payload(raw)) + assert "g1" in table + assert "m1" in table + assert "warming" in table + assert "min_points" in table + + +# ---------------------------------------------------------------------- +# Endpoint wiring +# ---------------------------------------------------------------------- +def _yaml_config(tmp_path: Path, expose: bool) -> Path: + cfg = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://stub"}, + "server": {"listen": ":0", "expose_warmup_endpoint": expose}, + "defaults": {"window": "5m", "step": "15s", "min_points": 5}, + "groups": [ + { + "name": "g1", + "queries": [{"id": "m1", "promql": "up", "detectors": [{"name": "MAD"}]}], + } + ], + } + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(cfg)) + return path + + +def test_warmup_endpoint_404_when_disabled(tmp_path: Path, monkeypatch: Any) -> None: + stub = StubSource() + stub.respond(lambda _: StubResponse(series=[make_series({"i": "h"}, clean_baseline(n=120))])) + monkeypatch.setattr("promanomaly.main.PromQLSource", lambda *a, **k: stub) + path = _yaml_config(tmp_path, expose=False) + app = Application(Config(**yaml.safe_load(path.read_text())), tmp_path) + with TestClient(app.build_app()) as client: + resp = client.get("/warmup") + assert resp.status_code == 404 + + +def test_warmup_endpoint_returns_report_when_enabled(tmp_path: Path, monkeypatch: Any) -> None: + stub = StubSource() + stub.respond(lambda _: StubResponse(series=[make_series({"i": "h"}, clean_baseline(n=120))])) + monkeypatch.setattr("promanomaly.main.PromQLSource", lambda *a, **k: stub) + path = _yaml_config(tmp_path, expose=True) + app = Application(Config(**yaml.safe_load(path.read_text())), tmp_path) + built = app.build_app() + with TestClient(built) as client: + resp = client.get("/warmup") + assert resp.status_code == 200 + body = resp.json() + assert body["groups"][0]["name"] == "g1" + assert body["groups"][0]["queries"][0]["status"] == "ready" + + +# ---------------------------------------------------------------------- +# CLI +# ---------------------------------------------------------------------- +def test_warmup_cli_renders_table(monkeypatch: Any) -> None: + payload = { + "groups": [ + { + "name": "g1", + "queries": [ + { + "id": "m1", + "status": "ready", + "points_available": 60, + "points_needed": 30, + "blocked_by": "none", + "eta_seconds": 0.0, + } + ], + } + ] + } + + class _Resp: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return payload + + monkeypatch.setattr("httpx.get", lambda *a, **k: _Resp()) + result = CliRunner().invoke(cli, ["warmup", "--target", "http://x"]) + assert result.exit_code == 0 + assert "m1" in result.output + + +def test_warmup_cli_exit_1_when_still_warming(monkeypatch: Any) -> None: + payload = { + "groups": [ + { + "name": "g1", + "queries": [ + { + "id": "m1", + "status": "warming", + "points_available": 5, + "points_needed": 30, + "blocked_by": "min_points", + "eta_seconds": 375.0, + } + ], + } + ] + } + + class _Resp: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return payload + + monkeypatch.setattr("httpx.get", lambda *a, **k: _Resp()) + result = CliRunner().invoke(cli, ["warmup", "--target", "http://x"]) + assert result.exit_code == 1 diff --git a/docs/architecture/multi-cluster.md b/docs/architecture/multi-cluster.md new file mode 100644 index 0000000..f65673e --- /dev/null +++ b/docs/architecture/multi-cluster.md @@ -0,0 +1,88 @@ +# Multi-Cluster Reference Architectures + +Running promanomaly across many Kubernetes clusters raises one key question the single-cluster docs don’t answer: **where should the detector live relative to the workloads it watches, and where should its analytics-plane TSDB live?** + +There is no single right answer — the trade-off is between cardinality, cross-cluster network cost, and blast radius/autonomy. + +This page documents the three patterns that cover essentially every deployment. It mirrors promforecast’s multi-cluster guide so a platform team can deploy both tools the same way. + +All patterns assume the standard flow: production Prometheus `remote_write`s into a long-term VictoriaMetrics (analytics plane), and the detector reads a short rolling window from that VM. + +## At a Glance + +| Pattern | Detectors | Analytics TSDB | Best when | +|---------|----------------------------|---------------------------------|-----------| +| **A. Per-cluster detector + central aggregation** | One per cluster | Per-cluster VM → anomaly scores `remote_write` to central VM | You want per-cluster autonomy + one central anomaly view | +| **B. Central detector + federated VM** | One (or HA set) centrally | One central/federated VM | Clusters are small/homogeneous and central bandwidth is cheap | +| **C. Fully isolated per-cluster** | One per cluster | Per-cluster VM (no aggregation) | Hard tenancy, data-residency, or regulatory isolation | + +## Pattern A — Per-Cluster Detector with Central Aggregation (Recommended Default) + +``` +cluster-1: prod-prom ─► VM-1 ─► detector-1 ─┐ +cluster-2: prod-prom ─► VM-2 ─► detector-2 ─┼─ remote_write anomaly_* ─► central VM ─► Grafana / Alertmanager +cluster-N: prod-prom ─► VM-N ─► detector-N ─┘ +``` + +Each cluster runs its own detector against its local VM. Only the bounded anomaly output (`anomaly_*` metrics) crosses the boundary to a central VM (via `remote_write` or federated scrape). Add a `cluster` label upstream for slicing. + +**Trade-offs** + +- **Cardinality**: Best — each detector sees only its own cluster. +- **Network**: Low — only anomaly scores cross clusters. +- **Blast radius / autonomy**: Best — one cluster’s outage never affects others. +- **Operational cost**: N detector deployments (easy to template with GitOps). + +**Use when** you have more than a handful of clusters and want both autonomy and a single pane of glass. This is the recommended default for growing fleets. + +## Pattern B — Central Detector Reading a Federated VictoriaMetrics + +``` +cluster-1: prod-prom ─┐ +cluster-2: prod-prom ─┼─ remote_write ─► central/federated VM ─► detector (HA) ─► Grafana / Alertmanager +cluster-N: prod-prom ─┘ +``` + +All clusters write into one central VM. A single (usually HA) detector reads everything from it. + +**Trade-offs** + +- **Cardinality**: Worst — the detector must handle the entire fleet at once. +- **Network**: High — all PromQL reads hit the central VM. +- **Blast radius / autonomy**: Worst — central VM or detector outage blinds the whole fleet. +- **Operational cost**: Lowest — one config, one deployment. + +**Use when** clusters are small/homogeneous, the central VM already exists for other reasons, and total cardinality comfortably fits one detector. Switch to Pattern A once you approach the single-replica sizing limit. + +## Pattern C — Fully Isolated Per-Cluster Detectors + +``` +cluster-1: prod-prom ─► VM-1 ─► detector-1 ─► cluster-1 Grafana / Alertmanager +cluster-2: prod-prom ─► VM-2 ─► detector-2 ─► cluster-2 Grafana / Alertmanager +cluster-N: prod-prom ─► VM-N ─► detector-N ─► cluster-N Grafana / Alertmanager +``` + +Every cluster is completely self-contained. Nothing crosses boundaries. + +**Trade-offs** + +- **Cardinality**: Best (same as A). +- **Network**: None. +- **Blast radius / autonomy**: Best — total isolation. +- **Operational cost**: Highest (N of everything). + +**Use when** hard multi-tenancy, data-residency, or regulatory rules forbid any cross-cluster dependency. If you need isolation for raw metrics but still want a central anomaly view, prefer Pattern A. + +## Choosing Between Them + +1. **Start with blast radius**: If clusters must be independent → A or C. If a central control plane is acceptable → consider B. +2. **Check cardinality**: If aggregate fleet exceeds one detector’s budget → B is out; use A. +3. **Decide on central view**: Want one pane of glass → A. Must not have one → C. + +**When in doubt, start with Pattern A** — it gives you the best of both worlds: edge-local detection (great cardinality and blast radius) plus a central anomaly view, with only the already-bounded anomaly signal crossing the wire. + +## Cross-References + +- [Operations guide](../operations.md) — HA, `remote_write` sink, dynamic discovery +- [Production sizing & readiness checklist](../production-checklist.md) +- [Degraded-mode operations playbook](../operations/degraded-modes.md) \ No newline at end of file diff --git a/docs/cli.md b/docs/cli.md index afb1e28..2faa186 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -15,16 +15,20 @@ promanomaly ships with a small, focused set of CLI commands for validation, test | `detectors list` | List registered detectors | | `inspect` | Inspect one running detector's state for an (id, label) series | | `top` | List the currently-firing series, ranked by severity | +| `warmup` | Report which series are still warming up, with an ETA to readiness | +| `diagnose` | Flag mis-tuned detectors (never-fires, fires-often, stuck warming, empty) | All commands accept `--log-level`. `--config` is required only for server, `validate`, and `dry-run`. ## `validate` -Validates the YAML configuration against the schema. With `--probe` it also executes the queries against the live datasource. +Validates the YAML configuration against the schema. With `--probe` it also executes the queries against the live datasource; with `--estimate-cost` it statically projects the config's cardinality and query/compute cost without touching the datasource; with `--lint-metadata` it checks each query's metric type against `/api/v1/metadata`. All three compose — pass them together to check that the queries return data, fit the caps, *and* aren't feeding a raw counter to a detector. ```bash promanomaly validate --config config.yaml promanomaly validate --config config.yaml --probe --strict +promanomaly validate --config config.yaml --estimate-cost --strict +promanomaly validate --config config.yaml --lint-metadata --strict ``` **Flags** @@ -33,9 +37,19 @@ promanomaly validate --config config.yaml --probe --strict |---------------------|--------------|-------------| | `--config ` | required | Path to YAML config | | `--probe` | off | Execute queries against datasource | -| `--strict` | off | Exit non-zero if any query returns no data or errors | +| `--estimate-cost` | off | Statically project cardinality, TSDB query load, and a coarse CPU/memory estimate (no datasource access) | +| `--lint-metadata` | off | Query `/api/v1/metadata` and warn when a query feeds a raw counter to a detector without `rate()`/`increase()` | +| `--strict` | off | With `--probe`: exit non-zero if any query returns no data or errors. With `--estimate-cost`: exit non-zero if projected series exceed `safety.max_total_series`. With `--lint-metadata`: exit non-zero if any mismatch is found | | `--datasource-url` | from config | Override datasource URL | +### `--lint-metadata` output + +One `{"status": "lint", ...}` JSON line per finding (`group`, `query`, `metric`, `metric_type`, `issue`, `detectors`, `message`) plus a `{"summary": ...}` line with `metrics_with_metadata` and `findings` counts. The only check today is `counter_not_rated`: a metric the TSDB reports as a `counter` that appears in a query without a rate-like wrapper (`rate`, `irate`, `increase`, `delta`, `idelta`, `deriv`, `resets`). A counter used only inside a series-identity aggregation (`count`, `count_values`, `group`, `absent`, …) is *not* flagged — e.g. `node_load1 / count(node_cpu_seconds_total)` counts cores and never scores the raw counter. **Lint only** — it never rewrites the query. Absent metadata is treated as "no opinion", so a metric the TSDB carries no metadata for never produces a false positive (handles VictoriaMetrics/Mimir/Thanos endpoint quirks). The same finding is surfaced live by `inspect` and `top --lint`. Flag-name parity with promforecast. + +### `--estimate-cost` output + +One JSON line per group plus a summary line. Per group: `projected_series` (worst case at `safety.max_series_per_query`), `short_queries_per_refresh`, `discovery_probe_queries_per_refresh`, `stratified_baseline_queries_per_day`, `estimated_memory_bytes`, and `estimated_cpu_cores`. The summary aggregates the totals and flags `over_budget` against `safety.max_total_series`. The CPU figure is anchored to the documented "~10k series at a 1-minute refresh on one 1-CPU replica" model — an order-of-magnitude sizing aid, not a benchmark. Pairs with `--probe` (which confirms the queries actually return data); see [production-checklist.md](production-checklist.md). + ## `dry-run` Full validation + one complete detection run for every group. No HTTP server is started. @@ -155,7 +169,61 @@ Answers "what is anomalous right now, ranked?" without opening Grafana — like ```bash promanomaly top --target http://localhost:9092 promanomaly top --target http://localhost:9092 --group node_cpu --min-severity 0.5 +promanomaly top --target http://localhost:9092 --no-lint # skip the extra metadata query promanomaly top --target http://localhost:9092 --output json ``` -Pairs with `inspect` as a two-step triage flow: `top` to find the worst series, `inspect` to understand *why* it scored. The `/debug/anomalies` endpoint is part of the `/debug/*` family the bundled NetworkPolicy denies by default. See [triage.md](triage.md) for the full surface. \ No newline at end of file +By default `top` also prints metadata-lint hints (the same counter-not-rated warning `validate --lint-metadata` reports), which costs one extra `/api/v1/metadata` query against the TSDB; pass `--no-lint` to skip it. Pairs with `inspect` as a two-step triage flow: `top` to find the worst series, `inspect` to understand *why* it scored. The `/debug/anomalies` endpoint is part of the `/debug/*` family the bundled NetworkPolicy denies by default. See [triage.md](triage.md) for the full surface. + +## `warmup` + +Answers "the dashboard is empty after `helm install` — do I wait or debug?" Queries a running detector's opt-in `/warmup` endpoint (enable it with `server.expose_warmup_endpoint: true`) and reports, per group and query, how many rolling-window samples are available versus `min_points`, an ETA to readiness at the current step, and the blocker. + +```bash +promanomaly warmup --target http://localhost:9092 +promanomaly warmup --target http://localhost:9092 --output json +``` + +**Flags** + +| Flag | Default | Description | +|-------------------|---------|-------------| +| `--target ` | required | Running detector base URL | +| `--output` | text | `text` (aligned table) or `json` (raw `/warmup` response) | +| `--timeout` | 30 | HTTP timeout in seconds | + +**`blocked_by` values**: `none` (ready), `min_points` (some samples, below the threshold), `lookback` (no samples yet), `baseline` (a stratified detector still filling its multi-week lookback), `discovery` (a templated query that can't be probed until discovery resolves). + +The exit code is **0** when every query in every group is `ready` (or there are no queries to warm up) and **1** when at least one is still warming or errored — so a CI pre-flight can gate "is the install done?" on `until promanomaly warmup --target ...`. Network failures exit **2**. The endpoint is in-namespace only (the bundled NetworkPolicy denies external ingress alongside `/debug/*`). Mirrors `promforecast warmup`. + +## `diagnose` + +Answers "which of my detectors are mis-tuned?" from evidence, not guesswork. Two modes share one report shape: + +- **TSDB lookback** (`--datasource-url`) — aggregates the detector's own scraped metrics over `--window` to flag the firing-rate bands (**never fires** = dead config; **fires often** = threshold too low / wrong detector), **chronically-warming** series, and **empty queries**. With `--config` it also flags **silent detectors** — a configured detector that emitted no score over the window, which includes a cohort detector dropping every cohort below `min_cohort_size`. +- **live snapshot** (`--target`) — one `/metrics` scrape for a current-state read (firing now, warming now, empty now). A single snapshot can't establish a rate over time, so it never reports `never_fires` — use the TSDB mode for that. + +Pass exactly one of `--target` / `--datasource-url`. + +```bash +# Over a 24h lookback against the TSDB scraping the detector: +promanomaly diagnose --datasource-url http://victoriametrics:8428 --window 24h +promanomaly diagnose --datasource-url http://victoriametrics:8428 --config config.yaml --output json + +# Quick current-state read straight off a running detector: +promanomaly diagnose --target http://localhost:9092 +``` + +**Flags** + +| Flag | Default | Description | +|---------------------|---------|-------------| +| `--target ` | — | Running detector base URL (snapshot mode) | +| `--datasource-url` | — | TSDB holding the detector's scraped metrics (lookback mode) | +| `--window ` | 24h | Lookback for the TSDB analysis (Prometheus duration) | +| `--config ` | none | With `--datasource-url`, cross-check configured detectors for silent ones | +| `--fire-high ` | 0.5 | Firing-rate fraction at/above which a detector is flagged too noisy | +| `--output` | text | `text` (grouped by category) or `json` (for CI) | +| `--timeout ` | 30 | HTTP timeout | + +Pure analysis, no persisted state. Pairs with `backtest` (precision/recall on injected anomalies) and `calibrate-buckets` (stratified bucketing) to close the tuning loop. Parity with `promforecast diagnose`. \ No newline at end of file diff --git a/docs/operations.md b/docs/operations.md index 1b29a6f..d9de34c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -323,7 +323,7 @@ The same `AnomalySignalMissing` reference alert covers both the static `expect:` **Probe failure under serve_stale.** A probe that hits a TSDB error (timeout, server_error, …) honours `safety.on_source_failure` the same way a regular query failure does — under the default `serve_stale` the previous snapshot is preserved instead of being overwritten with empty data. The original TSDB reason flows into `anomaly_source_failure_total{reason}` so it shares a bucket with regular-query failures; the discovery-specific `anomaly_discovery_failures_total{reason="probe_query_failed"}` counter is bumped separately for the discovery dashboard. -**HA-mode caveat.** The discovery tracker is in-memory on the leader; on Lease failover, the new leader rebuilds it from zero. Series that disappeared right before the handover only get an `anomaly_signal_missing` from the new leader once they have missed `expect_grace_runs` more runs after takeover. For most workloads this is invisible; for workloads with a high churn rate close to failover, treat the first `(grace_runs × refresh_interval)` after a leader transition as a soft window. +**HA mode.** When `highAvailability.enabled: true`, the discovery miss-counter tracker lives in the shared Redis cache (under `{key_prefix}:discovery::`) instead of leader-local memory, so a Lease failover no longer resets it: the newly-elected leader inherits each series' consecutive-miss count and a series that disappeared right before the handover still trips `anomaly_signal_missing` on schedule rather than restarting its `expect_grace_runs` window from zero. Single-replica deployments keep the in-memory tracker unchanged. The Redis state is bounded by the same `safety.discovery.forget_after_runs` cap as the in-memory tracker, and the entries carry a TTL (refreshed every run) so a fully-removed deployment's keys expire rather than leaking. A transient Redis outage degrades gracefully — that run records no new observations and emits no missing signal — rather than crashing the detection run. ## Validating Configs Before Deploy @@ -436,4 +436,70 @@ defaults: - namespace # one anomaly_density{by="namespace", namespace=} series per distinct value ``` -Each listed label adds one series per distinct value seen in the run, so partition by **low-cardinality** dimensions (namespace, job, region) — never by `instance` or `pod` on a large fleet. Partitioning by a canonical detector label (`id`, `group`, `detector`, `detector_instance`) is rejected at config-load time. As a backstop against a high-cardinality mistake, a dimension that produces more than 200 distinct slice values in a run is **skipped for that run** and logged as `density_by_cardinality_capped` (the group rollup and per-detector partition are unaffected) — point the dimension at a lower-cardinality label and the slices return. Cohort divergence rolls up for free: `anomaly_density{group, detector="Cohort"}` is "members currently diverging ÷ cohort members scored". \ No newline at end of file +Each listed label adds one series per distinct value seen in the run, so partition by **low-cardinality** dimensions (namespace, job, region) — never by `instance` or `pod` on a large fleet. Partitioning by a canonical detector label (`id`, `group`, `detector`, `detector_instance`) is rejected at config-load time. As a backstop against a high-cardinality mistake, a dimension that produces more than 200 distinct slice values in a run is **skipped for that run** and logged as `density_by_cardinality_capped` (the group rollup and per-detector partition are unaffected) — point the dimension at a lower-cardinality label and the slices return. Cohort divergence rolls up for free: `anomaly_density{group, detector="Cohort"}` is "members currently diverging ÷ cohort members scored". +## Self-Observability + +The detector emits telemetry about its **own** health so a silently degrading component is caught before it stops catching anomalies. All four families are bounded by the configured group / detector cardinality — none touch the per-series budget. + +| Metric | Labels | Meaning | +|--------|--------|---------| +| `anomaly_detect_success_ratio` | `group, detector, window` | Fraction of attempted detector computations that completed without raising (timeout / exception), over a rolling window. Two windows are emitted: `1h` (spots a fresh regression) and `1d` (the slower trend). | +| `anomaly_group_cpu_seconds_total` | `group` | Cumulative detector compute time attributed to a group (sum of per-detector wall-clock). Use `rate()` to see which group dominates the compute budget. | +| `anomaly_group_memory_bytes` | `group` | Best-effort size of the per-group output snapshot held by the exporter — answers "which group's snapshot is dominant", not an exact RSS breakdown. | +| `anomaly_snapshot_age_seconds` | `group` | Age of the group's most recent successful snapshot. Recomputed on every scrape, so it keeps climbing between runs and resets on each successful run. | +| `anomaly_series_staleness_seconds` | `group` | Age of the *freshest per-series score* in a group. Diverges from snapshot age when a group keeps writing snapshots that carry no scores (all series warming up, or scores dropped under `drop_scores`); absent until the group has produced at least one score. | + +The success ratio is recorded once per group-run per detector (aggregating the run's per-series attempts), so the rolling ledger stays bounded at runs-per-window regardless of fleet size. CPU is the per-run sum of detector durations; memory is sampled at the end of each successful run. Snapshot-age and series-staleness are derived at scrape time from the snapshot-store timestamps. + +Two reference alerts ship for this surface in `examples/alerts/promanomaly-rules.yaml` (and the bundled Helm `PrometheusRule`): + +- **`AnomalyDetectorDegraded`** — `anomaly_detect_success_ratio{window="1h"} < 0.9 for 10m`. A detector is failing or timing out on too many series; its scores are silently missing before any per-series anomaly alert can fire. +- **`AnomalySnapshotStale`** — `anomaly_snapshot_age_seconds > 600 for 5m`. The served `/metrics` snapshot has aged out. Distinct from `AnomalyStale`, which keys off the last *successful-run* timestamp — under `serve_stale` the run timestamp can stop advancing while `/metrics` keeps returning an ever-older snapshot. + +The `dashboards/grafana/promanomaly-self-observability.json` dashboard graphs all five families. See the [degraded-mode playbook](operations/degraded-modes.md) for the full failure-mode matrix. + +## Warm-Up Status Endpoint + +On a fresh install or after a restart, `/metrics` is empty until each series fills its rolling window (`min_points`). The opt-in `GET /warmup` endpoint (enable with `server.expose_warmup_endpoint: true`) makes that self-explaining: per group and query it reports points available versus needed, an ETA at the current `refresh_interval`, and the blocker (`min_points`, `lookback`, a stratified `baseline` still filling its multi-week lookback, or `discovery`). + +```bash +promanomaly warmup --target http://localhost:9092 +``` + +Each call issues one cheap probe per query — off by default because that cost is wasted once past the first install. The endpoint is in-namespace only (the bundled NetworkPolicy denies external ingress alongside `/debug/*`). The `promanomaly warmup` CLI exits non-zero while anything is still warming, so `until promanomaly warmup --target ...` is a valid post-install gate. See [`cli.md`](cli.md) for the command reference. + +## End-to-End Detection Self-Test (dead-man's-switch) + +The self-observability success ratios prove each detector *runs*; they do not prove the pipeline actually *catches* anomalies. A global misconfiguration — an absurd `alert_thresholds.score`, an exporter regression, a threshold mistake — can leave the process up and emitting nothing while every success ratio still reads 1.0. Enable the self-test to guard against exactly that: + +```yaml +server: + selftest: + enabled: true # off by default + detector: MAD # detector run through the synthetic path + threshold: 3.0 # threshold the synthetic score is checked against +``` + +Each scheduled run drives one synthetic series carrying a known injected point-spike anomaly through the **real** detect → threshold → export path (the sample is even re-rendered through the exporter, so an exporter regression that rejects it also trips the test) and asserts it is caught. It publishes: + +| Metric | Labels | Meaning | +|--------|--------|---------| +| `anomaly_selftest_ok` | `detector` | 1 when the injected anomaly was caught on the last run, 0 when the detect/threshold/export path failed to surface it. | +| `anomaly_selftest_failures_total` | `detector` | Cumulative failed self-test runs. | + +Both metrics carry a `detector` label, so the series is **absent** (and the `AnomalyPipelineDead` alert dormant) on a deployment that hasn't opted in — a disabled self-test never looks like a dead pipeline. Stateless and bounded to one synthetic series; leader-gated in HA mode so the gauge is published once per cluster. The `AnomalyPipelineDead` reference alert (`anomaly_selftest_ok == 0 for 5m`, critical) ships in `examples/alerts/promanomaly-rules.yaml` and the Helm `PrometheusRule`. Distinct from the self-monitoring example config, which watches the detector's *operational* metrics rather than re-driving the detection path. See the [degraded-mode playbook](operations/degraded-modes.md). + +## Config Health and Tuning Diagnostics + +`promanomaly diagnose` answers "which of my detectors are mis-tuned?" from evidence. Against the TSDB scraping the detector's own metrics it flags, over a lookback: detectors that **never fire** (dead config), detectors that **fire too often** (threshold too low / wrong detector), **chronically-warming** series, **empty queries**, and — with `--config` — **silent detectors** (a configured detector emitting nothing, which includes a cohort below `min_cohort_size`). A lighter `--target` mode reads one `/metrics` scrape for a current-state view. + +```bash +promanomaly diagnose --datasource-url http://victoriametrics:8428 --window 24h +promanomaly diagnose --target http://localhost:9092 +``` + +Pure analysis, no persisted state; pairs with `backtest` and `calibrate-buckets` to close the tuning loop. See [`cli.md`](cli.md) for the full flag reference. + +## Metadata-Aware Validation Lint + +`promanomaly validate --lint-metadata` queries the datasource's `/api/v1/metadata` and warns when a query feeds a raw counter to a detector without a `rate()` / `increase()` wrapper — a monotonic counter scored directly produces nonsense. Lint only (it never rewrites the query); under `--strict` any finding fails CI, and it composes with `--probe` / `--estimate-cost`. The same hint surfaces live in `inspect` and `promanomaly top --lint`. Absent metadata is treated as "no opinion", so the lint never manufactures a false positive from a metric the TSDB carries no type for. See [`cli.md`](cli.md). diff --git a/docs/operations/degraded-modes.md b/docs/operations/degraded-modes.md new file mode 100644 index 0000000..cefbdf4 --- /dev/null +++ b/docs/operations/degraded-modes.md @@ -0,0 +1,127 @@ +# Degraded-Mode Operations Playbook + +promanomaly isolates failures at every layer and keeps serving the last good snapshot during transient TSDB issues. This resilience is intentional — but it also means several degraded states are **invisible** if you only monitor whether the pod is Running. + +This playbook lists every failure mode, the exact signal that detects it, and the recovery steps. The shipped alerts in [`examples/alerts/promanomaly-rules.yaml`](../../examples/alerts/promanomaly-rules.yaml) cover all of them. + +## Failure-Isolation Layers + +A single fault never takes down more than its own layer: + +| Layer | Caught by | Continues running? | +|----------------|------------------------------------|-------------------------------------| +| Per-detector | `try/except` in scoring loop | Yes — other detectors on the series emit | +| Per-series | `try/except` inside a query | Yes — sibling series continue | +| Per-query | `try/except` per query | Yes — other queries in the group continue | +| Per-group | `try/except` at group level | Yes — sibling groups continue | +| TSDB-level | `safety.on_source_failure` policy | Yes — last good snapshot served | + +All failures increment `anomaly_failures_total{group, detector, reason}` (bounded cardinality). + +## `safety.on_source_failure` Policy + +| Policy | Snapshot on failure | Readiness | Use when | +|----------------------|----------------------------------------------|----------------------------|----------| +| `serve_stale` (default) | Last good snapshot kept unchanged | Unaffected | You prefer stale data over blank charts | +| `drop_scores` | Failing series dropped; others stay fresh (full-group failure degrades to `serve_stale`) | Unaffected | You prefer staleness handling over stale values | +| `fail_ready` | Same as `serve_stale` | `anomaly_group_ready` → 0 after N failures | You want the pod to shed traffic on sustained TSDB outage | + +`anomaly_source_failure_streak{group}` is always published so you can alert *before* `fail_ready` trips. + +## Failure-Mode Matrix + +| Failure mode | Detection signal | Recovery | +|-------------------------------|-------------------------------------------------------|----------| +| TSDB unreachable | `rate(anomaly_source_failure_total[15m]) > 0` | Check TSDB reachability / auth / network | +| Partial group failure | `anomaly_failures_total{group, reason} > 0` | Fix PromQL or cardinality, then reload | +| Sustained query timeout | `rate(anomaly_failures_total{reason="timeout"}[15m])` | Raise timeouts or lighten config | +| Stale snapshot | `anomaly_snapshot_age_seconds > 600` or `time() - anomaly_last_run_timestamp_seconds > 600` | Check pod health + rows above | +| Detector degrading | `anomaly_detect_success_ratio{window="1h"} < 0.9` | Inspect failures by detector | +| Dead detection pipeline | `anomaly_selftest_ok == 0` (opt-in self-test) | Check threshold / exporter regression / global misconfig | +| Leader churn (HA) | `rate(anomaly_leader_transitions_total[15m]) > 2` | Check Lease contention / pod restarts | + +## TSDB Read Failure + +**Detection** +`rate(anomaly_source_failure_total[15m]) > 0` + +**Recovery** +Check VictoriaMetrics/Mimir/Thanos health, network policies, and auth. Under `serve_stale` the `/metrics` endpoint still serves the last good snapshot. + +**Shipped alert:** `AnomalySourceFailing` + +## Partial Group Failure + +**Detection** +Non-zero `anomaly_failures_total{group, reason}` for one group while others are healthy. + +**Recovery** +Check JSON logs for the group + reason, fix the PromQL or cardinality issue, then `SIGHUP` or `POST /-/reload`. + +## Sustained Query Timeout + +**Detection** +Rising `rate(anomaly_failures_total{reason="timeout"}[15m])` or `anomaly_detect_duration_seconds` approaching the timeout. + +**Recovery** +Increase `safety.query_timeout` / `safety.detect_timeout`, switch to a lighter detector, or split heavy queries into their own group. + +## Stale Snapshot + +**Detection** +```promql +anomaly_snapshot_age_seconds > 600 +or +time() - anomaly_last_run_timestamp_seconds > 600 +``` + +**Recovery** +Check pod health (`OOMKilled`, `Recreate` deploy gap) and the failure modes above. + +**Shipped alerts:** `AnomalyStale`, `AnomalySnapshotStale` + +## Detector Degrading + +**Detection** +`anomaly_detect_success_ratio{window="1h"} < 0.9` + +**Recovery** +Inspect `anomaly_failures_total` by detector and `anomaly_detect_duration_seconds`. Common causes: new detector version, parameter change, or pathological signal shape. + +**Shipped alert:** `AnomalyDetectorDegraded` + +## Dead Detection Pipeline (opt-in self-test) + +Per-detector success ratios prove each detector *runs* without raising — they do **not** prove the pipeline actually catches anomalies. A global misconfiguration (an absurd `alert_thresholds.score`, an exporter regression, a threshold mistake) can leave the process up and emitting nothing while every success ratio still reads 1.0. The end-to-end self-test is the dead-man's-switch for exactly that. + +**Enable** +Set `server.selftest.enabled: true`. Each run drives a synthetic series with a known injected anomaly through the real detect → threshold → export path and asserts it is caught, publishing `anomaly_selftest_ok` (1/0) and `anomaly_selftest_failures_total`. + +**Detection** +`anomaly_selftest_ok == 0` + +**Recovery** +The detect/threshold/export path is failing to surface a known anomaly: check for a recently-changed global threshold, a detector parameter mistake, or an exporter regression. `anomaly_selftest_failures_total` confirms it is failing repeatedly rather than flapping. + +**Shipped alert:** `AnomalyPipelineDead` (dormant unless the self-test is enabled — the series only exists when `server.selftest.enabled`). + +## Leader Churn (HA mode only) + +**Detection** +`rate(anomaly_leader_transitions_total[15m]) > 2` + +**Recovery** +Check Lease contention, pod restarts, and `lease_duration` / `retry_period` tuning. Brief churn is safe; sustained churn wastes runs. + +## Putting It Together + +- **Page** on `AnomalySourceFailing`, `AnomalyStale`, `AnomalySnapshotStale`, and (if the self-test is enabled) `AnomalyPipelineDead`. +- **Ticket** on `AnomalyDetectorDegraded`. + +All other silencing, deploy damping, and flap suppression belongs in Alertmanager (silences + `for:` clauses). + +## Cross-References + +- [Production-readiness checklist](../production-checklist.md) +- [Operations guide](../operations.md) +- [Configuration schema](../config-schema.md) \ No newline at end of file diff --git a/docs/production-checklist.md b/docs/production-checklist.md new file mode 100644 index 0000000..53f0f63 --- /dev/null +++ b/docs/production-checklist.md @@ -0,0 +1,72 @@ +# Production-Readiness Checklist & Sizing Guide + +This single page answers the questions every operator must resolve before going live: how much CPU/memory to request, which deployment shape to choose, and which knobs to verify. + +Detailed guidance lives in the linked pages; this is the integrated checklist. + +## Sizing (Rule of Thumb) + +Detection is cheap (`O(n log n)` per series on the rolling window). The bottleneck is **TSDB query throughput**, not compute. + +A single 1-CPU replica comfortably handles **~10 000 series at 1-minute refresh** with default detectors. + +| Series count | Refresh | CPU | Memory | Replicas | +|------------------|---------|-------|--------|-------------------| +| < 1 000 | 1 m | 100m | 128 Mi | 1 (no HA) | +| 1 000 – 10 000 | 1 m | 1 | 512 Mi | 1 or 2 (HA) | +| 10 000 – 25 000 | 1 m | 2 | 1 Gi | 2 (HA) | +| 25 000 – 50 000 | 2 m | 4 | 2 Gi | 2–3 (HA) | +| > 50 000 | varies | — | — | Shard by group | + +**Always run** `promanomaly validate --config --estimate-cost` for a config-specific projection (projected series, TSDB queries per refresh, and coarse CPU/memory estimate). + +**Refresh-interval impact** +CPU scales linearly with `1 / refresh_interval`. Keep `refresh_interval` no smaller than `defaults.window / 6` (the detector warns on aggressive settings). + +**Stratified detectors** add TSDB reads (not CPU). The cost estimator reports `stratified_baseline_queries_per_day` separately. + +## Reference Architectures + +| Shape | When to use | Helm values | +|----------------------------|--------------------------------------------------|-------------| +| **A. Single-replica Recreate** | < 10 k series, brief `/metrics` gap OK during deploys | `replicaCount: 1`
`strategy: Recreate`
`highAvailability.enabled: false` | +| **B. HA leader-elected** | Need continuous `/metrics` availability | `replicaCount: 2`
`highAvailability.enabled: true`
`safety.redis.url: …` | +| **C. Multi-cluster** | Many clusters (per-cluster or federated) | See [multi-cluster reference architectures](architecture/multi-cluster.md) | + +## Production-Readiness Checklist + +| Item | Expected end state | Reference | +|------|--------------------|---------| +| Cardinality caps set | `safety.max_series_per_query` and `safety.max_total_series` sized for your fleet; `--estimate-cost --strict` passes in CI | [config schema](config-schema.md) | +| Source-failure policy chosen | `safety.on_source_failure` deliberately set (`serve_stale` is safe default) | [degraded modes](operations/degraded-modes.md) | +| Queries validated | `promanomaly validate --config --probe --strict` passes against real TSDB | [CLI](cli.md) | +| Cost estimate run | `promanomaly validate --config --estimate-cost --strict` passes | [CLI](cli.md) | +| ServiceMonitor present | promanomaly’s own `/metrics` is scraped | chart `values.yaml` | +| `honor_labels: true` on scrape | Source labels pass through unchanged | [LABELS_CONTRACT.md](LABELS_CONTRACT.md) | +| NetworkPolicy reviewed | `/-/reload` and `/debug/*` denied by default | chart `NetworkPolicy` | +| Alerts loaded | Shipped `promanomaly-rules.yaml` deployed (including self-observability) | [examples/alerts/](examples/alerts/promanomaly-rules.yaml) | +| Dashboards provisioned | Shipped Grafana dashboards installed | [dashboards/grafana/](dashboards/grafana/) | +| Degraded-mode alerts wired | `AnomalySourceFailing`, `AnomalyStale`, etc. on-call | [degraded modes](operations/degraded-modes.md) | +| Warm-up understood | `promanomaly warmup --target ` explains empty dashboards on first install | [CLI](cli.md) | + +## What You Can Defer + +- HA mode (single-replica `Recreate` is fully supported) +- `/warmup` endpoint (leave `expose_warmup_endpoint: false` after initial rollout) +- Stratified detectors (add later for diurnal signals) + +## What You Cannot Defer + +- `safety.max_total_series` (re-evaluate after every new detector or `discover:` block) +- Choosing a `safety.on_source_failure` policy +- Wiring the degraded-mode alerts + +## Cross-References + +- [Degraded-mode operations playbook](operations/degraded-modes.md) +- [Multi-cluster reference architectures](architecture/multi-cluster.md) +- [Operations guide](operations.md) +- [Configuration schema](config-schema.md) +- [CLI reference](cli.md) + +Run the checklist, run the two `validate` commands, and you’re ready for production. \ No newline at end of file diff --git a/examples/alerts/promanomaly-rules.yaml b/examples/alerts/promanomaly-rules.yaml index 2c2a252..2135753 100644 --- a/examples/alerts/promanomaly-rules.yaml +++ b/examples/alerts/promanomaly-rules.yaml @@ -217,3 +217,63 @@ spec: No promanomaly group has completed a successful run in the last 5 minutes. The pod will fail readiness if the fail_ready policy is configured. + + - alert: AnomalyDetectorDegraded + # Self-observability: a detector's compute success ratio has + # dropped — it is timing out or raising on a growing fraction + # of series, so its scores are silently missing before any + # downstream anomaly alert can fire. The 1h window catches a + # fresh regression; switch to window="1d" for the slower trend. + expr: anomaly_detect_success_ratio{window="1h"} < 0.9 + for: 10m + labels: + severity: warning + annotations: + summary: "Detector {{ $labels.detector }} degraded in {{ $labels.group }}" + description: | + anomaly_detect_success_ratio for detector + {{ $labels.detector }} in group {{ $labels.group }} has been + below 0.9 over the last hour — the detector is failing or + timing out on too many series and its scores are missing. + Check anomaly_failures_total and anomaly_detect_duration_seconds. + + - alert: AnomalySnapshotStale + # The /metrics snapshot a group is serving has aged past the + # threshold. Distinct from AnomalyStale (which keys off the + # last *successful run* timestamp): under serve_stale the run + # timestamp can stop advancing while /metrics keeps returning an + # ever-older snapshot — this rule catches that the served data + # itself is stale. Tune 600s relative to refresh_interval. + expr: anomaly_snapshot_age_seconds > 600 + for: 5m + labels: + severity: warning + annotations: + summary: "promanomaly snapshot stale for {{ $labels.group }}" + description: | + The most recent snapshot for group {{ $labels.group }} is + over 10 minutes old. /metrics is serving stale scores; + cross-check AnomalySourceFailing and AnomalyStale. + + - alert: AnomalyPipelineDead + # Dead-man's-switch: the opt-in end-to-end self-test + # (server.selftest.enabled) drives a synthetic series with a + # known injected anomaly through the real detect/threshold/ + # export path each run. When anomaly_selftest_ok flips to 0 the + # whole detection pipeline is silently emitting nothing — a + # global misconfig, an exporter regression, a threshold mistake. + # The series only exists when the self-test is enabled, so this + # rule is dormant on opted-out deployments. + expr: anomaly_selftest_ok == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "promanomaly detection pipeline is dead (self-test failing)" + description: | + The end-to-end self-test for detector + {{ $labels.detector }} has failed to catch its injected + synthetic anomaly for over 5 minutes. The detect → threshold + → export pipeline is not surfacing anomalies even though the + process is up — check for a config/threshold mistake or an + exporter regression. See anomaly_selftest_failures_total.