From ba5b42e1a6e72e01e29dcad5e6e121456c2e6dd9 Mon Sep 17 00:00:00 2001 From: Werner Dijkerman Date: Fri, 29 May 2026 19:25:13 +0200 Subject: [PATCH] Improved the operator expeirence a bit --- .github/workflows/release.yml | 102 +++++++ README.md | 16 +- .../promanomaly/templates/prometheusrule.yaml | 31 +++ charts/promanomaly/values.yaml | 17 +- .../grafana/promanomaly-fleet-density.json | 118 ++++++++ .../grafana/promanomaly-node-overlay.json | 160 +++++++++++ detector/src/promanomaly/anomalies.py | 142 ++++++++++ detector/src/promanomaly/anomaly_type.py | 183 +++++++++++++ detector/src/promanomaly/cli/__init__.py | 86 ++++++ detector/src/promanomaly/cli/_top.py | 56 ++++ detector/src/promanomaly/config.py | 49 +++- detector/src/promanomaly/exporter.py | 51 +++- detector/src/promanomaly/runner.py | 246 ++++++++++++++++- detector/src/promanomaly/server.py | 27 ++ detector/src/promanomaly/severity.py | 129 +++++++++ detector/tests/test_anomaly_type.py | 114 ++++++++ detector/tests/test_documented_examples.py | 76 ++++++ detector/tests/test_fleet_density.py | 252 ++++++++++++++++++ detector/tests/test_runner.py | 5 + detector/tests/test_severity.py | 112 ++++++++ detector/tests/test_source_failure_policy.py | 11 +- detector/tests/test_triage.py | 169 ++++++++++++ docs/cli.md | 15 +- docs/cookbook/README.md | 41 +++ docs/cookbook/applications.md | 163 +++++++++++ docs/cookbook/infrastructure.md | 189 +++++++++++++ docs/cookbook/kubernetes.md | 134 ++++++++++ docs/detectors.md | 39 +++ docs/operations.md | 45 +++- docs/severity.md | 95 +++++++ docs/triage.md | 72 +++++ examples/alerts/promanomaly-rules.yaml | 41 +++ examples/configs/applications.yaml | 92 +++++++ examples/configs/cadvisor.yaml | 95 +++++++ examples/configs/kube-state-metrics.yaml | 109 ++++++++ examples/configs/kubelet.yaml | 88 ++++++ examples/configs/node-exporter.yaml | 188 +++++++++++++ 37 files changed, 3538 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 dashboards/grafana/promanomaly-fleet-density.json create mode 100644 dashboards/grafana/promanomaly-node-overlay.json create mode 100644 detector/src/promanomaly/anomalies.py create mode 100644 detector/src/promanomaly/anomaly_type.py create mode 100644 detector/src/promanomaly/cli/_top.py create mode 100644 detector/src/promanomaly/severity.py create mode 100644 detector/tests/test_anomaly_type.py create mode 100644 detector/tests/test_documented_examples.py create mode 100644 detector/tests/test_fleet_density.py create mode 100644 detector/tests/test_severity.py create mode 100644 detector/tests/test_triage.py create mode 100644 docs/cookbook/README.md create mode 100644 docs/cookbook/applications.md create mode 100644 docs/cookbook/infrastructure.md create mode 100644 docs/cookbook/kubernetes.md create mode 100644 docs/severity.md create mode 100644 docs/triage.md create mode 100644 examples/configs/applications.yaml create mode 100644 examples/configs/cadvisor.yaml create mode 100644 examples/configs/kube-state-metrics.yaml create mode 100644 examples/configs/kubelet.yaml create mode 100644 examples/configs/node-exporter.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a3df3af --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,102 @@ +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: read + +jobs: + image: + name: Container image + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + id-token: write + outputs: + image: ${{ steps.meta.outputs.tags }} + digest: ${{ steps.build.outputs.digest }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: docker/setup-qemu-action@v4 + - uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - id: build + uses: docker/build-push-action@v7 + with: + context: detector + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + provenance: true + sbom: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Generate SBOM (Syft) + uses: anchore/sbom-action@v0 + with: + image: ghcr.io/${{ github.repository }}@${{ steps.build.outputs.digest }} + format: spdx-json + artifact-name: promanomaly.spdx.json + upload-artifact: true + + - name: Install Trivy + run: | + mkdir -p "$HOME/.local/bin" + curl -sSfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh \ + | sh -s -- -b "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Trivy scan (release) + # SARIF upload intentionally not wired up — requires GitHub Advanced + # Security on private repos. Findings surface in the job log. + run: | + trivy image \ + --format table \ + --severity HIGH,CRITICAL \ + --ignore-unfixed \ + --exit-code 0 \ + ghcr.io/${{ github.repository }}@${{ steps.build.outputs.digest }} + + github-release: + name: GitHub Release + needs: [image] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Create GitHub release with auto-generated notes + uses: softprops/action-gh-release@v3 + with: + generate_release_notes: true + fail_on_unmatched_files: false + body: | + Container image: `${{ needs.image.outputs.image }}` + Digest: `${{ needs.image.outputs.digest }}` diff --git a/README.md b/README.md index e91e78c..14231a4 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,20 @@ helm install promanomaly promanomaly/promanomaly-stack This installs VictoriaMetrics, the detector, and example dashboards. For an existing TSDB, install only `promanomaly/promanomaly`. +For a complete day-one starter next to node-exporter, point the detector +at [`examples/configs/node-exporter.yaml`](examples/configs/node-exporter.yaml) — +a ready-made multi-group config watching the host golden signals (CPU, +memory, filesystem fill + fill-rate, disk I/O, network errors, load) with +detectors already matched to each signal's shape. Sibling starters cover +[kube-state-metrics](examples/configs/kube-state-metrics.yaml), +[cAdvisor](examples/configs/cadvisor.yaml), +[kubelet](examples/configs/kubelet.yaml), and +[application/middleware](examples/configs/applications.yaml) signals, and +the [`docs/cookbook/`](docs/cookbook/) recipes explain which detector fits +which signal. Once running, `promanomaly top --target ` ranks what is +anomalous right now by a normalised severity signal — see +[`docs/triage.md`](docs/triage.md). + Editing the YAML config does **not** require a pod restart: the detector watches the mounted ConfigMap, validates the new config in-place, and rolls back on validation failure (`kubectl apply` to the ConfigMap is @@ -90,7 +104,7 @@ rare events, and per-instance outliers. ## Status -The configuration schema is stable as of the v1.0 release: +The configuration schema is stable as of the first stable release: `apiVersion: promanomaly.io/v1` and the `values.schema.json` `$id` both move from alpha to v1. The previous `promanomaly.io/v1alpha1` alias keeps loading as a byte-identical form with a deprecation diff --git a/charts/promanomaly/templates/prometheusrule.yaml b/charts/promanomaly/templates/prometheusrule.yaml index 7038eba..5aec116 100644 --- a/charts/promanomaly/templates/prometheusrule.yaml +++ b/charts/promanomaly/templates/prometheusrule.yaml @@ -98,6 +98,21 @@ spec: series for at least the configured grace runs. {{- end }} {{- end }} + {{- with $rules.anomalyHighSeverity }} + {{- if .enabled }} + - alert: AnomalyHighSeverity + expr: anomaly_severity >= {{ .threshold }} + for: {{ .for }} + labels: + severity: {{ .severity }} + annotations: + summary: "High-severity anomaly on {{ "{{" }} $labels.id {{ "}}" }} ({{ "{{" }} $labels.group {{ "}}" }})" + description: | + Detector {{ "{{" }} $labels.detector {{ "}}" }} reports + anomaly_severity >= {{ .threshold }} for at least {{ .for }}. + Use `promanomaly top` to rank against other current firings. + {{- end }} + {{- end }} {{- with $rules.anomalyConfidenceLow }} {{- if .enabled }} - alert: AnomalyConfidenceLow @@ -165,6 +180,22 @@ spec: last good snapshot — scores are stale, not fresh. {{- end }} {{- end }} + {{- with $rules.fleetAnomalyDensityHigh }} + {{- if .enabled }} + - alert: FleetAnomalyDensityHigh + expr: anomaly_density{detector="", by=""} > {{ .threshold }} + for: {{ .for }} + labels: + severity: {{ .severity }} + annotations: + summary: "High anomaly density in group {{ "{{" }} $labels.group {{ "}}" }}" + description: | + More than {{ .threshold }} (fraction) of series in group + {{ "{{" }} $labels.group {{ "}}" }} are currently anomalous. Use + `promanomaly top --group {{ "{{" }} $labels.group {{ "}}" }}` to see + the worst series ranked by severity. + {{- end }} + {{- end }} {{- with $rules.anomalyNotReady }} {{- if .enabled }} - alert: AnomalyNotReady diff --git a/charts/promanomaly/values.yaml b/charts/promanomaly/values.yaml index e79e03a..c2fcd68 100644 --- a/charts/promanomaly/values.yaml +++ b/charts/promanomaly/values.yaml @@ -75,7 +75,7 @@ safety: key_prefix: promanomaly timeout: 2s - # Dynamic series discovery (v0.4). Bounds the in-memory tracker so + # Dynamic series discovery. Bounds the in-memory tracker so # workloads with ephemeral identities (transient pod names, replica # IDs that never recycle) don't grow the discovery dict without end. # Default of 1000 runs is ~16 hours at the 1m refresh — plenty of @@ -261,6 +261,13 @@ prometheusRule: threshold: 0.5 for: 10m severity: warning + # Normalised 0-1 severity gate. Pure signal enrichment ranked across + # detectors with incomparable raw-score units. See docs/severity.md. + anomalyHighSeverity: + enabled: true + threshold: 0.7 + for: 5m + severity: warning anomalyStale: enabled: true maxStaleSeconds: 600 @@ -281,3 +288,11 @@ prometheusRule: enabled: true for: 5m severity: critical + # Fleet anomaly-density rollup gate ("is anything weird right now"). + # Bounded-cardinality — fires once per group, not per series. Tune + # the threshold (fraction outside) per fleet size and SLO. + fleetAnomalyDensityHigh: + enabled: true + threshold: 0.1 + for: 5m + severity: warning diff --git a/dashboards/grafana/promanomaly-fleet-density.json b/dashboards/grafana/promanomaly-fleet-density.json new file mode 100644 index 0000000..c42b8ab --- /dev/null +++ b/dashboards/grafana/promanomaly-fleet-density.json @@ -0,0 +1,118 @@ +{ + "annotations": {"list": []}, + "editable": true, + "schemaVersion": 38, + "title": "promanomaly fleet density", + "description": "Is anything weird right now? Bounded-cardinality fleet rollups driven by anomaly_density / anomaly_active_series / anomaly_severity_density.", + "tags": ["promanomaly", "anomaly-detection", "triage"], + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "label": "Datasource", + "query": "prometheus", + "current": {"text": "VictoriaMetrics", "value": "victoriametrics"} + }, + { + "name": "group", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "query": "label_values(anomaly_density, group)", + "refresh": 2, + "multi": true, + "includeAll": true + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Anomaly density by group", + "description": "Fraction of each group currently outside threshold (group rollup only; detector/by partitions excluded).", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}, + "targets": [ + { + "expr": "anomaly_density{group=~\"$group\", detector=\"\", by=\"\"}", + "legendFormat": "{{group}}", + "instant": true + } + ], + "gridPos": {"h": 6, "w": 12, "x": 0, "y": 0} + }, + { + "id": 2, + "type": "stat", + "title": "Active (firing) series by group", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "anomaly_active_series{group=~\"$group\"}", + "legendFormat": "{{group}}", + "instant": true + } + ], + "gridPos": {"h": 6, "w": 12, "x": 12, "y": 0} + }, + { + "id": 3, + "type": "timeseries", + "title": "Anomaly density over time", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}, + "targets": [ + { + "expr": "anomaly_density{group=~\"$group\", detector=\"\", by=\"\"}", + "legendFormat": "{{group}}" + } + ], + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 6} + }, + { + "id": 4, + "type": "timeseries", + "title": "Severity-weighted density (how bad, not just how many)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}, + "targets": [ + { + "expr": "anomaly_severity_density{group=~\"$group\", by=\"\"}", + "legendFormat": "{{group}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14} + }, + { + "id": 5, + "type": "timeseries", + "title": "Density by slice (defaults.density_by)", + "description": "Per-slice partition — e.g. by namespace. Only present when defaults.density_by is configured.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}}, + "targets": [ + { + "expr": "anomaly_density{group=~\"$group\", by!=\"\"}", + "legendFormat": "{{group}} {{by}}={{namespace}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14} + }, + { + "id": 6, + "type": "table", + "title": "Most severe firings right now", + "description": "Mirrors `promanomaly top` — the currently-firing series ranked by severity.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "topk(20, anomaly_severity{group=~\"$group\"} > 0)", + "instant": true, + "format": "table" + } + ], + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 22} + } + ] +} diff --git a/dashboards/grafana/promanomaly-node-overlay.json b/dashboards/grafana/promanomaly-node-overlay.json new file mode 100644 index 0000000..c524c81 --- /dev/null +++ b/dashboards/grafana/promanomaly-node-overlay.json @@ -0,0 +1,160 @@ +{ + "annotations": { + "list": [ + { + "name": "change-points", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "enable": true, + "iconColor": "red", + "expr": "increase(anomaly_change_point_total{group=~\"node_.*\", instance=~\"$instance\"}[5m]) > 0", + "titleFormat": "change-point {{id}}", + "tagKeys": "detector,id" + } + ] + }, + "editable": true, + "schemaVersion": 38, + "title": "promanomaly node overlay", + "tags": ["promanomaly", "anomaly-detection", "node-exporter"], + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "label": "Datasource", + "query": "prometheus", + "current": {"text": "VictoriaMetrics", "value": "victoriametrics"} + }, + { + "name": "group", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "query": "label_values(anomaly_score{group=~\"node_.*\"}, group)", + "refresh": 2, + "multi": true, + "includeAll": true + }, + { + "name": "instance", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "query": "label_values(anomaly_score{group=~\"$group\"}, instance)", + "refresh": 2, + "multi": true, + "includeAll": true + } + ] + }, + "panels": [ + { + "id": 1, + "type": "table", + "title": "Most anomalous hosts right now", + "description": "Top hosts ranked by the normalised 0-1 severity signal. The same ranking promanomaly top surfaces from the CLI.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "topk(10, max by (instance, id, group, detector) (anomaly_severity{group=~\"$group\", instance=~\"$instance\"}))", + "instant": true, + "format": "table" + } + ], + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 0} + }, + { + "id": 2, + "type": "timeseries", + "title": "CPU busy — actual + baseline (firing shaded)", + "description": "Raw signal overlaid with anomaly_baseline; anomaly_outside_threshold==1 regions and change-point annotations mark where the detector flagged the host.", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$instance\"}[5m]))", + "legendFormat": "actual {{instance}}" + }, + { + "expr": "anomaly_baseline{group=\"node_cpu\", id=\"node_cpu_busy\", detector=\"MAD\", instance=~\"$instance\"}", + "legendFormat": "baseline {{instance}}" + }, + { + "expr": "anomaly_outside_threshold{group=\"node_cpu\", id=\"node_cpu_busy\", instance=~\"$instance\"}", + "legendFormat": "firing {{instance}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8} + }, + { + "id": 3, + "type": "timeseries", + "title": "Memory available fraction — actual + baseline", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes{instance=~\"$instance\"} / node_memory_MemTotal_bytes{instance=~\"$instance\"}", + "legendFormat": "actual {{instance}}" + }, + { + "expr": "anomaly_baseline{group=\"node_memory\", id=\"node_memory_available_fraction\", instance=~\"$instance\"}", + "legendFormat": "baseline {{instance}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8} + }, + { + "id": 4, + "type": "timeseries", + "title": "Filesystem used fraction — actual + baseline (firing shaded)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "anomaly_baseline{group=\"node_filesystem\", id=\"node_filesystem_used_fraction\", detector=\"MAD\", instance=~\"$instance\"}", + "legendFormat": "baseline {{instance}}" + }, + { + "expr": "anomaly_outside_threshold{group=\"node_filesystem\", id=\"node_filesystem_used_fraction\", instance=~\"$instance\"}", + "legendFormat": "firing {{instance}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16} + }, + { + "id": 5, + "type": "timeseries", + "title": "Disk I/O saturation — score", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "anomaly_score{group=\"node_disk_io\", id=\"node_disk_io_saturation\", instance=~\"$instance\"}", + "legendFormat": "score {{instance}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16} + }, + { + "id": 6, + "type": "timeseries", + "title": "Network errors/drops — score", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "anomaly_score{group=\"node_network\", id=\"node_network_errs_drops\", instance=~\"$instance\"}", + "legendFormat": "score {{instance}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 24} + }, + { + "id": 7, + "type": "timeseries", + "title": "Load per core — score (HourOfDayMAD)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "targets": [ + { + "expr": "anomaly_score{group=\"node_load\", id=\"node_load1_per_core\", detector=\"HourOfDayMAD\", instance=~\"$instance\"}", + "legendFormat": "score {{instance}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 24} + } + ] +} diff --git a/detector/src/promanomaly/anomalies.py b/detector/src/promanomaly/anomalies.py new file mode 100644 index 0000000..1df4753 --- /dev/null +++ b/detector/src/promanomaly/anomalies.py @@ -0,0 +1,142 @@ +"""Live triage — assemble the currently-firing series, ranked by severity. + +Powers the ``/debug/anomalies`` endpoint and the ``promanomaly top`` CLI. +The on-call question is "what is anomalous right now, ranked?" — answered +without opening Grafana or hand-writing PromQL, the same way ``top`` +answers "what is eating this host right now". + +The data is read straight from the in-memory snapshot store — the same +samples ``/metrics`` is serving — so there are no extra TSDB queries and +the answer is exactly consistent with what a scrape would show. A firing +is one ``(id, group, detector, source-labels)`` series whose +``anomaly_outside_threshold == 1``; its score / severity / duration / +type are joined from the sibling samples sharing the same label set. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: # pragma: no cover - import only for type hints + from .state import SnapshotStore + +# Canonical labels stamped by the detector; everything else on a sample +# is a source label echoed from the query. +_CANONICAL = frozenset({"id", "group", "detector", "detector_instance"}) + + +@dataclass(frozen=True) +class FiringSeries: + """One currently-firing detector series, with its triage fields.""" + + id: str + group: str + detector: str + detector_instance: str | None + score: float + severity: float + duration_seconds: float + type: str | None + labels: dict[str, str] + + def as_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "group": self.group, + "detector": self.detector, + "detector_instance": self.detector_instance, + "score": self.score, + "severity": self.severity, + "duration_seconds": self.duration_seconds, + "type": self.type, + "labels": dict(self.labels), + } + + +def _detector_key(labels: dict[str, str]) -> tuple[tuple[str, str], ...]: + """Full per-detector identity (includes detector + detector_instance).""" + return tuple(sorted(labels.items())) + + +def collect_firing( + store: SnapshotStore, + *, + group: str | None = None, + min_severity: float = 0.0, + limit: int | None = None, +) -> list[FiringSeries]: + """Return currently-firing series across all groups, ranked by severity. + + Args: + store: the live snapshot store. + group: restrict to one group when set. + min_severity: drop firings below this severity (0-1). + limit: keep at most this many after ranking. + + Ranking is by severity descending, then score descending, then + ``(group, id, detector)`` for a stable order so two scrapes with + equal severities don't shuffle the table. + """ + # Index sibling samples by full per-detector label key so we can join + # score / severity / duration / type onto each firing. + scores: dict[tuple[tuple[str, str], ...], float] = {} + severities: dict[tuple[tuple[str, str], ...], float] = {} + durations: dict[tuple[tuple[str, str], ...], float] = {} + types: dict[tuple[tuple[str, str], ...], str] = {} + firing_keys: list[tuple[tuple[str, str], ...]] = [] + firing_labels: dict[tuple[tuple[str, str], ...], dict[str, str]] = {} + + for snap in store.all_snapshots(): + for sample in snap.samples: + labels = dict(sample.labels) + if group is not None and labels.get("group") != group: + continue + metric = sample.metric + if metric == "anomaly_outside_threshold": + if sample.value >= 1.0: + key = _detector_key(labels) + firing_keys.append(key) + firing_labels[key] = labels + elif metric == "anomaly_score": + scores[_detector_key(labels)] = sample.value + elif metric == "anomaly_severity": + severities[_detector_key(labels)] = sample.value + elif metric == "anomaly_duration_seconds": + durations[_detector_key(labels)] = sample.value + elif metric == "anomaly_type_score": + # The type metric carries an extra ``type`` label; strip it + # so the key joins onto the per-detector series. + anomaly_kind = labels.get("type") + joinable = {k: v for k, v in labels.items() if k != "type"} + if anomaly_kind is not None: + types[_detector_key(joinable)] = anomaly_kind + + results: list[FiringSeries] = [] + for key in firing_keys: + labels = firing_labels[key] + severity = severities.get(key, 0.0) + if severity < min_severity: + continue + source_labels = {k: v for k, v in labels.items() if k not in _CANONICAL} + results.append( + FiringSeries( + id=labels.get("id", ""), + group=labels.get("group", ""), + detector=labels.get("detector", ""), + detector_instance=labels.get("detector_instance"), + score=scores.get(key, 0.0), + severity=severity, + duration_seconds=durations.get(key, 0.0), + type=types.get(key), + labels=source_labels, + ) + ) + + results.sort(key=lambda f: (-f.severity, -f.score, f.group, f.id, f.detector)) + if limit is not None: + results = results[:limit] + return results + + +__all__ = ["FiringSeries", "collect_firing"] diff --git a/detector/src/promanomaly/anomaly_type.py b/detector/src/promanomaly/anomaly_type.py new file mode 100644 index 0000000..cd5be15 --- /dev/null +++ b/detector/src/promanomaly/anomaly_type.py @@ -0,0 +1,183 @@ +"""Anomaly typing — classify *what kind* of anomaly is firing. + +When a series fires, on-call wants to know whether it is a point spike, +a step up/down, a slow level drift, a variance/distribution shift, or a +cohort divergence — so they can route, runbook, and filter without first +opening the signal. The detectors already compute most of this internally +(CUSUM direction, the cohort delta, the deviation shape inside the +rolling window); this module surfaces it as a frozen, public value-set. + +The ``type`` value-set is **public API**, enumerated in +``docs/detectors.md`` and treated like a metric name: adding a value is +non-breaking, renaming or removing one is breaking. The classifier is +deterministic — same window in, same type out — so the +``anomaly_type_score`` gauge and the triage endpoint never disagree. + +The classification is heuristic by design: it reads the rolling window +the detector already scored against, so it adds no extra TSDB load and +no per-detector protocol changes. Detectors that want to override the +heuristic can populate an ``anomaly_type`` column in their score row and +it is used verbatim (after validation against the frozen set). +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +# Frozen, public value-set. See module docstring for the stability +# contract. ``shape_discord`` is reserved for the future shape/discord +# detectors and is listed here so the enum is stable when they land; +# nothing emits it today. +ANOMALY_TYPES: frozenset[str] = frozenset( + { + "point_spike", + "step_up", + "step_down", + "level_drift", + "variance_shift", + "cohort_divergence", + "shape_discord", + } +) + +# Tuning constants for the window heuristic. Expressed in robust-sigma +# (MAD-scaled) units so they are detector-agnostic. +_STEP_LEVEL_SIGMA: float = 2.0 # half-to-half median shift that reads as a step +# Fraction of the half-to-half delta that one consecutive jump must +# carry for the shift to read as an abrupt step rather than a drift. +_STEP_JUMP_FRACTION: float = 0.5 +_VARIANCE_RATIO: float = 2.5 # tail-spread / head-spread that reads as variance shift +_DRIFT_MONOTONIC_FRACTION: float = 0.7 # |rank-correlation| above this reads as drift +_MAD_TO_SIGMA: float = 1.4826 # MAD -> Gaussian sigma scale factor + + +def _robust_scale(values: np.ndarray[Any, Any]) -> float: + """MAD-derived sigma, falling back to std then to 1.0 on a flatline.""" + median = float(np.median(values)) + mad = float(np.median(np.abs(values - median))) + if mad > 0.0: + return mad * _MAD_TO_SIGMA + std = float(np.std(values)) + return std if std > 0.0 else 1.0 + + +def _monotonic_fraction(values: np.ndarray[Any, Any]) -> float: + """|Spearman-style rank correlation with the index|, in ``[0, 1]``. + + A clean monotonic drift correlates near ±1 with the time index; a + one-off step or spike does not. Computed on ranks so a single outlier + can't dominate the slope the way a raw-value regression would. + """ + n = values.size + if n < 3: + return 0.0 + index_ranks = np.arange(n, dtype=float) + order = np.argsort(values, kind="stable") + value_ranks = np.empty(n, dtype=float) + value_ranks[order] = np.arange(n, dtype=float) + # Pearson correlation of the two rank vectors == Spearman's rho. + idx_c = index_ranks - index_ranks.mean() + val_c = value_ranks - value_ranks.mean() + denom = float(np.sqrt(np.sum(idx_c**2) * np.sum(val_c**2))) + if denom == 0.0: + return 0.0 + return abs(float(np.sum(idx_c * val_c) / denom)) + + +def classify_window( + *, + detector_name: str, + values: np.ndarray[Any, Any], + baseline: float, + latest_y: float, + change_point_fired: bool, + cohort_aware: bool, +) -> str: + """Return the anomaly type for a firing series from its rolling window. + + Priority order: + + 1. Cohort detectors → ``cohort_divergence`` (the anomaly is + cross-member, not intra-series). + 2. A change-point detector that fired → directional step + (``step_up`` / ``step_down``) by the sign of the latest value vs. + its baseline. + 3. Otherwise read the window shape: a sustained half-to-half level + shift reads as a step (or, if monotonic across the whole window, + ``level_drift``); an elevated recent spread with a stable level + reads as ``variance_shift``; a lone deviating latest sample reads + as ``point_spike`` (the default). + """ + if cohort_aware: + return "cohort_divergence" + if change_point_fired: + return "step_up" if latest_y >= baseline else "step_down" + + n = int(values.size) + if n < 4: + return "point_spike" + + mid = n // 2 + head = values[:mid] + tail = values[mid:] + head_med = float(np.median(head)) + tail_med = float(np.median(tail)) + raw_level_delta = abs(tail_med - head_med) + + head_spread = float(np.median(np.abs(head - head_med))) + tail_spread = float(np.median(np.abs(tail - tail_med))) + + # Scale by the *within-half* noise, not the whole-window spread: a + # step (or a strong drift) inflates the global MAD enough to mask + # itself, so a global scale would shrink level_delta below the + # threshold and the shift would never register. + noise_mad = max(head_spread, tail_spread) + scale = noise_mad * _MAD_TO_SIGMA if noise_mad > 0.0 else _robust_scale(values) + level_delta = raw_level_delta / scale + + if level_delta >= _STEP_LEVEL_SIGMA: + # A sustained shift between halves. Distinguish an abrupt step + # from a gradual drift by where the movement happens: a step + # concentrates it in one consecutive jump, while a drift spreads + # it evenly across the window. The largest single sample-to-sample + # jump as a fraction of the half-to-half delta separates them + # cleanly — a rank-correlation test can't, because a clean step + # is also perfectly monotonic in rank. + diffs = np.abs(np.diff(values)) + max_jump = float(diffs.max()) if diffs.size else 0.0 + if max_jump >= _STEP_JUMP_FRACTION * raw_level_delta: + return "step_up" if tail_med >= head_med else "step_down" + return "level_drift" + + # Variance shift: comparable level, materially wider recent spread. + if head_spread > 0.0 and (tail_spread / head_spread) >= _VARIANCE_RATIO: + return "variance_shift" + + # A monotonic crawl that never reached a full half-to-half step is + # still a drift if the trend is strong. + if _monotonic_fraction(values) >= _DRIFT_MONOTONIC_FRACTION: + return "level_drift" + + return "point_spike" + + +def coerce_declared_type(value: Any) -> str | None: + """Validate a detector-declared ``anomaly_type`` against the frozen set. + + Returns the value when it is a recognised type, else ``None`` so the + caller falls back to the window heuristic. Keeps a typo in a + third-party detector from leaking an unbounded ``type`` label value + onto the monitoring plane. + """ + if isinstance(value, str) and value in ANOMALY_TYPES: + return value + return None + + +__all__ = [ + "ANOMALY_TYPES", + "classify_window", + "coerce_declared_type", +] diff --git a/detector/src/promanomaly/cli/__init__.py b/detector/src/promanomaly/cli/__init__.py index d789f4e..9be6bcc 100644 --- a/detector/src/promanomaly/cli/__init__.py +++ b/detector/src/promanomaly/cli/__init__.py @@ -12,6 +12,8 @@ stratified detector + bucket width for a given signal. - ``inspect`` — query a running detector's /debug/inspect for one (id, label) series. +- ``top`` — query a running detector's /debug/anomalies for the + currently-firing series, ranked by severity. - ``detectors list`` — dump the entry-point registry. Running without a subcommand boots the server (used by the container's @@ -72,6 +74,7 @@ from ._probe import ( stratified_summary_for_query as _stratified_summary_for_query, ) +from ._top import _print_top_text @click.group(invoke_without_command=True, context_settings={"help_option_names": ["-h", "--help"]}) @@ -669,6 +672,88 @@ def inspect_cmd( _print_inspect_text(payload) +@cli.command(name="top") +@click.option( + "--target", + "target_url", + required=True, + help="Running detector base URL (e.g. http://localhost:9092).", +) +@click.option( + "--group", + "group", + default=None, + help="Restrict to one group.", +) +@click.option( + "--min-severity", + "min_severity", + default=0.0, + show_default=True, + type=float, + help="Drop firings below this severity (0-1).", +) +@click.option( + "--limit", + "limit", + default=20, + show_default=True, + type=int, + help="Show at most this many series (ranked by severity).", +) +@click.option( + "--output", + type=click.Choice(["text", "json"]), + default="text", + show_default=True, + help="Output format. 'json' is the raw /debug/anomalies response.", +) +@click.option( + "--timeout", + default=10.0, + show_default=True, + help="HTTP timeout.", +) +def top_cmd( + target_url: str, + group: str | None, + min_severity: float, + limit: int, + output: str, + timeout: float, +) -> None: + """List what is anomalous right now, ranked by severity. + + Wraps a running detector's /debug/anomalies endpoint — like ``top`` + on a host, but for the fleet. Pairs with ``inspect`` as a two-step + triage flow: ``top`` to find the worst series, ``inspect`` to + understand *why* it scored. + """ + import httpx + + params: dict[str, str] = {"min_severity": str(min_severity), "limit": str(limit)} + if group: + params["group"] = group + url = target_url.rstrip("/") + "/debug/anomalies" + try: + resp = httpx.get(url, params=params, timeout=timeout) + except httpx.HTTPError as exc: + click.echo(f"could not reach {target_url}: {exc}", err=True) + sys.exit(2) + try: + payload = resp.json() + except ValueError: + click.echo(f"non-JSON response from {url}: {resp.text[:200]}", err=True) + sys.exit(2) + if resp.status_code >= 400: + click.echo(json.dumps(payload, indent=2, sort_keys=True), err=True) + sys.exit(1) + if output == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + return + _print_top_text(payload) + + # Register the backtest command imported from promanomaly.backtest so # operators can invoke it as ``promanomaly backtest`` rather than the # longer ``python -m promanomaly.backtest`` (which still works because @@ -724,6 +809,7 @@ def main() -> None: "_print_analyze_text", "_print_calibrate_text", "_print_inspect_text", + "_print_top_text", "_probe_discover_query", "_recommend_detector", "_round_to_dividing_bucket", diff --git a/detector/src/promanomaly/cli/_top.py b/detector/src/promanomaly/cli/_top.py new file mode 100644 index 0000000..a5f5bb0 --- /dev/null +++ b/detector/src/promanomaly/cli/_top.py @@ -0,0 +1,56 @@ +"""Text rendering for the ``top`` subcommand.""" + +from __future__ import annotations + +from typing import Any + +import click + + +def _format_duration(seconds: float) -> str: + """Compact ``1h2m`` / ``3m`` / ``45s`` duration for the table.""" + total = int(seconds) + if total <= 0: + return "-" + hours, rem = divmod(total, 3600) + minutes, secs = divmod(rem, 60) + parts: list[str] = [] + if hours: + parts.append(f"{hours}h") + if minutes: + parts.append(f"{minutes}m") + if secs and not hours: + parts.append(f"{secs}s") + return "".join(parts) or "0s" + + +def _print_top_text(payload: dict[str, Any]) -> None: + anomalies = payload.get("anomalies", []) + if not anomalies: + click.echo("no firing anomalies") + return + header = f"{'SEV':>5} {'SCORE':>8} {'DUR':>6} {'TYPE':<16} {'GROUP':<16} ID / LABELS" + click.echo(header) + click.echo("-" * len(header)) + for a in anomalies: + labels = a.get("labels") or {} + labels_repr = ",".join(f"{k}={v}" for k, v in sorted(labels.items())) + detector = a.get("detector", "") + inst = a.get("detector_instance") + det_repr = f"{detector}[{inst}]" if inst else detector + id_repr = a.get("id", "") + if det_repr: + id_repr = f"{id_repr} ({det_repr})" + if labels_repr: + id_repr = f"{id_repr} {{{labels_repr}}}" + click.echo( + f"{a.get('severity', 0.0):>5.2f} " + f"{a.get('score', 0.0):>8.3f} " + f"{_format_duration(a.get('duration_seconds', 0.0)):>6} " + f"{(a.get('type') or '-'):<16} " + f"{a.get('group', ''):<16} " + f"{id_repr}" + ) + + +__all__ = ["_print_top_text"] diff --git a/detector/src/promanomaly/config.py b/detector/src/promanomaly/config.py index 399ec00..e4a48fc 100644 --- a/detector/src/promanomaly/config.py +++ b/detector/src/promanomaly/config.py @@ -25,7 +25,7 @@ model_validator, ) -# Graduated from ``v1alpha1`` to ``v1`` in the v1.0 release. The previous +# Graduated from ``v1alpha1`` to ``v1`` in the stable release. The previous # alpha alias remains valid (byte-identical schema) but loads with a # ``DeprecationWarning`` so operators get a nudge to bump their configs # without their pods CrashLoopBackOff'ing on the next image pull. A future @@ -207,7 +207,7 @@ class QueryCacheConfig(_ModelBase): LRU. ``backend: redis`` plugs into the same call site but stores entries in the configured Redis (set ``safety.redis.url``) so a leader and its followers share a single TSDB hit per (promql, - window) bucket — the HA-mode variant from v0.4. + window) bucket — the HA-mode variant. """ enabled: bool = True @@ -322,6 +322,49 @@ class DefaultsConfig(_ModelBase): emit_change_points: bool = True emit_baseline_stability: bool = True emit_duration: bool = True + # Operator-experience signals. + # + # ``emit_severity`` (default ``true``) emits the normalised 0-1 + # ``anomaly_severity`` gauge per detector series — a single + # operator-facing "how much should I care" number that folds + # statistical strength, calibration confidence, baseline stability, + # sustained duration, and practical-significance into one ranked + # value. The formula and fixed weights live in docs/severity.md and + # are deliberately not user-tunable (operators who want a different + # weighting compose via recording rules). + emit_severity: bool = True + # ``emit_anomaly_type`` (default ``false`` — opt-in to cardinality) + # emits ``anomaly_type_score{...,type}`` for firing series, exposing + # the detector's classification of the current anomaly (point spike, + # step up/down, level drift, variance shift, change-point, cohort + # divergence). A brand-new metric with a frozen ``type`` value-set; + # never a new label silently added to the existing per-detector + # series. Off by default because the ``type`` dimension multiplies + # the firing-series cardinality. + emit_anomaly_type: bool = False + # ``density_by`` lists the source labels the fleet anomaly-density + # rollup partitions by (e.g. ``["namespace"]``) so operators can + # answer "which slice of the fleet is unhealthy" without per-series + # cardinality. Empty (default) emits only the group-level rollup. + # Each listed label adds one ``anomaly_density{group, by=