Self-observability and production operations

README.md

@@ -79,7 +79,14 @@ monitoring, deploy-time silencing, cross-tool joins with promforecast).
 [`docs/when-to-use.md`](docs/when-to-use.md) walks through the
 "promforecast, promanomaly, or both?" decision tree, and
 [`docs/config-schema.md`](docs/config-schema.md) documents the v1
-schema stability commitments.
+schema stability commitments. Before going to production, work through
+the [`docs/production-checklist.md`](docs/production-checklist.md)
+sizing guide and readiness checklist, and keep the
+[degraded-mode playbook](docs/operations/degraded-modes.md) handy for
+on-call. Running many clusters? The
+[multi-cluster reference architectures](docs/architecture/multi-cluster.md)
+spell out the three topologies and their cardinality / network /
+blast-radius trade-offs.
 
 A self-monitoring example config is bundled at
 [`examples/configs/self-monitoring.yaml`](examples/configs/self-monitoring.yaml),

charts/promanomaly/ci/all-features-values.yaml

@@ -6,6 +6,15 @@ serviceMonitor:
   enabled: true
   interval: 30s
 
+# End-to-end detection self-test (dead-man's-switch): exercises the
+# server.selftest configmap branch and keeps the AnomalyPipelineDead
+# rule wired under the matrix render.
+server:
+  selftest:
+    enabled: true
+    detector: MAD
+    threshold: 3.0
+
 prometheusRule:
   enabled: true
   labels:
@@ -28,7 +37,7 @@ defaults:
   min_abs_delta: 0.0
   min_relative_delta: 0.05
 
-# Output-label filtering: covers the v0.4 exporter.output_labels
+# Output-label filtering: covers the exporter.output_labels
 # surface so the configmap.yaml conditional block renders under the
 # CI matrix. ``allow`` keeps an explicit set of instance/region labels
 # on the wire; ``drop`` strips a privacy-sensitive customer id even

charts/promanomaly/templates/configmap.yaml

@@ -22,6 +22,15 @@ data:
     server:
       listen: {{ .Values.server.listen | quote }}
       refresh_interval: {{ .Values.server.refresh_interval | quote }}
+      {{- with .Values.server.expose_warmup_endpoint }}
+      expose_warmup_endpoint: {{ . }}
+      {{- end }}
+      {{- if .Values.server.selftest.enabled }}
+      selftest:
+        enabled: true
+        detector: {{ .Values.server.selftest.detector | quote }}
+        threshold: {{ .Values.server.selftest.threshold }}
+      {{- end }}
       reload:
         enabled: {{ .Values.server.reload.enabled }}
         watch_configmap: {{ .Values.server.reload.watch_configmap }}

charts/promanomaly/templates/networkpolicy.yaml

@@ -11,9 +11,10 @@ spec:
     - Ingress
     - Egress
   ingress:
-    # Allow /metrics scraping. The detector serves /-/reload and the
-    # debug endpoints from the same pod & port, so this rule
-    # deliberately allows the *namespace*, not the world.
+    # Allow /metrics scraping. The detector serves /-/reload, the debug
+    # endpoints, and the opt-in /warmup endpoint from the same pod &
+    # port, so this rule deliberately allows the *namespace*, not the
+    # world.
     - from:
         - podSelector: {}
         {{- range .Values.networkPolicy.allowedScrapeNamespaces }}

charts/promanomaly/templates/prometheusrule.yaml

@@ -211,4 +211,54 @@ spec:
               fail_ready policy is configured.
         {{- end }}
         {{- end }}
+        {{- with $rules.anomalyDetectorDegraded }}
+        {{- if .enabled }}
+        - alert: AnomalyDetectorDegraded
+          expr: anomaly_detect_success_ratio{window="{{ .window }}"} < {{ .threshold }}
+          for: {{ .for }}
+          labels:
+            severity: {{ .severity }}
+          annotations:
+            summary: "Detector {{ "{{" }} $labels.detector {{ "}}" }} degraded in {{ "{{" }} $labels.group {{ "}}" }}"
+            description: |
+              anomaly_detect_success_ratio for detector
+              {{ "{{" }} $labels.detector {{ "}}" }} in group {{ "{{" }} $labels.group {{ "}}" }}
+              has been below {{ .threshold }} over the {{ .window }} window —
+              the detector is failing or timing out on too many series and
+              its scores are missing.
+        {{- end }}
+        {{- end }}
+        {{- with $rules.anomalySnapshotStale }}
+        {{- if .enabled }}
+        - alert: AnomalySnapshotStale
+          expr: anomaly_snapshot_age_seconds > {{ .maxAgeSeconds }}
+          for: {{ .for }}
+          labels:
+            severity: {{ .severity }}
+          annotations:
+            summary: "promanomaly snapshot stale for {{ "{{" }} $labels.group {{ "}}" }}"
+            description: |
+              The most recent snapshot for group {{ "{{" }} $labels.group {{ "}}" }}
+              is older than {{ .maxAgeSeconds }}s. /metrics is serving stale
+              scores; cross-check AnomalySourceFailing and AnomalyStale.
+        {{- end }}
+        {{- end }}
+        {{- with $rules.anomalyPipelineDead }}
+        {{- if .enabled }}
+        - alert: AnomalyPipelineDead
+          expr: anomaly_selftest_ok == 0
+          for: {{ .for }}
+          labels:
+            severity: {{ .severity }}
+          annotations:
+            summary: "promanomaly detection pipeline is dead (self-test failing)"
+            description: |
+              The end-to-end self-test for detector
+              {{ "{{" }} $labels.detector {{ "}}" }} has failed to catch its injected
+              synthetic anomaly for over {{ .for }}. The detect → threshold → export
+              pipeline is not surfacing anomalies even though the process is up —
+              check for a config/threshold mistake or an exporter regression.
+              See anomaly_selftest_failures_total. Requires server.selftest.enabled.
+        {{- end }}
+        {{- end }}
 {{- end }}

charts/promanomaly/values.schema.json

@@ -53,7 +53,16 @@
       "type": "object",
       "properties": {
         "listen": {"type": "string"},
-        "refresh_interval": {"type": "string"}
+        "refresh_interval": {"type": "string"},
+        "expose_warmup_endpoint": {"type": "boolean"},
+        "selftest": {
+          "type": "object",
+          "properties": {
+            "enabled": {"type": "boolean"},
+            "detector": {"type": "string"},
+            "threshold": {"type": "number", "exclusiveMinimum": 0}
+          }
+        }
       }
     },
     "safety": {

charts/promanomaly/values.yaml

@@ -45,6 +45,20 @@ server:
     auth:
       type: none       # none | bearer | mtls
       existingSecret: ""
+  # Opt-in GET /warmup endpoint (off by default). When true the
+  # detector reports per-(group, query) warm-up status; the shipped
+  # NetworkPolicy keeps it in-namespace alongside /debug/*.
+  expose_warmup_endpoint: false
+  # End-to-end detection self-test (dead-man's-switch). Off by default.
+  # When enabled, each run drives a synthetic series with a known
+  # injected anomaly through the real detect/threshold/export path and
+  # emits anomaly_selftest_ok (1/0); the AnomalyPipelineDead alert fires
+  # when it stops passing. Proves the detection path itself is live, not
+  # merely that the process is up.
+  selftest:
+    enabled: false
+    detector: MAD
+    threshold: 3.0
 
 safety:
   max_series_per_query: 1000
@@ -296,3 +310,34 @@ prometheusRule:
       threshold: 0.1
       for: 5m
       severity: warning
+    # Self-observability: the detector's own health.
+    # AnomalyDetectorDegraded fires when a detector's rolling compute
+    # success ratio drops — it is silently missing scores before any
+    # downstream anomaly alert can fire.
+    anomalyDetectorDegraded:
+      enabled: true
+      threshold: 0.9
+      # Must be one of the emitted window labels: "1h" or "1d". The
+      # detector emits anomaly_detect_success_ratio only at these two
+      # windows, so any other value matches no series and the alert
+      # silently never fires.
+      window: 1h
+      for: 10m
+      severity: warning
+    # AnomalySnapshotStale fires when the served /metrics snapshot ages
+    # past maxAgeSeconds — distinct from AnomalyStale, which keys off the
+    # last successful-run timestamp.
+    anomalySnapshotStale:
+      enabled: true
+      maxAgeSeconds: 600
+      for: 5m
+      severity: warning
+    # AnomalyPipelineDead (dead-man's-switch) fires when the opt-in
+    # end-to-end self-test stops catching its injected synthetic anomaly —
+    # the whole detect/threshold/export pipeline is silently dead. The
+    # underlying anomaly_selftest_ok series only exists when
+    # server.selftest.enabled, so this rule is dormant otherwise.
+    anomalyPipelineDead:
+      enabled: true
+      for: 5m
+      severity: critical

dashboards/grafana/promanomaly-self-observability.json

@@ -0,0 +1,120 @@
+{
+  "annotations": {"list": []},
+  "editable": true,
+  "schemaVersion": 38,
+  "title": "promanomaly self-observability",
+  "description": "Is the detector itself healthy? Per-detector compute success ratio, per-group CPU/memory attribution, and per-series/snapshot freshness. Pairs with the AnomalyDetectorDegraded and AnomalySnapshotStale alerts.",
+  "tags": ["promanomaly", "anomaly-detection", "self-observability"],
+  "templating": {
+    "list": [
+      {
+        "name": "datasource",
+        "type": "datasource",
+        "label": "Datasource",
+        "query": "prometheus",
+        "current": {"text": "VictoriaMetrics", "value": "victoriametrics"}
+      },
+      {
+        "name": "group",
+        "type": "query",
+        "datasource": {"type": "prometheus", "uid": "${datasource}"},
+        "query": "label_values(anomaly_snapshot_age_seconds, group)",
+        "refresh": 2,
+        "multi": true,
+        "includeAll": true
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "timeseries",
+      "title": "Detector compute success ratio (1h)",
+      "description": "Fraction of attempted detector computations that completed without raising, over the rolling 1h window. A drop below ~0.9 means the detector is timing out or erroring on a growing fraction of series — its scores are silently missing.",
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}},
+      "targets": [
+        {
+          "expr": "anomaly_detect_success_ratio{group=~\"$group\", window=\"1h\"}",
+          "legendFormat": "{{group}}/{{detector}}"
+        }
+      ],
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "Detector compute success ratio (1d)",
+      "description": "The slower 24h trend of the same signal — distinguishes a transient blip from a sustained regression.",
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}},
+      "targets": [
+        {
+          "expr": "anomaly_detect_success_ratio{group=~\"$group\", window=\"1d\"}",
+          "legendFormat": "{{group}}/{{detector}}"
+        }
+      ],
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Snapshot age by group",
+      "description": "Age of each group's most recent successful snapshot. Climbs between runs and resets on each successful run; a monotonic climb past refresh_interval means the group has stopped producing fresh snapshots (drives AnomalySnapshotStale).",
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "fieldConfig": {"defaults": {"unit": "s"}},
+      "targets": [
+        {
+          "expr": "anomaly_snapshot_age_seconds{group=~\"$group\"}",
+          "legendFormat": "{{group}}"
+        }
+      ],
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Series staleness by group",
+      "description": "Age of the freshest per-series score in each group. Diverges from snapshot age when a group keeps writing snapshots that carry no scores (all series warming up, or scores dropped under drop_scores).",
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "fieldConfig": {"defaults": {"unit": "s"}},
+      "targets": [
+        {
+          "expr": "anomaly_series_staleness_seconds{group=~\"$group\"}",
+          "legendFormat": "{{group}}"
+        }
+      ],
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Per-group CPU rate",
+      "description": "Detector compute time attributed to each group, as a per-second rate of the cumulative counter. Surfaces which group dominates the detection compute budget.",
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "fieldConfig": {"defaults": {"unit": "s"}},
+      "targets": [
+        {
+          "expr": "rate(anomaly_group_cpu_seconds_total{group=~\"$group\"}[5m])",
+          "legendFormat": "{{group}}"
+        }
+      ],
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Per-group snapshot memory",
+      "description": "Best-effort size of each group's output snapshot held by the exporter. Answers 'which group's snapshot is dominant' — not an exact process RSS breakdown.",
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "fieldConfig": {"defaults": {"unit": "bytes"}},
+      "targets": [
+        {
+          "expr": "anomaly_group_memory_bytes{group=~\"$group\"}",
+          "legendFormat": "{{group}}"
+        }
+      ],
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}
+    }
+  ]
+}