Horizontal scale-out and resilience

charts/promanomaly/templates/NOTES.txt

@@ -10,6 +10,19 @@ hot-reloads on change (validation failure rolls back automatically).
 
 Datasource:
   URL: {{ .Values.datasource.url }}
+{{- if .Values.scaleOut.enabled }}
+
+Scale-out (active/active sharding) is ENABLED:
+  - A StatefulSet of {{ .Values.scaleOut.shards }} shard(s) is running; each pod
+    scores a disjoint slice of the series (shard index from its ordinal).
+  - safety.max_total_series is a PER-SHARD cap, so the estate ceiling is
+    max_total_series x shards. Size it per shard, not for the whole estate.
+  - To resize, change scaleOut.shards (the StatefulSet replicas follow it);
+    do NOT ``kubectl scale`` the StatefulSet directly.
+  - Set max_series_per_query above your largest single query's cardinality:
+    a query truncated at that cap shards non-deterministically.
+  See docs/operations.md (Horizontal Scale-Out).
+{{- end }}
 
 Config schema: this chart renders ``apiVersion: promanomaly.io/v1``
 (stable since the v1.0 release). The legacy ``promanomaly.io/v1alpha1``

charts/promanomaly/templates/configmap.yaml

@@ -105,6 +105,9 @@ data:
       {{- with .Values.safety.fail_ready_after }}
       fail_ready_after: {{ . }}
       {{- end }}
+      {{- with .Values.safety.maxConcurrentGroups }}
+      max_concurrent_groups: {{ . }}
+      {{- end }}
       {{- with .Values.safety.query_cache }}
       query_cache:
         enabled: {{ .enabled }}
@@ -134,6 +137,19 @@ data:
       retry_period: {{ .Values.highAvailability.retry_period | quote }}
       snapshot_ttl: {{ .Values.highAvailability.snapshot_ttl | quote }}
     {{- end }}
+    {{- if .Values.scaleOut.enabled }}
+    scaleOut:
+      enabled: true
+      shards: {{ .Values.scaleOut.shards }}
+      {{- /*
+        shard_index is normally left unset: each replica self-assigns from
+        its StatefulSet pod ordinal (POD_NAME, injected via the Downward
+        API). Only rendered when an operator pins it explicitly.
+      */}}
+      {{- with .Values.scaleOut.shard_index }}
+      shard_index: {{ . }}
+      {{- end }}
+    {{- end }}
     defaults:
       window: {{ .Values.defaults.window | quote }}
       step: {{ .Values.defaults.step | quote }}

charts/promanomaly/templates/deployment.yaml

@@ -1,3 +1,9 @@
+{{- /*
+  Single-replica / active-passive HA topology renders a Deployment.
+  Active/active scale-out (``scaleOut.enabled``) renders a StatefulSet
+  instead (statefulset.yaml) so each replica gets a stable shard ordinal.
+*/ -}}
+{{- if not .Values.scaleOut.enabled -}}
 {{- /*
   Bearer-token Secret mounts for the active push sink(s). The scalar
   ``sink:`` form mounts a single Secret at /etc/promanomaly-sink-auth; the
@@ -151,3 +157,4 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- end }}

charts/promanomaly/templates/prometheusrule.yaml

@@ -261,4 +261,22 @@ spec:
               See anomaly_selftest_failures_total. Requires server.selftest.enabled.
         {{- end }}
         {{- end }}
+        {{- with $rules.anomalyGroupStarved }}
+        {{- if .enabled }}
+        - alert: AnomalyGroupStarved
+          expr: rate(anomaly_group_skipped_total[15m]) > 0
+          for: {{ .for }}
+          labels:
+            severity: {{ .severity }}
+          annotations:
+            summary: "promanomaly is shedding runs for {{ "{{" }} $labels.group {{ "}}" }} ({{ "{{" }} $labels.reason {{ "}}" }})"
+            description: |
+              Group {{ "{{" }} $labels.group {{ "}}" }} has had runs skipped
+              (reason={{ "{{" }} $labels.reason {{ "}}" }}) over the last 15 minutes.
+              The detector cannot keep every group on its cadence under the current
+              load. Scores for this group are delayed, not suppressed. Raise
+              safety.max_concurrent_groups, lengthen the group's refresh_interval,
+              or split its heavy queries.
+        {{- end }}
+        {{- end }}
 {{- end }}

charts/promanomaly/templates/rbac-ha.yaml

@@ -1,5 +1,6 @@
-{{- if .Values.highAvailability.enabled -}}
-# Lease-API permissions required by the HA leader-election loop.
+{{- if and .Values.highAvailability.enabled (not .Values.scaleOut.enabled) -}}
+# Lease-API permissions required by the HA leader-election loop. Never
+# emitted in scale-out mode (active/active sharding needs no Lease).
 # Only the get/list/watch/create/update verbs on coordination.k8s.io
 # Leases are required — no cluster-scoped permissions, scoped to a
 # single Lease resource by name. Renders only when HA is enabled so

charts/promanomaly/templates/statefulset.yaml

@@ -0,0 +1,177 @@
+{{- /*
+  Active/active scale-out (``scaleOut.enabled``) renders a StatefulSet so
+  every replica gets a stable ordinal (pod-0..pod-N). The detector reads
+  that ordinal from POD_NAME and self-assigns its shard index; each shard
+  scores a disjoint slice of the series via rendezvous hashing. The
+  Deployment (deployment.yaml) renders in every other topology. The pod
+  spec mirrors the Deployment's; the only differences are the StatefulSet
+  wrapper (serviceName, ordinal-derived replicas, parallel pod management)
+  and the absence of a rollout strategy.
+*/ -}}
+{{- if .Values.scaleOut.enabled -}}
+{{- /* Fail fast on the mutually-exclusive combination rather than letting
+  the pods CrashLoop at boot on the pydantic validation (and granting a
+  scale-out workload Lease RBAC it can never use). Mirrors the sink/sinks
+  fail guard in configmap.yaml. */ -}}
+{{- if .Values.highAvailability.enabled -}}
+{{- fail "scaleOut.enabled and highAvailability.enabled are mutually exclusive: scale-out is active/active sharding, HA is active/passive leader election. Enable exactly one." -}}
+{{- end -}}
+{{- /* Bearer-token Secret mounts for the active push sink(s) — identical
+  to deployment.yaml so the two topologies mount sinks the same way. */ -}}
+{{- $sinkMounts := list -}}
+{{- if .Values.sinks -}}
+  {{- range $sink := .Values.sinks -}}
+    {{- $secret := dig $sink.type "auth" "existingSecret" "" $sink -}}
+    {{- if $secret -}}
+      {{- $name := printf "sink-auth-%s" (replace "_" "-" $sink.type) -}}
+      {{- $sinkMounts = append $sinkMounts (dict "name" $name "secret" $secret "path" (include "promanomaly.sinkAuthDir" $sink.type)) -}}
+    {{- end -}}
+  {{- end -}}
+{{- else if .Values.sink.type -}}
+  {{- $scalarSecret := dig .Values.sink.type "auth" "existingSecret" "" .Values.sink -}}
+  {{- if $scalarSecret -}}
+    {{- $sinkMounts = append $sinkMounts (dict "name" "sink-auth" "secret" $scalarSecret "path" "/etc/promanomaly-sink-auth") -}}
+  {{- end -}}
+{{- end -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "promanomaly.fullname" . }}
+  labels: {{- include "promanomaly.labels" . | nindent 4 }}
+spec:
+  # One replica per shard; replicaCount is ignored in scale-out mode.
+  replicas: {{ .Values.scaleOut.shards }}
+  # Stable headless Service backing the pods' DNS identities.
+  serviceName: {{ include "promanomaly.fullname" . }}-headless
+  # Parallel: every shard owns a disjoint slice, so there is no ordering
+  # dependency between pods — start and reschedule them together.
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels: {{- include "promanomaly.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        checksum/config: {{ toYaml .Values | sha256sum }}
+      labels:
+        {{- include "promanomaly.selectorLabels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ include "promanomaly.serviceAccountName" . }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.securityContext | nindent 8 }}
+      containers:
+        - name: promanomaly
+          image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - "--config"
+            - "/etc/promanomaly/config.yaml"
+          env:
+            # Downward API: POD_NAME carries the StatefulSet ordinal, which
+            # the detector parses into its shard index at boot. No per-pod
+            # config — the ordinal self-assigns shards 0..N-1.
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          ports:
+            - name: http
+              containerPort: 9092
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          securityContext:
+            {{- toYaml .Values.containerSecurityContext | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/promanomaly
+              readOnly: true
+            {{- if .Values.datasource.auth.existingSecret }}
+            - name: datasource-auth
+              mountPath: /etc/promanomaly-datasource-auth
+              readOnly: true
+            {{- end }}
+            {{- range $sinkMounts }}
+            - name: {{ .name }}
+              mountPath: {{ .path }}
+              readOnly: true
+            {{- end }}
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "promanomaly.configMapName" . }}
+        {{- if .Values.datasource.auth.existingSecret }}
+        - name: datasource-auth
+          secret:
+            secretName: {{ .Values.datasource.auth.existingSecret | quote }}
+        {{- end }}
+        {{- range $sinkMounts }}
+        - name: {{ .name }}
+          secret:
+            secretName: {{ .secret | quote }}
+        {{- end }}
+        - name: tmp
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+---
+{{- /*
+  Headless Service required by the StatefulSet's ``serviceName``. The
+  detector derives its shard index from the POD_NAME ordinal alone (no
+  peer discovery), so this Service exists only to satisfy the StatefulSet
+  contract and give the pods stable DNS should anything ever need it.
+  Scraping goes through the regular Service (service.yaml), which selects
+  all shards' pods, so the ServiceMonitor collects every shard's
+  /metrics — the union covers the whole estate.
+*/}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "promanomaly.fullname" . }}-headless
+  labels: {{- include "promanomaly.labels" . | nindent 4 }}
+spec:
+  clusterIP: None
+  ports:
+    - name: http
+      port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+  selector: {{- include "promanomaly.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/promanomaly/values.schema.json

@@ -20,7 +20,7 @@
     "replicaCount": {
       "type": "integer",
       "minimum": 1,
-      "description": "Replica count. Single-replica mode requires 1; bump only when highAvailability.enabled=true."
+      "description": "Replica count. Single-replica mode requires 1; bump only when highAvailability.enabled=true. Ignored when scaleOut.enabled=true (the StatefulSet runs scaleOut.shards replicas)."
     },
     "highAvailability": {
       "type": "object",
@@ -34,6 +34,15 @@
         "snapshot_ttl": {"type": "string"}
       }
     },
+    "scaleOut": {
+      "type": "object",
+      "description": "Active/active horizontal sharding. Mutually exclusive with highAvailability; renders a StatefulSet of `shards` replicas, each scoring a disjoint slice of the series.",
+      "properties": {
+        "enabled": {"type": "boolean"},
+        "shards": {"type": "integer", "minimum": 1},
+        "shard_index": {"type": ["integer", "null"], "minimum": 0}
+      }
+    },
     "datasource": {
       "type": "object",
       "required": ["url"],
@@ -128,6 +137,7 @@
         "query_timeout": {"type": "string"},
         "on_source_failure": {"type": "string", "enum": ["serve_stale", "drop_scores", "fail_ready"]},
         "fail_ready_after": {"type": "integer", "minimum": 1},
+        "maxConcurrentGroups": {"type": ["integer", "null"], "minimum": 1},
         "max_stratified_cache_entries": {"type": "integer", "minimum": 1},
         "query_cache": {
           "type": "object",

charts/promanomaly/values.yaml

@@ -17,6 +17,8 @@ replicaCount: 1        # in single-replica mode this MUST stay at 1.
                        # to bump (typically to 2 or 3) and the chart
                        # automatically flips to a RollingUpdate strategy
                        # via the override below.
+                       # Ignored when scaleOut.enabled=true — the
+                       # StatefulSet runs exactly scaleOut.shards replicas.
 
 strategy:
   # Recreate is correct for the single-replica MVP. The chart's helper
@@ -105,6 +107,15 @@ safety:
   # Only honoured when on_source_failure: fail_ready. A single
   # transient TSDB blip must not take the pod out of the service.
   fail_ready_after: 3
+  # Priority-aware load shedding. Unset (null) leaves group scheduling
+  # unbounded — every group runs on its own cadence with no admission
+  # control. Set to an integer to cap how many groups run at once; under
+  # TSDB backpressure the scheduler then prefers higher-priority groups
+  # and sheds (skips) low-priority runs that can't get a slot within
+  # their refresh interval, surfacing them on
+  # anomaly_group_skipped_total{reason="backpressure"}. See
+  # docs/operations/degraded-modes.md.
+  maxConcurrentGroups: null
   # In-process LRU+TTL cache for overlapping PromQL responses. The
   # Redis-backed HA variant is selected with ``backend: redis`` and
   # requires the ``redis:`` block below. See docs/operations.md for
@@ -150,6 +161,24 @@ highAvailability:
   retry_period: 2s
   snapshot_ttl: 5m
 
+# Active/active horizontal scale-out. Orthogonal to (and mutually
+# exclusive with) highAvailability: HA is active/passive (one leader does
+# all detection), scale-out is active/active (``shards`` replicas each
+# score a disjoint slice of the series via rendezvous hashing and run
+# concurrently). When enabled the chart renders a StatefulSet instead of
+# the Deployment so each pod gets a stable ordinal — the shard index,
+# self-assigned from POD_NAME with no per-pod config — plus a headless
+# Service for stable pod DNS. ``replicaCount`` is ignored in this mode;
+# the StatefulSet runs exactly ``shards`` replicas. Remember that
+# ``safety.max_total_series`` becomes a PER-SHARD cap, so the estate
+# ceiling is ``max_total_series x shards`` (see docs/operations.md).
+scaleOut:
+  enabled: false
+  shards: 1
+  # Leave unset so each pod derives its index from its StatefulSet
+  # ordinal. Pin only for non-StatefulSet topologies.
+  shard_index: null
+
 defaults:
   window: 1h
   step: 15s
@@ -437,3 +466,11 @@ prometheusRule:
       enabled: true
       for: 5m
       severity: critical
+    # AnomalyGroupStarved fires when the priority-aware scheduler sheds
+    # group runs under sustained backpressure (safety.max_concurrent_groups)
+    # or a group overruns its refresh interval. A skip delays a score, it
+    # never suppresses a detected anomaly. Dormant unless shedding occurs.
+    anomalyGroupStarved:
+      enabled: true
+      for: 15m
+      severity: warning