esops-dev · dj-wasabi · May 30, 2026 · May 30, 2026
@@ -64,14 +64,32 @@ jobs:
           path: detector/coverage.xml
           if-no-files-found: ignore
 
+  validate-action:
+    name: Config-validation Action (dogfood)
+    # Exercises the composite action.yml on an example config so the
+    # published GitOps validation Action can't regress. No datasource is
+    # available in CI, so this runs the static checks only (schema +
+    # --estimate-cost --strict); --probe / --lint-metadata are covered by
+    # the unit tests, which stub the datasource.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Validate an example config via the action
+        uses: ./
+        with:
+          config: examples/configs/node-exporter.yaml
+          estimate-cost: "true"
+          strict: "true"
+
   community-prophet:
     name: Community ProphetResidual (lint + test with Prophet stubbed)
     # Prophet's real install pulls cmdstan / pystan and is multi-minute on
     # cold CI runners. The detector itself only exercises Prophet through
     # ``Prophet(...).fit(...).predict(...)``, so we lint + test the
     # community package against a tiny Prophet stub (tests/conftest.py)
     # without installing the real thing — the third-party plug-in path
-    # documented in the v0.4 roadmap.
+    # documented in the roadmap.
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -137,7 +155,7 @@ jobs:
       - name: ct lint
         run: |
           ct lint \
-            --charts charts/promanomaly,charts/promanomaly-stack \
+            --charts charts/promanomaly,charts/promanomaly-stack,charts/promanomaly-metrics-adapter \
             --validate-maintainers=false \
             --check-version-increment=false \
             --target-branch ${{ github.event.repository.default_branch }}
@@ -146,6 +164,7 @@ jobs:
         run: |
           helm lint --strict charts/promanomaly
           helm lint --strict charts/promanomaly-stack
+          helm lint --strict charts/promanomaly-metrics-adapter
 
       - name: Install kubeconform
         run: |
@@ -156,8 +175,9 @@ jobs:
 
       - name: kubeconform
         run: |
-          helm template charts/promanomaly       | kubeconform -strict -summary -ignore-missing-schemas
-          helm template charts/promanomaly-stack | kubeconform -strict -summary -ignore-missing-schemas
+          helm template charts/promanomaly                  | kubeconform -strict -summary -ignore-missing-schemas
+          helm template charts/promanomaly-stack            | kubeconform -strict -summary -ignore-missing-schemas
+          helm template charts/promanomaly-metrics-adapter  | kubeconform -strict -summary -ignore-missing-schemas
 
       - name: chart-testing values matrix (lint + render + kubeconform)
         # ct lint already exercises the ci/ values files, but we
@@ -170,7 +190,7 @@ jobs:
           set -euo pipefail
           shopt -s nullglob
           fail=0
-          for chart in charts/promanomaly charts/promanomaly-stack; do
+          for chart in charts/promanomaly charts/promanomaly-stack charts/promanomaly-metrics-adapter; do
             for vf in "$chart"/ci/*-values.yaml; do
               echo "::group::render $chart with $(basename "$vf")"
               if ! helm template "$chart" --values "$vf" \

@@ -85,16 +85,21 @@ chart-deps: ## Resolve umbrella subchart dependencies.
 chart-lint: chart-deps ## helm lint + kubeconform on rendered manifests.
 	helm lint $(CHART_DIR)/promanomaly
 	helm lint $(CHART_DIR)/promanomaly-stack
-	helm template $(CHART_DIR)/promanomaly       | kubeconform -strict -summary -ignore-missing-schemas
-	helm template $(CHART_DIR)/promanomaly-stack | kubeconform -strict -summary -ignore-missing-schemas
+	helm lint $(CHART_DIR)/promanomaly-metrics-adapter
+	helm template $(CHART_DIR)/promanomaly                  | kubeconform -strict -summary -ignore-missing-schemas
+	helm template $(CHART_DIR)/promanomaly-stack            | kubeconform -strict -summary -ignore-missing-schemas
+	helm template $(CHART_DIR)/promanomaly-metrics-adapter  | kubeconform -strict -summary -ignore-missing-schemas
 	@if command -v yamllint >/dev/null; then \
 	  yamllint -c .chart-testing-lintconf.yaml \
 	    $(CHART_DIR)/promanomaly/Chart.yaml \
 	    $(CHART_DIR)/promanomaly/values.yaml \
 	    $(CHART_DIR)/promanomaly/ci \
 	    $(CHART_DIR)/promanomaly-stack/Chart.yaml \
 	    $(CHART_DIR)/promanomaly-stack/values.yaml \
-	    $(CHART_DIR)/promanomaly-stack/ci; \
+	    $(CHART_DIR)/promanomaly-stack/ci \
+	    $(CHART_DIR)/promanomaly-metrics-adapter/Chart.yaml \
+	    $(CHART_DIR)/promanomaly-metrics-adapter/values.yaml \
+	    $(CHART_DIR)/promanomaly-metrics-adapter/ci; \
 	else \
 	  echo "yamllint not installed — skipping (ct lint will run it in CI)"; \
 	fi
@@ -104,7 +109,7 @@ chart-lint: chart-deps ## helm lint + kubeconform on rendered manifests.
 .PHONY: chart-matrix
 chart-matrix: chart-deps ## Render each chart with every ci/*-values.yaml + kubeconform.
 	@set -e; \
-	for chart in $(CHART_DIR)/promanomaly $(CHART_DIR)/promanomaly-stack; do \
+	for chart in $(CHART_DIR)/promanomaly $(CHART_DIR)/promanomaly-stack $(CHART_DIR)/promanomaly-metrics-adapter; do \
 	  for vf in $$chart/ci/*-values.yaml; do \
 	    echo "--- $$chart :: $$(basename $$vf) ---"; \
 	    helm template $$chart --values $$vf \

@@ -96,6 +96,29 @@ The full multi-group production reference deployment — values, NetworkPolicy
 egress, ArgoCD `Application`, and Flux `HelmRepository`/`HelmRelease` —
 lives at [`examples/production/`](examples/production/).
 
+## Feeding anomaly signal to the platform
+
+Anomaly scores are most useful when the rest of your stack can act on
+them. promanomaly stays the signal *provider*; the systems that react
+(autoscalers, rollout controllers, dashboards, long-term storage) stay
+the decision-makers.
+
+- **Push sinks** ship each snapshot beyond `/metrics`: a Prometheus
+  `remote_write` sink for long-horizon anomaly history, and a Grafana
+  annotations sink that surfaces firings across every dashboard. See
+  [`docs/sinks.md`](docs/sinks.md).
+- **Kubernetes metrics adapter** re-serves anomaly signal through the
+  external/custom metrics APIs so HPA and KEDA can consume it. Opt-in via
+  [`charts/promanomaly-metrics-adapter/`](charts/promanomaly-metrics-adapter/);
+  see [`docs/adapter.md`](docs/adapter.md).
+- **Reaction recipes** — KEDA, HPA, and Argo Rollouts cohort-gated
+  canaries, with the autoscaling guard-rails baked in — live at
+  [`examples/k8s/`](examples/k8s/).
+- **GitOps config validation** — a composite GitHub Action validates your
+  config on every PR (schema, cardinality/cost, optional live probe), and
+  `promanomaly generate-rules` scaffolds a PrometheusRule from it. See
+  [`docs/gitops.md`](docs/gitops.md).
+
 ## Relationship with promforecast
 
 | | promforecast | promanomaly |

@@ -0,0 +1,90 @@
+name: promanomaly config validation
+description: >-
+  Validate a promanomaly detector config on every PR — schema, optional live
+  probe, static cardinality/cost estimate, and counter-metadata lint — so a
+  broken or over-expensive config never reaches the cluster.
+branding:
+  icon: activity
+  color: purple
+
+inputs:
+  config:
+    description: Path to the promanomaly config file to validate.
+    required: true
+  probe:
+    description: >-
+      Execute every query against a live datasource and fail on empty/erroring
+      queries. Requires network access to the TSDB from the runner; usually a
+      mock or staging datasource in CI. Defaults to off.
+    required: false
+    default: "false"
+  estimate-cost:
+    description: >-
+      Statically project cardinality, TSDB query load, and a coarse CPU/memory
+      estimate from the config (no datasource access). On by default — it is
+      cheap and needs nothing external.
+    required: false
+    default: "true"
+  lint-metadata:
+    description: >-
+      Query the datasource's /api/v1/metadata and warn when a query feeds a raw
+      counter to a detector without rate()/increase(). Requires datasource
+      access. Defaults to off.
+    required: false
+    default: "false"
+  strict:
+    description: >-
+      Fail the job on a soft finding from any enabled check: empty/erroring
+      queries (probe), projected series over safety.max_total_series
+      (estimate-cost), or a counter-not-rated finding (lint-metadata). On by
+      default so the action gates a PR. Set to "false" to report only.
+    required: false
+    default: "true"
+  datasource-url:
+    description: >-
+      Override the datasource URL used by --probe / --lint-metadata, e.g. a
+      staging or mock TSDB. Leave empty to use the URL from the config.
+    required: false
+    default: ""
+  python-version:
+    description: Python version used to run the validator.
+    required: false
+    default: "3.12"
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install promanomaly
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        # Install the detector from this action's own checkout so the
+        # validator version always matches the action ref the caller pinned.
+        pip install "${GITHUB_ACTION_PATH}/detector"
+
+    - name: Validate config
+      shell: bash
+      env:
+        PROMANOMALY_CONFIG: ${{ inputs.config }}
+        PROMANOMALY_PROBE: ${{ inputs.probe }}
+        PROMANOMALY_ESTIMATE_COST: ${{ inputs.estimate-cost }}
+        PROMANOMALY_LINT_METADATA: ${{ inputs.lint-metadata }}
+        PROMANOMALY_STRICT: ${{ inputs.strict }}
+        PROMANOMALY_DATASOURCE_URL: ${{ inputs.datasource-url }}
+      run: |
+        set -euo pipefail
+        args=(validate --config "${PROMANOMALY_CONFIG}")
+        if [ "${PROMANOMALY_PROBE}" = "true" ]; then args+=(--probe); fi
+        if [ "${PROMANOMALY_ESTIMATE_COST}" = "true" ]; then args+=(--estimate-cost); fi
+        if [ "${PROMANOMALY_LINT_METADATA}" = "true" ]; then args+=(--lint-metadata); fi
+        if [ "${PROMANOMALY_STRICT}" = "true" ]; then args+=(--strict); fi
+        if [ -n "${PROMANOMALY_DATASOURCE_URL}" ]; then
+          args+=(--datasource-url "${PROMANOMALY_DATASOURCE_URL}")
+        fi
+        echo "+ promanomaly ${args[*]}"
+        promanomaly "${args[@]}"
@@ -0,0 +1,32 @@
+apiVersion: v2
+name: promanomaly-metrics-adapter
+description: |
+  Opt-in Kubernetes external/custom metrics adapter for promanomaly. A
+  stateless aggregated apiserver that re-serves the anomaly metrics the
+  detector already wrote to the TSDB through external.metrics.k8s.io /
+  custom.metrics.k8s.io, so existing HPA and KEDA tooling can consume
+  anomaly signal like any other metric. promanomaly stays the metrics
+  provider; the autoscaler stays the decision-maker.
+
+  Off by default — install this chart only when you intend to scale or
+  gate on anomaly signal, and read the guard-rails in the README first
+  (scaling raw on anomaly_score AMPLIFIES incidents).
+type: application
+icon: https://raw.githubusercontent.com/esops-dev/promanomaly/main/.github/logo.png
+home: https://github.com/esops-dev/promanomaly
+sources:
+  - https://github.com/esops-dev/promanomaly
+maintainers:
+  - name: promanomaly maintainers
+    url: https://github.com/esops-dev/promanomaly
+keywords:
+  - prometheus
+  - anomaly-detection
+  - kubernetes
+  - autoscaling
+  - hpa
+  - keda
+# Chart and app versions are placeholders intentionally — the release
+# pipeline rewrites both at tag time.
+version: 0.0.0
+appVersion: "0.0.0"
@@ -0,0 +1,96 @@
+# promanomaly-metrics-adapter
+
+An **opt-in** Kubernetes external/custom metrics adapter for promanomaly.
+
+It is a stateless aggregated apiserver that re-serves the anomaly metrics
+the detector already wrote to the TSDB through
+`external.metrics.k8s.io` / `custom.metrics.k8s.io`, so your existing HPA
+and KEDA tooling can consume anomaly signal like any other metric.
+
+promanomaly stays the metrics **provider**; the autoscaler stays the
+**decision-maker**. The adapter never makes a reaction decision itself.
+
+## Guard-rails (read first)
+
+Anomaly signal is a *sharp* scaling input. Wire it carefully:
+
+- **Scale on `anomaly_density`**, not raw `anomaly_score`. "Add workers
+  when the backlog is anomalously deep" is sound; "scale on how
+  anomalous a latency metric looks" *amplifies* incidents — a latency
+  spike scales you up, which can deepen the spike.
+- **Gate on `anomaly_confidence_score` and `anomaly_duration_seconds`.**
+  React to sustained, high-confidence anomalies, not single-tick blips.
+- **Always pair an anomaly-driven scaler with a reactive fallback HPA**
+  (CPU/memory or queue length). If the detector or TSDB is unavailable,
+  the fallback keeps the workload safe.
+
+The default `metrics` list deliberately omits scaling presets for
+`anomaly_score`. Worked recipes with these guard-rails baked in live in
+[`examples/k8s/`](../../examples/k8s/).
+
+## What it exposes
+
+| Metric | Good for |
+| --- | --- |
+| `anomaly_density` | Fraction of a group currently anomalous — the recommended scaling input. |
+| `anomaly_severity` | Operator-facing 0-1 severity; gate scaling on it. |
+| `anomaly_active_series` | Count of firing series in a group. |
+| `anomaly_outside_threshold` | Per-series firing flag (0/1). |
+
+Both API groups are served:
+
+- **`external.metrics.k8s.io`** — cluster-scoped, the path KEDA's
+  `external` trigger and HPA `External` metric source use. Recommended.
+- **`custom.metrics.k8s.io`** — the same signals associated with objects
+  (pods, namespaces) for HPA `Object`/`Pods` rules.
+
+## Install
+
+```bash
+helm install promanomaly-adapter charts/promanomaly-metrics-adapter \
+  --namespace monitoring \
+  --set datasource.url=http://victoriametrics.monitoring.svc:8428/
+```
+
+Verify:
+
+```bash
+kubectl get apiservices | grep metrics.k8s.io
+kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" | jq .
+```
+
+## TLS
+
+The aggregation layer only talks HTTPS to an APIService backend, so a
+serving cert is mandatory.
+
+- **Default**: the chart mints a self-signed CA + serving cert and wires
+  the APIService `caBundle` to it. Simple, but it **regenerates on every
+  `helm upgrade`** (a brief reconcile blip).
+- **Rotation-free**: set `tls.existingSecret` to a `kubernetes.io/tls`
+  Secret (e.g. from cert-manager) and `tls.caBundle` to its CA. No churn.
+- **Skip verification**: `apiService.insecureSkipTLSVerify: true` if you
+  accept the aggregation layer not verifying the adapter cert.
+
+## Key values
+
+| Key | Default | Description |
+| --- | --- | --- |
+| `datasource.url` | `http://victoriametrics:8428/` | TSDB the detector writes to. |
+| `datasource.auth.type` | `none` | `none`/`bearer`/`basic`/`mtls`. |
+| `datasource.auth.existingSecret` | `""` | Secret with the credential (keys: `token`/`password`/`ca.crt`+`tls.crt`+`tls.key`), mounted and referenced by file. |
+| `metrics` | the four above | Anomaly metrics to expose. |
+| `customResources` | `[pods, namespaces]` | Object kinds for custom metrics. |
+| `tls.existingSecret` | `""` | Use a managed serving cert instead of generating one. |
+| `apiService.enabled` | `true` | Register the two APIService objects. |
+| `apiService.insecureSkipTLSVerify` | `false` | Skip aggregation-layer cert verification. |
+| `rbac.create` | `true` | auth-delegator + auth-reader + HPA metrics-reader RBAC. |
+| `networkPolicy.enabled` | `true` | Restrict ingress to the serving port. |
+
+## How it differs from k8s-prometheus-adapter
+
+This adapter is purpose-built for the bounded set of anomaly metrics
+promanomaly emits, with the guard-rails documented above. It reads the
+same TSDB the detector writes to and translates label selectors directly
+into PromQL matchers. For arbitrary Prometheus metrics, use the
+general-purpose [prometheus-adapter](https://github.com/kubernetes-sigs/prometheus-adapter).
@@ -0,0 +1,3 @@
+# Default install: chart-generated self-signed serving cert + APIServices.
+datasource:
+  url: http://victoriametrics.monitoring.svc:8428/
@@ -0,0 +1,20 @@
+# Operator-managed TLS (e.g. cert-manager) + explicit caBundle, RBAC and
+# NetworkPolicy off (managed out-of-band), single replica.
+datasource:
+  url: http://victoriametrics.monitoring.svc:8428/
+  auth:
+    type: bearer
+    existingSecret: tsdb-credentials
+replicaCount: 1
+tls:
+  existingSecret: adapter-serving-cert
+  caBundle: "TEST_CA_BUNDLE_BASE64"
+rbac:
+  create: false
+networkPolicy:
+  enabled: false
+metrics:
+  - anomaly_density
+  - anomaly_severity
+customResources:
+  - pods
@@ -0,0 +1,6 @@
+# APIService registration with insecureSkipTLSVerify (chart still mints a
+# serving cert; the aggregation layer just doesn't verify it).
+datasource:
+  url: http://victoriametrics.monitoring.svc:8428/
+apiService:
+  insecureSkipTLSVerify: true