From 31ddb0544d700067ae3286b74911a9a40f4d6c18 Mon Sep 17 00:00:00 2001 From: Werner Dijkerman Date: Sat, 30 May 2026 17:33:49 +0200 Subject: [PATCH] feeding anomaly signal to the platform --- .github/workflows/ci.yml | 30 +- Makefile | 13 +- README.md | 23 ++ action.yml | 90 +++++ charts/promanomaly-metrics-adapter/Chart.yaml | 32 ++ charts/promanomaly-metrics-adapter/README.md | 96 +++++ .../ci/default-values.yaml | 3 + .../ci/existing-secret-values.yaml | 20 + .../ci/insecure-skip-tls-values.yaml | 6 + .../templates/NOTES.txt | 27 ++ .../templates/_helpers.tpl | 103 +++++ .../templates/apiservice.yaml | 30 ++ .../templates/configmap.yaml | 45 +++ .../templates/deployment.yaml | 100 +++++ .../templates/networkpolicy.yaml | 23 ++ .../templates/rbac.yaml | 65 +++ .../templates/service.yaml | 13 + .../templates/serviceaccount.yaml | 7 + .../templates/tls-secret.yaml | 13 + .../values.schema.json | 102 +++++ .../promanomaly-metrics-adapter/values.yaml | 127 ++++++ charts/promanomaly/README.md | 7 + .../ci/datasource-auth-values.yaml | 16 + .../ci/multi-sink-noauth-values.yaml | 20 + charts/promanomaly/ci/multi-sink-values.yaml | 30 ++ charts/promanomaly/ci/sink-values.yaml | 19 + charts/promanomaly/templates/_helpers.tpl | 65 +++ charts/promanomaly/templates/configmap.yaml | 40 ++ charts/promanomaly/templates/deployment.yaml | 44 ++ charts/promanomaly/values.schema.json | 44 ++ charts/promanomaly/values.yaml | 70 +++- detector/Dockerfile | 4 +- detector/pyproject.toml | 13 + detector/src/promanomaly/adapter/__init__.py | 24 ++ detector/src/promanomaly/adapter/app.py | 380 ++++++++++++++++++ detector/src/promanomaly/adapter/config.py | 114 ++++++ detector/src/promanomaly/adapter/resolver.py | 169 ++++++++ detector/src/promanomaly/adapter/serve.py | 55 +++ detector/src/promanomaly/cli/__init__.py | 87 ++++ .../src/promanomaly/cli/_generate_rules.py | 201 +++++++++ detector/src/promanomaly/config.py | 180 +++++++++ detector/src/promanomaly/exporter.py | 17 + detector/src/promanomaly/httpauth.py | 43 ++ detector/src/promanomaly/main.py | 64 ++- detector/src/promanomaly/sinks/__init__.py | 51 +++ detector/src/promanomaly/sinks/base.py | 70 ++++ .../promanomaly/sinks/grafana_annotations.py | 212 ++++++++++ .../src/promanomaly/sinks/remote_write.py | 179 +++++++++ detector/src/promanomaly/source.py | 13 +- detector/tests/test_adapter.py | 257 ++++++++++++ detector/tests/test_generate_rules.py | 208 ++++++++++ detector/tests/test_httpauth.py | 65 +++ detector/tests/test_sinks_config.py | 282 +++++++++++++ .../tests/test_sinks_grafana_annotations.py | 219 ++++++++++ detector/tests/test_sinks_remote_write.py | 214 ++++++++++ detector/uv.lock | 76 +++- docs/adapter.md | 93 +++++ docs/cli.md | 40 +- docs/gitops.md | 76 ++++ docs/patterns.md | 18 + docs/sinks.md | 89 ++++ examples/k8s/README.md | 49 +++ .../k8s/argo-rollouts-analysistemplate.yaml | 95 +++++ examples/k8s/hpa-custom-metrics.yaml | 55 +++ examples/k8s/keda-scaledobject.yaml | 62 +++ 65 files changed, 5066 insertions(+), 31 deletions(-) create mode 100644 action.yml create mode 100644 charts/promanomaly-metrics-adapter/Chart.yaml create mode 100644 charts/promanomaly-metrics-adapter/README.md create mode 100644 charts/promanomaly-metrics-adapter/ci/default-values.yaml create mode 100644 charts/promanomaly-metrics-adapter/ci/existing-secret-values.yaml create mode 100644 charts/promanomaly-metrics-adapter/ci/insecure-skip-tls-values.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/NOTES.txt create mode 100644 charts/promanomaly-metrics-adapter/templates/_helpers.tpl create mode 100644 charts/promanomaly-metrics-adapter/templates/apiservice.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/configmap.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/deployment.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/networkpolicy.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/rbac.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/service.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/serviceaccount.yaml create mode 100644 charts/promanomaly-metrics-adapter/templates/tls-secret.yaml create mode 100644 charts/promanomaly-metrics-adapter/values.schema.json create mode 100644 charts/promanomaly-metrics-adapter/values.yaml create mode 100644 charts/promanomaly/ci/datasource-auth-values.yaml create mode 100644 charts/promanomaly/ci/multi-sink-noauth-values.yaml create mode 100644 charts/promanomaly/ci/multi-sink-values.yaml create mode 100644 charts/promanomaly/ci/sink-values.yaml create mode 100644 detector/src/promanomaly/adapter/__init__.py create mode 100644 detector/src/promanomaly/adapter/app.py create mode 100644 detector/src/promanomaly/adapter/config.py create mode 100644 detector/src/promanomaly/adapter/resolver.py create mode 100644 detector/src/promanomaly/adapter/serve.py create mode 100644 detector/src/promanomaly/cli/_generate_rules.py create mode 100644 detector/src/promanomaly/httpauth.py create mode 100644 detector/src/promanomaly/sinks/__init__.py create mode 100644 detector/src/promanomaly/sinks/base.py create mode 100644 detector/src/promanomaly/sinks/grafana_annotations.py create mode 100644 detector/src/promanomaly/sinks/remote_write.py create mode 100644 detector/tests/test_adapter.py create mode 100644 detector/tests/test_generate_rules.py create mode 100644 detector/tests/test_httpauth.py create mode 100644 detector/tests/test_sinks_config.py create mode 100644 detector/tests/test_sinks_grafana_annotations.py create mode 100644 detector/tests/test_sinks_remote_write.py create mode 100644 docs/adapter.md create mode 100644 docs/gitops.md create mode 100644 docs/sinks.md create mode 100644 examples/k8s/README.md create mode 100644 examples/k8s/argo-rollouts-analysistemplate.yaml create mode 100644 examples/k8s/hpa-custom-metrics.yaml create mode 100644 examples/k8s/keda-scaledobject.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6a7301..4349173 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,24 @@ jobs: path: detector/coverage.xml if-no-files-found: ignore + validate-action: + name: Config-validation Action (dogfood) + # Exercises the composite action.yml on an example config so the + # published GitOps validation Action can't regress. No datasource is + # available in CI, so this runs the static checks only (schema + + # --estimate-cost --strict); --probe / --lint-metadata are covered by + # the unit tests, which stub the datasource. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Validate an example config via the action + uses: ./ + with: + config: examples/configs/node-exporter.yaml + estimate-cost: "true" + strict: "true" + community-prophet: name: Community ProphetResidual (lint + test with Prophet stubbed) # Prophet's real install pulls cmdstan / pystan and is multi-minute on @@ -71,7 +89,7 @@ jobs: # ``Prophet(...).fit(...).predict(...)``, so we lint + test the # community package against a tiny Prophet stub (tests/conftest.py) # without installing the real thing — the third-party plug-in path - # documented in the v0.4 roadmap. + # documented in the roadmap. runs-on: ubuntu-latest defaults: run: @@ -137,7 +155,7 @@ jobs: - name: ct lint run: | ct lint \ - --charts charts/promanomaly,charts/promanomaly-stack \ + --charts charts/promanomaly,charts/promanomaly-stack,charts/promanomaly-metrics-adapter \ --validate-maintainers=false \ --check-version-increment=false \ --target-branch ${{ github.event.repository.default_branch }} @@ -146,6 +164,7 @@ jobs: run: | helm lint --strict charts/promanomaly helm lint --strict charts/promanomaly-stack + helm lint --strict charts/promanomaly-metrics-adapter - name: Install kubeconform run: | @@ -156,8 +175,9 @@ jobs: - name: kubeconform run: | - helm template charts/promanomaly | kubeconform -strict -summary -ignore-missing-schemas - helm template charts/promanomaly-stack | kubeconform -strict -summary -ignore-missing-schemas + helm template charts/promanomaly | kubeconform -strict -summary -ignore-missing-schemas + helm template charts/promanomaly-stack | kubeconform -strict -summary -ignore-missing-schemas + helm template charts/promanomaly-metrics-adapter | kubeconform -strict -summary -ignore-missing-schemas - name: chart-testing values matrix (lint + render + kubeconform) # ct lint already exercises the ci/ values files, but we @@ -170,7 +190,7 @@ jobs: set -euo pipefail shopt -s nullglob fail=0 - for chart in charts/promanomaly charts/promanomaly-stack; do + for chart in charts/promanomaly charts/promanomaly-stack charts/promanomaly-metrics-adapter; do for vf in "$chart"/ci/*-values.yaml; do echo "::group::render $chart with $(basename "$vf")" if ! helm template "$chart" --values "$vf" \ diff --git a/Makefile b/Makefile index 05deaf6..87f8878 100644 --- a/Makefile +++ b/Makefile @@ -85,8 +85,10 @@ chart-deps: ## Resolve umbrella subchart dependencies. chart-lint: chart-deps ## helm lint + kubeconform on rendered manifests. helm lint $(CHART_DIR)/promanomaly helm lint $(CHART_DIR)/promanomaly-stack - helm template $(CHART_DIR)/promanomaly | kubeconform -strict -summary -ignore-missing-schemas - helm template $(CHART_DIR)/promanomaly-stack | kubeconform -strict -summary -ignore-missing-schemas + helm lint $(CHART_DIR)/promanomaly-metrics-adapter + helm template $(CHART_DIR)/promanomaly | kubeconform -strict -summary -ignore-missing-schemas + helm template $(CHART_DIR)/promanomaly-stack | kubeconform -strict -summary -ignore-missing-schemas + helm template $(CHART_DIR)/promanomaly-metrics-adapter | kubeconform -strict -summary -ignore-missing-schemas @if command -v yamllint >/dev/null; then \ yamllint -c .chart-testing-lintconf.yaml \ $(CHART_DIR)/promanomaly/Chart.yaml \ @@ -94,7 +96,10 @@ chart-lint: chart-deps ## helm lint + kubeconform on rendered manifests. $(CHART_DIR)/promanomaly/ci \ $(CHART_DIR)/promanomaly-stack/Chart.yaml \ $(CHART_DIR)/promanomaly-stack/values.yaml \ - $(CHART_DIR)/promanomaly-stack/ci; \ + $(CHART_DIR)/promanomaly-stack/ci \ + $(CHART_DIR)/promanomaly-metrics-adapter/Chart.yaml \ + $(CHART_DIR)/promanomaly-metrics-adapter/values.yaml \ + $(CHART_DIR)/promanomaly-metrics-adapter/ci; \ else \ echo "yamllint not installed — skipping (ct lint will run it in CI)"; \ fi @@ -104,7 +109,7 @@ chart-lint: chart-deps ## helm lint + kubeconform on rendered manifests. .PHONY: chart-matrix chart-matrix: chart-deps ## Render each chart with every ci/*-values.yaml + kubeconform. @set -e; \ - for chart in $(CHART_DIR)/promanomaly $(CHART_DIR)/promanomaly-stack; do \ + for chart in $(CHART_DIR)/promanomaly $(CHART_DIR)/promanomaly-stack $(CHART_DIR)/promanomaly-metrics-adapter; do \ for vf in $$chart/ci/*-values.yaml; do \ echo "--- $$chart :: $$(basename $$vf) ---"; \ helm template $$chart --values $$vf \ diff --git a/README.md b/README.md index 9f7b747..a634c06 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,29 @@ The full multi-group production reference deployment — values, NetworkPolicy egress, ArgoCD `Application`, and Flux `HelmRepository`/`HelmRelease` — lives at [`examples/production/`](examples/production/). +## Feeding anomaly signal to the platform + +Anomaly scores are most useful when the rest of your stack can act on +them. promanomaly stays the signal *provider*; the systems that react +(autoscalers, rollout controllers, dashboards, long-term storage) stay +the decision-makers. + +- **Push sinks** ship each snapshot beyond `/metrics`: a Prometheus + `remote_write` sink for long-horizon anomaly history, and a Grafana + annotations sink that surfaces firings across every dashboard. See + [`docs/sinks.md`](docs/sinks.md). +- **Kubernetes metrics adapter** re-serves anomaly signal through the + external/custom metrics APIs so HPA and KEDA can consume it. Opt-in via + [`charts/promanomaly-metrics-adapter/`](charts/promanomaly-metrics-adapter/); + see [`docs/adapter.md`](docs/adapter.md). +- **Reaction recipes** — KEDA, HPA, and Argo Rollouts cohort-gated + canaries, with the autoscaling guard-rails baked in — live at + [`examples/k8s/`](examples/k8s/). +- **GitOps config validation** — a composite GitHub Action validates your + config on every PR (schema, cardinality/cost, optional live probe), and + `promanomaly generate-rules` scaffolds a PrometheusRule from it. See + [`docs/gitops.md`](docs/gitops.md). + ## Relationship with promforecast | | promforecast | promanomaly | diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..7139f7d --- /dev/null +++ b/action.yml @@ -0,0 +1,90 @@ +name: promanomaly config validation +description: >- + Validate a promanomaly detector config on every PR — schema, optional live + probe, static cardinality/cost estimate, and counter-metadata lint — so a + broken or over-expensive config never reaches the cluster. +branding: + icon: activity + color: purple + +inputs: + config: + description: Path to the promanomaly config file to validate. + required: true + probe: + description: >- + Execute every query against a live datasource and fail on empty/erroring + queries. Requires network access to the TSDB from the runner; usually a + mock or staging datasource in CI. Defaults to off. + required: false + default: "false" + estimate-cost: + description: >- + Statically project cardinality, TSDB query load, and a coarse CPU/memory + estimate from the config (no datasource access). On by default — it is + cheap and needs nothing external. + required: false + default: "true" + lint-metadata: + description: >- + Query the datasource's /api/v1/metadata and warn when a query feeds a raw + counter to a detector without rate()/increase(). Requires datasource + access. Defaults to off. + required: false + default: "false" + strict: + description: >- + Fail the job on a soft finding from any enabled check: empty/erroring + queries (probe), projected series over safety.max_total_series + (estimate-cost), or a counter-not-rated finding (lint-metadata). On by + default so the action gates a PR. Set to "false" to report only. + required: false + default: "true" + datasource-url: + description: >- + Override the datasource URL used by --probe / --lint-metadata, e.g. a + staging or mock TSDB. Leave empty to use the URL from the config. + required: false + default: "" + python-version: + description: Python version used to run the validator. + required: false + default: "3.12" + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + + - name: Install promanomaly + shell: bash + run: | + python -m pip install --upgrade pip + # Install the detector from this action's own checkout so the + # validator version always matches the action ref the caller pinned. + pip install "${GITHUB_ACTION_PATH}/detector" + + - name: Validate config + shell: bash + env: + PROMANOMALY_CONFIG: ${{ inputs.config }} + PROMANOMALY_PROBE: ${{ inputs.probe }} + PROMANOMALY_ESTIMATE_COST: ${{ inputs.estimate-cost }} + PROMANOMALY_LINT_METADATA: ${{ inputs.lint-metadata }} + PROMANOMALY_STRICT: ${{ inputs.strict }} + PROMANOMALY_DATASOURCE_URL: ${{ inputs.datasource-url }} + run: | + set -euo pipefail + args=(validate --config "${PROMANOMALY_CONFIG}") + if [ "${PROMANOMALY_PROBE}" = "true" ]; then args+=(--probe); fi + if [ "${PROMANOMALY_ESTIMATE_COST}" = "true" ]; then args+=(--estimate-cost); fi + if [ "${PROMANOMALY_LINT_METADATA}" = "true" ]; then args+=(--lint-metadata); fi + if [ "${PROMANOMALY_STRICT}" = "true" ]; then args+=(--strict); fi + if [ -n "${PROMANOMALY_DATASOURCE_URL}" ]; then + args+=(--datasource-url "${PROMANOMALY_DATASOURCE_URL}") + fi + echo "+ promanomaly ${args[*]}" + promanomaly "${args[@]}" diff --git a/charts/promanomaly-metrics-adapter/Chart.yaml b/charts/promanomaly-metrics-adapter/Chart.yaml new file mode 100644 index 0000000..3c3813c --- /dev/null +++ b/charts/promanomaly-metrics-adapter/Chart.yaml @@ -0,0 +1,32 @@ +apiVersion: v2 +name: promanomaly-metrics-adapter +description: | + Opt-in Kubernetes external/custom metrics adapter for promanomaly. A + stateless aggregated apiserver that re-serves the anomaly metrics the + detector already wrote to the TSDB through external.metrics.k8s.io / + custom.metrics.k8s.io, so existing HPA and KEDA tooling can consume + anomaly signal like any other metric. promanomaly stays the metrics + provider; the autoscaler stays the decision-maker. + + Off by default — install this chart only when you intend to scale or + gate on anomaly signal, and read the guard-rails in the README first + (scaling raw on anomaly_score AMPLIFIES incidents). +type: application +icon: https://raw.githubusercontent.com/esops-dev/promanomaly/main/.github/logo.png +home: https://github.com/esops-dev/promanomaly +sources: + - https://github.com/esops-dev/promanomaly +maintainers: + - name: promanomaly maintainers + url: https://github.com/esops-dev/promanomaly +keywords: + - prometheus + - anomaly-detection + - kubernetes + - autoscaling + - hpa + - keda +# Chart and app versions are placeholders intentionally — the release +# pipeline rewrites both at tag time. +version: 0.0.0 +appVersion: "0.0.0" diff --git a/charts/promanomaly-metrics-adapter/README.md b/charts/promanomaly-metrics-adapter/README.md new file mode 100644 index 0000000..21a7099 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/README.md @@ -0,0 +1,96 @@ +# promanomaly-metrics-adapter + +An **opt-in** Kubernetes external/custom metrics adapter for promanomaly. + +It is a stateless aggregated apiserver that re-serves the anomaly metrics +the detector already wrote to the TSDB through +`external.metrics.k8s.io` / `custom.metrics.k8s.io`, so your existing HPA +and KEDA tooling can consume anomaly signal like any other metric. + +promanomaly stays the metrics **provider**; the autoscaler stays the +**decision-maker**. The adapter never makes a reaction decision itself. + +## Guard-rails (read first) + +Anomaly signal is a *sharp* scaling input. Wire it carefully: + +- **Scale on `anomaly_density`**, not raw `anomaly_score`. "Add workers + when the backlog is anomalously deep" is sound; "scale on how + anomalous a latency metric looks" *amplifies* incidents — a latency + spike scales you up, which can deepen the spike. +- **Gate on `anomaly_confidence_score` and `anomaly_duration_seconds`.** + React to sustained, high-confidence anomalies, not single-tick blips. +- **Always pair an anomaly-driven scaler with a reactive fallback HPA** + (CPU/memory or queue length). If the detector or TSDB is unavailable, + the fallback keeps the workload safe. + +The default `metrics` list deliberately omits scaling presets for +`anomaly_score`. Worked recipes with these guard-rails baked in live in +[`examples/k8s/`](../../examples/k8s/). + +## What it exposes + +| Metric | Good for | +| --- | --- | +| `anomaly_density` | Fraction of a group currently anomalous — the recommended scaling input. | +| `anomaly_severity` | Operator-facing 0-1 severity; gate scaling on it. | +| `anomaly_active_series` | Count of firing series in a group. | +| `anomaly_outside_threshold` | Per-series firing flag (0/1). | + +Both API groups are served: + +- **`external.metrics.k8s.io`** — cluster-scoped, the path KEDA's + `external` trigger and HPA `External` metric source use. Recommended. +- **`custom.metrics.k8s.io`** — the same signals associated with objects + (pods, namespaces) for HPA `Object`/`Pods` rules. + +## Install + +```bash +helm install promanomaly-adapter charts/promanomaly-metrics-adapter \ + --namespace monitoring \ + --set datasource.url=http://victoriametrics.monitoring.svc:8428/ +``` + +Verify: + +```bash +kubectl get apiservices | grep metrics.k8s.io +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" | jq . +``` + +## TLS + +The aggregation layer only talks HTTPS to an APIService backend, so a +serving cert is mandatory. + +- **Default**: the chart mints a self-signed CA + serving cert and wires + the APIService `caBundle` to it. Simple, but it **regenerates on every + `helm upgrade`** (a brief reconcile blip). +- **Rotation-free**: set `tls.existingSecret` to a `kubernetes.io/tls` + Secret (e.g. from cert-manager) and `tls.caBundle` to its CA. No churn. +- **Skip verification**: `apiService.insecureSkipTLSVerify: true` if you + accept the aggregation layer not verifying the adapter cert. + +## Key values + +| Key | Default | Description | +| --- | --- | --- | +| `datasource.url` | `http://victoriametrics:8428/` | TSDB the detector writes to. | +| `datasource.auth.type` | `none` | `none`/`bearer`/`basic`/`mtls`. | +| `datasource.auth.existingSecret` | `""` | Secret with the credential (keys: `token`/`password`/`ca.crt`+`tls.crt`+`tls.key`), mounted and referenced by file. | +| `metrics` | the four above | Anomaly metrics to expose. | +| `customResources` | `[pods, namespaces]` | Object kinds for custom metrics. | +| `tls.existingSecret` | `""` | Use a managed serving cert instead of generating one. | +| `apiService.enabled` | `true` | Register the two APIService objects. | +| `apiService.insecureSkipTLSVerify` | `false` | Skip aggregation-layer cert verification. | +| `rbac.create` | `true` | auth-delegator + auth-reader + HPA metrics-reader RBAC. | +| `networkPolicy.enabled` | `true` | Restrict ingress to the serving port. | + +## How it differs from k8s-prometheus-adapter + +This adapter is purpose-built for the bounded set of anomaly metrics +promanomaly emits, with the guard-rails documented above. It reads the +same TSDB the detector writes to and translates label selectors directly +into PromQL matchers. For arbitrary Prometheus metrics, use the +general-purpose [prometheus-adapter](https://github.com/kubernetes-sigs/prometheus-adapter). diff --git a/charts/promanomaly-metrics-adapter/ci/default-values.yaml b/charts/promanomaly-metrics-adapter/ci/default-values.yaml new file mode 100644 index 0000000..86d04e8 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/ci/default-values.yaml @@ -0,0 +1,3 @@ +# Default install: chart-generated self-signed serving cert + APIServices. +datasource: + url: http://victoriametrics.monitoring.svc:8428/ diff --git a/charts/promanomaly-metrics-adapter/ci/existing-secret-values.yaml b/charts/promanomaly-metrics-adapter/ci/existing-secret-values.yaml new file mode 100644 index 0000000..70e4304 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/ci/existing-secret-values.yaml @@ -0,0 +1,20 @@ +# Operator-managed TLS (e.g. cert-manager) + explicit caBundle, RBAC and +# NetworkPolicy off (managed out-of-band), single replica. +datasource: + url: http://victoriametrics.monitoring.svc:8428/ + auth: + type: bearer + existingSecret: tsdb-credentials +replicaCount: 1 +tls: + existingSecret: adapter-serving-cert + caBundle: "TEST_CA_BUNDLE_BASE64" +rbac: + create: false +networkPolicy: + enabled: false +metrics: + - anomaly_density + - anomaly_severity +customResources: + - pods diff --git a/charts/promanomaly-metrics-adapter/ci/insecure-skip-tls-values.yaml b/charts/promanomaly-metrics-adapter/ci/insecure-skip-tls-values.yaml new file mode 100644 index 0000000..7f55707 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/ci/insecure-skip-tls-values.yaml @@ -0,0 +1,6 @@ +# APIService registration with insecureSkipTLSVerify (chart still mints a +# serving cert; the aggregation layer just doesn't verify it). +datasource: + url: http://victoriametrics.monitoring.svc:8428/ +apiService: + insecureSkipTLSVerify: true diff --git a/charts/promanomaly-metrics-adapter/templates/NOTES.txt b/charts/promanomaly-metrics-adapter/templates/NOTES.txt new file mode 100644 index 0000000..a4722fb --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/NOTES.txt @@ -0,0 +1,27 @@ +promanomaly-metrics-adapter is installed. + +The adapter serves anomaly signal through the Kubernetes metrics APIs: + + external.metrics.k8s.io/v1beta1 (recommended for autoscaling) + custom.metrics.k8s.io/v1beta1 + +Verify the API is registered and answering: + + kubectl get apiservices | grep metrics.k8s.io + kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" | jq . + +Query a metric (cluster-scoped external metric example): + + kubectl get --raw \ + "/apis/external.metrics.k8s.io/v1beta1/namespaces/{{ .Release.Namespace }}/anomaly_density?labelSelector=group%3Dqueue_depths" | jq . + +GUARD-RAILS — read before wiring an autoscaler: + + * Scale on anomaly_density (e.g. "add workers when the backlog is + anomalously deep"), gated on anomaly_confidence_score and + anomaly_duration_seconds. + * Do NOT scale raw on anomaly_score — it AMPLIFIES incidents. + * Always pair an anomaly-driven scaler with a reactive fallback HPA. + +Worked KEDA / HPA / Argo Rollouts recipes (with the guard-rails baked +in) live in examples/k8s/ in the promanomaly repository. diff --git a/charts/promanomaly-metrics-adapter/templates/_helpers.tpl b/charts/promanomaly-metrics-adapter/templates/_helpers.tpl new file mode 100644 index 0000000..6dd03b6 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/_helpers.tpl @@ -0,0 +1,103 @@ +{{/* Expand the name of the chart. */}} +{{- define "adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Create a default fully qualified app name. */}} +{{- define "adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Chart label string. */}} +{{- define "adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Common labels applied to every resource. */}} +{{- define "adapter.labels" -}} +helm.sh/chart: {{ include "adapter.chart" . }} +{{ include "adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* Selector labels — must be stable across upgrades. */}} +{{- define "adapter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* ServiceAccount name to use. */}} +{{- define "adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "adapter.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{/* ConfigMap name: either the user-supplied existing CM or the one we render. */}} +{{- define "adapter.configMapName" -}} +{{- if .Values.existingConfigMap -}} +{{- .Values.existingConfigMap -}} +{{- else -}} +{{- printf "%s-config" (include "adapter.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* TLS secret name. */}} +{{- define "adapter.tlsSecretName" -}} +{{- if .Values.tls.existingSecret -}} +{{- .Values.tls.existingSecret -}} +{{- else -}} +{{- printf "%s-tls" (include "adapter.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Memoized self-signed serving certificate. Generated once on first call +and stashed on the shared root context so the Secret and both APIService +caBundles read the *same* pair within one `helm template` render (a fresh +genSignedCert per file would make the caBundles disagree with the served +cert). Access the result via `.adapterCerts.cert.Cert` / +`.adapterCerts.cert.Key` / `.adapterCerts.ca.Cert`. + +This regenerates on every `helm upgrade`; for a long-lived rotation-free +cert use `tls.existingSecret` (e.g. cert-manager) — see the README. +*/}} +{{- define "adapter.certs" -}} +{{- if not .adapterCerts -}} +{{- $svc := include "adapter.fullname" . -}} +{{- $ns := .Release.Namespace -}} +{{- $cn := printf "%s.%s.svc" $svc $ns -}} +{{- $altNames := list $cn (printf "%s.%s.svc.cluster.local" $svc $ns) -}} +{{- $ca := genCA (printf "%s-ca" $svc) 3650 -}} +{{- $cert := genSignedCert $cn nil $altNames 3650 $ca -}} +{{- $_ := set . "adapterCerts" (dict "ca" $ca "cert" $cert) -}} +{{- end -}} +{{- end -}} + +{{/* +The CA bundle (base64) the APIService should trust. Empty when +insecureSkipTLSVerify is set or when the operator supplied their own +secret without a caBundle value. +*/}} +{{- define "adapter.caBundle" -}} +{{- if .Values.tls.existingSecret -}} +{{- .Values.tls.caBundle -}} +{{- else -}} +{{- include "adapter.certs" . -}} +{{- b64enc .adapterCerts.ca.Cert -}} +{{- end -}} +{{- end -}} diff --git a/charts/promanomaly-metrics-adapter/templates/apiservice.yaml b/charts/promanomaly-metrics-adapter/templates/apiservice.yaml new file mode 100644 index 0000000..5e7dc15 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/apiservice.yaml @@ -0,0 +1,30 @@ +{{- if .Values.apiService.enabled }} +{{- $caBundle := "" }} +{{- if not .Values.apiService.insecureSkipTLSVerify }} +{{- $caBundle = include "adapter.caBundle" . }} +{{- end }} +{{- range $group := list "external.metrics.k8s.io" "custom.metrics.k8s.io" }} +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.{{ $group }} + labels: {{- include "adapter.labels" $ | nindent 4 }} +spec: + service: + name: {{ include "adapter.fullname" $ }} + namespace: {{ $.Release.Namespace }} + port: {{ $.Values.service.port }} + group: {{ $group }} + version: v1beta1 + {{- if $.Values.apiService.insecureSkipTLSVerify }} + insecureSkipTLSVerify: true + {{- else if $caBundle }} + caBundle: {{ $caBundle }} + {{- else }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: {{ $.Values.apiService.groupPriorityMinimum }} + versionPriority: {{ $.Values.apiService.versionPriority }} +{{- end }} +{{- end }} diff --git a/charts/promanomaly-metrics-adapter/templates/configmap.yaml b/charts/promanomaly-metrics-adapter/templates/configmap.yaml new file mode 100644 index 0000000..5826446 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/configmap.yaml @@ -0,0 +1,45 @@ +{{- if not .Values.existingConfigMap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "adapter.configMapName" . }} + labels: {{- include "adapter.labels" . | nindent 4 }} +data: + adapter.yaml: | + datasource: + url: {{ .Values.datasource.url | quote }} + timeout: {{ .Values.datasource.timeout | quote }} + auth: + type: {{ .Values.datasource.auth.type | quote }} + {{- if .Values.datasource.auth.existingSecret }} + {{- /* + Credentials mounted from datasource.auth.existingSecret at + /etc/promanomaly-adapter/datasource-auth and referenced by file. + Secret keys per type: bearer -> token; basic -> password + (+ auth.username here); mtls -> ca.crt / tls.crt / tls.key. + */}} + {{- if eq .Values.datasource.auth.type "bearer" }} + token_file: /etc/promanomaly-adapter/datasource-auth/token + {{- else if eq .Values.datasource.auth.type "basic" }} + username: {{ .Values.datasource.auth.username | quote }} + password_file: /etc/promanomaly-adapter/datasource-auth/password + {{- else if eq .Values.datasource.auth.type "mtls" }} + ca_file: /etc/promanomaly-adapter/datasource-auth/ca.crt + cert_file: /etc/promanomaly-adapter/datasource-auth/tls.crt + key_file: /etc/promanomaly-adapter/datasource-auth/tls.key + {{- end }} + {{- end }} + listen: ":{{ .Values.containerPort }}" + tls: + cert_file: /etc/promanomaly-adapter/tls/tls.crt + key_file: /etc/promanomaly-adapter/tls/tls.key + timeout: {{ .Values.datasource.timeout | quote }} + metrics: + {{- range .Values.metrics }} + - {{ . | quote }} + {{- end }} + custom_resources: + {{- range .Values.customResources }} + - {{ . | quote }} + {{- end }} +{{- end }} diff --git a/charts/promanomaly-metrics-adapter/templates/deployment.yaml b/charts/promanomaly-metrics-adapter/templates/deployment.yaml new file mode 100644 index 0000000..80bad15 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/deployment.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "adapter.fullname" . }} + labels: {{- include "adapter.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: {{- include "adapter.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/config: {{ toYaml .Values | sha256sum }} + labels: + {{- include "adapter.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "adapter.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.securityContext | nindent 8 }} + containers: + - name: adapter + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "adapter" + - "--config" + - "/etc/promanomaly-adapter/adapter.yaml" + ports: + - name: https + containerPort: {{ .Values.containerPort }} + protocol: TCP + livenessProbe: + httpGet: + path: /livez + port: https + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/promanomaly-adapter/adapter.yaml + subPath: adapter.yaml + readOnly: true + - name: tls + mountPath: /etc/promanomaly-adapter/tls + readOnly: true + {{- if .Values.datasource.auth.existingSecret }} + - name: datasource-auth + mountPath: /etc/promanomaly-adapter/datasource-auth + readOnly: true + {{- end }} + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: {{ include "adapter.configMapName" . }} + - name: tls + secret: + secretName: {{ include "adapter.tlsSecretName" . }} + {{- if .Values.datasource.auth.existingSecret }} + - name: datasource-auth + secret: + secretName: {{ .Values.datasource.auth.existingSecret | quote }} + {{- end }} + - name: tmp + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/promanomaly-metrics-adapter/templates/networkpolicy.yaml b/charts/promanomaly-metrics-adapter/templates/networkpolicy.yaml new file mode 100644 index 0000000..44b9880 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "adapter.fullname" . }} + labels: {{- include "adapter.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: {{- include "adapter.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + # The aggregation layer reaches the adapter through the kube-apiserver, + # which is host-networked, so ingress can't be pinned to a pod + # selector. Restrict to the serving port instead. + - ports: + - protocol: TCP + port: {{ .Values.containerPort }} + egress: + # Allow the adapter to reach the TSDB and DNS. + - {} +{{- end }} diff --git a/charts/promanomaly-metrics-adapter/templates/rbac.yaml b/charts/promanomaly-metrics-adapter/templates/rbac.yaml new file mode 100644 index 0000000..149a201 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/rbac.yaml @@ -0,0 +1,65 @@ +{{- if .Values.rbac.create }} +# Delegated authentication/authorization: the aggregated apiserver asks +# the core apiserver to authenticate and authorize incoming requests. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "adapter.fullname" . }}:system:auth-delegator + labels: {{- include "adapter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: {{ include "adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +# Read the extension-apiserver-authentication ConfigMap in kube-system so +# the adapter can validate client certs presented by the aggregation layer. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "adapter.fullname" . }}-auth-reader + namespace: kube-system + labels: {{- include "adapter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ include "adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +# Let the HPA controller read the metrics APIs this adapter serves. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "adapter.fullname" . }}-metrics-reader + labels: {{- include "adapter.labels" . | nindent 4 }} +rules: + - apiGroups: + - external.metrics.k8s.io + - custom.metrics.k8s.io + resources: + - "*" + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "adapter.fullname" . }}-hpa-metrics-reader + labels: {{- include "adapter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "adapter.fullname" . }}-metrics-reader +subjects: + - kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end }} diff --git a/charts/promanomaly-metrics-adapter/templates/service.yaml b/charts/promanomaly-metrics-adapter/templates/service.yaml new file mode 100644 index 0000000..2966d58 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "adapter.fullname" . }} + labels: {{- include "adapter.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: https + port: {{ .Values.service.port }} + targetPort: https + protocol: TCP + selector: {{- include "adapter.selectorLabels" . | nindent 4 }} diff --git a/charts/promanomaly-metrics-adapter/templates/serviceaccount.yaml b/charts/promanomaly-metrics-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000..2f06c5b --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "adapter.serviceAccountName" . }} + labels: {{- include "adapter.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/promanomaly-metrics-adapter/templates/tls-secret.yaml b/charts/promanomaly-metrics-adapter/templates/tls-secret.yaml new file mode 100644 index 0000000..4d3c639 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/templates/tls-secret.yaml @@ -0,0 +1,13 @@ +{{- if not .Values.tls.existingSecret }} +{{- include "adapter.certs" . }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: {{ include "adapter.tlsSecretName" . }} + labels: {{- include "adapter.labels" . | nindent 4 }} +data: + tls.crt: {{ .adapterCerts.cert.Cert | b64enc }} + tls.key: {{ .adapterCerts.cert.Key | b64enc }} + ca.crt: {{ .adapterCerts.ca.Cert | b64enc }} +{{- end }} diff --git a/charts/promanomaly-metrics-adapter/values.schema.json b/charts/promanomaly-metrics-adapter/values.schema.json new file mode 100644 index 0000000..b1227c6 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/values.schema.json @@ -0,0 +1,102 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://promanomaly.io/schemas/v1/metrics-adapter.values.schema.json", + "title": "promanomaly-metrics-adapter chart values", + "description": "Stable v1 schema for the opt-in Kubernetes metrics adapter chart.", + "type": "object", + "additionalProperties": true, + "required": ["image", "datasource"], + "properties": { + "image": { + "type": "object", + "required": ["repository"], + "properties": { + "repository": {"type": "string", "minLength": 1}, + "tag": {"type": "string"}, + "pullPolicy": {"type": "string", "enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "imagePullSecrets": {"type": "array"}, + "nameOverride": {"type": "string"}, + "fullnameOverride": {"type": "string"}, + "replicaCount": {"type": "integer", "minimum": 1}, + "datasource": { + "type": "object", + "required": ["url"], + "properties": { + "url": {"type": "string", "minLength": 1}, + "timeout": {"type": "string"}, + "auth": { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["none", "bearer", "basic", "mtls"]}, + "username": {"type": "string"}, + "existingSecret": {"type": "string"} + } + } + } + }, + "metrics": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$"} + }, + "customResources": { + "type": "array", + "items": {"type": "string"} + }, + "tls": { + "type": "object", + "properties": { + "existingSecret": {"type": "string"}, + "caBundle": {"type": "string"} + } + }, + "apiService": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "insecureSkipTLSVerify": {"type": "boolean"}, + "groupPriorityMinimum": {"type": "integer", "minimum": 1}, + "versionPriority": {"type": "integer", "minimum": 1} + } + }, + "service": { + "type": "object", + "properties": { + "type": {"type": "string"}, + "port": {"type": "integer"}, + "targetPort": {"type": "integer"} + } + }, + "containerPort": {"type": "integer", "minimum": 1, "maximum": 65535}, + "serviceAccount": { + "type": "object", + "properties": { + "create": {"type": "boolean"}, + "name": {"type": "string"} + } + }, + "existingConfigMap": {"type": "string"}, + "rbac": { + "type": "object", + "properties": { + "create": {"type": "boolean"} + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"} + } + }, + "resources": {"type": "object"}, + "securityContext": {"type": "object"}, + "containerSecurityContext": {"type": "object"}, + "podAnnotations": {"type": "object"}, + "podLabels": {"type": "object"}, + "nodeSelector": {"type": "object"}, + "tolerations": {"type": "array"}, + "affinity": {"type": "object"} + } +} diff --git a/charts/promanomaly-metrics-adapter/values.yaml b/charts/promanomaly-metrics-adapter/values.yaml new file mode 100644 index 0000000..f7d7e41 --- /dev/null +++ b/charts/promanomaly-metrics-adapter/values.yaml @@ -0,0 +1,127 @@ +# Default values for promanomaly-metrics-adapter. +# +# This chart is OFF by default in the sense that you install it +# deliberately. Read the README guard-rails before wiring an autoscaler +# to anomaly signal: scaling raw on anomaly_score amplifies incidents. +# Prefer anomaly_density (gated on confidence + duration) and always pair +# an anomaly-driven scaler with a reactive fallback HPA. + +image: + repository: ghcr.io/esops-dev/promanomaly + tag: "" # overrides Chart.appVersion when set + pullPolicy: IfNotPresent +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +# The adapter is stateless and horizontally scalable; 2 replicas give the +# aggregation layer an HA backend without any coordination. +replicaCount: 2 + +# Datasource: the same PromQL-compatible TSDB the detector writes to. +datasource: + url: http://victoriametrics:8428/ + timeout: 10s + auth: + # none | bearer | basic | mtls — mirrors the detector chart. + type: none + username: "" # basic-auth username (password comes from the Secret) + # Secret carrying the credential, mounted as files and referenced by + # path. Expected keys per type: + # bearer -> token + # basic -> password (username above) + # mtls -> ca.crt, tls.crt, tls.key + existingSecret: "" + +# Anomaly metrics exposed through the metrics APIs. The defaults are the +# four the project supports; trim the list to reduce the discovery +# surface. Scaling raw on anomaly_score is intentionally NOT a default. +metrics: + - anomaly_severity + - anomaly_density + - anomaly_active_series + - anomaly_outside_threshold + +# Kubernetes object kinds exposed through custom.metrics.k8s.io. The +# object name is matched against the same-named PromQL label. +customResources: + - pods + - namespaces + +# Serving TLS. The aggregation layer only talks HTTPS to an APIService +# backend, so a serving cert is mandatory in-cluster. +tls: + # generate: chart mints a self-signed CA + serving cert and wires the + # APIService caBundle to it. Simple, but regenerates on every upgrade. + # For rotation-free certs set existingSecret (e.g. cert-manager). + existingSecret: "" # name of a kubernetes.io/tls Secret to use + caBundle: "" # base64 CA bundle for the APIService (existingSecret only) + +apiService: + # Register the two APIService objects with the aggregation layer. + # Disable only if you manage them out-of-band. + enabled: true + # Skip TLS verification of the adapter from the aggregation layer. + # Leave false; the chart wires the generated CA into caBundle. + insecureSkipTLSVerify: false + # APIService priority knobs (aggregation layer ordering). + groupPriorityMinimum: 100 + versionPriority: 100 + +service: + type: ClusterIP + port: 443 + targetPort: 6443 + +# The container serves HTTPS on this port. +containerPort: 6443 + +serviceAccount: + create: true + name: "" + +# Manage the adapter config out-of-band (Argo CD, Flux) instead of +# letting the chart render it. +existingConfigMap: "" + +rbac: + # Create the auth-delegator binding and the extension-apiserver + # authentication reader the aggregated apiserver needs, plus the + # ClusterRole letting the HPA controller read the metrics APIs. + create: true + +resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + +securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + +containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +podAnnotations: {} +podLabels: {} +nodeSelector: {} +tolerations: [] +affinity: {} + +# Deny ingress except from the aggregation layer (kube-apiserver). The +# adapter has no /debug surface; this keeps the metrics API reachable +# only via the cluster's apiserver proxy. +networkPolicy: + enabled: true diff --git a/charts/promanomaly/README.md b/charts/promanomaly/README.md index 124046b..6501c27 100644 --- a/charts/promanomaly/README.md +++ b/charts/promanomaly/README.md @@ -34,6 +34,13 @@ notable bits: - ``existingConfigMap`` lets GitOps tooling (Argo CD, Flux) manage the detector configuration out-of-band; the chart then skips its own ConfigMap rendering. +- ``datasource.auth`` supports ``none``/``bearer``/``basic``/``mtls``. + For anything other than ``none``, set + ``datasource.auth.existingSecret`` to a Secret carrying the credential + — the chart mounts it as files and references it by path so it never + lands in the ConfigMap. Expected Secret keys: ``token`` (bearer), + ``password`` (basic; set ``datasource.auth.username`` too), or + ``ca.crt``/``tls.crt``/``tls.key`` (mTLS). - ``networkPolicy.enabled`` defaults on; turn it off if your cluster does not use NetworkPolicies. - ``serviceMonitor.honorLabels: true`` is the default and required diff --git a/charts/promanomaly/ci/datasource-auth-values.yaml b/charts/promanomaly/ci/datasource-auth-values.yaml new file mode 100644 index 0000000..71bfc75 --- /dev/null +++ b/charts/promanomaly/ci/datasource-auth-values.yaml @@ -0,0 +1,16 @@ +# Exercise the datasource-auth Secret mount + file-referenced credential +# rendering (bearer here; basic/mtls render the analogous *_file paths). +datasource: + url: http://victoriametrics.monitoring.svc:8428/ + auth: + type: bearer + existingSecret: tsdb-credentials + +groups: + - name: ci_auth + priority: 1 + queries: + - id: up_probe + promql: up + detectors: + - name: MAD diff --git a/charts/promanomaly/ci/multi-sink-noauth-values.yaml b/charts/promanomaly/ci/multi-sink-noauth-values.yaml new file mode 100644 index 0000000..4d38964 --- /dev/null +++ b/charts/promanomaly/ci/multi-sink-noauth-values.yaml @@ -0,0 +1,20 @@ +# Regression guard: a ``sinks:`` list entry that omits the optional +# ``auth`` / ``timeout`` / ``tags`` / ``max_annotations_per_run`` fields. +# These default in the detector's config schema, so the chart must render +# them away rather than nil-pointer on the missing sub-objects. +sinks: + - type: remote_write + remote_write: + url: http://victoriametrics.monitoring.svc:8428/api/v1/write + - type: grafana_annotations + grafana_annotations: + url: http://grafana.monitoring.svc:3000 + +groups: + - name: ci_multi_sink_noauth + priority: 1 + queries: + - id: queue_depth + promql: max(queue_depth) by (queue) + detectors: + - name: MAD diff --git a/charts/promanomaly/ci/multi-sink-values.yaml b/charts/promanomaly/ci/multi-sink-values.yaml new file mode 100644 index 0000000..9b46c4a --- /dev/null +++ b/charts/promanomaly/ci/multi-sink-values.yaml @@ -0,0 +1,30 @@ +# Exercise the multi-sink list (``sinks:``) configmap + per-sink Secret +# mount branches. Both sink types are active at once, each with its own +# bearer-token Secret mounted under /etc/promanomaly-sink-auth/. +sinks: + - type: remote_write + remote_write: + url: http://victoriametrics.monitoring.svc:8428/api/v1/write + timeout: 10s + auth: + type: bearer + existingSecret: remote-write-credentials + - type: grafana_annotations + grafana_annotations: + url: http://grafana.monitoring.svc:3000 + timeout: 10s + auth: + type: bearer + existingSecret: grafana-credentials + tags: + - promanomaly + max_annotations_per_run: 50 + +groups: + - name: ci_multi_sink + priority: 1 + queries: + - id: queue_depth + promql: max(queue_depth) by (queue) + detectors: + - name: MAD diff --git a/charts/promanomaly/ci/sink-values.yaml b/charts/promanomaly/ci/sink-values.yaml new file mode 100644 index 0000000..56e6111 --- /dev/null +++ b/charts/promanomaly/ci/sink-values.yaml @@ -0,0 +1,19 @@ +# Exercise the optional push-sink configmap branch. Uses the +# remote_write sink (the grafana_annotations branch is symmetric). +sink: + type: remote_write + remote_write: + url: http://victoriametrics.monitoring.svc:8428/api/v1/write + timeout: 10s + auth: + type: bearer + existingSecret: remote-write-credentials + +groups: + - name: ci_sink + priority: 1 + queries: + - id: queue_depth + promql: max(queue_depth) by (queue) + detectors: + - name: MAD diff --git a/charts/promanomaly/templates/_helpers.tpl b/charts/promanomaly/templates/_helpers.tpl index 55c1f5c..2ce498f 100644 --- a/charts/promanomaly/templates/_helpers.tpl +++ b/charts/promanomaly/templates/_helpers.tpl @@ -55,3 +55,68 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- printf "%s-config" (include "promanomaly.fullname" .) -}} {{- end -}} {{- end -}} + +{{/* + Render the body of one sink (the fields below ``type:``) at column 0; + the caller indents the result for the scalar ``sink:`` mapping or each + ``sinks:`` list item. Shared by configmap.yaml so the two forms can + never drift. Input dict: ``sink`` (the sink values) and ``tokenPath`` + (where the bearer token Secret is mounted, when one is set). + + Optional fields (``timeout``, ``auth``, ``max_annotations_per_run``, + ``tags``) are emitted only when set: ``sinks:`` list entries are + user-supplied and may omit them, and the detector's config schema fills + the documented defaults. Only the typed block and its ``url`` are + required (the config schema enforces a block matching ``type``). +*/}} +{{- define "promanomaly.sinkFields" -}} +{{- $sink := .sink -}} +{{- $tokenPath := .tokenPath -}} +type: {{ $sink.type | quote }} +{{- if eq $sink.type "remote_write" }} +{{- $rw := $sink.remote_write }} +remote_write: + url: {{ $rw.url | quote }} + {{- with $rw.timeout }} + timeout: {{ . | quote }} + {{- end }} + {{- with $rw.auth }} + auth: + type: {{ .type | default "none" | quote }} + {{- if .existingSecret }} + token_file: {{ $tokenPath }} + {{- end }} + {{- end }} +{{- else if eq $sink.type "grafana_annotations" }} +{{- $ga := $sink.grafana_annotations }} +grafana_annotations: + url: {{ $ga.url | quote }} + {{- with $ga.timeout }} + timeout: {{ . | quote }} + {{- end }} + {{- with $ga.auth }} + auth: + type: {{ .type | default "none" | quote }} + {{- if .existingSecret }} + token_file: {{ $tokenPath }} + {{- end }} + {{- end }} + {{- with $ga.max_annotations_per_run }} + max_annotations_per_run: {{ . }} + {{- end }} + {{- with $ga.tags }} + tags: +{{ toYaml . | indent 4 }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* + Mount path for one sink's bearer-token Secret in the list (``sinks:``) + form. The sink ``type`` is unique per config, so it makes a stable + per-sink directory; ``_`` is not valid in a Kubernetes name so it is + rewritten to ``-`` for the volume name (the mount path keeps it). +*/}} +{{- define "promanomaly.sinkAuthDir" -}} +{{- printf "/etc/promanomaly-sink-auth/%s" (. | replace "_" "-") -}} +{{- end -}} diff --git a/charts/promanomaly/templates/configmap.yaml b/charts/promanomaly/templates/configmap.yaml index f5bc000..aff7253 100644 --- a/charts/promanomaly/templates/configmap.yaml +++ b/charts/promanomaly/templates/configmap.yaml @@ -19,6 +19,25 @@ data: timeout: {{ .Values.datasource.timeout | quote }} auth: type: {{ .Values.datasource.auth.type | quote }} + {{- if .Values.datasource.auth.existingSecret }} + {{- /* + Credentials are mounted from datasource.auth.existingSecret at + /etc/promanomaly-datasource-auth and referenced by file, so they + never land in this ConfigMap. Expected Secret keys per type: + bearer -> token; basic -> password (+ auth.username here); + mtls -> ca.crt / tls.crt / tls.key. + */}} + {{- if eq .Values.datasource.auth.type "bearer" }} + token_file: /etc/promanomaly-datasource-auth/token + {{- else if eq .Values.datasource.auth.type "basic" }} + username: {{ .Values.datasource.auth.username | quote }} + password_file: /etc/promanomaly-datasource-auth/password + {{- else if eq .Values.datasource.auth.type "mtls" }} + ca_file: /etc/promanomaly-datasource-auth/ca.crt + cert_file: /etc/promanomaly-datasource-auth/tls.crt + key_file: /etc/promanomaly-datasource-auth/tls.key + {{- end }} + {{- end }} server: listen: {{ .Values.server.listen | quote }} refresh_interval: {{ .Values.server.refresh_interval | quote }} @@ -119,6 +138,27 @@ data: insecure: {{ default true .otlp.insecure }} service_name: {{ default "promanomaly" .otlp.service_name | quote }} {{- end }} + {{- end }} + {{- /* + Optional push sink(s) (remote_write / grafana_annotations) active + alongside the always-on /metrics exporter. The ``sinks:`` list takes + precedence when non-empty; otherwise the scalar ``sink:`` form is + rendered when ``sink.type`` is set; pull-only when neither is set. + Setting both is rejected here (mirrors the detector's own config + validation) so the scalar isn't silently dropped by precedence. + */ -}} + {{- if and .Values.sinks .Values.sink.type }} + {{- fail "set either .Values.sink (scalar) or .Values.sinks (list), not both" }} + {{- end }} + {{- if .Values.sinks }} + sinks: + {{- range $sink := .Values.sinks }} + - +{{ include "promanomaly.sinkFields" (dict "sink" $sink "tokenPath" (printf "%s/token" (include "promanomaly.sinkAuthDir" $sink.type))) | indent 8 }} + {{- end }} + {{- else if .Values.sink.type }} + sink: +{{ include "promanomaly.sinkFields" (dict "sink" .Values.sink "tokenPath" "/etc/promanomaly-sink-auth/token") | indent 6 }} {{- end }} groups: {{ toYaml .Values.groups | indent 6 }} diff --git a/charts/promanomaly/templates/deployment.yaml b/charts/promanomaly/templates/deployment.yaml index 7e64600..57798dd 100644 --- a/charts/promanomaly/templates/deployment.yaml +++ b/charts/promanomaly/templates/deployment.yaml @@ -1,3 +1,27 @@ +{{- /* + Bearer-token Secret mounts for the active push sink(s). The scalar + ``sink:`` form mounts a single Secret at /etc/promanomaly-sink-auth; the + ``sinks:`` list form mounts each sink's Secret under its own + /etc/promanomaly-sink-auth/ directory (the sink type is unique). + ``$sinkMounts`` is a list of {name, secret, path} consumed by the + volumeMounts and volumes blocks below so the two never drift. +*/ -}} +{{- $sinkMounts := list -}} +{{- if .Values.sinks -}} + {{- range $sink := .Values.sinks -}} + {{- /* ``auth`` is optional on a list entry, so traverse with dig. */ -}} + {{- $secret := dig $sink.type "auth" "existingSecret" "" $sink -}} + {{- if $secret -}} + {{- $name := printf "sink-auth-%s" (replace "_" "-" $sink.type) -}} + {{- $sinkMounts = append $sinkMounts (dict "name" $name "secret" $secret "path" (include "promanomaly.sinkAuthDir" $sink.type)) -}} + {{- end -}} + {{- end -}} +{{- else if .Values.sink.type -}} + {{- $scalarSecret := dig .Values.sink.type "auth" "existingSecret" "" .Values.sink -}} + {{- if $scalarSecret -}} + {{- $sinkMounts = append $sinkMounts (dict "name" "sink-auth" "secret" $scalarSecret "path" "/etc/promanomaly-sink-auth") -}} + {{- end -}} +{{- end -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -87,12 +111,32 @@ spec: - name: config mountPath: /etc/promanomaly readOnly: true + {{- if .Values.datasource.auth.existingSecret }} + - name: datasource-auth + mountPath: /etc/promanomaly-datasource-auth + readOnly: true + {{- end }} + {{- range $sinkMounts }} + - name: {{ .name }} + mountPath: {{ .path }} + readOnly: true + {{- end }} - name: tmp mountPath: /tmp volumes: - name: config configMap: name: {{ include "promanomaly.configMapName" . }} + {{- if .Values.datasource.auth.existingSecret }} + - name: datasource-auth + secret: + secretName: {{ .Values.datasource.auth.existingSecret | quote }} + {{- end }} + {{- range $sinkMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secret | quote }} + {{- end }} - name: tmp emptyDir: {} {{- with .Values.nodeSelector }} diff --git a/charts/promanomaly/values.schema.json b/charts/promanomaly/values.schema.json index 3a000cf..2f36c1f 100644 --- a/charts/promanomaly/values.schema.json +++ b/charts/promanomaly/values.schema.json @@ -44,6 +44,7 @@ "type": "object", "properties": { "type": {"type": "string", "enum": ["none", "bearer", "basic", "mtls"]}, + "username": {"type": "string"}, "existingSecret": {"type": "string"} } } @@ -138,6 +139,11 @@ } } }, + "sink": {"$ref": "#/$defs/sink"}, + "sinks": { + "type": "array", + "items": {"$ref": "#/$defs/sink"} + }, "groups": {"type": "array"}, "existingConfigMap": {"type": "string"}, "service": {"type": "object"}, @@ -164,5 +170,43 @@ "strategy": {"type": "object"}, "nameOverride": {"type": "string"}, "fullnameOverride": {"type": "string"} + }, + "$defs": { + "sink": { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["", "remote_write", "grafana_annotations"]}, + "remote_write": { + "type": "object", + "properties": { + "url": {"type": "string"}, + "timeout": {"type": "string"}, + "auth": { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["none", "bearer"]}, + "existingSecret": {"type": "string"} + } + } + } + }, + "grafana_annotations": { + "type": "object", + "properties": { + "url": {"type": "string"}, + "timeout": {"type": "string"}, + "auth": { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["none", "bearer"]}, + "existingSecret": {"type": "string"} + } + }, + "tags": {"type": "array", "items": {"type": "string"}}, + "max_annotations_per_run": {"type": "integer", "minimum": 1} + } + } + } + } } } diff --git a/charts/promanomaly/values.yaml b/charts/promanomaly/values.yaml index 8ca9458..a0e9cc5 100644 --- a/charts/promanomaly/values.yaml +++ b/charts/promanomaly/values.yaml @@ -31,7 +31,14 @@ datasource: timeout: 10s auth: type: none # none | bearer | basic | mtls - existingSecret: "" # secret name carrying the configured credential + username: "" # basic-auth username (not sensitive; the password + # comes from existingSecret) + # Secret carrying the credential, mounted as files and referenced by + # path so it never lands in the ConfigMap. Expected keys per type: + # bearer -> token + # basic -> password (username above) + # mtls -> ca.crt, tls.crt, tls.key + existingSecret: "" # Reload endpoint protection. The shipped NetworkPolicy denies ingress # to /-/reload and the debug endpoints by default; auth is defence in @@ -164,6 +171,67 @@ telemetry: insecure: true # set false when the collector terminates TLS service_name: promanomaly +# Optional push sink active alongside the always-on /metrics exporter. +# Leave ``type`` empty for pull-only (the default). This scalar form +# configures a single sink and is kept for backward compatibility; to run +# several sinks at once, use the ``sinks:`` list below instead (set one or +# the other, never both). A sink failure is isolated and counted on +# ``anomaly_sink_failures_total``; it never blocks /metrics or readiness. +sink: + # "" | remote_write | grafana_annotations + type: "" + # Push each successful snapshot to a Prometheus remote-write endpoint — + # the recommended path for long-horizon anomaly_density history. + remote_write: + url: "" # e.g. http://victoriametrics:8428/api/v1/write + timeout: 10s + auth: + type: none # none | bearer + # Secret with a `token` key for bearer auth, mounted and referenced + # by file. The chart wires only the bearer (token_file) shape here; + # for basic/mtls remote-write auth, manage the sink config via + # `existingConfigMap` instead. + existingSecret: "" + # Post anomaly firings and change-points to the Grafana annotations API + # so they surface across all dashboards. + grafana_annotations: + url: "" # e.g. http://grafana.monitoring.svc:3000 + timeout: 10s + auth: + type: none # none | bearer + existingSecret: "" # Secret with a `token` key (Grafana service-account token) + tags: + - promanomaly + max_annotations_per_run: 50 + +# Several push sinks at once. When non-empty this takes precedence over the +# scalar ``sink:`` above (set one or the other, never both). ``/metrics`` +# stays available regardless. Each entry has the same shape as ``sink:``; +# a sink ``type`` may appear at most once so the +# ``anomaly_sink_failures_total{sink=...}`` label stays unambiguous. Each +# entry's bearer-token Secret is mounted under its own +# ``/etc/promanomaly-sink-auth/`` directory. +# +# sinks: +# - type: remote_write +# remote_write: +# url: http://victoriametrics:8428/api/v1/write +# timeout: 10s +# auth: +# type: bearer +# existingSecret: remote-write-credentials +# - type: grafana_annotations +# grafana_annotations: +# url: http://grafana.monitoring.svc:3000 +# timeout: 10s +# auth: +# type: bearer +# existingSecret: grafana-credentials +# tags: +# - promanomaly +# max_annotations_per_run: 50 +sinks: [] + # Inline detector groups. Mutually exclusive with ``existingConfigMap``. # # REQUIRED: the schema validator rejects a config without at least one diff --git a/detector/Dockerfile b/detector/Dockerfile index ad283cd..584c3b0 100644 --- a/detector/Dockerfile +++ b/detector/Dockerfile @@ -12,7 +12,7 @@ COPY src ./src RUN pip install --upgrade pip build && \ pip wheel --no-deps -w /wheels . && \ - pip wheel -w /wheels '.[ha]' + pip wheel -w /wheels '.[ha,sinks]' FROM python:3.14-slim AS runtime @@ -28,7 +28,7 @@ RUN groupadd --system --gid 1000 promanomaly && \ WORKDIR /app COPY --from=builder /wheels /wheels -RUN pip install --no-cache-dir --no-index --find-links=/wheels 'promanomaly[ha]' && \ +RUN pip install --no-cache-dir --no-index --find-links=/wheels 'promanomaly[ha,sinks]' && \ rm -rf /wheels USER promanomaly:promanomaly diff --git a/detector/pyproject.toml b/detector/pyproject.toml index 5813bae..15cbf46 100644 --- a/detector/pyproject.toml +++ b/detector/pyproject.toml @@ -53,6 +53,15 @@ otel = [ "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-grpc>=1.27.0", ] +# Push sinks (remote_write / grafana_annotations). Only the remote-write +# sink needs an extra dependency: ``cramjam`` provides the snappy block +# compression the remote-write 1.0 spec mandates, as a wheel-distributed +# Rust binding with no system libsnappy. Install with +# ``pip install promanomaly[sinks]`` when configuring ``sink:``. The +# WriteRequest protobuf is hand-encoded, so no ``protobuf`` / ``protoc``. +sinks = [ + "cramjam>=2.8.0", +] dev = [ "pytest>=8.3.0", "pytest-asyncio>=0.24.0", @@ -70,6 +79,9 @@ dev = [ # with the dev extras means ``uv sync --all-extras`` covers both # single-replica and HA test paths. "kubernetes>=31.0.0", + # Snappy compression for the remote-write sink test suite; matches + # the ``sinks`` extra so ``uv sync --all-extras`` exercises it. + "cramjam>=2.8.0", ] [project.scripts] @@ -124,6 +136,7 @@ module = [ "kubernetes.*", "fakeredis.*", "opentelemetry.*", + "cramjam.*", ] ignore_missing_imports = true diff --git a/detector/src/promanomaly/adapter/__init__.py b/detector/src/promanomaly/adapter/__init__.py new file mode 100644 index 0000000..79fa2a7 --- /dev/null +++ b/detector/src/promanomaly/adapter/__init__.py @@ -0,0 +1,24 @@ +"""Kubernetes custom/external metrics adapter. + +A stateless, opt-in process that re-serves the anomaly metrics the +detector already wrote to the TSDB through the Kubernetes +``external.metrics.k8s.io`` / ``custom.metrics.k8s.io`` APIs, so existing +HPA and KEDA tooling can consume anomaly signal like any other metric. + +promanomaly stays a metrics *provider*; the autoscaler remains the +decision-maker. Shipped via the opt-in ``promanomaly-metrics-adapter`` +Helm chart and the ``promanomaly adapter`` CLI subcommand. +""" + +from __future__ import annotations + +from .app import build_adapter_app +from .config import AdapterConfig, load_adapter_config +from .serve import serve_adapter + +__all__ = [ + "AdapterConfig", + "build_adapter_app", + "load_adapter_config", + "serve_adapter", +] diff --git a/detector/src/promanomaly/adapter/app.py b/detector/src/promanomaly/adapter/app.py new file mode 100644 index 0000000..72f860f --- /dev/null +++ b/detector/src/promanomaly/adapter/app.py @@ -0,0 +1,380 @@ +"""FastAPI app implementing the Kubernetes external/custom metrics API. + +The Kubernetes aggregation layer (kube-aggregator) registers two +``APIService`` objects pointing at this backend and proxies +``/apis///...`` requests to it. This module serves the +discovery documents those services expect plus the value endpoints HPA / +KEDA call: + +- ``external.metrics.k8s.io/v1beta1`` — cluster-scoped anomaly signals + (the recommended path for scaling on ``anomaly_density``). +- ``custom.metrics.k8s.io/v1beta1`` — the same signals associated with + Kubernetes objects (pods, namespaces) for HPA ``Object``/``Pods`` rules. + +Every value is read live from the TSDB via :mod:`.resolver`; the adapter +holds no state. +""" + +from __future__ import annotations + +import datetime +from typing import Any + +import structlog +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, PlainTextResponse, Response +from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Counter, generate_latest + +from .config import AdapterConfig +from .resolver import AdapterQueryError, MetricResolver, MetricValue, TSDBClient + +logger = structlog.get_logger(__name__) + +EXTERNAL_GROUP = "external.metrics.k8s.io" +CUSTOM_GROUP = "custom.metrics.k8s.io" +VERSION = "v1beta1" + +# Request outcomes for the adapter's own /metrics. Closed enumeration so +# anomaly_adapter_requests_total stays bounded-cardinality. +_OUTCOME_OK = "ok" +_OUTCOME_NOT_FOUND = "not_found" +_OUTCOME_QUERY_ERROR = "query_error" + + +class AdapterMetrics: + """Bounded operational metrics for the adapter process itself. + + Cardinality is bounded: ``api`` is ``external``/``custom``, ``metric`` + is drawn from the configured allow-list, and ``outcome`` is a closed + enum. + """ + + def __init__(self) -> None: + self.registry = CollectorRegistry() + self.requests_total = Counter( + "anomaly_adapter_requests_total", + "Adapter metric-API requests by API group, metric, and outcome.", + ["api", "metric", "outcome"], + registry=self.registry, + ) + self.tsdb_failures_total = Counter( + "anomaly_adapter_tsdb_failures_total", + "Adapter TSDB instant-query failures by metric.", + ["metric"], + registry=self.registry, + ) + + def record(self, api: str, metric: str, outcome: str) -> None: + self.requests_total.labels(api=api, metric=metric, outcome=outcome).inc() + if outcome == _OUTCOME_QUERY_ERROR: + self.tsdb_failures_total.labels(metric=metric).inc() + + def render(self) -> bytes: + return generate_latest(self.registry) + + +# Map a custom-metrics resource path segment to (Kind, PromQL label the +# object name is matched against, describedObject apiVersion). Core-group +# kinds report "v1"; grouped kinds report "/v1". +_RESOURCE_KIND = { + "pods": ("Pod", "pod", "v1"), + "namespaces": ("Namespace", "namespace", "v1"), + "nodes": ("Node", "node", "v1"), + "services": ("Service", "service", "v1"), + "deployments": ("Deployment", "deployment", "apps/v1"), +} + + +def format_quantity(value: float) -> str: + """Render a float as a Kubernetes resource.Quantity string. + + Integers render without a decimal point; fractional values render as a + trimmed fixed-point decimal (never scientific notation, which Quantity + rejects). + """ + if value == int(value): + return str(int(value)) + text = f"{value:.6f}".rstrip("0").rstrip(".") + return text or "0" + + +def _rfc3339(timestamp: float) -> str: + dt = datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC) + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def build_adapter_app(config: AdapterConfig, resolver: MetricResolver | None = None) -> FastAPI: + """Build the adapter FastAPI app. + + ``resolver`` may be injected (tests); otherwise a TSDB-backed resolver + is built from ``config`` and its client lifecycle is managed by the + app's lifespan. + """ + owns_client = resolver is None + client = TSDBClient( + base_url=config.datasource.url, + timeout=config.timeout_seconds, + auth=config.datasource.auth, + ) + active_resolver = resolver or MetricResolver(client, config.metrics) + metrics = AdapterMetrics() + + async def lifespan(_: FastAPI) -> Any: + if owns_client: + await client.start() + try: + yield + finally: + if owns_client: + await client.aclose() + + app = FastAPI(lifespan=lifespan, title="promanomaly-metrics-adapter") + + # ------------------------------------------------------------------ + # Health / liveness / readiness (kubelet + aggregation layer). + # ------------------------------------------------------------------ + @app.get("/healthz", response_class=PlainTextResponse) + @app.get("/livez", response_class=PlainTextResponse) + @app.get("/readyz", response_class=PlainTextResponse) + async def health() -> str: + return "ok" + + @app.get("/metrics") + async def adapter_metrics() -> Response: + return Response(content=metrics.render(), media_type=CONTENT_TYPE_LATEST) + + # ------------------------------------------------------------------ + # Top-level discovery. kube-aggregator merges these with the + # APIService registrations. + # ------------------------------------------------------------------ + @app.get("/apis") + async def api_groups() -> JSONResponse: + return JSONResponse( + { + "kind": "APIGroupList", + "apiVersion": "v1", + "groups": [_api_group(EXTERNAL_GROUP), _api_group(CUSTOM_GROUP)], + } + ) + + @app.get("/apis/{group}") + async def api_group(group: str) -> Response: + if group not in (EXTERNAL_GROUP, CUSTOM_GROUP): + return _not_found(f"group {group!r}") + return JSONResponse(_api_group(group)) + + # ------------------------------------------------------------------ + # external.metrics.k8s.io + # ------------------------------------------------------------------ + @app.get(f"/apis/{EXTERNAL_GROUP}/{VERSION}") + async def external_resources() -> JSONResponse: + resources = [ + { + "name": metric, + "singularName": "", + "namespaced": True, + "kind": "ExternalMetricValueList", + "verbs": ["get"], + } + for metric in config.metrics + ] + return JSONResponse( + { + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": f"{EXTERNAL_GROUP}/{VERSION}", + "resources": resources, + } + ) + + @app.get(f"/apis/{EXTERNAL_GROUP}/{VERSION}/namespaces/{{namespace}}/{{metric}}") + async def external_metric(namespace: str, metric: str, request: Request) -> Response: + if not active_resolver.supports(metric): + metrics.record("external", metric, _OUTCOME_NOT_FOUND) + return _not_found(f"external metric {metric!r}") + selector = request.query_params.get("labelSelector") + try: + values = await active_resolver.resolve(metric, label_selector=selector) + except AdapterQueryError as exc: + metrics.record("external", metric, _OUTCOME_QUERY_ERROR) + return _query_error(exc) + metrics.record("external", metric, _OUTCOME_OK) + items = [_external_item(metric, v) for v in values] + return JSONResponse( + { + "kind": "ExternalMetricValueList", + "apiVersion": f"{EXTERNAL_GROUP}/{VERSION}", + "metadata": {}, + "items": items, + } + ) + + # ------------------------------------------------------------------ + # custom.metrics.k8s.io + # ------------------------------------------------------------------ + @app.get(f"/apis/{CUSTOM_GROUP}/{VERSION}") + async def custom_resources() -> JSONResponse: + resources = [ + { + "name": f"{resource}/{metric}", + "singularName": "", + "namespaced": True, + "kind": "MetricValueList", + "verbs": ["get"], + } + for resource in config.custom_resources + for metric in config.metrics + ] + return JSONResponse( + { + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": f"{CUSTOM_GROUP}/{VERSION}", + "resources": resources, + } + ) + + async def _serve_custom( + resource: str, name: str, metric: str, namespace: str | None, selector: str | None + ) -> Response: + if resource not in config.custom_resources: + return _not_found(f"custom resource {resource!r}") + if not active_resolver.supports(metric): + metrics.record("custom", metric, _OUTCOME_NOT_FOUND) + return _not_found(f"custom metric {metric!r}") + kind, label, api_version = _RESOURCE_KIND.get( + resource, (resource.rstrip("s").title(), resource, "v1") + ) + matchers: list[str] = [] + if namespace is not None and resource != "namespaces": + matchers.append(f'namespace="{_escape(namespace)}"') + if name != "*": + matchers.append(f'{label}="{_escape(name)}"') + try: + values = await active_resolver.resolve( + metric, label_selector=selector, extra_matchers=",".join(matchers) + ) + except AdapterQueryError as exc: + metrics.record("custom", metric, _OUTCOME_QUERY_ERROR) + return _query_error(exc) + metrics.record("custom", metric, _OUTCOME_OK) + items = [_custom_item(metric, kind, api_version, namespace, label, name, v) for v in values] + return JSONResponse( + { + "kind": "MetricValueList", + "apiVersion": f"{CUSTOM_GROUP}/{VERSION}", + "metadata": {}, + "items": items, + } + ) + + # Namespaced object metrics (pods, namespaces, ...). + @app.get( + f"/apis/{CUSTOM_GROUP}/{VERSION}/namespaces/{{namespace}}/{{resource}}/{{name}}/{{metric}}" + ) + async def custom_metric_namespaced( + namespace: str, resource: str, name: str, metric: str, request: Request + ) -> Response: + return await _serve_custom( + resource, name, metric, namespace, request.query_params.get("labelSelector") + ) + + # Cluster-scoped object metrics (e.g. nodes). Fewer path segments, so + # it never shadows the namespaced route above. + @app.get(f"/apis/{CUSTOM_GROUP}/{VERSION}/{{resource}}/{{name}}/{{metric}}") + async def custom_metric_cluster( + resource: str, name: str, metric: str, request: Request + ) -> Response: + return await _serve_custom( + resource, name, metric, None, request.query_params.get("labelSelector") + ) + + return app + + +# ---------------------------------------------------------------------- +# Response builders +# ---------------------------------------------------------------------- +def _api_group(group: str) -> dict[str, Any]: + gv = {"groupVersion": f"{group}/{VERSION}", "version": VERSION} + return { + "kind": "APIGroup", + "apiVersion": "v1", + "name": group, + "versions": [gv], + "preferredVersion": gv, + } + + +def _external_item(metric: str, value: MetricValue) -> dict[str, Any]: + return { + "metricName": metric, + "metricLabels": value.labels, + "timestamp": _rfc3339(value.timestamp), + "value": format_quantity(value.value), + } + + +def _custom_item( + metric: str, + kind: str, + api_version: str, + namespace: str | None, + label: str, + requested_name: str, + value: MetricValue, +) -> dict[str, Any]: + # Prefer the object name carried on the series; fall back to the + # requested name (single-object queries). + object_name = value.labels.get(label, requested_name if requested_name != "*" else "") + described: dict[str, Any] = { + "kind": kind, + "name": object_name, + "apiVersion": api_version, + } + # Cluster-scoped objects (e.g. nodes) carry no namespace. + if namespace is not None: + described["namespace"] = namespace + return { + "describedObject": described, + "metricName": metric, + "timestamp": _rfc3339(value.timestamp), + "value": format_quantity(value.value), + "selector": None, + } + + +def _escape(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def _not_found(what: str) -> JSONResponse: + return JSONResponse( + { + "kind": "Status", + "apiVersion": "v1", + "status": "Failure", + "message": f"{what} not found", + "reason": "NotFound", + "code": 404, + }, + status_code=404, + ) + + +def _query_error(exc: AdapterQueryError) -> JSONResponse: + logger.warning("adapter_query_error", error=str(exc)) + return JSONResponse( + { + "kind": "Status", + "apiVersion": "v1", + "status": "Failure", + "message": str(exc), + "reason": "ServiceUnavailable", + "code": 503, + }, + status_code=503, + ) + + +__all__ = ["build_adapter_app", "format_quantity"] diff --git a/detector/src/promanomaly/adapter/config.py b/detector/src/promanomaly/adapter/config.py new file mode 100644 index 0000000..556f32d --- /dev/null +++ b/detector/src/promanomaly/adapter/config.py @@ -0,0 +1,114 @@ +"""Configuration schema for the Kubernetes custom-metrics adapter. + +The adapter is a *separate*, stateless process from the detector: it reads +the anomaly metrics the detector already wrote to the TSDB and re-serves +them through the Kubernetes external/custom metrics API so existing HPA / +KEDA tooling can consume them. It therefore has its own small config +(its own YAML, its own ConfigMap), reusing the detector's +:class:`~promanomaly.config.DatasourceConfig` and +:class:`~promanomaly.config.AuthConfig` so operators configure TSDB access +the same way in both. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from pydantic import Field, field_validator, model_validator + +from ..config import ( + DatasourceConfig, + Duration, + _ModelBase, + _validate_prom_name, + parse_duration, +) + +# The four anomaly signals the roadmap names as the adapter's surface. +# Severity / density / active-series are sensible scaling and gating +# inputs; outside-threshold is exposed for completeness. Scaling raw on +# anomaly_score is a documented anti-pattern, so it is deliberately absent +# from the default set. +DEFAULT_METRICS: tuple[str, ...] = ( + "anomaly_severity", + "anomaly_density", + "anomaly_active_series", + "anomaly_outside_threshold", +) + + +class AdapterTLSConfig(_ModelBase): + """Serving TLS for the aggregated apiserver. + + The Kubernetes aggregation layer only talks to an APIService backend + over HTTPS, so a serving cert is required in-cluster. Both unset + (the default) serves plain HTTP — useful only for local testing and + unit tests; the Helm chart always wires a self-signed pair. + """ + + cert_file: str | None = None + key_file: str | None = None + + @property + def enabled(self) -> bool: + return bool(self.cert_file and self.key_file) + + +class AdapterConfig(_ModelBase): + """Top-level adapter config. + + ``metrics`` is the closed set of anomaly metric names the adapter will + serve; each is validated against the Prometheus-name rules (and the + no-colons rule) exactly like detector output ids. ``custom_resources`` + lists the Kubernetes object kinds the custom-metrics API associates + metrics with (mapping the object name to a same-named PromQL label). + """ + + datasource: DatasourceConfig + # Aggregated apiservers conventionally serve on 6443. + listen: str = ":6443" + tls: AdapterTLSConfig = Field(default_factory=AdapterTLSConfig) + timeout: Duration = "10s" + metrics: list[str] = Field(default_factory=lambda: list(DEFAULT_METRICS)) + # Object kinds exposed through custom.metrics.k8s.io. The object name + # is matched against the same-named PromQL label (pod -> pod="...", + # namespace -> namespace="..."). External metrics need no object and + # are always served. + custom_resources: list[str] = Field(default_factory=lambda: ["pods", "namespaces"]) + + @property + def timeout_seconds(self) -> float: + return parse_duration(self.timeout) + + @field_validator("metrics") + @classmethod + def _validate_metric_names(cls, value: list[str]) -> list[str]: + if not value: + raise ValueError("adapter.metrics must list at least one metric") + for name in value: + _validate_prom_name(name, field="adapter.metrics[]") + return value + + @model_validator(mode="after") + def _validate_tls_pair(self) -> AdapterConfig: + tls = self.tls + if bool(tls.cert_file) != bool(tls.key_file): + raise ValueError("adapter.tls requires both cert_file and key_file, or neither") + return self + + +def load_adapter_config(path: str | Path) -> AdapterConfig: + """Parse and validate an adapter YAML config file.""" + raw = yaml.safe_load(Path(path).read_text()) + if not isinstance(raw, dict): + raise ValueError(f"adapter config {path}: top level must be a mapping") + return AdapterConfig.model_validate(raw) + + +__all__ = [ + "DEFAULT_METRICS", + "AdapterConfig", + "AdapterTLSConfig", + "load_adapter_config", +] diff --git a/detector/src/promanomaly/adapter/resolver.py b/detector/src/promanomaly/adapter/resolver.py new file mode 100644 index 0000000..acf5429 --- /dev/null +++ b/detector/src/promanomaly/adapter/resolver.py @@ -0,0 +1,169 @@ +"""TSDB-backed resolution of anomaly metrics into k8s metric values. + +The adapter answers a metrics-API request by running an *instant* PromQL +query for the requested metric (optionally constrained by a translated +label selector) and returning the matching series. It holds no state — +every request is a fresh read of what the detector already wrote. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any + +import httpx +import structlog + +from ..config import AuthConfig +from ..httpauth import build_httpx_auth + +logger = structlog.get_logger(__name__) + + +class AdapterQueryError(RuntimeError): + """Raised when the TSDB instant query fails or returns an error envelope.""" + + +@dataclass(frozen=True) +class MetricValue: + """One resolved series: its (non-reserved) labels and latest value.""" + + labels: dict[str, str] + value: float + timestamp: float + + +class TSDBClient: + """Minimal async client for the PromQL instant-query endpoint.""" + + def __init__(self, base_url: str, timeout: float, auth: AuthConfig | None = None) -> None: + self._base_url = base_url.rstrip("/") + self._timeout = timeout + self._auth = auth or AuthConfig() + self._client: httpx.AsyncClient | None = None + + async def start(self) -> None: + if self._client is not None: + return + kwargs: dict[str, Any] = { + "timeout": self._timeout, + "base_url": self._base_url, + } + kwargs.update(build_httpx_auth(self._auth)) + self._client = httpx.AsyncClient(**kwargs) + + async def aclose(self) -> None: + if self._client is None: + return + try: + await self._client.aclose() + finally: + self._client = None + + async def instant_query(self, promql: str) -> list[MetricValue]: + """Run an instant query and return one :class:`MetricValue` per series.""" + if self._client is None: + await self.start() + assert self._client is not None + try: + response = await self._client.get("/api/v1/query", params={"query": promql}) + response.raise_for_status() + except httpx.HTTPError as exc: + raise AdapterQueryError(f"instant query failed: {exc}") from exc + payload = response.json() + if payload.get("status") != "success": + raise AdapterQueryError(f"query error: {payload.get('error', 'unknown')}") + result = payload.get("data", {}).get("result", []) + values: list[MetricValue] = [] + for series in result: + metric = dict(series.get("metric", {})) + metric.pop("__name__", None) + raw = series.get("value") + if not raw or len(raw) != 2: + continue + try: + ts = float(raw[0]) + val = float(raw[1]) + except (TypeError, ValueError): + continue + # A Kubernetes resource.Quantity cannot represent NaN/±Inf, and + # rendering one would raise mid-response and surface as a 500 to + # the HPA. Such a value is never a usable scaling signal, so skip + # the series rather than fail the whole request. + if not math.isfinite(val): + continue + values.append(MetricValue(labels=metric, value=val, timestamp=ts)) + return values + + +def selector_to_matchers(label_selector: str | None) -> str: + """Translate a Kubernetes label selector into PromQL label matchers. + + Supports the equality-based forms HPA / KEDA emit: + ``key=value`` / ``key==value`` / ``key!=value``, comma-separated. + Unparseable clauses are skipped with a warning rather than failing the + whole request — a malformed selector yields a broader query, never a + 500. Set-based selectors (``key in (...)``) are not translated. + """ + if not label_selector: + return "" + matchers: list[str] = [] + for clause in label_selector.split(","): + clause = clause.strip() + if not clause: + continue + if "!=" in clause: + key, _, value = clause.partition("!=") + op = "!=" + elif "==" in clause: + key, _, value = clause.partition("==") + op = "=" + elif "=" in clause: + key, _, value = clause.partition("=") + op = "=" + else: + logger.warning("adapter_selector_skipped", clause=clause) + continue + key = key.strip() + value = value.strip().strip('"') + if not key: + continue + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + matchers.append(f'{key}{op}"{escaped}"') + return ",".join(matchers) + + +class MetricResolver: + """Resolves a metric name + selector into series values via the TSDB.""" + + def __init__(self, client: TSDBClient, allowed_metrics: list[str]) -> None: + self._client = client + self._allowed = set(allowed_metrics) + + def supports(self, metric: str) -> bool: + return metric in self._allowed + + async def resolve( + self, metric: str, label_selector: str | None = None, extra_matchers: str = "" + ) -> list[MetricValue]: + """Resolve ``metric`` constrained by ``label_selector`` and any + adapter-supplied ``extra_matchers`` (e.g. the object name match for + custom metrics). Raises :class:`KeyError` for an unsupported metric. + """ + if metric not in self._allowed: + raise KeyError(metric) + selector = selector_to_matchers(label_selector) + parts = [p for p in (selector, extra_matchers) if p] + matcher_str = ",".join(parts) + promql = f"{metric}{{{matcher_str}}}" if matcher_str else metric + return await self._client.instant_query(promql) + + +__all__ = [ + "AdapterQueryError", + "MetricResolver", + "MetricValue", + "TSDBClient", + "selector_to_matchers", +] diff --git a/detector/src/promanomaly/adapter/serve.py b/detector/src/promanomaly/adapter/serve.py new file mode 100644 index 0000000..c2e7439 --- /dev/null +++ b/detector/src/promanomaly/adapter/serve.py @@ -0,0 +1,55 @@ +"""Adapter process entrypoint: load config, run uvicorn (TLS in-cluster).""" + +from __future__ import annotations + +from pathlib import Path + +import uvicorn + +from ..logging import configure_logging, get_logger +from ..main import _parse_listen +from .app import build_adapter_app +from .config import load_adapter_config + +logger = get_logger(__name__) + + +def serve_adapter(config_path: str | Path) -> None: + """Boot the metrics adapter. Used by the ``promanomaly adapter`` CLI.""" + config_path = Path(config_path) + config = load_adapter_config(config_path) + configure_logging("INFO") + logger.info( + "adapter_boot", + config_path=str(config_path), + metrics=config.metrics, + tls=config.tls.enabled, + ) + app = build_adapter_app(config) + host, port = _parse_listen(config.listen) + # The aggregation layer only speaks HTTPS to an APIService backend, so + # in-cluster the Helm chart always supplies a cert pair; the plain-HTTP + # fallback exists only for local testing. + cert_file: str | None = None + key_file: str | None = None + if config.tls.enabled: + cert_file = config.tls.cert_file + key_file = config.tls.key_file + else: + logger.warning("adapter_tls_disabled", note="serving plain HTTP; in-cluster requires TLS") + server = uvicorn.Server( + uvicorn.Config( + app, + host=host, + port=port, + log_config=None, + access_log=False, + lifespan="on", + ssl_certfile=cert_file, + ssl_keyfile=key_file, + ) + ) + server.run() + + +__all__ = ["serve_adapter"] diff --git a/detector/src/promanomaly/cli/__init__.py b/detector/src/promanomaly/cli/__init__.py index 7c6882e..8a6d6bb 100644 --- a/detector/src/promanomaly/cli/__init__.py +++ b/detector/src/promanomaly/cli/__init__.py @@ -31,6 +31,7 @@ import json import sys import time +from pathlib import Path from typing import Any import click @@ -64,6 +65,7 @@ from ._cost import run_estimate_cost as _run_estimate_cost from ._diagnose import run_diagnose_target as _run_diagnose_target from ._diagnose import run_diagnose_tsdb as _run_diagnose_tsdb +from ._generate_rules import generate_prometheus_rules as _generate_prometheus_rules from ._inspect import _print_inspect_text from ._metadata import run_metadata_lint as _run_metadata_lint from ._probe import ( @@ -240,6 +242,43 @@ def validate( # monkeypatch hooks (``promanomaly.cli._run_probe``, etc.) keep working. +@cli.command(name="generate-rules") +@click.option( + "--config", + "config_path", + type=click.Path(exists=True, dir_okay=False), + required=True, +) +@click.option( + "--output", + "output_path", + type=click.Path(dir_okay=False), + default=None, + help="Write the generated PrometheusRule here instead of stdout.", +) +@click.option( + "--name", + "rule_name", + default="promanomaly-generated", + show_default=True, + help="metadata.name for the generated PrometheusRule.", +) +def generate_rules(config_path: str, output_path: str | None, rule_name: str) -> None: + """Scaffold a PrometheusRule from a config (selectors filled, thresholds TODO).""" + try: + cfg = load_config(config_path) + except Exception as exc: + click.echo(f"INVALID: {exc}", err=True) + sys.exit(1) + + manifest = _generate_prometheus_rules(cfg, name=rule_name) + if output_path is not None: + Path(output_path).write_text(manifest) + click.echo(f"wrote {output_path}", err=True) + else: + click.echo(manifest, nl=False) + + @cli.command(name="dry-run") @click.option( "--config", @@ -1008,6 +1047,54 @@ def diagnose_cmd( cli.add_command(backtest_cmd, name="backtest") +@cli.command(name="adapter") +@click.option( + "--config", + "config_path", + type=click.Path(dir_okay=False, exists=True), + required=True, + help="Path to the adapter YAML configuration file.", +) +def adapter(config_path: str) -> None: + """Run the Kubernetes custom/external metrics adapter. + + Serves the anomaly metrics already written to the TSDB through the + external.metrics.k8s.io / custom.metrics.k8s.io APIs so HPA / KEDA + can consume them. promanomaly stays the provider; the autoscaler + stays the decision-maker. + """ + from ..adapter import serve_adapter # local import: avoids uvicorn for CLI utilities + + serve_adapter(config_path) + + +@cli.command(name="adapter-validate") +@click.option( + "--config", + "config_path", + type=click.Path(dir_okay=False, exists=True), + required=True, + help="Path to the adapter YAML configuration file.", +) +def adapter_validate(config_path: str) -> None: + """Validate an adapter config file (schema only).""" + from ..adapter import load_adapter_config + + cfg = load_adapter_config(config_path) + click.echo( + json.dumps( + { + "status": "ok", + "listen": cfg.listen, + "tls": cfg.tls.enabled, + "metrics": cfg.metrics, + "custom_resources": cfg.custom_resources, + }, + indent=2, + ) + ) + + @cli.group() def detectors() -> None: """Introspect installed detectors.""" diff --git a/detector/src/promanomaly/cli/_generate_rules.py b/detector/src/promanomaly/cli/_generate_rules.py new file mode 100644 index 0000000..253e6fe --- /dev/null +++ b/detector/src/promanomaly/cli/_generate_rules.py @@ -0,0 +1,201 @@ +"""``generate-rules`` — scaffold PrometheusRules from a detector config. + +Emits a ``PrometheusRule`` skeleton covering every configured group and +query, with the ``id`` / ``group`` selectors already filled in from the +config so an operator never hand-copies a label set. Severities, ``for:`` +windows, and any other policy knobs are left as clearly-marked ``TODO`` +placeholders — the project deliberately ships no opinionated thresholds, +so this is scaffolding, not a recommendation. + +Each group also gets the ``max``-ensemble recording-rule fallback +(``anomaly_outside_threshold_any``) described in ``docs/patterns.md``, so +operators who don't use the built-in ``ensemble:`` block still get the +group-level OR rollup for free. + +The output is plain YAML (not a Helm template), written with inline +comments, so the generated file is immediately reviewable and appliable. +The intended workflow pairs with the GitOps validation Action: generate, +edit the ``TODO`` placeholders, validate, commit. +""" + +from __future__ import annotations + +import re + +from ..config import Config, GroupConfig, QueryConfig + +_NON_ALNUM_RE = re.compile(r"[^0-9A-Za-z]+") + + +def _camel(text: str) -> str: + """PascalCase a snake/space/template-ish string for use in an alert name. + + Non-alphanumeric runs are treated as separators, so a discover id like + ``cpu_{{ instance }}`` keeps its variable name (``CpuInstance``) — more + descriptive than dropping it, and less collision-prone. Exact + uniqueness is still guaranteed by the dedup pass in + :func:`generate_prometheus_rules`, since camel-casing alone can map two + distinct ids (``a_b`` and ``a__b``) to the same token. + """ + return "".join(part[:1].upper() + part[1:] for part in _NON_ALNUM_RE.split(text) if part) + + +# Default placeholder values. These are deliberately generic starting +# points, NOT opinions — every one carries a ``# TODO`` marker in the +# output so a reviewer can't mistake them for tuned values. ``for:`` has +# to be a real duration for the manifest to apply, hence a value rather +# than a bare ``TODO`` token. +_TODO_FOR = "5m" +_TODO_SEVERITY = "warning" + + +def _is_templated(query: QueryConfig) -> bool: + """True when the query id is a discover template (no literal id).""" + return "{{" in query.id + + +def _outside_threshold_alert(group: GroupConfig, query: QueryConfig, name: str) -> list[str]: + """Render the per-query outside-threshold alert lines. + + ``name`` is the caller-resolved, de-duplicated alert name so the + generated file is duplicate-rule-lint clean and each alert is + individually routable / silenceable in Alertmanager. + """ + lines = [ + f" # Outside-threshold alert for query {query.id!r} in group {group.name!r}.", + ] + if _is_templated(query): + # A discover-templated id has no single literal value, so scope by + # group only and leave a marker for the operator to narrow it. + selector = f'anomaly_outside_threshold{{group="{group.name}"}} == 1' + lines.append( + " # NOTE: id is discover-templated; this fires for every expanded " + 'series. Narrow with an id=~"..." matcher once the expansions are known.' + ) + else: + selector = f'anomaly_outside_threshold{{group="{group.name}", id="{query.id}"}} == 1' + lines.extend( + [ + f" - alert: {name}", + f" expr: {selector}", + f" for: {_TODO_FOR} # TODO: set the flap-suppression window for your SLO", + " labels:", + f" severity: {_TODO_SEVERITY} # TODO: set severity", + " annotations:", + ' summary: "Anomaly above threshold for {{ $labels.id }} ' + '({{ $labels.group }})"', + " description: | # TODO: tailor / add a runbook_url", + " Detector {{ $labels.detector }} reported anomaly_outside_threshold=1", + f" for at least {_TODO_FOR} in group {group.name}.", + ] + ) + return lines + + +def _ensemble_alert(group: GroupConfig, name: str) -> list[str]: + """Render the per-group ensemble-agreement alert (when configured). + + ``name`` is the caller-resolved, de-duplicated alert name. + """ + return [ + f" # Ensemble agreement for group {group.name!r} " + f"(ensemble: {group.ensemble.method} configured).", # type: ignore[union-attr] + f" - alert: {name}", + f' expr: anomaly_composite_outside_threshold{{group="{group.name}"}} == 1', + f" for: {_TODO_FOR} # TODO: set the flap-suppression window for your SLO", + " labels:", + f" severity: {_TODO_SEVERITY} # TODO: set severity", + " annotations:", + ' summary: "Ensemble agreement on anomaly for {{ $labels.id }} ' + '({{ $labels.group }})"', + " description: |", + f" Multiple detectors agree the signal is anomalous in group {group.name}.", + ] + + +def _recording_rules() -> list[str]: + """Render the global ``max``-ensemble recording-rule fallback. + + One rule covering every group: ``by (id, group)`` already partitions + the output per series, so a single rule is correct (and avoids the + duplicate-recording-rule lint that per-group copies would trip). + """ + return [ + " # max-ensemble OR rollup — the recording-rule equivalent of an", + " # `ensemble: { method: max }` block (see docs/patterns.md). Covers", + " # every group: `by (id, group)` partitions the output per series.", + " - record: anomaly_outside_threshold_any", + " expr: max by (id, group) (anomaly_outside_threshold)", + ] + + +def _make_unique(base: str, used: set[str]) -> str: + """Return ``base``, or ``base2``/``base3``/… if it's already taken. + + Guarantees globally-unique alert names even when two distinct query + ids camel-case to the same token, keeping the manifest + duplicate-rule-lint clean. + """ + candidate = base + suffix = 2 + while candidate in used: + candidate = f"{base}{suffix}" + suffix += 1 + used.add(candidate) + return candidate + + +def generate_prometheus_rules(cfg: Config, *, name: str = "promanomaly-generated") -> str: + """Build a PrometheusRule manifest scaffolding from a validated config. + + Returns the YAML document as a string. One alerting group per detector + group (with one ``AnomalyOutsideThreshold`` alert per query, plus an + ``AnomalyEnsembleAgreement`` alert when the group sets ``ensemble:``), + and one recording group holding the global ``max``-ensemble fallback. + """ + # Rule-group names must be unique within a file. Alert groups are + # named ``promanomaly.``; pick a recording-group name that can't + # collide even with a detector group literally named ``recording``. + alert_group_names = {f"promanomaly.{g.name}" for g in cfg.groups} + recording_group = _make_unique("promanomaly.recording", set(alert_group_names)) + + lines: list[str] = [ + "# Generated by `promanomaly generate-rules` — scaffolding, not opinions.", + "#", + "# Selectors (id / group) are filled in from your config. Severities and", + "# `for:` windows are TODO placeholders: the project ships no opinionated", + "# thresholds, so tune every `# TODO` below for your own SLOs before wiring", + "# this into Alertmanager. Re-run after config changes and diff.", + "apiVersion: monitoring.coreos.com/v1", + "kind: PrometheusRule", + "metadata:", + f" name: {name}", + " labels:", + " app.kubernetes.io/name: promanomaly", + " app.kubernetes.io/part-of: promanomaly", + " # TODO: add the label your Prometheus Operator selects on, e.g.", + " # release: prometheus", + "spec:", + " groups:", + f" - name: {recording_group}", + " rules:", + ] + lines.extend(_recording_rules()) + + used_names: set[str] = set() + for group in cfg.groups: + lines.append(f" - name: promanomaly.{group.name}") + lines.append(" rules:") + for query in group.queries: + name = _make_unique( + f"Anomaly{_camel(group.name)}{_camel(query.id)}OutsideThreshold", used_names + ) + lines.extend(_outside_threshold_alert(group, query, name)) + if group.ensemble is not None: + name = _make_unique(f"Anomaly{_camel(group.name)}EnsembleAgreement", used_names) + lines.extend(_ensemble_alert(group, name)) + + return "\n".join(lines) + "\n" + + +__all__ = ["generate_prometheus_rules"] diff --git a/detector/src/promanomaly/config.py b/detector/src/promanomaly/config.py index 0b9e64a..dafcea9 100644 --- a/detector/src/promanomaly/config.py +++ b/detector/src/promanomaly/config.py @@ -138,10 +138,36 @@ class AuthConfig(_ModelBase): token: str | None = None username: str | None = None password: str | None = None + # File-based credential sources, mirroring the existing ``*_file`` + # mTLS fields. Mounting a Secret as a file and pointing ``token_file`` + # / ``password_file`` at it keeps the credential out of the rendered + # ConfigMap and out of the process environment — the path the Helm + # charts wire up for ``datasource.auth.existingSecret``. When both an + # inline value and a ``*_file`` are set, the file wins. + token_file: str | None = None + password_file: str | None = None ca_file: str | None = None cert_file: str | None = None key_file: str | None = None + def resolved_token(self) -> str | None: + """Bearer token, read from ``token_file`` when set, else inline.""" + return _read_secret_file(self.token_file) if self.token_file else self.token + + def resolved_password(self) -> str | None: + """Basic-auth password, read from ``password_file`` when set, else inline.""" + return _read_secret_file(self.password_file) if self.password_file else self.password + + +def _read_secret_file(path: str) -> str: + """Read a credential from a mounted Secret file, trimming trailing newline. + + Kubernetes Secret files and ``echo``-created tokens commonly carry a + trailing newline; a stray ``\\n`` in a bearer header silently breaks + auth, so it is stripped here. + """ + return Path(path).read_text().strip() + class DatasourceConfig(_ModelBase): url: str @@ -513,6 +539,108 @@ class TelemetryConfig(_ModelBase): otlp: OTLPConfig | None = None +class RemoteWriteSinkConfig(_ModelBase): + """Push the snapshot to a Prometheus remote-write endpoint. + + Runs after each successful group run, in addition to (never instead + of) ``/metrics`` — the pull exporter is always available. The + recommended path for long-horizon ``anomaly_density`` history that + outlives scrape retention and feeds recurrence analysis. + + Auth mirrors :class:`AuthConfig` so operators reuse the datasource + auth shapes (bearer / basic / mTLS) they already know. + """ + + url: str + timeout: Duration = "10s" + auth: AuthConfig = Field(default_factory=AuthConfig) + + @property + def timeout_seconds(self) -> float: + return parse_duration(self.timeout) + + +class GrafanaAnnotationsSinkConfig(_ModelBase): + """Post anomaly firings and change-points to the Grafana annotations API. + + Emits an annotation on each ``anomaly_outside_threshold`` 0->1 + transition and each ``anomaly_change_point_total`` increment, tagged + with ``id`` / ``group`` / ``detector`` so they surface across *all* + dashboards (Grafana shows tag-matched annotations globally), even on + dashboards that don't query promanomaly metrics directly. + + Rate-bounded by ``max_annotations_per_run`` so a fleet-wide event + can't flood the Grafana API; the overflow is dropped and counted via + ``anomaly_sink_failures_total{reason="rate_limited"}``. + """ + + # Grafana base URL, e.g. ``http://grafana.monitoring.svc:3000``. + url: str + timeout: Duration = "10s" + # Service-account / API token. Only ``bearer`` is supported here + # because that is the single auth shape the Grafana HTTP API accepts + # for the annotations endpoint; basic / mTLS are intentionally not + # offered to avoid implying support Grafana doesn't give us. + auth: AuthConfig = Field(default_factory=AuthConfig) + # Tags stamped on every annotation in addition to the per-event + # id / group / detector tags. A dashboard configured with a matching + # annotation query (e.g. tag ``promanomaly``) then shows them. + tags: list[str] = Field(default_factory=lambda: ["promanomaly"]) + # Hard cap on annotations posted per run; protects the Grafana API + # from a fleet-wide firing storm. Overflow is dropped and counted. + max_annotations_per_run: int = Field(default=50, ge=1) + + @property + def timeout_seconds(self) -> float: + return parse_duration(self.timeout) + + @field_validator("auth") + @classmethod + def _only_bearer_or_none(cls, value: AuthConfig) -> AuthConfig: + if value.type not in ("none", "bearer"): + raise ValueError( + "sink.grafana_annotations.auth.type must be 'none' or 'bearer' " + "(the Grafana annotations API accepts only token auth)" + ) + return value + + +class SinkConfig(_ModelBase): + """One push sink active alongside the always-on ``/metrics`` exporter. + + A single sink may be configured via the scalar ``sink:`` field, or + several via the ``sinks:`` list (see :class:`Config`); the scalar form + is retained for backward compatibility. The ``type`` selects which + nested block is consulted; the matching block is required and the + others must be absent. + + A push-sink failure is always isolated from the detection pipeline: + it is logged and counted on ``anomaly_sink_failures_total`` but never + blocks ``/metrics`` or readiness. + """ + + type: Literal["remote_write", "grafana_annotations"] + remote_write: RemoteWriteSinkConfig | None = None + grafana_annotations: GrafanaAnnotationsSinkConfig | None = None + + @model_validator(mode="after") + def _require_matching_block(self) -> SinkConfig: + blocks = { + "remote_write": self.remote_write, + "grafana_annotations": self.grafana_annotations, + } + selected = blocks[self.type] + if selected is None: + raise ValueError(f"sink.type={self.type!r} requires a sink.{self.type} block") + extras = sorted(name for name, block in blocks.items() if name != self.type and block) + if extras: + raise ValueError( + f"sink.type={self.type!r} but unrelated sink block(s) {extras} are also set; " + "configure exactly one sink" + ) + return self + + _DISCOVER_VARIABLE_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") @@ -703,8 +831,57 @@ class Config(_ModelBase): exporter: ExporterConfig = Field(default_factory=ExporterConfig) highAvailability: HighAvailabilityConfig = Field(default_factory=HighAvailabilityConfig) telemetry: TelemetryConfig = Field(default_factory=TelemetryConfig) + # Optional push sink(s) active alongside the always-on ``/metrics`` + # exporter. Both default to pull-only. + # + # ``sink`` is the scalar single-sink form, retained for backward + # compatibility. ``sinks`` is the list form: ``/metrics`` is always + # available and each configured push sink runs after every successful + # run. The two are mutually exclusive — set one or neither, never + # both — and :meth:`effective_sinks` normalises whichever was given + # into a single list the runtime consumes. + sink: SinkConfig | None = None + sinks: list[SinkConfig] | None = None groups: list[GroupConfig] + @model_validator(mode="after") + def _validate_sinks(self) -> Config: + if self.sink is not None and self.sinks is not None: + raise ValueError( + "set either 'sink' (single) or 'sinks' (list), not both; " + "'sink' is the backward-compatible scalar form of 'sinks'" + ) + if self.sinks is not None: + if not self.sinks: + raise ValueError("'sinks' must list at least one sink, or be omitted for pull-only") + # Each sink's ``type`` is its ``sink`` label value on + # ``anomaly_sink_failures_total``; duplicate types would + # collide on that label and make per-sink failure counts + # ambiguous, so a type may appear at most once. Two sinks of + # the same kind to different targets is intentionally not + # supported (it drifts toward unbounded sink fan-out). + types = [s.type for s in self.sinks] + dupes = sorted({t for t in types if types.count(t) > 1}) + if dupes: + raise ValueError( + f"duplicate sink type(s) {dupes} in 'sinks'; each sink type may " + "appear at most once so anomaly_sink_failures_total{sink=...} stays unambiguous" + ) + return self + + def effective_sinks(self) -> list[SinkConfig]: + """Normalise the scalar and list sink forms into one list. + + Returns ``[]`` when pull-only. The runtime only ever consults + this, so the scalar / list distinction stays confined to config + parsing. + """ + if self.sinks is not None: + return list(self.sinks) + if self.sink is not None: + return [self.sink] + return [] + @model_validator(mode="after") def _validate_ha_requires_redis(self) -> Config: # HA mode needs the Redis snapshot cache; without it followers @@ -886,6 +1063,7 @@ def _validate_detector_params(cfg: Config) -> None: "EnsembleConfig", "ExporterConfig", "ExporterLabelsConfig", + "GrafanaAnnotationsSinkConfig", "GroupConfig", "HighAvailabilityConfig", "OTLPConfig", @@ -894,9 +1072,11 @@ def _validate_detector_params(cfg: Config) -> None: "RedisConfig", "ReloadAuthConfig", "ReloadConfig", + "RemoteWriteSinkConfig", "SafetyConfig", "SelfTestConfig", "ServerConfig", + "SinkConfig", "TelemetryConfig", "load_config", "parse_duration", diff --git a/detector/src/promanomaly/exporter.py b/detector/src/promanomaly/exporter.py index 953a38d..f34858a 100644 --- a/detector/src/promanomaly/exporter.py +++ b/detector/src/promanomaly/exporter.py @@ -493,6 +493,23 @@ def _declare_selftest_metrics(self) -> None: ["detector"], registry=self.registry, ) + self._declare_sink_metrics() + + # ------------------------------------------------------------------ + # Push-sink metrics. Cardinality is bounded — one series per + # (sink, reason) and both label sets are closed enumerations. + # ------------------------------------------------------------------ + def _declare_sink_metrics(self) -> None: + self.sink_failures_total = Counter( + "anomaly_sink_failures_total", + ( + "Total push-sink failures bucketed by sink and reason. A sink " + "failure is isolated from the detection pipeline and never " + "blocks /metrics or readiness." + ), + ["sink", "reason"], + registry=self.registry, + ) def render(self) -> bytes: return generate_latest(self.registry) diff --git a/detector/src/promanomaly/httpauth.py b/detector/src/promanomaly/httpauth.py new file mode 100644 index 0000000..39905c9 --- /dev/null +++ b/detector/src/promanomaly/httpauth.py @@ -0,0 +1,43 @@ +"""Shared httpx client kwargs for the :class:`~promanomaly.config.AuthConfig` +auth shapes (bearer / basic / mTLS). + +Used by the remote-write sink and the metrics adapter so the four auth +shapes are wired identically wherever promanomaly talks to an +authenticated HTTP endpoint. +""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from .config import AuthConfig + + +def build_httpx_auth(auth: AuthConfig) -> dict[str, Any]: + """Return httpx ``AsyncClient`` kwargs implementing ``auth``. + + Returns an empty dict for ``type: none``. Bearer sets the + Authorization header; basic sets httpx BasicAuth; mTLS sets the client + cert pair and (optionally) the CA bundle to verify against. Bearer + tokens and basic passwords are resolved from their ``*_file`` Secret + mount when configured (the path the Helm charts wire up), else the + inline value. + """ + kwargs: dict[str, Any] = {} + if auth.type == "bearer": + token = auth.resolved_token() + if token: + kwargs["headers"] = {"Authorization": f"Bearer {token}"} + elif auth.type == "basic" and auth.username: + kwargs["auth"] = httpx.BasicAuth(auth.username, auth.resolved_password() or "") + elif auth.type == "mtls": + if auth.cert_file and auth.key_file: + kwargs["cert"] = (auth.cert_file, auth.key_file) + if auth.ca_file: + kwargs["verify"] = auth.ca_file + return kwargs + + +__all__ = ["build_httpx_auth"] diff --git a/detector/src/promanomaly/main.py b/detector/src/promanomaly/main.py index 43cac6a..1d6c585 100644 --- a/detector/src/promanomaly/main.py +++ b/detector/src/promanomaly/main.py @@ -41,6 +41,7 @@ from .runner import Runner from .selftest import run_selftest from .server import build_app as _build_app +from .sinks import Sink, build_sinks from .source import PromQLSource, QueryResult from .state import SnapshotStore from .telemetry import Telemetry @@ -81,6 +82,11 @@ def __init__(self, config: Config, config_path: Path) -> None: telemetry=self._telemetry, ) self._exporter = Exporter(self._store, self._ops, self._source) + # Optional push sinks (remote_write / grafana_annotations) running + # alongside the always-on /metrics exporter. Empty when pull-only. + # Started in startup(); swapped on reload only when the sink config + # actually changed. + self._sinks: list[Sink] = build_sinks(config, self._ops) self._scheduler: AsyncIOScheduler | None = None # Anchor background tasks so the GC can't drop them mid-run. self._background: set[asyncio.Task[Any]] = set() @@ -123,6 +129,8 @@ def __init__(self, config: Config, config_path: Path) -> None: # ------------------------------------------------------------------ async def startup(self) -> None: await self._source.start() + for sink in self._sinks: + await sink.start() self._scheduler = AsyncIOScheduler() self._reschedule_jobs(self._scheduler) self._scheduler.start() @@ -147,6 +155,8 @@ async def shutdown(self) -> None: if self._scheduler is not None: self._scheduler.shutdown(wait=False) self._scheduler = None + for sink in self._sinks: + await sink.aclose() await self._source.close() # Shut down the runner's thread pool so pending detector # invocations don't outlive the process. wait=False mirrors the @@ -192,6 +202,7 @@ async def reload(self) -> tuple[bool, str]: self._runner.replace_config(new_config) self._source.replace_cache(_build_query_cache(new_config, self._redis)) + await self._swap_sinks(new_config) self._config = new_config self._stamp_config_hash(new_config) if self._scheduler is not None: @@ -287,13 +298,21 @@ async def _safe_group_run(self, group_name: str) -> None: # ``_on_stopped_leading``: once demotion has flipped # ``_is_leader`` under the lock, any task waiting here on # the same lock observes the new value and skips the publish. - if result.succeeded and self._ha is not None: + if result.succeeded: snapshot = self._store.get(group_name) if snapshot is not None: - lock = self._ensure_publish_lock() - async with lock: - if not self._ha_enabled or self._is_leader: - self._ha.snapshot_cache.publish(snapshot) + if self._ha is not None: + lock = self._ensure_publish_lock() + async with lock: + if not self._ha_enabled or self._is_leader: + self._ha.snapshot_cache.publish(snapshot) + # Push to the configured sinks. In HA mode only the + # leader emits, so a snapshot is pushed once per + # cluster. A brief duplicate during failover is + # harmless (remote-write samples and Grafana + # annotations are effectively idempotent). + if self._sinks and (not self._ha_enabled or self._is_leader): + await self._emit_to_sinks(snapshot) finally: if task is not None: tasks = self._running_group_tasks.get(group_name) @@ -302,6 +321,41 @@ async def _safe_group_run(self, group_name: str) -> None: if not tasks: self._running_group_tasks.pop(group_name, None) + async def _swap_sinks(self, new_config: Config) -> None: + """Rebuild the push sinks on reload only when their config changed. + + Comparing the effective sink lists avoids churning the httpx + clients (and resetting the Grafana-annotations transition + baselines) on reloads that don't touch the sinks. A genuine change + closes the old sinks and starts the new ones; resetting the + baselines on a real change is the safe direction — already-firing + series re-baseline silently rather than replaying as fresh + annotations. + """ + if new_config.effective_sinks() == self._config.effective_sinks(): + return + for sink in self._sinks: + await sink.aclose() + self._sinks = build_sinks(new_config, self._ops) + for sink in self._sinks: + await sink.start() + + async def _emit_to_sinks(self, snapshot: Any) -> None: + """Push a snapshot to every configured sink with an outer safety net. + + Concrete sinks already swallow and count their own failures + (the "never blocks /metrics or readiness" contract), so this + guard only catches programming errors that escape that net; it + must never let one sink's fault propagate into the scheduler task + or starve a later sink — each emit is isolated. + """ + for sink in self._sinks: + try: + await sink.emit(snapshot) + except Exception as exc: # pragma: no cover - defensive + logger.exception("sink_emit_failed", sink=sink.name, error=str(exc)) + self._ops.sink_failures_total.labels(sink=sink.name, reason="exception").inc() + async def _selftest_tick(self) -> None: # Leader-gated like ``_tick``: the self-test fires on every replica # but only the elected leader actually runs it in HA mode, so the diff --git a/detector/src/promanomaly/sinks/__init__.py b/detector/src/promanomaly/sinks/__init__.py new file mode 100644 index 0000000..36c05fb --- /dev/null +++ b/detector/src/promanomaly/sinks/__init__.py @@ -0,0 +1,51 @@ +"""Optional push sinks active alongside the always-on ``/metrics`` exporter. + +A sink pushes each successful group snapshot to an external system after +the run completes. One or several sinks may be configured; ``/metrics`` +is always available regardless. Failures are isolated and counted on +``anomaly_sink_failures_total`` and never block ``/metrics`` or readiness. +""" + +from __future__ import annotations + +from ..config import Config, SinkConfig +from ..exporter import OperationalMetrics +from .base import Sink +from .grafana_annotations import GrafanaAnnotationsSink +from .remote_write import RemoteWriteSink + + +def build_sink(config: SinkConfig | None, operational: OperationalMetrics) -> Sink | None: + """Construct one configured sink, or ``None`` when pull-only. + + Raises ``ValueError`` if the sink type is unknown — config validation + already constrains ``type`` to the supported set, so this only trips + on an internal mismatch. + """ + if config is None: + return None + if config.type == "remote_write": + assert config.remote_write is not None # enforced by SinkConfig validation + return RemoteWriteSink(config.remote_write, operational) + if config.type == "grafana_annotations": + assert config.grafana_annotations is not None + return GrafanaAnnotationsSink(config.grafana_annotations, operational) + raise ValueError(f"unknown sink type {config.type!r}") + + +def build_sinks(config: Config, operational: OperationalMetrics) -> list[Sink]: + """Construct every configured push sink from a :class:`Config`. + + Normalises the scalar ``sink:`` and list ``sinks:`` forms via + :meth:`Config.effective_sinks`, so the runtime never has to know which + form the operator used. Returns ``[]`` when pull-only. + """ + sinks: list[Sink] = [] + for sink_config in config.effective_sinks(): + sink = build_sink(sink_config, operational) + assert sink is not None # effective_sinks never yields a None entry + sinks.append(sink) + return sinks + + +__all__ = ["GrafanaAnnotationsSink", "RemoteWriteSink", "Sink", "build_sink", "build_sinks"] diff --git a/detector/src/promanomaly/sinks/base.py b/detector/src/promanomaly/sinks/base.py new file mode 100644 index 0000000..2d7f091 --- /dev/null +++ b/detector/src/promanomaly/sinks/base.py @@ -0,0 +1,70 @@ +"""Push-sink abstraction. + +Sinks run *after* a successful group run, in addition to the always-on +``/metrics`` pull exporter. The contract is strict: a sink failure is +isolated and counted on ``anomaly_sink_failures_total`` but never blocks +``/metrics`` or readiness (see the project guide). Concrete sinks +therefore swallow their own errors — :meth:`Sink.emit` must not raise — +and the caller in :mod:`promanomaly.main` keeps an outer guard as +defence in depth. +""" + +from __future__ import annotations + +import abc + +from ..exporter import OperationalMetrics +from ..state import GroupSnapshot + +# Bounded reason taxonomy for ``anomaly_sink_failures_total{reason=...}``. +# Keep this tight — the whole point of the metric is bounded cardinality. +REASON_HTTP = "http_error" +REASON_TIMEOUT = "timeout" +REASON_CONNECT = "connect_error" +REASON_SERIALIZE = "serialize_error" +REASON_RATE_LIMITED = "rate_limited" +REASON_EXCEPTION = "exception" + + +class Sink(abc.ABC): + """A push destination for anomaly snapshots. + + Each configured sink instance is long-lived: it is shared across the + scheduler for the process lifetime (several may run at once). + :meth:`start` opens any network client; :meth:`emit` is invoked once + per successful group run with that group's snapshot; :meth:`aclose` + releases resources on shutdown. + """ + + #: Bounded-cardinality identifier used as the ``sink`` label value on + #: ``anomaly_sink_failures_total``. + name: str + + def __init__(self, operational: OperationalMetrics) -> None: + self._ops = operational + + @abc.abstractmethod + async def start(self) -> None: + """Open the network client. Idempotent.""" + + @abc.abstractmethod + async def emit(self, snapshot: GroupSnapshot) -> None: + """Push one group's snapshot. Must never raise.""" + + @abc.abstractmethod + async def aclose(self) -> None: + """Release resources. Idempotent.""" + + def _count_failure(self, reason: str) -> None: + self._ops.sink_failures_total.labels(sink=self.name, reason=reason).inc() + + +__all__ = [ + "REASON_CONNECT", + "REASON_EXCEPTION", + "REASON_HTTP", + "REASON_RATE_LIMITED", + "REASON_SERIALIZE", + "REASON_TIMEOUT", + "Sink", +] diff --git a/detector/src/promanomaly/sinks/grafana_annotations.py b/detector/src/promanomaly/sinks/grafana_annotations.py new file mode 100644 index 0000000..9593dcd --- /dev/null +++ b/detector/src/promanomaly/sinks/grafana_annotations.py @@ -0,0 +1,212 @@ +"""Grafana annotations sink. + +Posts an annotation to the Grafana HTTP API on each +``anomaly_outside_threshold`` 0->1 transition and each +``anomaly_change_point_total`` increment, so anomalies surface across +*all* dashboards (Grafana renders tag-matched annotations globally), +even ones that never query promanomaly metrics directly. + +Transition detection is in-memory and stateless across restarts: the +previous per-series value is held on the instance and rebuilt from the +next run. On a cold start, series already firing are recorded as a +baseline and do *not* annotate — only observed 0->1 edges do — so a +restart can't replay a storm of stale firings into every dashboard. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx +import structlog + +from ..config import GrafanaAnnotationsSinkConfig +from ..exporter import OperationalMetrics +from ..httpauth import build_httpx_auth +from ..state import GroupSnapshot, Sample +from .base import ( + REASON_CONNECT, + REASON_HTTP, + REASON_RATE_LIMITED, + REASON_TIMEOUT, + Sink, +) + +logger = structlog.get_logger(__name__) + +_OUTSIDE_METRIC = "anomaly_outside_threshold" +_CHANGE_POINT_METRIC = "anomaly_change_point_total" +_ANNOTATIONS_PATH = "/api/annotations" + +# Labels lifted onto annotation tags (Grafana matches dashboards by tag). +_TAG_LABELS = ("id", "group", "detector", "detector_instance") + + +class GrafanaAnnotationsSink(Sink): + """Posts firing / change-point events to the Grafana annotations API.""" + + name = "grafana_annotations" + + def __init__( + self, config: GrafanaAnnotationsSinkConfig, operational: OperationalMetrics + ) -> None: + super().__init__(operational) + self._config = config + self._client: httpx.AsyncClient | None = None + # series identity -> last seen value. ``None`` (absent key) means + # never observed, so the first sighting records a baseline without + # annotating — a cold start can't replay stale firings. + self._last_outside: dict[tuple[tuple[str, str], ...], float] = {} + self._last_change_point: dict[tuple[tuple[str, str], ...], float] = {} + + async def start(self) -> None: + if self._client is not None: + return + kwargs: dict[str, Any] = { + "timeout": self._config.timeout_seconds, + "base_url": self._config.url.rstrip("/"), + } + # Bearer token resolved via the shared helper so a Secret mounted at + # ``token_file`` (the path the Helm chart wires) works identically to + # an inline token. Config validation already constrains the auth type + # to none / bearer, so only the bearer branch can fire here. + kwargs.update(build_httpx_auth(self._config.auth)) + self._client = httpx.AsyncClient(**kwargs) + + async def emit(self, snapshot: GroupSnapshot) -> None: + if self._client is None: + await self.start() + assert self._client is not None + events = self._collect_events(snapshot) + if not events: + return + budget = self._config.max_annotations_per_run + if len(events) > budget: + # Drop the overflow rather than flood Grafana; count it so the + # operator can alert on a rate-limited fleet-wide event storm. + dropped = len(events) - budget + self._count_failure(REASON_RATE_LIMITED) + logger.warning( + "grafana_annotations_rate_limited", + group=snapshot.group, + dropped=dropped, + budget=budget, + ) + events = events[:budget] + # Post concurrently — bounded by max_annotations_per_run, so this + # never opens more than ``budget`` in-flight requests. Each _post + # swallows and counts its own failures, so gather never raises. + await asyncio.gather(*(self._post(event, snapshot.group) for event in events)) + + def _collect_events(self, snapshot: GroupSnapshot) -> list[dict[str, Any]]: + """Diff this snapshot against the previous to find annotatable edges.""" + timestamp_ms = int(snapshot.timestamp * 1000) + events: list[dict[str, Any]] = [] + seen_outside: set[tuple[tuple[str, str], ...]] = set() + seen_change_point: set[tuple[tuple[str, str], ...]] = set() + for sample in snapshot.samples: + if sample.metric == _OUTSIDE_METRIC: + seen_outside.add(sample.labels) + event = self._outside_event(sample, timestamp_ms) + elif sample.metric == _CHANGE_POINT_METRIC: + seen_change_point.add(sample.labels) + event = self._change_point_event(sample, timestamp_ms) + else: + continue + if event is not None: + events.append(event) + # Bound the transition-tracking state: drop entries for *this* group + # whose series was absent from this run, so a workload that churns + # through ephemeral series identities (discovery) can't grow the + # dicts without bound. Keys for other groups are untouched — emit() + # is called once per group. A series that vanishes and later + # reappears firing re-baselines (no annotation), matching the + # cold-start guard. + self._prune(self._last_outside, snapshot.group, seen_outside) + self._prune(self._last_change_point, snapshot.group, seen_change_point) + return events + + @staticmethod + def _prune( + state: dict[tuple[tuple[str, str], ...], float], + group: str, + seen: set[tuple[tuple[str, str], ...]], + ) -> None: + stale = [key for key in state if dict(key).get("group") == group and key not in seen] + for key in stale: + del state[key] + + def _outside_event(self, sample: Sample, timestamp_ms: int) -> dict[str, Any] | None: + key = sample.labels + previous = self._last_outside.get(key) + self._last_outside[key] = sample.value + # First sighting establishes a baseline (no annotation); only an + # observed 0->1 edge between two runs annotates. + if previous is not None and previous < 1.0 <= sample.value: + labels = dict(sample.labels) + return { + "time": timestamp_ms, + "tags": self._tags(labels), + "text": self._describe(labels, "anomaly fired (outside threshold)"), + } + return None + + def _change_point_event(self, sample: Sample, timestamp_ms: int) -> dict[str, Any] | None: + key = sample.labels + previous = self._last_change_point.get(key) + self._last_change_point[key] = sample.value + # First sighting establishes a baseline; a counter reset (restart) + # lowers the value and is not treated as an increment. + if previous is not None and sample.value > previous: + labels = dict(sample.labels) + return { + "time": timestamp_ms, + "tags": self._tags(labels), + "text": self._describe(labels, "change-point detected"), + } + return None + + async def _post(self, event: dict[str, Any], group: str) -> None: + assert self._client is not None + try: + response = await self._client.post(_ANNOTATIONS_PATH, json=event) + response.raise_for_status() + except httpx.TimeoutException: + logger.warning("grafana_annotations_timeout", group=group) + self._count_failure(REASON_TIMEOUT) + except httpx.HTTPStatusError as exc: + logger.warning( + "grafana_annotations_http_error", group=group, status=exc.response.status_code + ) + self._count_failure(REASON_HTTP) + except httpx.HTTPError as exc: + logger.warning("grafana_annotations_connect_error", group=group, error=str(exc)) + self._count_failure(REASON_CONNECT) + + def _tags(self, labels: dict[str, str]) -> list[str]: + tags = list(self._config.tags) + for name in _TAG_LABELS: + value = labels.get(name) + if value: + tags.append(f"{name}:{value}") + return tags + + @staticmethod + def _describe(labels: dict[str, str], what: str) -> str: + ident = labels.get("id", "?") + group = labels.get("group", "?") + detector = labels.get("detector") + suffix = f" [{detector}]" if detector else "" + return f"promanomaly: {what} — {group}/{ident}{suffix}" + + async def aclose(self) -> None: + if self._client is None: + return + try: + await self._client.aclose() + finally: + self._client = None + + +__all__ = ["GrafanaAnnotationsSink"] diff --git a/detector/src/promanomaly/sinks/remote_write.py b/detector/src/promanomaly/sinks/remote_write.py new file mode 100644 index 0000000..7511b15 --- /dev/null +++ b/detector/src/promanomaly/sinks/remote_write.py @@ -0,0 +1,179 @@ +"""Prometheus remote-write sink. + +Serialises a group snapshot into a Prometheus ``WriteRequest`` (protobuf, +snappy-compressed) and POSTs it to a remote-write endpoint after each +successful run. The recommended path for long-horizon ``anomaly_density`` +history that outlives scrape retention. + +The protobuf ``WriteRequest`` schema is small and stable, so it is hand +encoded here (pure-Python wire format) rather than pulling in ``protobuf`` ++ ``protoc``. Snappy block compression — mandatory per the remote-write +1.0 spec — is provided by ``cramjam`` (a wheel-distributed Rust binding, +no system libsnappy), imported lazily so the dependency is only required +when this sink is actually configured. +""" + +from __future__ import annotations + +import asyncio +import struct +from typing import Any + +import httpx +import structlog + +from ..config import RemoteWriteSinkConfig +from ..exporter import OperationalMetrics +from ..httpauth import build_httpx_auth +from ..state import GroupSnapshot, Sample +from .base import ( + REASON_CONNECT, + REASON_HTTP, + REASON_SERIALIZE, + REASON_TIMEOUT, + Sink, +) + +logger = structlog.get_logger(__name__) + +# Remote-write 1.0 request headers (Content-Encoding/Type fixed by spec). +_HEADERS = { + "Content-Type": "application/x-protobuf", + "Content-Encoding": "snappy", + "X-Prometheus-Remote-Write-Version": "0.1.0", + "User-Agent": "promanomaly", +} + + +def _varint(value: int) -> bytes: + """Encode a non-negative integer as a protobuf base-128 varint.""" + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _tag(field: int, wire_type: int) -> bytes: + return _varint((field << 3) | wire_type) + + +def _len_delim(field: int, payload: bytes) -> bytes: + return _tag(field, 2) + _varint(len(payload)) + payload + + +def _string_field(field: int, value: str) -> bytes: + return _len_delim(field, value.encode("utf-8")) + + +def _double_field(field: int, value: float) -> bytes: + # wire type 1 (fixed64), little-endian IEEE-754 double. + return _tag(field, 1) + struct.pack(" bytes: + # protobuf int64 uses a plain varint; remote-write timestamps are + # positive milliseconds so the negative-value 10-byte case never hits. + return _tag(field, 0) + _varint(value) + + +def _encode_label(name: str, value: str) -> bytes: + return _string_field(1, name) + _string_field(2, value) + + +def _encode_sample(value: float, timestamp_ms: int) -> bytes: + return _double_field(1, value) + _int64_field(2, timestamp_ms) + + +def _encode_timeseries(labels: list[tuple[str, str]], value: float, timestamp_ms: int) -> bytes: + body = b"".join(_len_delim(1, _encode_label(name, val)) for name, val in labels) + body += _len_delim(2, _encode_sample(value, timestamp_ms)) + return body + + +def encode_write_request(samples: list[Sample], timestamp_ms: int) -> bytes: + """Encode samples into a snappy-compressed Prometheus ``WriteRequest``. + + Each :class:`Sample` becomes one ``TimeSeries`` with a single point. + The metric name is carried in the reserved ``__name__`` label as the + remote-write protocol requires. The remote-write spec requires every + series' labels sorted by name, so the *full* set (including + ``__name__``) is sorted together — prepending ``__name__`` and sorting + only the rest would emit out-of-order labels for any upper-case label + name (``_`` sorts after upper-case ASCII), which strict receivers + reject. + """ + import cramjam # lazy: only needed when this sink is configured + + body = bytearray() + for sample in samples: + labels: list[tuple[str, str]] = sorted([("__name__", sample.metric), *sample.labels]) + body += _len_delim(1, _encode_timeseries(labels, sample.value, timestamp_ms)) + return bytes(cramjam.snappy.compress_raw(bytes(body))) + + +class RemoteWriteSink(Sink): + """POSTs each successful group snapshot to a remote-write endpoint.""" + + name = "remote_write" + + def __init__(self, config: RemoteWriteSinkConfig, operational: OperationalMetrics) -> None: + super().__init__(operational) + self._config = config + self._client: httpx.AsyncClient | None = None + + async def start(self) -> None: + if self._client is not None: + return + kwargs: dict[str, Any] = {"timeout": self._config.timeout_seconds} + kwargs.update(build_httpx_auth(self._config.auth)) + self._client = httpx.AsyncClient(**kwargs) + + async def emit(self, snapshot: GroupSnapshot) -> None: + if not snapshot.samples: + return + if self._client is None: + await self.start() + assert self._client is not None + timestamp_ms = int(snapshot.timestamp * 1000) + try: + # Protobuf encoding is pure-Python and snappy compression is + # CPU-bound; on a 10k-series snapshot this is tens of + # milliseconds. Run it in a worker thread so a concurrent + # /metrics scrape on the event loop isn't blocked by the encode. + payload = await asyncio.to_thread(encode_write_request, snapshot.samples, timestamp_ms) + except Exception as exc: # serialization / missing cramjam + logger.error("remote_write_serialize_failed", group=snapshot.group, error=str(exc)) + self._count_failure(REASON_SERIALIZE) + return + try: + response = await self._client.post(self._config.url, content=payload, headers=_HEADERS) + response.raise_for_status() + except httpx.TimeoutException: + logger.warning("remote_write_timeout", group=snapshot.group, url=self._config.url) + self._count_failure(REASON_TIMEOUT) + except httpx.HTTPStatusError as exc: + logger.warning( + "remote_write_http_error", + group=snapshot.group, + status=exc.response.status_code, + ) + self._count_failure(REASON_HTTP) + except httpx.HTTPError as exc: + logger.warning("remote_write_connect_error", group=snapshot.group, error=str(exc)) + self._count_failure(REASON_CONNECT) + + async def aclose(self) -> None: + if self._client is None: + return + try: + await self._client.aclose() + finally: + self._client = None + + +__all__ = ["RemoteWriteSink", "encode_write_request"] diff --git a/detector/src/promanomaly/source.py b/detector/src/promanomaly/source.py index 7ec6a26..51793fe 100644 --- a/detector/src/promanomaly/source.py +++ b/detector/src/promanomaly/source.py @@ -20,6 +20,7 @@ from .cache import TTLCache from .config import AuthConfig +from .httpauth import build_httpx_auth # Shared connection pool sized for the typical 10k-series workload. The # httpx default of 10 concurrent connections bottlenecks against a fast @@ -121,17 +122,7 @@ async def start(self) -> None: "base_url": self._base_url, "limits": _HTTP_LIMITS, } - if self._auth.type == "bearer" and self._auth.token: - kwargs["headers"] = {"Authorization": f"Bearer {self._auth.token}"} - elif self._auth.type == "basic" and self._auth.username: - kwargs["auth"] = httpx.BasicAuth(self._auth.username, self._auth.password or "") - elif self._auth.type == "mtls": - cert = self._auth.cert_file - key = self._auth.key_file - if cert and key: - kwargs["cert"] = (cert, key) - if self._auth.ca_file: - kwargs["verify"] = self._auth.ca_file + kwargs.update(build_httpx_auth(self._auth)) self._client = httpx.AsyncClient(**kwargs) async def close(self) -> None: diff --git a/detector/tests/test_adapter.py b/detector/tests/test_adapter.py new file mode 100644 index 0000000..5f097bd --- /dev/null +++ b/detector/tests/test_adapter.py @@ -0,0 +1,257 @@ +"""Tests for the Kubernetes metrics adapter. + +Covers config validation, the PromQL-instant-query client + label-selector +translation, the discovery documents, and the external/custom metric +value endpoints (via FastAPI TestClient with an injected resolver). +""" + +from __future__ import annotations + +import httpx +import pytest +from fastapi.testclient import TestClient + +from promanomaly.adapter.app import build_adapter_app, format_quantity +from promanomaly.adapter.config import AdapterConfig, load_adapter_config +from promanomaly.adapter.resolver import ( + MetricResolver, + MetricValue, + TSDBClient, + selector_to_matchers, +) + + +# -------------------------------------------------------------------------- +# Config +# -------------------------------------------------------------------------- +def test_config_defaults() -> None: + cfg = AdapterConfig.model_validate({"datasource": {"url": "http://vm:8428/"}}) + assert cfg.listen == ":6443" + assert "anomaly_density" in cfg.metrics + assert cfg.tls.enabled is False + + +def test_config_rejects_colon_metric() -> None: + with pytest.raises(ValueError, match="colons"): + AdapterConfig.model_validate( + {"datasource": {"url": "http://vm/"}, "metrics": ["anomaly:bad"]} + ) + + +def test_config_requires_both_tls_files() -> None: + with pytest.raises(ValueError, match="cert_file and key_file"): + AdapterConfig.model_validate( + {"datasource": {"url": "http://vm/"}, "tls": {"cert_file": "/x.crt"}} + ) + + +def test_load_adapter_config(tmp_path) -> None: # type: ignore[no-untyped-def] + path = tmp_path / "adapter.yaml" + path.write_text("datasource:\n url: http://vm:8428/\nmetrics:\n - anomaly_density\n") + cfg = load_adapter_config(path) + assert cfg.metrics == ["anomaly_density"] + + +# -------------------------------------------------------------------------- +# Selector translation + quantity formatting +# -------------------------------------------------------------------------- +@pytest.mark.parametrize( + "selector,expected", + [ + (None, ""), + ("", ""), + ("group=q", 'group="q"'), + ("group==q,ns!=kube-system", 'group="q",ns!="kube-system"'), + ("bare-no-op", ""), + ], +) +def test_selector_to_matchers(selector, expected) -> None: # type: ignore[no-untyped-def] + assert selector_to_matchers(selector) == expected + + +@pytest.mark.parametrize( + "value,expected", + [(5.0, "5"), (0.5, "0.5"), (4.7, "4.7"), (0.0, "0"), (0.25, "0.25")], +) +def test_format_quantity(value, expected) -> None: # type: ignore[no-untyped-def] + assert format_quantity(value) == expected + + +# -------------------------------------------------------------------------- +# TSDB instant-query client +# -------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_tsdb_instant_query_parses_vector() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/query" + return httpx.Response( + 200, + json={ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {"__name__": "anomaly_density", "group": "q"}, + "value": [1700000000, "0.4"], + } + ], + }, + }, + ) + + client = TSDBClient(base_url="http://stub", timeout=1.0) + client._client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://stub" + ) + values = await client.instant_query("anomaly_density") + assert len(values) == 1 + assert values[0].labels == {"group": "q"} # __name__ stripped + assert values[0].value == pytest.approx(0.4) + + +@pytest.mark.asyncio +async def test_tsdb_instant_query_skips_non_finite_values() -> None: + # A NaN/Inf value cannot be a Kubernetes resource.Quantity; rendering + # one would 500 the whole metrics-API request, so such series are + # dropped rather than served. + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {"__name__": "anomaly_density", "p": "nan"}, + "value": [1, "NaN"], + }, + { + "metric": {"__name__": "anomaly_density", "p": "inf"}, + "value": [1, "+Inf"], + }, + {"metric": {"__name__": "anomaly_density", "p": "ok"}, "value": [1, "0.7"]}, + ], + }, + }, + ) + + client = TSDBClient(base_url="http://stub", timeout=1.0) + client._client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://stub" + ) + values = await client.instant_query("anomaly_density") + assert [v.labels["p"] for v in values] == ["ok"] + + +# -------------------------------------------------------------------------- +# FastAPI app with an injected resolver +# -------------------------------------------------------------------------- +class _FakeResolver(MetricResolver): + def __init__(self) -> None: + super().__init__(client=None, allowed_metrics=["anomaly_density", "anomaly_severity"]) # type: ignore[arg-type] + self.calls: list[tuple[str, str | None, str]] = [] + + async def resolve(self, metric, label_selector=None, extra_matchers=""): # type: ignore[no-untyped-def] + self.calls.append((metric, label_selector, extra_matchers)) + return [MetricValue(labels={"group": "q", "pod": "p1"}, value=0.4, timestamp=1700000000.0)] + + +def _client() -> tuple[TestClient, _FakeResolver]: + cfg = AdapterConfig.model_validate({"datasource": {"url": "http://vm/"}}) + resolver = _FakeResolver() + app = build_adapter_app(cfg, resolver=resolver) + return TestClient(app), resolver + + +def test_health_endpoints() -> None: + client, _ = _client() + for path in ("/healthz", "/livez", "/readyz"): + assert client.get(path).text == "ok" + + +def test_apis_discovery_lists_both_groups() -> None: + client, _ = _client() + body = client.get("/apis").json() + names = {g["name"] for g in body["groups"]} + assert names == {"external.metrics.k8s.io", "custom.metrics.k8s.io"} + + +def test_external_resource_list() -> None: + client, _ = _client() + body = client.get("/apis/external.metrics.k8s.io/v1beta1").json() + assert body["kind"] == "APIResourceList" + names = {r["name"] for r in body["resources"]} + assert "anomaly_density" in names + + +def test_external_metric_value() -> None: + client, resolver = _client() + resp = client.get( + "/apis/external.metrics.k8s.io/v1beta1/namespaces/monitoring/anomaly_density", + params={"labelSelector": "group=q"}, + ) + body = resp.json() + assert body["kind"] == "ExternalMetricValueList" + assert body["items"][0]["metricName"] == "anomaly_density" + assert body["items"][0]["value"] == "400m" or body["items"][0]["value"] == "0.4" + assert resolver.calls[0] == ("anomaly_density", "group=q", "") + + +def test_external_metric_unknown_is_404() -> None: + client, _ = _client() + resp = client.get("/apis/external.metrics.k8s.io/v1beta1/namespaces/monitoring/anomaly_score") + assert resp.status_code == 404 + + +def test_custom_resource_list_pairs_resource_and_metric() -> None: + client, _ = _client() + body = client.get("/apis/custom.metrics.k8s.io/v1beta1").json() + names = {r["name"] for r in body["resources"]} + assert "pods/anomaly_density" in names + + +def test_custom_metric_single_object_builds_matcher() -> None: + client, resolver = _client() + resp = client.get( + "/apis/custom.metrics.k8s.io/v1beta1/namespaces/monitoring/pods/p1/anomaly_severity" + ) + body = resp.json() + assert body["kind"] == "MetricValueList" + obj = body["items"][0]["describedObject"] + assert obj["kind"] == "Pod" + assert obj["apiVersion"] == "v1" + assert obj["namespace"] == "monitoring" + # extra_matchers carries the namespace + pod name match. + _metric, _sel, extra = resolver.calls[0] + assert 'namespace="monitoring"' in extra + assert 'pod="p1"' in extra + + +def test_custom_metric_cluster_scoped_node() -> None: + cfg = AdapterConfig.model_validate( + {"datasource": {"url": "http://vm/"}, "custom_resources": ["nodes"]} + ) + resolver = _FakeResolver() + client = TestClient(build_adapter_app(cfg, resolver=resolver)) + resp = client.get("/apis/custom.metrics.k8s.io/v1beta1/nodes/node-17/anomaly_density") + body = resp.json() + assert resp.status_code == 200 + obj = body["items"][0]["describedObject"] + assert obj["kind"] == "Node" + # Cluster-scoped objects carry no namespace. + assert "namespace" not in obj + # Only the node-name matcher, no namespace matcher. + _metric, _sel, extra = resolver.calls[0] + assert extra == 'node="node-17"' + + +def test_metrics_endpoint_counts_requests() -> None: + client, _ = _client() + client.get("/apis/external.metrics.k8s.io/v1beta1/namespaces/ns/anomaly_density") + client.get("/apis/external.metrics.k8s.io/v1beta1/namespaces/ns/anomaly_score") # not found + body = client.get("/metrics").text + assert "anomaly_adapter_requests_total" in body + assert 'outcome="ok"' in body + assert 'outcome="not_found"' in body diff --git a/detector/tests/test_generate_rules.py b/detector/tests/test_generate_rules.py new file mode 100644 index 0000000..36adc47 --- /dev/null +++ b/detector/tests/test_generate_rules.py @@ -0,0 +1,208 @@ +"""`generate-rules` — PrometheusRule scaffolding from a config.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml +from click.testing import CliRunner + +from promanomaly.cli import cli +from promanomaly.cli._generate_rules import generate_prometheus_rules +from promanomaly.config import CURRENT_API_VERSION, Config + + +def _config(**extra: Any) -> Config: + base: dict[str, Any] = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://vm"}, + "groups": [ + { + "name": "errors", + "queries": [ + {"id": "http_5xx", "promql": "rate(http_errors[5m])", "detectors": ["MAD"]}, + ], + } + ], + } + base.update(extra) + return Config.model_validate(base) + + +def _rule_groups(manifest: str) -> dict[str, Any]: + doc = yaml.safe_load(manifest) + assert doc["kind"] == "PrometheusRule" + return {g["name"]: g for g in doc["spec"]["groups"]} + + +def test_generates_valid_yaml_with_recording_and_alert_groups() -> None: + groups = _rule_groups(generate_prometheus_rules(_config())) + assert "promanomaly.recording" in groups + assert "promanomaly.errors" in groups + + +def test_outside_threshold_selector_is_prefilled() -> None: + groups = _rule_groups(generate_prometheus_rules(_config())) + alert = groups["promanomaly.errors"]["rules"][0] + # Unique, descriptive alert name per (group, query) so the file is + # duplicate-rule-lint clean and each alert is individually routable. + assert alert["alert"] == "AnomalyErrorsHttp5xxOutsideThreshold" + assert alert["expr"] == 'anomaly_outside_threshold{group="errors", id="http_5xx"} == 1' + + +def test_alert_names_are_unique_across_queries_and_groups() -> None: + cfg = _config( + groups=[ + { + "name": "errors", + "queries": [ + {"id": "http_5xx", "promql": "a", "detectors": ["MAD"]}, + {"id": "http_4xx", "promql": "b", "detectors": ["MAD"]}, + ], + }, + { + "name": "latency", + "queries": [{"id": "http_5xx", "promql": "c", "detectors": ["MAD"]}], + }, + { + # Pathological: two distinct ids that camel-case to the same + # token ("AB"). The dedup pass must still keep names unique. + "name": "edge", + "queries": [ + {"id": "a_b", "promql": "d", "detectors": ["MAD"]}, + {"id": "a__b", "promql": "e", "detectors": ["MAD"]}, + ], + }, + ] + ) + doc = yaml.safe_load(generate_prometheus_rules(cfg)) + names = [r["alert"] for g in doc["spec"]["groups"] for r in g["rules"] if "alert" in r] + assert len(names) == len(set(names)) # no duplicates + # The collision was resolved with a numeric suffix, not dropped. + assert "AnomalyEdgeABOutsideThreshold" in names + assert "AnomalyEdgeABOutsideThreshold2" in names + + +def test_recording_rule_is_the_max_ensemble_fallback() -> None: + groups = _rule_groups(generate_prometheus_rules(_config())) + rules = groups["promanomaly.recording"]["rules"] + # A single global rule covers every group; by (id, group) partitions + # the output per series, so per-group copies would only trip the + # duplicate-recording-rule lint. + assert len(rules) == 1 + assert rules[0]["record"] == "anomaly_outside_threshold_any" + assert rules[0]["expr"] == "max by (id, group) (anomaly_outside_threshold)" + + +def test_recording_group_name_does_not_collide_with_a_user_group() -> None: + # A detector group literally named "recording" would otherwise produce + # two rule groups both named "promanomaly.recording" — which Prometheus + # rejects as a repeated group name in the same file. + cfg = _config( + groups=[ + {"name": "recording", "queries": [{"id": "q", "promql": "up", "detectors": ["MAD"]}]} + ] + ) + doc = yaml.safe_load(generate_prometheus_rules(cfg)) + group_names = [g["name"] for g in doc["spec"]["groups"]] + assert len(group_names) == len(set(group_names)) # no repeated group names + + +def test_thresholds_are_marked_todo_not_baked_in() -> None: + manifest = generate_prometheus_rules(_config()) + # Severity / for: / runbook are placeholders, not opinions. + assert "# TODO: set severity" in manifest + assert "# TODO: set the flap-suppression window" in manifest + + +def test_templated_id_falls_back_to_group_scope() -> None: + cfg = _config( + groups=[ + { + "name": "per_node", + "queries": [ + { + "id": "cpu_{{ instance }}", + "promql": 'rate(cpu{instance="{{ instance }}"}[5m])', + "detectors": ["MAD"], + "discover": [{"variable": "instance", "probe": "up", "label": "instance"}], + } + ], + } + ] + ) + manifest = generate_prometheus_rules(cfg) + groups = _rule_groups(manifest) + alert = groups["promanomaly.per_node"]["rules"][0] + # No literal id selector — scoped by group only. + assert alert["expr"] == 'anomaly_outside_threshold{group="per_node"} == 1' + assert "discover-templated" in manifest + + +def test_ensemble_group_gets_an_ensemble_alert() -> None: + cfg = _config( + groups=[ + { + "name": "errors", + "ensemble": {"method": "voting", "min_detectors": 2}, + "queries": [ + { + "id": "http_5xx", + "promql": "rate(http_errors[5m])", + "detectors": [ + {"name": "MAD", "instance": "short"}, + {"name": "Hampel", "instance": "long"}, + ], + }, + ], + } + ] + ) + groups = _rule_groups(generate_prometheus_rules(cfg)) + alert_names = [r.get("alert") for r in groups["promanomaly.errors"]["rules"]] + assert "AnomalyErrorsEnsembleAgreement" in alert_names + + +def _write_config(tmp_path: Path) -> Path: + path = tmp_path / "c.yaml" + path.write_text( + yaml.safe_dump( + { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://vm"}, + "groups": [ + { + "name": "errors", + "queries": [{"id": "q", "promql": "up", "detectors": ["MAD"]}], + } + ], + } + ) + ) + return path + + +def test_cli_generate_rules_to_stdout(tmp_path: Path) -> None: + result = CliRunner().invoke(cli, ["generate-rules", "--config", str(_write_config(tmp_path))]) + assert result.exit_code == 0 + assert "kind: PrometheusRule" in result.output + + +def test_cli_generate_rules_to_file(tmp_path: Path) -> None: + out = tmp_path / "rules.yaml" + result = CliRunner().invoke( + cli, + ["generate-rules", "--config", str(_write_config(tmp_path)), "--output", str(out)], + ) + assert result.exit_code == 0 + assert yaml.safe_load(out.read_text())["kind"] == "PrometheusRule" + + +def test_cli_generate_rules_custom_name(tmp_path: Path) -> None: + result = CliRunner().invoke( + cli, + ["generate-rules", "--config", str(_write_config(tmp_path)), "--name", "my-rules"], + ) + assert result.exit_code == 0 + assert "name: my-rules" in result.output diff --git a/detector/tests/test_httpauth.py b/detector/tests/test_httpauth.py new file mode 100644 index 0000000..908c3e6 --- /dev/null +++ b/detector/tests/test_httpauth.py @@ -0,0 +1,65 @@ +"""File-based credential resolution shared across datasource, sinks, adapter.""" + +from __future__ import annotations + +from pathlib import Path + +from promanomaly.config import AuthConfig +from promanomaly.httpauth import build_httpx_auth + + +def test_resolved_token_prefers_file(tmp_path: Path) -> None: + token_path = tmp_path / "token" + token_path.write_text("sekret-from-file\n") # trailing newline trimmed + auth = AuthConfig(type="bearer", token="inline", token_file=str(token_path)) + assert auth.resolved_token() == "sekret-from-file" + + +def test_resolved_token_falls_back_to_inline() -> None: + auth = AuthConfig(type="bearer", token="inline") + assert auth.resolved_token() == "inline" + + +def test_resolved_password_prefers_file(tmp_path: Path) -> None: + pw = tmp_path / "password" + pw.write_text("hunter2\n") + auth = AuthConfig(type="basic", username="u", password="x", password_file=str(pw)) + assert auth.resolved_password() == "hunter2" + + +def test_build_httpx_auth_bearer_from_file(tmp_path: Path) -> None: + token_path = tmp_path / "token" + token_path.write_text("filetoken") + auth = AuthConfig(type="bearer", token_file=str(token_path)) + kwargs = build_httpx_auth(auth) + assert kwargs["headers"]["Authorization"] == "Bearer filetoken" + + +def test_build_httpx_auth_basic_password_from_file(tmp_path: Path) -> None: + pw = tmp_path / "password" + pw.write_text("pw") + auth = AuthConfig(type="basic", username="user", password_file=str(pw)) + kwargs = build_httpx_auth(auth) + # httpx BasicAuth carries the credential; verify it encodes user:pw. + import base64 + + basic = kwargs["auth"] + request = next(basic.auth_flow(_DummyRequest())) + expected = "Basic " + base64.b64encode(b"user:pw").decode() + assert request.headers["Authorization"] == expected + + +def test_build_httpx_auth_none_is_empty() -> None: + assert build_httpx_auth(AuthConfig(type="none")) == {} + + +def test_build_httpx_auth_mtls() -> None: + auth = AuthConfig(type="mtls", cert_file="/c.crt", key_file="/c.key", ca_file="/ca.crt") + kwargs = build_httpx_auth(auth) + assert kwargs["cert"] == ("/c.crt", "/c.key") + assert kwargs["verify"] == "/ca.crt" + + +class _DummyRequest: + def __init__(self) -> None: + self.headers: dict[str, str] = {} diff --git a/detector/tests/test_sinks_config.py b/detector/tests/test_sinks_config.py new file mode 100644 index 0000000..518c26c --- /dev/null +++ b/detector/tests/test_sinks_config.py @@ -0,0 +1,282 @@ +"""SinkConfig validation, the sink factory, and Application wiring.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from promanomaly.config import ( + CURRENT_API_VERSION, + Config, + SinkConfig, + load_config, +) +from promanomaly.exporter import OperationalMetrics +from promanomaly.sinks import ( + GrafanaAnnotationsSink, + RemoteWriteSink, + build_sink, + build_sinks, +) + + +def _base_config(**extra: Any) -> dict[str, Any]: + cfg: dict[str, Any] = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://vm"}, + "groups": [{"name": "g1", "queries": [{"id": "q", "promql": "up", "detectors": ["MAD"]}]}], + } + cfg.update(extra) + return cfg + + +def test_sink_requires_matching_block() -> None: + with pytest.raises(ValueError, match=r"requires a sink\.remote_write block"): + SinkConfig.model_validate({"type": "remote_write"}) + + +def test_sink_rejects_unrelated_block() -> None: + with pytest.raises(ValueError, match="configure exactly one sink"): + SinkConfig.model_validate( + { + "type": "remote_write", + "remote_write": {"url": "http://x/api/v1/write"}, + "grafana_annotations": {"url": "http://g"}, + } + ) + + +def test_build_sink_none_is_pull_only() -> None: + assert build_sink(None, OperationalMetrics()) is None + + +def test_build_sink_remote_write() -> None: + cfg = SinkConfig.model_validate( + {"type": "remote_write", "remote_write": {"url": "http://x/api/v1/write"}} + ) + sink = build_sink(cfg, OperationalMetrics()) + assert isinstance(sink, RemoteWriteSink) + assert sink.name == "remote_write" + + +def test_build_sink_grafana_annotations() -> None: + cfg = SinkConfig.model_validate( + {"type": "grafana_annotations", "grafana_annotations": {"url": "http://g"}} + ) + sink = build_sink(cfg, OperationalMetrics()) + assert isinstance(sink, GrafanaAnnotationsSink) + + +def test_config_accepts_sink_block(tmp_path: Path) -> None: + cfg = { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://vm"}, + "sink": { + "type": "remote_write", + "remote_write": {"url": "http://vm/api/v1/write", "auth": {"type": "bearer"}}, + }, + "groups": [{"name": "g1", "queries": [{"id": "q", "promql": "up", "detectors": ["MAD"]}]}], + } + path = tmp_path / "c.yaml" + path.write_text(yaml.safe_dump(cfg)) + loaded = load_config(path) + assert loaded.sink is not None + assert loaded.sink.type == "remote_write" + + +def test_config_default_sink_is_none() -> None: + cfg = Config.model_validate(_base_config()) + assert cfg.sink is None + assert cfg.sinks is None + assert cfg.effective_sinks() == [] + + +def test_effective_sinks_normalises_scalar_form() -> None: + cfg = Config.model_validate( + _base_config( + sink={"type": "remote_write", "remote_write": {"url": "http://x/api/v1/write"}} + ) + ) + effective = cfg.effective_sinks() + assert [s.type for s in effective] == ["remote_write"] + + +def test_config_accepts_sinks_list() -> None: + cfg = Config.model_validate( + _base_config( + sinks=[ + {"type": "remote_write", "remote_write": {"url": "http://x/api/v1/write"}}, + {"type": "grafana_annotations", "grafana_annotations": {"url": "http://g"}}, + ] + ) + ) + assert [s.type for s in cfg.effective_sinks()] == ["remote_write", "grafana_annotations"] + + +def test_build_sinks_constructs_every_configured_sink() -> None: + cfg = Config.model_validate( + _base_config( + sinks=[ + {"type": "remote_write", "remote_write": {"url": "http://x/api/v1/write"}}, + {"type": "grafana_annotations", "grafana_annotations": {"url": "http://g"}}, + ] + ) + ) + sinks = build_sinks(cfg, OperationalMetrics()) + assert isinstance(sinks[0], RemoteWriteSink) + assert isinstance(sinks[1], GrafanaAnnotationsSink) + + +def test_build_sinks_pull_only_is_empty() -> None: + assert build_sinks(Config.model_validate(_base_config()), OperationalMetrics()) == [] + + +def test_sink_and_sinks_are_mutually_exclusive() -> None: + with pytest.raises(ValueError, match="not both"): + Config.model_validate( + _base_config( + sink={"type": "remote_write", "remote_write": {"url": "http://x/api/v1/write"}}, + sinks=[{"type": "grafana_annotations", "grafana_annotations": {"url": "http://g"}}], + ) + ) + + +def test_sinks_rejects_duplicate_type() -> None: + with pytest.raises(ValueError, match="duplicate sink type"): + Config.model_validate( + _base_config( + sinks=[ + {"type": "remote_write", "remote_write": {"url": "http://a/api/v1/write"}}, + {"type": "remote_write", "remote_write": {"url": "http://b/api/v1/write"}}, + ] + ) + ) + + +def test_sinks_rejects_empty_list() -> None: + with pytest.raises(ValueError, match="at least one sink"): + Config.model_validate(_base_config(sinks=[])) + + +class _RecordingSink: + name = "recording" + + def __init__(self) -> None: + self.started = False + self.closed = False + self.snapshots: list[Any] = [] + + async def start(self) -> None: + self.started = True + + async def emit(self, snapshot: Any) -> None: + self.snapshots.append(snapshot) + + async def aclose(self) -> None: + self.closed = True + + +@pytest.mark.asyncio +async def test_application_emits_to_sink_after_successful_run(tmp_path: Path) -> None: + from promanomaly.main import Application + from promanomaly.state import GroupSnapshot, Sample + + cfg = Config.model_validate( + { + "apiVersion": CURRENT_API_VERSION, + "datasource": {"url": "http://vm"}, + "groups": [ + {"name": "g1", "queries": [{"id": "q", "promql": "up", "detectors": ["MAD"]}]} + ], + } + ) + app = Application(cfg, tmp_path / "c.yaml") + sink = _RecordingSink() + app._sinks = [sink] # type: ignore[list-item] + + # Seed a snapshot and stub the runner so _safe_group_run succeeds. + snapshot = GroupSnapshot( + group="g1", timestamp=1.0, samples=[Sample(metric="anomaly_score", labels=(), value=1.0)] + ) + app._store.write(snapshot) + + class _Result: + succeeded = True + + async def _fake_run_group(name: str) -> Any: + return _Result() + + app._runner.run_group = _fake_run_group # type: ignore[assignment] + + await app._safe_group_run("g1") + assert sink.snapshots and sink.snapshots[0].group == "g1" + + +@pytest.mark.asyncio +async def test_emit_to_sinks_isolates_a_failing_sink(tmp_path: Path) -> None: + """One sink raising must not starve the others, and is counted.""" + from promanomaly.main import Application + from promanomaly.state import GroupSnapshot, Sample + + app = Application(Config.model_validate(_base_config()), tmp_path / "c.yaml") + + class _BoomSink: + name = "boom" + + async def start(self) -> None: ... + + async def emit(self, snapshot: Any) -> None: + raise RuntimeError("kaboom") + + async def aclose(self) -> None: ... + + good = _RecordingSink() + app._sinks = [_BoomSink(), good] # type: ignore[list-item] + + snapshot = GroupSnapshot( + group="g1", timestamp=1.0, samples=[Sample(metric="anomaly_score", labels=(), value=1.0)] + ) + await app._emit_to_sinks(snapshot) + + # The later sink still received the snapshot despite the earlier one + # raising, and the failure was counted under the failing sink's name. + assert good.snapshots and good.snapshots[0].group == "g1" + metric = app._ops.sink_failures_total.labels(sink="boom", reason="exception") + assert metric._value.get() == 1.0 + + +@pytest.mark.asyncio +async def test_swap_sinks_rebuilds_on_change_and_noops_when_unchanged(tmp_path: Path) -> None: + from promanomaly.main import Application + + # Start pull-only; swap in a two-sink list. The old sinks must be + # closed and the new ones built and started. + app = Application(Config.model_validate(_base_config()), tmp_path / "c.yaml") + old = _RecordingSink() + app._sinks = [old] # type: ignore[list-item] + + new_cfg = Config.model_validate( + _base_config( + sinks=[ + {"type": "remote_write", "remote_write": {"url": "http://x/api/v1/write"}}, + {"type": "grafana_annotations", "grafana_annotations": {"url": "http://g"}}, + ] + ) + ) + await app._swap_sinks(new_cfg) + try: + assert old.closed + assert [s.name for s in app._sinks] == ["remote_write", "grafana_annotations"] + + # A second swap to a config with identical effective sinks is a + # no-op — the live sink instances are preserved (not rebuilt). + app._config = new_cfg + before = app._sinks + await app._swap_sinks(new_cfg) + assert app._sinks is before + finally: + for sink in app._sinks: + await sink.aclose() diff --git a/detector/tests/test_sinks_grafana_annotations.py b/detector/tests/test_sinks_grafana_annotations.py new file mode 100644 index 0000000..8044a13 --- /dev/null +++ b/detector/tests/test_sinks_grafana_annotations.py @@ -0,0 +1,219 @@ +"""Tests for the Grafana annotations sink. + +Covers 0->1 transition detection, change-point increment detection, the +cold-start baseline guard, rate-bounding, and failure isolation, mocked +with ``httpx.MockTransport``. +""" + +from __future__ import annotations + +import httpx +import pytest + +from promanomaly.config import AuthConfig, GrafanaAnnotationsSinkConfig +from promanomaly.exporter import OperationalMetrics +from promanomaly.sinks.grafana_annotations import GrafanaAnnotationsSink +from promanomaly.state import GroupSnapshot, Sample + + +def _sink_with_handler(handler, **cfg_kwargs): # type: ignore[no-untyped-def] + ops = OperationalMetrics() + config = GrafanaAnnotationsSinkConfig(url="http://grafana", **cfg_kwargs) + sink = GrafanaAnnotationsSink(config, ops) + sink._client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://grafana" + ) + return sink, ops + + +def _snapshot(outside: float, *, ts: float = 100.0, extra=()) -> GroupSnapshot: + samples = [ + Sample( + metric="anomaly_outside_threshold", + labels=(("id", "errors"), ("group", "svc"), ("detector", "MAD")), + value=outside, + ), + *extra, + ] + return GroupSnapshot(group="svc", timestamp=ts, samples=samples) + + +@pytest.mark.asyncio +async def test_zero_to_one_transition_posts_annotation() -> None: + posts: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + import json + + posts.append(json.loads(request.content)) + return httpx.Response(200) + + sink, _ops = _sink_with_handler(handler) + # First run establishes baseline (value 0) — no annotation. + await sink.emit(_snapshot(0.0, ts=100.0)) + assert posts == [] + # Second run: 0 -> 1 transition annotates. + await sink.emit(_snapshot(1.0, ts=160.0)) + assert len(posts) == 1 + ann = posts[0] + assert ann["time"] == 160_000 + assert "id:errors" in ann["tags"] + assert "group:svc" in ann["tags"] + assert "detector:MAD" in ann["tags"] + assert "promanomaly" in ann["tags"] + + +@pytest.mark.asyncio +async def test_already_firing_on_cold_start_does_not_annotate() -> None: + posts: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + posts.append({}) + return httpx.Response(200) + + sink, _ops = _sink_with_handler(handler) + # First-ever observation is value 1 -> baseline only, no replay storm. + await sink.emit(_snapshot(1.0)) + assert posts == [] + # Staying at 1 also does not annotate. + await sink.emit(_snapshot(1.0)) + assert posts == [] + + +@pytest.mark.asyncio +async def test_change_point_increment_annotates() -> None: + posts: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + import json + + posts.append(json.loads(request.content)) + return httpx.Response(200) + + sink, _ops = _sink_with_handler(handler) + cp = lambda v: Sample( # noqa: E731 + metric="anomaly_change_point_total", + labels=(("id", "q"), ("group", "svc"), ("detector", "BOCPD")), + value=v, + ) + await sink.emit(GroupSnapshot(group="svc", timestamp=10.0, samples=[cp(3.0)])) + assert posts == [] # baseline + await sink.emit(GroupSnapshot(group="svc", timestamp=20.0, samples=[cp(4.0)])) + assert len(posts) == 1 + assert "change-point" in posts[0]["text"] + # Counter reset (restart) lowers the value — not an increment. + await sink.emit(GroupSnapshot(group="svc", timestamp=30.0, samples=[cp(1.0)])) + assert len(posts) == 1 + + +@pytest.mark.asyncio +async def test_rate_bounding_drops_overflow_and_counts() -> None: + posts: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + posts.append({}) + return httpx.Response(200) + + sink, ops = _sink_with_handler(handler, max_annotations_per_run=2) + # Baseline run with three series at 0. + base = [ + Sample( + metric="anomaly_outside_threshold", + labels=(("id", f"s{i}"), ("group", "svc"), ("detector", "MAD")), + value=0.0, + ) + for i in range(3) + ] + await sink.emit(GroupSnapshot(group="svc", timestamp=1.0, samples=base)) + # All three fire at once -> 3 events, budget 2 -> 1 dropped. + firing = [ + Sample( + metric="anomaly_outside_threshold", + labels=(("id", f"s{i}"), ("group", "svc"), ("detector", "MAD")), + value=1.0, + ) + for i in range(3) + ] + await sink.emit(GroupSnapshot(group="svc", timestamp=2.0, samples=firing)) + assert len(posts) == 2 + assert _failure_total(ops, reason="rate_limited") == 1.0 + + +@pytest.mark.asyncio +async def test_http_error_is_isolated_and_counted() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500) + + sink, ops = _sink_with_handler(handler) + await sink.emit(_snapshot(0.0)) + await sink.emit(_snapshot(1.0)) # transition -> post -> 500 + assert _failure_total(ops, reason="http_error") == 1.0 + + +@pytest.mark.asyncio +async def test_state_is_pruned_per_group_to_bound_memory() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200) + + sink, _ops = _sink_with_handler(handler) + + def series(group: str, ident: str, value: float) -> Sample: + return Sample( + metric="anomaly_outside_threshold", + labels=(("id", ident), ("group", group), ("detector", "MAD")), + value=value, + ) + + # Group A churns through ephemeral ids; group B has one stable id. + await sink.emit(GroupSnapshot(group="A", timestamp=1.0, samples=[series("A", "x0", 0.0)])) + await sink.emit(GroupSnapshot(group="B", timestamp=1.0, samples=[series("B", "stable", 0.0)])) + for i in range(1, 20): + await sink.emit( + GroupSnapshot(group="A", timestamp=float(i + 1), samples=[series("A", f"x{i}", 0.0)]) + ) + + # Group A state is bounded to the latest run's series (not 20 ephemerals). + a_keys = [k for k in sink._last_outside if dict(k).get("group") == "A"] + assert len(a_keys) == 1 + # Group B's entry is untouched by group A's runs. + b_keys = [k for k in sink._last_outside if dict(k).get("group") == "B"] + assert len(b_keys) == 1 + + +def test_auth_must_be_none_or_bearer() -> None: + with pytest.raises(ValueError, match="token auth"): + GrafanaAnnotationsSinkConfig(url="http://grafana", auth=AuthConfig(type="basic")) + + +@pytest.mark.asyncio +async def test_start_sets_bearer_header_from_token_file(tmp_path) -> None: # type: ignore[no-untyped-def] + # The Helm chart mounts the Grafana token as a file and wires + # ``token_file``; the sink must resolve it (not just the inline token) + # or annotations post unauthenticated. + token_path = tmp_path / "token" + token_path.write_text("grafana-sa-token\n") # trailing newline trimmed + config = GrafanaAnnotationsSinkConfig( + url="http://grafana", + auth=AuthConfig(type="bearer", token_file=str(token_path)), + ) + sink = GrafanaAnnotationsSink(config, OperationalMetrics()) + await sink.start() + try: + assert sink._client is not None + assert sink._client.headers["Authorization"] == "Bearer grafana-sa-token" + finally: + await sink.aclose() + + +def _failure_total(ops: OperationalMetrics, reason: str | None = None) -> float: + total = 0.0 + for metric in ops.registry.collect(): + if metric.name != "anomaly_sink_failures": + continue + for sample in metric.samples: + if not sample.name.endswith("_total"): + continue + if reason is not None and sample.labels.get("reason") != reason: + continue + total += sample.value + return total diff --git a/detector/tests/test_sinks_remote_write.py b/detector/tests/test_sinks_remote_write.py new file mode 100644 index 0000000..8d2fac9 --- /dev/null +++ b/detector/tests/test_sinks_remote_write.py @@ -0,0 +1,214 @@ +"""Tests for the Prometheus remote-write sink. + +Covers the hand-rolled protobuf ``WriteRequest`` encoding (round-tripped +through a minimal decoder), snappy framing, and the emit path's HTTP +behaviour + failure isolation, mocked with ``httpx.MockTransport``. +""" + +from __future__ import annotations + +import struct + +import cramjam +import httpx +import pytest + +from promanomaly.config import AuthConfig, RemoteWriteSinkConfig +from promanomaly.exporter import OperationalMetrics +from promanomaly.sinks.remote_write import RemoteWriteSink, encode_write_request +from promanomaly.state import GroupSnapshot, Sample + + +# -------------------------------------------------------------------------- +# Minimal protobuf decoder (test-only) to verify the hand-rolled encoder. +# -------------------------------------------------------------------------- +def _read_varint(buf: bytes, i: int) -> tuple[int, int]: + shift = 0 + result = 0 + while True: + byte = buf[i] + i += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + return result, i + shift += 7 + + +def _parse(buf: bytes) -> list[tuple[int, int, object]]: + i = 0 + fields: list[tuple[int, int, object]] = [] + while i < len(buf): + tag, i = _read_varint(buf, i) + field, wire = tag >> 3, tag & 7 + if wire == 2: + length, i = _read_varint(buf, i) + val: object = buf[i : i + length] + i += length + elif wire == 0: + val, i = _read_varint(buf, i) + elif wire == 1: + val = struct.unpack(" list[dict[str, object]]: + raw = bytes(cramjam.snappy.decompress_raw(payload)) + series: list[dict[str, object]] = [] + for field, _wire, val in _parse(raw): + assert field == 1 + ts_fields = _parse(val) # type: ignore[arg-type] + labels: dict[str, str] = {} + sample: dict[str, float] = {} + for f, _w, v in ts_fields: + if f == 1: # label + pair = _parse(v) # type: ignore[arg-type] + name = (pair[0][2]).decode() # type: ignore[union-attr] + value = (pair[1][2]).decode() # type: ignore[union-attr] + labels[name] = value + elif f == 2: # sample + sp = _parse(v) # type: ignore[arg-type] + sample = {"value": sp[0][2], "timestamp": sp[1][2]} # type: ignore[dict-item] + series.append({"labels": labels, "sample": sample}) + return series + + +def test_encode_write_request_roundtrips() -> None: + samples = [ + Sample(metric="anomaly_score", labels=(("id", "cpu"), ("group", "fleet")), value=4.5), + Sample(metric="anomaly_density", labels=(("group", "fleet"),), value=0.25), + ] + payload = encode_write_request(samples, timestamp_ms=1_700_000_000_000) + decoded = _decode_write_request(payload) + + assert len(decoded) == 2 + first = decoded[0] + # Metric name carried as __name__, labels sorted with __name__ first. + assert first["labels"] == {"__name__": "anomaly_score", "id": "cpu", "group": "fleet"} + assert first["sample"]["value"] == pytest.approx(4.5) + assert first["sample"]["timestamp"] == 1_700_000_000_000 + + +def _ordered_label_names(payload: bytes) -> list[str]: + """Wire-order label names of the first series (order is lost by the dict + decoder, but remote-write requires labels sorted by name).""" + raw = bytes(cramjam.snappy.decompress_raw(payload)) + first_ts = _parse(raw)[0][2] + names: list[str] = [] + for f, _w, v in _parse(first_ts): # type: ignore[arg-type] + if f == 1: + names.append((_parse(v)[0][2]).decode()) # type: ignore[arg-type,union-attr] + return names + + +def test_encode_write_request_sorts_labels_including_uppercase() -> None: + # An upper-case label name sorts before ``__name__`` ('_' is 0x5F, + # after upper-case ASCII). The full set must be sorted together or a + # strict remote-write receiver rejects the out-of-order series. + samples = [ + Sample(metric="anomaly_score", labels=(("Region", "eu"), ("id", "cpu")), value=1.0), + ] + payload = encode_write_request(samples, timestamp_ms=1) + assert _ordered_label_names(payload) == ["Region", "__name__", "id"] + + +def _sink_with_handler(handler, **cfg_kwargs): # type: ignore[no-untyped-def] + ops = OperationalMetrics() + config = RemoteWriteSinkConfig(url="http://stub/api/v1/write", **cfg_kwargs) + sink = RemoteWriteSink(config, ops) + sink._client = httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="http://stub") + return sink, ops + + +def _snapshot() -> GroupSnapshot: + return GroupSnapshot( + group="g1", + timestamp=1_700_000_000.0, + samples=[Sample(metric="anomaly_score", labels=(("id", "cpu"),), value=2.0)], + ) + + +@pytest.mark.asyncio +async def test_emit_posts_snappy_protobuf() -> None: + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["headers"] = dict(request.headers) + captured["content"] = request.content + return httpx.Response(200) + + sink, ops = _sink_with_handler(handler) + await sink.emit(_snapshot()) + + headers = captured["headers"] + assert headers["content-encoding"] == "snappy" + assert headers["content-type"] == "application/x-protobuf" + assert headers["x-prometheus-remote-write-version"] == "0.1.0" + decoded = _decode_write_request(captured["content"]) # type: ignore[arg-type] + assert decoded[0]["labels"]["__name__"] == "anomaly_score" + # No failures counted. + assert _failure_total(ops) == 0.0 + + +@pytest.mark.asyncio +async def test_emit_empty_snapshot_skips_post() -> None: + called = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + called["n"] += 1 + return httpx.Response(200) + + sink, _ops = _sink_with_handler(handler) + await sink.emit(GroupSnapshot(group="g1", timestamp=1.0, samples=[])) + assert called["n"] == 0 + + +@pytest.mark.asyncio +async def test_emit_http_error_is_isolated_and_counted() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500) + + sink, ops = _sink_with_handler(handler) + # Must not raise. + await sink.emit(_snapshot()) + assert _failure_total(ops, reason="http_error") == 1.0 + + +@pytest.mark.asyncio +async def test_emit_timeout_is_counted() -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.TimeoutException("slow") + + sink, ops = _sink_with_handler(handler) + await sink.emit(_snapshot()) + assert _failure_total(ops, reason="timeout") == 1.0 + + +def test_bearer_auth_sets_header() -> None: + ops = OperationalMetrics() + config = RemoteWriteSinkConfig( + url="http://stub/api/v1/write", + auth=AuthConfig(type="bearer", token="secret"), + ) + RemoteWriteSink(config, ops) # constructs without error + # build_httpx_auth is exercised via start(); inspect the kwargs path. + from promanomaly.httpauth import build_httpx_auth + + assert build_httpx_auth(config.auth)["headers"]["Authorization"] == "Bearer secret" + + +def _failure_total(ops: OperationalMetrics, reason: str | None = None) -> float: + total = 0.0 + for metric in ops.registry.collect(): + if metric.name != "anomaly_sink_failures": + continue + for sample in metric.samples: + if not sample.name.endswith("_total"): + continue + if reason is not None and sample.labels.get("reason") != reason: + continue + total += sample.value + return total diff --git a/detector/uv.lock b/detector/uv.lock index 2753068..24c8abb 100644 --- a/detector/uv.lock +++ b/detector/uv.lock @@ -402,6 +402,74 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl", hash = "sha256:8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1", size = 211764, upload-time = "2026-05-10T18:02:29.538Z" }, ] +[[package]] +name = "cramjam" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/12/34bf6e840a79130dfd0da7badfb6f7810b8fcfd60e75b0539372667b41b6/cramjam-2.11.0.tar.gz", hash = "sha256:5c82500ed91605c2d9781380b378397012e25127e89d64f460fea6aeac4389b4", size = 99100, upload-time = "2025-07-27T21:25:07.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/0d/7c84c913a5fae85b773a9dcf8874390f9d68ba0fcc6630efa7ff1541b950/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:dba5c14b8b4f73ea1e65720f5a3fe4280c1d27761238378be8274135c60bbc6e", size = 3553368, upload-time = "2025-07-27T21:22:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cc/4f6d185d8a744776f53035e72831ff8eefc2354f46ab836f4bd3c4f6c138/cramjam-2.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:11eb40722b3fcf3e6890fba46c711bf60f8dc26360a24876c85e52d76c33b25b", size = 1860014, upload-time = "2025-07-27T21:22:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a8/626c76263085c6d5ded0e71823b411e9522bfc93ba6cc59855a5869296e7/cramjam-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aeb26e2898994b6e8319f19a4d37c481512acdcc6d30e1b5ecc9d8ec57e835cb", size = 1693512, upload-time = "2025-07-27T21:22:30.999Z" }, + { url = "https://files.pythonhosted.org/packages/e9/52/0851a16a62447532e30ba95a80e638926fdea869a34b4b5b9d0a020083ba/cramjam-2.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f8d82081ed7d8fe52c982bd1f06e4c7631a73fe1fb6d4b3b3f2404f87dc40fe", size = 2025285, upload-time = "2025-07-27T21:22:32.954Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/122e444f59dbc216451d8e3d8282c9665dc79eaf822f5f1470066be1b695/cramjam-2.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:092a3ec26e0a679305018380e4f652eae1b6dfe3fc3b154ee76aa6b92221a17c", size = 1761327, upload-time = "2025-07-27T21:22:34.484Z" }, + { url = "https://files.pythonhosted.org/packages/a3/bc/3a0189aef1af2b29632c039c19a7a1b752bc21a4053582a5464183a0ad3d/cramjam-2.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:529d6d667c65fd105d10bd83d1cd3f9869f8fd6c66efac9415c1812281196a92", size = 1854075, upload-time = "2025-07-27T21:22:36.157Z" }, + { url = "https://files.pythonhosted.org/packages/2e/80/8a6343b13778ce52d94bb8d5365a30c3aa951276b1857201fe79d7e2ad25/cramjam-2.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:555eb9c90c450e0f76e27d9ff064e64a8b8c6478ab1a5594c91b7bc5c82fd9f0", size = 2032710, upload-time = "2025-07-27T21:22:38.17Z" }, + { url = "https://files.pythonhosted.org/packages/df/6b/cd1778a207c29eda10791e3dfa018b588001928086e179fc71254793c625/cramjam-2.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5edf4c9e32493035b514cf2ba0c969d81ccb31de63bd05490cc8bfe3b431674e", size = 2068353, upload-time = "2025-07-27T21:22:39.615Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f0/5c2a5cd5711032f3b191ca50cb786c17689b4a9255f9f768866e6c9f04d9/cramjam-2.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa2fe41f48c4d58d923803383b0737f048918b5a0d10390de9628bb6272b107", size = 1978104, upload-time = "2025-07-27T21:22:41.106Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8b/b363a5fb2c3347504fe9a64f8d0f1e276844f0e532aa7162c061cd1ffee4/cramjam-2.11.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9ca14cf1cabdb0b77d606db1bb9e9ca593b1dbd421fcaf251ec9a5431ec449f3", size = 2030779, upload-time = "2025-07-27T21:22:42.969Z" }, + { url = "https://files.pythonhosted.org/packages/78/7b/d83dad46adb6c988a74361f81ad9c5c22642be53ad88616a19baedd06243/cramjam-2.11.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:309e95bf898829476bccf4fd2c358ec00e7ff73a12f95a3cdeeba4bb1d3683d5", size = 2155297, upload-time = "2025-07-27T21:22:44.6Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/60d9be4cb33d8740a4aa94c7513f2ef3c4eba4fd13536f086facbafade71/cramjam-2.11.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:86dca35d2f15ef22922411496c220f3c9e315d5512f316fe417461971cc1648d", size = 2169255, upload-time = "2025-07-27T21:22:46.534Z" }, + { url = "https://files.pythonhosted.org/packages/11/b0/4a595f01a243aec8ad272b160b161c44351190c35d98d7787919d962e9e5/cramjam-2.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:193c6488bd2f514cbc0bef5c18fad61a5f9c8d059dd56edf773b3b37f0e85496", size = 2155651, upload-time = "2025-07-27T21:22:48.46Z" }, + { url = "https://files.pythonhosted.org/packages/38/47/7776659aaa677046b77f527106e53ddd47373416d8fcdb1e1a881ec5dc06/cramjam-2.11.0-cp312-cp312-win32.whl", hash = "sha256:514e2c008a8b4fa823122ca3ecab896eac41d9aa0f5fc881bd6264486c204e32", size = 1603568, upload-time = "2025-07-27T21:22:50.084Z" }, + { url = "https://files.pythonhosted.org/packages/75/b1/d53002729cfd94c5844ddfaf1233c86d29f2dbfc1b764a6562c41c044199/cramjam-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:53fed080476d5f6ad7505883ec5d1ec28ba36c2273db3b3e92d7224fe5e463db", size = 1709287, upload-time = "2025-07-27T21:22:51.534Z" }, + { url = "https://files.pythonhosted.org/packages/0a/8b/406c5dc0f8e82385519d8c299c40fd6a56d97eca3fcd6f5da8dad48de75b/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2c289729cc1c04e88bafa48b51082fb462b0a57dbc96494eab2be9b14dca62af", size = 3553330, upload-time = "2025-07-27T21:22:53.124Z" }, + { url = "https://files.pythonhosted.org/packages/00/ad/4186884083d6e4125b285903e17841827ab0d6d0cffc86216d27ed91e91d/cramjam-2.11.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:045201ee17147e36cf43d8ae2fa4b4836944ac672df5874579b81cf6d40f1a1f", size = 1859756, upload-time = "2025-07-27T21:22:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/54/01/91b485cf76a7efef638151e8a7d35784dae2c4ff221b1aec2c083e4b106d/cramjam-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:619cd195d74c9e1d2a3ad78d63451d35379c84bd851aec552811e30842e1c67a", size = 1693609, upload-time = "2025-07-27T21:22:56.331Z" }, + { url = "https://files.pythonhosted.org/packages/cd/84/d0c80d279b2976870fc7d10f15dcb90a3c10c06566c6964b37c152694974/cramjam-2.11.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6eb3ae5ab72edb2ed68bdc0f5710f0a6cad7fd778a610ec2c31ee15e32d3921e", size = 2024912, upload-time = "2025-07-27T21:22:57.915Z" }, + { url = "https://files.pythonhosted.org/packages/d6/70/88f2a5cb904281ed5d3c111b8f7d5366639817a5470f059bcd26833fc870/cramjam-2.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df7da3f4b19e3078f9635f132d31b0a8196accb2576e3213ddd7a77f93317c20", size = 1760715, upload-time = "2025-07-27T21:22:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/cf5b02081132537d28964fb385fcef9ed9f8a017dd7d8c59d317e53ba50d/cramjam-2.11.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:57286b289cd557ac76c24479d8ecfb6c3d5b854cce54ccc7671f9a2f5e2a2708", size = 1853782, upload-time = "2025-07-27T21:23:01.07Z" }, + { url = "https://files.pythonhosted.org/packages/57/27/63525087ed40a53d1867021b9c4858b80cc86274ffe7225deed067d88d92/cramjam-2.11.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28952fbbf8b32c0cb7fa4be9bcccfca734bf0d0989f4b509dc7f2f70ba79ae06", size = 2032354, upload-time = "2025-07-27T21:23:03.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ef/dbba082c6ebfb6410da4dd39a64e654d7194fcfd4567f85991a83fa4ec32/cramjam-2.11.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78ed2e4099812a438b545dfbca1928ec825e743cd253bc820372d6ef8c3adff4", size = 2068007, upload-time = "2025-07-27T21:23:04.526Z" }, + { url = "https://files.pythonhosted.org/packages/35/ce/d902b9358a46a086938feae83b2251720e030f06e46006f4c1fc0ac9da20/cramjam-2.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9aecd5c3845d415bd6c9957c93de8d93097e269137c2ecb0e5a5256374bdc8", size = 1977485, upload-time = "2025-07-27T21:23:06.058Z" }, + { url = "https://files.pythonhosted.org/packages/e8/03/982f54553244b0afcbdb2ad2065d460f0ab05a72a96896a969a1ca136a1e/cramjam-2.11.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:362fcf4d6f5e1242a4540812455f5a594949190f6fbc04f2ffbfd7ae0266d788", size = 2030447, upload-time = "2025-07-27T21:23:07.679Z" }, + { url = "https://files.pythonhosted.org/packages/74/5f/748e54cdb665ec098ec519e23caacc65fc5ae58718183b071e33fc1c45b4/cramjam-2.11.0-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:13240b3dea41b1174456cb9426843b085dc1a2bdcecd9ee2d8f65ac5703374b0", size = 2154949, upload-time = "2025-07-27T21:23:09.366Z" }, + { url = "https://files.pythonhosted.org/packages/69/81/c4e6cb06ed69db0dc81f9a8b1dc74995ebd4351e7a1877143f7031ff2700/cramjam-2.11.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:c54eed83726269594b9086d827decc7d2015696e31b99bf9b69b12d9063584fe", size = 2168925, upload-time = "2025-07-27T21:23:10.976Z" }, + { url = "https://files.pythonhosted.org/packages/13/5b/966365523ce8290a08e163e3b489626c5adacdff2b3da9da1b0823dfb14e/cramjam-2.11.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f8195006fdd0fc0a85b19df3d64a3ef8a240e483ae1dfc7ac6a4316019eb5df2", size = 2154950, upload-time = "2025-07-27T21:23:12.514Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7d/7f8eb5c534b72b32c6eb79d74585bfee44a9a5647a14040bb65c31c2572d/cramjam-2.11.0-cp313-cp313-win32.whl", hash = "sha256:ccf30e3fe6d770a803dcdf3bb863fa44ba5dc2664d4610ba2746a3c73599f2e4", size = 1603199, upload-time = "2025-07-27T21:23:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/37/05/47b5e0bf7c41a3b1cdd3b7c2147f880c93226a6bef1f5d85183040cbdece/cramjam-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:ee36348a204f0a68b03400f4736224e9f61d1c6a1582d7f875c1ca56f0254268", size = 1708924, upload-time = "2025-07-27T21:23:16.332Z" }, + { url = "https://files.pythonhosted.org/packages/de/07/a1051cdbbe6d723df16d756b97f09da7c1adb69e29695c58f0392bc12515/cramjam-2.11.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7ba5e38c9fbd06f086f4a5a64a1a5b7b417cd3f8fc07a20e5c03651f72f36100", size = 3554141, upload-time = "2025-07-27T21:23:17.938Z" }, + { url = "https://files.pythonhosted.org/packages/74/66/58487d2e16ef3d04f51a7c7f0e69823e806744b4c21101e89da4873074bc/cramjam-2.11.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8adeee57b41fe08e4520698a4b0bd3cc76dbd81f99424b806d70a5256a391d3", size = 1860353, upload-time = "2025-07-27T21:23:19.593Z" }, + { url = "https://files.pythonhosted.org/packages/67/b4/67f6254d166ffbcc9d5fa1b56876eaa920c32ebc8e9d3d525b27296b693b/cramjam-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b96a74fa03a636c8a7d76f700d50e9a8bc17a516d6a72d28711225d641e30968", size = 1693832, upload-time = "2025-07-27T21:23:21.185Z" }, + { url = "https://files.pythonhosted.org/packages/55/a3/4e0b31c0d454ae70c04684ed7c13d3c67b4c31790c278c1e788cb804fa4a/cramjam-2.11.0-cp314-cp314-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c3811a56fa32e00b377ef79121c0193311fd7501f0fb378f254c7f083cc1fbe0", size = 2027080, upload-time = "2025-07-27T21:23:23.303Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c7/5e8eed361d1d3b8be14f38a54852c5370cc0ceb2c2d543b8ba590c34f080/cramjam-2.11.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5d927e87461f8a0d448e4ab5eb2bca9f31ca5d8ea86d70c6f470bb5bc666d7e", size = 1761543, upload-time = "2025-07-27T21:23:24.991Z" }, + { url = "https://files.pythonhosted.org/packages/09/0c/06b7f8b0ce9fde89470505116a01fc0b6cb92d406c4fb1e46f168b5d3fa5/cramjam-2.11.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f1f5c450121430fd89cb5767e0a9728ecc65997768fd4027d069cb0368af62f9", size = 1854636, upload-time = "2025-07-27T21:23:26.987Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c6/6ebc02c9d5acdf4e5f2b1ec6e1252bd5feee25762246798ae823b3347457/cramjam-2.11.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:724aa7490be50235d97f07e2ca10067927c5d7f336b786ddbc868470e822aa25", size = 2032715, upload-time = "2025-07-27T21:23:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/a2/77/a122971c23f5ca4b53e4322c647ac7554626c95978f92d19419315dddd05/cramjam-2.11.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54c4637122e7cfd7aac5c1d3d4c02364f446d6923ea34cf9d0e8816d6e7a4936", size = 2069039, upload-time = "2025-07-27T21:23:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/19/0f/f6121b90b86b9093c066889274d26a1de3f29969d45c2ed1ecbe2033cb78/cramjam-2.11.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17eb39b1696179fb471eea2de958fa21f40a2cd8bf6b40d428312d5541e19dc4", size = 1979566, upload-time = "2025-07-27T21:23:32.002Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/f95bc57fd7f4166ce6da816cfa917fb7df4bb80e669eb459d85586498414/cramjam-2.11.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:36aa5a798aa34e11813a80425a30d8e052d8de4a28f27bfc0368cfc454d1b403", size = 2030905, upload-time = "2025-07-27T21:23:33.696Z" }, + { url = "https://files.pythonhosted.org/packages/fc/52/e429de4e8bc86ee65e090dae0f87f45abd271742c63fb2d03c522ffde28a/cramjam-2.11.0-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:449fca52774dc0199545fbf11f5128933e5a6833946707885cf7be8018017839", size = 2155592, upload-time = "2025-07-27T21:23:35.375Z" }, + { url = "https://files.pythonhosted.org/packages/6c/6c/65a7a0207787ad39ad804af4da7f06a60149de19481d73d270b540657234/cramjam-2.11.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:d87d37b3d476f4f7623c56a232045d25bd9b988314702ea01bd9b4a94948a778", size = 2170839, upload-time = "2025-07-27T21:23:37.197Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c5/5c5db505ba692bc844246b066e23901d5905a32baf2f33719c620e65887f/cramjam-2.11.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:26cb45c47d71982d76282e303931c6dd4baee1753e5d48f9a89b3a63e690b3a3", size = 2157236, upload-time = "2025-07-27T21:23:38.854Z" }, + { url = "https://files.pythonhosted.org/packages/b0/22/88e6693e60afe98901e5bbe91b8dea193e3aa7f42e2770f9c3339f5c1065/cramjam-2.11.0-cp314-cp314-win32.whl", hash = "sha256:4efe919d443c2fd112fe25fe636a52f9628250c9a50d9bddb0488d8a6c09acc6", size = 1604136, upload-time = "2025-07-27T21:23:40.56Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f8/01618801cd59ccedcc99f0f96d20be67d8cfc3497da9ccaaad6b481781dd/cramjam-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ccec3524ea41b9abd5600e3e27001fd774199dbb4f7b9cb248fcee37d4bda84c", size = 1710272, upload-time = "2025-07-27T21:23:42.236Z" }, + { url = "https://files.pythonhosted.org/packages/40/81/6cdb3ed222d13ae86bda77aafe8d50566e81a1169d49ed195b6263610704/cramjam-2.11.0-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:966ac9358b23d21ecd895c418c048e806fd254e46d09b1ff0cdad2eba195ea3e", size = 3559671, upload-time = "2025-07-27T21:23:44.504Z" }, + { url = "https://files.pythonhosted.org/packages/cb/43/52b7e54fe5ba1ef0270d9fdc43dabd7971f70ea2d7179be918c997820247/cramjam-2.11.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:387f09d647a0d38dcb4539f8a14281f8eb6bb1d3e023471eb18a5974b2121c86", size = 1867876, upload-time = "2025-07-27T21:23:46.987Z" }, + { url = "https://files.pythonhosted.org/packages/9d/28/30d5b8d10acd30db3193bc562a313bff722888eaa45cfe32aa09389f2b24/cramjam-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:665b0d8fbbb1a7f300265b43926457ec78385200133e41fef19d85790fc1e800", size = 1695562, upload-time = "2025-07-27T21:23:48.644Z" }, + { url = "https://files.pythonhosted.org/packages/d9/86/ec806f986e01b896a650655024ea52a13e25c3ac8a3a382f493089483cdc/cramjam-2.11.0-cp314-cp314t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ca905387c7a371531b9622d93471be4d745ef715f2890c3702479cd4fc85aa51", size = 2025056, upload-time = "2025-07-27T21:23:50.404Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/c2c17586b90848d29d63181f7d14b8bd3a7d00975ad46e3edf2af8af7e1f/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c1aa56aef2c8af55a21ed39040a94a12b53fb23beea290f94d19a76027e2ffb", size = 1764084, upload-time = "2025-07-27T21:23:52.265Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a9/68bc334fadb434a61df10071dc8606702aa4f5b6cdb2df62474fc21d2845/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e5db59c1cdfaa2ab85cc988e602d6919495f735ca8a5fd7603608eb1e23c26d5", size = 1854859, upload-time = "2025-07-27T21:23:54.085Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4e/b48e67835b5811ec5e9cb2e2bcba9c3fd76dab3e732569fe801b542c6ca9/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1f893014f00fe5e89a660a032e813bf9f6d91de74cd1490cdb13b2b59d0c9a3", size = 2035970, upload-time = "2025-07-27T21:23:55.758Z" }, + { url = "https://files.pythonhosted.org/packages/c4/70/d2ac33d572b4d90f7f0f2c8a1d60fb48f06b128fdc2c05f9b49891bb0279/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c26a1eb487947010f5de24943bd7c422dad955b2b0f8650762539778c380ca89", size = 2069320, upload-time = "2025-07-27T21:23:57.494Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4c/85cec77af4a74308ba5fca8e296c4e2f80ec465c537afc7ab1e0ca2f9a00/cramjam-2.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d5c8bfb438d94e7b892d1426da5fc4b4a5370cc360df9b8d9d77c33b896c37e", size = 1982668, upload-time = "2025-07-27T21:23:59.126Z" }, + { url = "https://files.pythonhosted.org/packages/55/45/938546d1629e008cc3138df7c424ef892719b1796ff408a2ab8550032e5e/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:cb1fb8c9337ab0da25a01c05d69a0463209c347f16512ac43be5986f3d1ebaf4", size = 2034028, upload-time = "2025-07-27T21:24:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/01/76/b5a53e20505555f1640e66dcf70394bcf51a1a3a072aa18ea35135a0f9ed/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:1f6449f6de52dde3e2f1038284910c8765a397a25e2d05083870f3f5e7fc682c", size = 2155513, upload-time = "2025-07-27T21:24:02.92Z" }, + { url = "https://files.pythonhosted.org/packages/84/12/8d3f6ceefae81bbe45a347fdfa2219d9f3ac75ebc304f92cd5fcb4fbddc5/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_i686.whl", hash = "sha256:382dec4f996be48ed9c6958d4e30c2b89435d7c2c4dbf32480b3b8886293dd65", size = 2170035, upload-time = "2025-07-27T21:24:04.558Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/3be6f0a1398f976070672be64f61895f8839857618a2d8cc0d3ab529d3dc/cramjam-2.11.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:d388bd5723732c3afe1dd1d181e4213cc4e1be210b080572e7d5749f6e955656", size = 2160229, upload-time = "2025-07-27T21:24:06.729Z" }, + { url = "https://files.pythonhosted.org/packages/57/5e/66cfc3635511b20014bbb3f2ecf0095efb3049e9e96a4a9e478e4f3d7b78/cramjam-2.11.0-cp314-cp314t-win32.whl", hash = "sha256:0a70ff17f8e1d13f322df616505550f0f4c39eda62290acb56f069d4857037c8", size = 1610267, upload-time = "2025-07-27T21:24:08.428Z" }, + { url = "https://files.pythonhosted.org/packages/ce/c6/c71e82e041c95ffe6a92ac707785500aa2a515a4339c2c7dd67e3c449249/cramjam-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:028400d699442d40dbda02f74158c73d05cb76587a12490d0bfedd958fd49188", size = 1713108, upload-time = "2025-07-27T21:24:10.147Z" }, +] + [[package]] name = "durationpy" version = "0.10" @@ -1165,6 +1233,7 @@ analyze = [ { name = "ruptures" }, ] dev = [ + { name = "cramjam" }, { name = "fakeredis" }, { name = "kubernetes" }, { name = "mypy" }, @@ -1184,11 +1253,16 @@ otel = [ { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-sdk" }, ] +sinks = [ + { name = "cramjam" }, +] [package.metadata] requires-dist = [ { name = "apscheduler", specifier = ">=3.10.4" }, { name = "click", specifier = ">=8.1.7" }, + { name = "cramjam", marker = "extra == 'dev'", specifier = ">=2.8.0" }, + { name = "cramjam", marker = "extra == 'sinks'", specifier = ">=2.8.0" }, { name = "fakeredis", marker = "extra == 'dev'", specifier = ">=2.26.0" }, { name = "fastapi", specifier = ">=0.115.0" }, { name = "httpx", specifier = ">=0.27.0" }, @@ -1215,7 +1289,7 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, ] -provides-extras = ["analyze", "ha", "otel", "dev"] +provides-extras = ["analyze", "ha", "otel", "sinks", "dev"] [[package]] name = "prometheus-client" diff --git a/docs/adapter.md b/docs/adapter.md new file mode 100644 index 0000000..154c6af --- /dev/null +++ b/docs/adapter.md @@ -0,0 +1,93 @@ +# Kubernetes Metrics Adapter + +The metrics adapter is an **opt-in**, stateless sidecar that exposes promanomaly’s anomaly metrics through the Kubernetes `external.metrics.k8s.io` and `custom.metrics.k8s.io` APIs. This lets HPA and KEDA treat anomaly signals exactly like any other metric. + +promanomaly remains the **metrics provider**; the autoscaler remains the **decision maker**. The adapter never makes scaling decisions itself. + +Deploy it with the [`promanomaly-metrics-adapter`](../charts/promanomaly-metrics-adapter/) Helm chart (disabled by default) or run it directly via `promanomaly adapter --config adapter.yaml`. + +## Architecture + +```mermaid +flowchart TD + HPA[KEDA / HPA] -->|queries external/custom metrics| KubeAPI[kube-apiserver
aggregation layer] + KubeAPI -->|proxies /apis/...| Adapter[promanomaly-metrics-adapter] + Adapter -->|PromQL instant query| TSDB[VictoriaMetrics / TSDB
written by the detector] +``` + +Two `APIService` objects register the adapter: +- `external.metrics.k8s.io/v1beta1` — for KEDA `external` triggers and HPA `External` metric source (recommended). +- `custom.metrics.k8s.io/v1beta1` — for HPA `Object`/`Pods` rules on Kubernetes objects. + +Every value is fetched live from the TSDB; the adapter is completely stateless. + +It also exposes its own operational metrics on `/metrics`: +- `anomaly_adapter_requests_total{api,metric,outcome}` +- `anomaly_adapter_tsdb_failures_total{metric}` + +## Exposed Metrics (default allow-list) + +| Metric | Typical use | +|----------------------------|------------------------------------------| +| `anomaly_density` | Fraction of a group that is anomalous (recommended scaling signal) | +| `anomaly_severity` | 0–1 normalized severity | +| `anomaly_active_series` | Count of currently firing series | +| `anomaly_outside_threshold`| Per-series 0/1 firing flag | + +You can trim this list. `anomaly_score` is deliberately **not** included by default. + +## Label Selectors + +HPA/KEDA `labelSelector` expressions are automatically turned into PromQL label matchers. Equality operators and comma-separated lists are supported. For custom metrics, the Kubernetes object name is matched against the corresponding PromQL label (e.g. `pod="..."`). + +## Guard-Rails for Autoscaling + +1. Scale on `anomaly_density`, not raw `anomaly_score`. +2. Gate on `anomaly_severity` (already includes confidence + duration). +3. Always pair an anomaly-driven scaler with a traditional reactive HPA fallback. +4. Set sensible `minReplicas` / `maxReplicas` and long scale-down stabilization windows. + +See ready-made recipes in [`examples/k8s/`](../examples/k8s/). + +## Configuration + +```yaml +datasource: + url: http://victoriametrics.monitoring.svc:8428/ + timeout: 10s + auth: + type: none # none | bearer | basic | mtls +listen: ":6443" +tls: + cert_file: /etc/promanomaly-adapter/tls/tls.crt + key_file: /etc/promanomaly-adapter/tls/tls.key +metrics: + - anomaly_severity + - anomaly_density + - anomaly_active_series + - anomaly_outside_threshold +custom_resources: + - pods + - namespaces +``` + +The chart automatically mounts TLS certificates and credentials. Use `datasource.auth.existingSecret` for bearer tokens, basic auth, or mTLS files. + +Validate with: + +```bash +promanomaly adapter-validate --config adapter.yaml +``` + +## Quick Verification + +```bash +kubectl get apiservices | grep metrics.k8s.io + +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" | jq . + +# Example query for a specific group +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/monitoring/anomaly_density?labelSelector=group%3Dqueue_depths" | jq . +``` + +That’s it — drop the adapter in, point HPA/KEDA at the anomaly signals, and you’re done. \ No newline at end of file diff --git a/docs/cli.md b/docs/cli.md index 2faa186..5157f9a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,6 +8,7 @@ promanomaly ships with a small, focused set of CLI commands for validation, test |---------------------|---------| | (default) | Start the production server | | `validate` | Validate config (optionally probe datasource) | +| `generate-rules` | Scaffold a PrometheusRule from a config (selectors filled, thresholds TODO) | | `dry-run` | Validate + run one full detection cycle, then exit | | `detect-once` | Run one detector on an ad-hoc PromQL query | | `analyze` | Offline change-point analysis over historical data | @@ -17,8 +18,10 @@ promanomaly ships with a small, focused set of CLI commands for validation, test | `top` | List the currently-firing series, ranked by severity | | `warmup` | Report which series are still warming up, with an ETA to readiness | | `diagnose` | Flag mis-tuned detectors (never-fires, fires-often, stuck warming, empty) | +| `adapter` | Run the Kubernetes external/custom metrics adapter | +| `adapter-validate` | Validate an adapter config file (schema only) | -All commands accept `--log-level`. `--config` is required only for server, `validate`, and `dry-run`. +All commands accept `--log-level`. `--config` is required only for server, `validate`, `generate-rules`, and `dry-run`. ## `validate` @@ -50,6 +53,28 @@ One `{"status": "lint", ...}` JSON line per finding (`group`, `query`, `metric`, One JSON line per group plus a summary line. Per group: `projected_series` (worst case at `safety.max_series_per_query`), `short_queries_per_refresh`, `discovery_probe_queries_per_refresh`, `stratified_baseline_queries_per_day`, `estimated_memory_bytes`, and `estimated_cpu_cores`. The summary aggregates the totals and flags `over_budget` against `safety.max_total_series`. The CPU figure is anchored to the documented "~10k series at a 1-minute refresh on one 1-CPU replica" model — an order-of-magnitude sizing aid, not a benchmark. Pairs with `--probe` (which confirms the queries actually return data); see [production-checklist.md](production-checklist.md). +## `generate-rules` + +Scaffolds a `PrometheusRule` manifest from a config so you don't hand-copy `id`/`group` label sets. Emits one `AnomalyOutsideThreshold`-style alert per group/query with the selectors already filled in, plus the global `max`-ensemble recording-rule fallback (`anomaly_outside_threshold_any`) described in [patterns.md](patterns.md). Groups that set an `ensemble:` block also get an ensemble-agreement alert. + +```bash +promanomaly generate-rules --config config.yaml # to stdout +promanomaly generate-rules --config config.yaml --output rules.yaml +promanomaly generate-rules --config config.yaml --name my-team-anomalies +``` + +Severities, `for:` windows, and any runbook links are emitted as clearly-marked `# TODO` placeholders — the project ships **no opinionated thresholds**, so this is scaffolding, not a recommendation. Each alert gets a unique, descriptive name (derived from group + query id) so the file is duplicate-rule-lint clean (`promtool check rules`) and each alert is individually routable / silenceable in Alertmanager. Discover-templated query ids can't be pinned to a single literal `id=`, so those alerts scope by `group` only and carry a note to narrow the selector once the expansions are known. + +The intended workflow pairs with the GitOps validation Action: **generate, edit the `TODO`s, validate, commit.** + +**Flags** + +| Flag | Default | Description | +|---------------------|------------------------|-------------| +| `--config ` | required | Path to YAML config | +| `--output ` | stdout | Write the manifest to a file instead of stdout | +| `--name ` | `promanomaly-generated`| `metadata.name` for the generated PrometheusRule | + ## `dry-run` Full validation + one complete detection run for every group. No HTTP server is started. @@ -226,4 +251,15 @@ promanomaly diagnose --target http://localhost:9092 | `--output` | text | `text` (grouped by category) or `json` (for CI) | | `--timeout ` | 30 | HTTP timeout | -Pure analysis, no persisted state. Pairs with `backtest` (precision/recall on injected anomalies) and `calibrate-buckets` (stratified bucketing) to close the tuning loop. Parity with `promforecast diagnose`. \ No newline at end of file +Pure analysis, no persisted state. Pairs with `backtest` (precision/recall on injected anomalies) and `calibrate-buckets` (stratified bucketing) to close the tuning loop. Parity with `promforecast diagnose`. + +## `adapter` + +Run the Kubernetes external/custom metrics adapter — a separate, stateless process from the detector that re-serves the anomaly metrics already in the TSDB through `external.metrics.k8s.io` / `custom.metrics.k8s.io`, so HPA and KEDA can consume anomaly signal. It is normally deployed via the [`promanomaly-metrics-adapter`](../charts/promanomaly-metrics-adapter/) chart, which generates this config and wires the serving TLS + APIService registrations. See [adapter.md](adapter.md) for the full surface and the autoscaling guard-rails. + +```bash +promanomaly adapter --config adapter.yaml +promanomaly adapter-validate --config adapter.yaml # schema only +``` + +The adapter takes its **own** config file (datasource, listen address, serving TLS, the metric allow-list). It shares the datasource auth shapes (`none`/`bearer`/`basic`/`mtls`) with the detector. `adapter-validate` checks the schema and prints the resolved listen address, TLS state, and exposed metrics — suitable as a CI pre-deploy gate. \ No newline at end of file diff --git a/docs/gitops.md b/docs/gitops.md new file mode 100644 index 0000000..b99631b --- /dev/null +++ b/docs/gitops.md @@ -0,0 +1,76 @@ +# GitOps Config Validation + +Keep your promanomaly configuration in Git and validate it on every pull request. This prevents broken or overly expensive configs from ever reaching production. + +The repository ships a composite GitHub Action (`action.yml`) that wraps [`promanomaly validate`](cli.md#validate). + +## What It Checks + +The action runs `promanomaly validate` with the checks you enable: + +- **Schema validation** (always on) — config parses and matches the pydantic schema. +- **`--estimate-cost`** (default: on) — statically projects cardinality, TSDB query load, and rough CPU/memory usage. With `strict`, exceeding `safety.max_total_series` fails the job. Requires no live datasource. +- **`--probe`** (default: off) — executes every query against a live TSDB and fails on empty or errored results. +- **`--lint-metadata`** (default: off) — warns when raw counters are fed to detectors without `rate()` or `increase()`. Requires datasource access. + +`strict` (default: `true`) turns any soft finding into a hard failure. + +## Action Inputs + +| Input | Default | Description | +|--------------------|--------------|-------------| +| `config` | required | Path to the config YAML file | +| `estimate-cost` | `true` | Enable static cardinality & cost projection | +| `probe` | `false` | Run live queries against datasource | +| `lint-metadata` | `false` | Enable counter-without-rate linting | +| `strict` | `true` | Fail on any issue | +| `datasource-url` | from config | Override datasource for probe/lint checks | +| `python-version` | `3.12` | Python version used by the validator | + +## Sample Workflows + +### Basic static validation (recommended for most PRs) + +```yaml +name: Validate promanomaly config +on: + pull_request: + paths: + - "promanomaly/**.yaml" + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: esops-dev/promanomaly@main + with: + config: promanomaly/config.yaml + estimate-cost: "true" + strict: "true" +``` + +### Full validation with live probe + +```yaml + - uses: esops-dev/promanomaly@main + with: + config: promanomaly/config.yaml + probe: "true" + lint-metadata: "true" + strict: "true" + datasource-url: https://staging-tsdb.internal/ +``` + +Pin the action to a specific release tag (instead of `@main`) once you adopt a stable version. + +## Pairing with `generate-rules` + +This action pairs perfectly with [`promanomaly generate-rules`](cli.md#generate-rules): + +1. Generate a PrometheusRule skeleton from your config. +2. Edit the `TODO` alert thresholds. +3. Validate the full config in CI with this action. +4. Commit the changes. + +The workflow mirrors the one used by promforecast, so teams running both tools use identical CI patterns. \ No newline at end of file diff --git a/docs/patterns.md b/docs/patterns.md index 6036e1c..a1ce5c5 100644 --- a/docs/patterns.md +++ b/docs/patterns.md @@ -53,6 +53,24 @@ Example alert: for: 5m ``` +## Scaffolding Rules From Your Config + +Rather than hand-copying `id`/`group` label sets into alert rules, generate +a starting-point `PrometheusRule` straight from your config: + +```bash +promanomaly generate-rules --config config.yaml --output rules.yaml +``` + +It emits one outside-threshold alert per group/query with the selectors +pre-filled, plus the global `max`-ensemble recording-rule fallback +(`anomaly_outside_threshold_any = max by (id, group) (anomaly_outside_threshold)`) +— the recording-rule equivalent of an `ensemble: { method: max }` block, for +operators who don't use the built-in `ensemble:`. Severities, `for:` windows, +and runbook links are left as `# TODO` placeholders (the project ships no +opinionated thresholds). The generated file is `promtool check rules` clean. +See [cli.md](cli.md#generate-rules). + ## Flap Suppression Use the standard alert `for:` clause: diff --git a/docs/sinks.md b/docs/sinks.md new file mode 100644 index 0000000..ca63e8a --- /dev/null +++ b/docs/sinks.md @@ -0,0 +1,89 @@ +# Sinks + +By default promanomaly is **pull-only**: Prometheus scrapes `/metrics`. + +You can optionally add **push sinks** that forward each successful group snapshot to external systems after every run. Sinks are *additional* — they never replace the `/metrics` endpoint. + +## Failure Isolation + +A sink failure is completely isolated: +- It is logged and counted in `anomaly_sink_failures_total{sink, reason}` (bounded cardinality). +- It **never** blocks `/metrics`, readiness, or the detection pipeline. +- Common reasons: `http_error`, `timeout`, `connect_error`, `serialize_error`, `rate_limited`, `exception`. + +In HA mode only the leader pushes, so each snapshot is sent exactly once (brief duplicates during failover are harmless). + +## Available Sinks + +### `remote_write` — Long-horizon anomaly history + +Pushes snapshots to a Prometheus remote-write endpoint. Ideal for keeping `anomaly_density` and other signals beyond scrape retention. + +```yaml +sink: + type: remote_write + remote_write: + url: http://victoriametrics:8428/api/v1/write + timeout: 10s + auth: + type: bearer # none | bearer | basic | mtls + token_file: /etc/promanomaly-sink-auth/remote-write/token +``` + +### `grafana_annotations` — Anomalies on every dashboard + +Posts an annotation to Grafana on every `0→1` transition of `anomaly_outside_threshold` and every `anomaly_change_point_total` increment. Annotations appear across **all** dashboards that match the tags. + +```yaml +sink: + type: grafana_annotations + grafana_annotations: + url: http://grafana.monitoring.svc:3000 + timeout: 10s + auth: + type: bearer + token_file: /etc/promanomaly-sink-auth/grafana-annotations/token + tags: + - promanomaly + max_annotations_per_run: 50 +``` + +**Key behaviours** +- Stateless across restarts (only new 0→1 transitions are annotated). +- Rate-limited per run to protect the Grafana API. +- Add a dashboard annotation query filtering on your tags to see vertical markers. + +## Multiple Sinks + +To push to several destinations at once, use the `sinks:` list: + +```yaml +sinks: + - type: remote_write + remote_write: + url: http://victoriametrics:8428/api/v1/write + auth: + type: bearer + token_file: /etc/promanomaly-sink-auth/remote-write/token + - type: grafana_annotations + grafana_annotations: + url: http://grafana.monitoring.svc:3000 + auth: + type: bearer + token_file: /etc/promanomaly-sink-auth/grafana-annotations/token +``` + +- `sink:` and `sinks:` are mutually exclusive. +- Each sink type may appear at most once. +- Failures are isolated per sink. + +## Helm Configuration + +Both single and multiple sinks are fully configurable in the detector chart via the `sink:` (scalar) or `sinks:` (list) block. +Credentials are mounted from Secrets — never inline tokens. See `charts/promanomaly/values.yaml`. + +## Why No Built-in Alerting Engine? + +Sinks only forward **signal**. Alerting decisions, flap suppression, deploy silencing, and routing belong in Alertmanager and the rest of the Prometheus stack — not the detector. + +This keeps promanomaly focused, stateless, and easy to operate. \ No newline at end of file diff --git a/examples/k8s/README.md b/examples/k8s/README.md new file mode 100644 index 0000000..6110bbd --- /dev/null +++ b/examples/k8s/README.md @@ -0,0 +1,49 @@ +# Reaction recipes: wiring anomaly signal into KEDA / HPA / Argo Rollouts + +Worked examples for feeding promanomaly's anomaly signal to the systems +that *act* on it — autoscalers and rollout controllers — with the +guard-rails baked in. promanomaly only emits signal; every decision here +is made by the reaction system, not the detector. + +| File | System | Pattern | +| --- | --- | --- | +| [`keda-scaledobject.yaml`](keda-scaledobject.yaml) | KEDA | Scale workers on anomalous backlog depth. | +| [`hpa-custom-metrics.yaml`](hpa-custom-metrics.yaml) | HPA | Same, via the external-metrics adapter. | +| [`argo-rollouts-analysistemplate.yaml`](argo-rollouts-analysistemplate.yaml) | Argo Rollouts | Cohort-gated canary: abort when the canary diverges from its peers. | + +## The guard-rails (why they're in every file) + +Anomaly signal is a *sharp* input. Wire it carelessly and it amplifies +incidents. Every recipe encodes these rules: + +1. **Scale on density, not score.** `anomaly_density` ("how much of the + backlog is anomalously deep") is a sound scaling input. + `anomaly_score` is **not** — scaling on how anomalous a latency metric + looks creates a feedback loop where the spike scales you, and the + scaling deepens the spike. +2. **Gate on confidence and duration.** React to sustained, + high-confidence anomalies (`anomaly_severity_density`, + `anomaly_duration_seconds`, `count`/`interval`), never a single tick. +3. **Always keep a reactive fallback.** Pair the anomaly-driven trigger + with a plain reactive one (queue length, CPU). If the detector or TSDB + is down, the fallback keeps the workload safe — the anomaly path is + never the only thing between you and an unbounded queue. +4. **Cap the blast radius.** `minReplicaCount`/`maxReplicaCount` floors + and ceilings, long scale-down stabilization windows. + +## Which path needs the metrics adapter? + +- **KEDA** and **Argo Rollouts** talk to the TSDB's Prometheus HTTP API + directly (`prometheus` scaler / metric provider) — no adapter required. +- **HPA** consumes Kubernetes metrics APIs, so the + [`promanomaly-metrics-adapter`](../../charts/promanomaly-metrics-adapter/) + chart must be installed for `hpa-custom-metrics.yaml`. + +## Why cohort gating for canaries is the standout + +The Argo Rollouts recipe is the strongest fit. Static canary thresholds +("p95 < 300ms") need per-service tuning and drift. The Cohort detector +instead asks "does the canary look *different from its stable peers right +now?*" — which needs no threshold and adapts to whatever the service's +current normal is. A canary that regresses relative to the fleet fails +the rollout automatically. diff --git a/examples/k8s/argo-rollouts-analysistemplate.yaml b/examples/k8s/argo-rollouts-analysistemplate.yaml new file mode 100644 index 0000000..5d81579 --- /dev/null +++ b/examples/k8s/argo-rollouts-analysistemplate.yaml @@ -0,0 +1,95 @@ +# Argo Rollouts AnalysisTemplate — anomaly-gated progressive delivery. +# +# This is the strongest fit for promanomaly in the reaction space: use the +# Cohort detector to compare the *canary* pods against the cohort of +# *stable* pods, and FAIL the rollout when the new version's golden +# signals look anomalous relative to its peers. No static thresholds to +# maintain — "different from the fleet" is the signal. +# +# How it works: +# 1. promanomaly runs a Cohort detector over a per-pod golden signal +# (e.g. error rate or p95 latency), with the pod's role +# (canary|stable) carried as a label. See the cohort example config. +# 2. Argo Rollouts runs this AnalysisTemplate during the canary steps. +# 3. The metric queries promanomaly's anomaly output for the CANARY +# pods. If a canary pod is outside threshold relative to the cohort, +# the analysis fails and Argo aborts/rolls back. +# +# Argo Rollouts' Prometheus provider talks to the TSDB directly, so no +# metrics adapter is needed for this path. +--- +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: anomaly-cohort-gate + namespace: workloads +spec: + args: + - name: service-name + - name: canary-role + value: canary + metrics: + - name: canary-cohort-anomaly + # GUARD-RAIL: require several consecutive clean measurements before + # promoting, and tolerate no anomalous readings. interval * count + # should cover at least one promanomaly refresh_interval. + interval: 1m + count: 5 + # Fail the analysis if the canary is anomalous relative to the + # cohort even once: failureLimit 0 means a single firing aborts. + failureLimit: 0 + successCondition: result == 0 + failureCondition: result > 0 + provider: + prometheus: + address: http://victoriametrics.monitoring.svc:8428 + # Count canary series that the Cohort detector flagged as + # outside threshold. anomaly_outside_threshold is 0/1 per + # series; summing over canary pods yields the number of + # canary pods diverging from the stable cohort right now. + query: | + sum( + anomaly_outside_threshold{ + group="{{args.service-name}}_cohort", + detector="Cohort", + role="{{args.canary-role}}" + } + ) + or vector(0) +--- +# Wiring snippet: reference the template from a Rollout's canary strategy. +# (Trimmed to the analysis wiring — keep your own replicas/template/etc.) +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: orders-api + namespace: workloads +spec: + strategy: + canary: + steps: + - setWeight: 20 + - pause: {duration: 2m} + # Run the cohort anomaly gate as a background analysis across the + # remaining canary steps. If a canary pod diverges from the stable + # cohort, the rollout aborts and rolls back automatically. + - analysis: + templates: + - templateName: anomaly-cohort-gate + args: + - name: service-name + value: orders_api + - setWeight: 50 + - pause: {duration: 2m} + - setWeight: 100 + selector: + matchLabels: + app: orders-api + template: + metadata: + labels: + app: orders-api + spec: + containers: + - name: orders-api + image: ghcr.io/example/orders-api:latest diff --git a/examples/k8s/hpa-custom-metrics.yaml b/examples/k8s/hpa-custom-metrics.yaml new file mode 100644 index 0000000..ac42d3c --- /dev/null +++ b/examples/k8s/hpa-custom-metrics.yaml @@ -0,0 +1,55 @@ +# HorizontalPodAutoscaler — scale on anomaly signal via the metrics +# adapter (charts/promanomaly-metrics-adapter). +# +# This path uses the Kubernetes external-metrics API served by the +# adapter, so the HPA consumes anomaly_density like any other external +# metric. Install the adapter chart first. +# +# GUARD-RAILS (same as the KEDA example, expressed in HPA terms): +# * External metric is anomaly_density (a bounded fraction), NOT +# anomaly_score. +# * A second `Resource` metric (CPU) is the reactive fallback — the HPA +# scales to the max desired across metrics, so CPU keeps the workload +# safe if anomaly signal is unavailable. +# * scaleDown stabilization is long to avoid flapping. +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: queue-workers + namespace: workloads +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: queue-workers + minReplicas: 3 # GUARD-RAIL: safe floor + maxReplicas: 30 # GUARD-RAIL: blast-radius ceiling + metrics: + # Anomaly-driven: target an average anomaly_density across the group. + # Value is a milli-quantity ("200m" == 0.2). The adapter serves this + # from external.metrics.k8s.io. + - type: External + external: + metric: + name: anomaly_density + selector: + matchLabels: + group: queue_depths + target: + type: AverageValue + averageValue: "200m" + # GUARD-RAIL: reactive fallback on CPU. The HPA takes the max desired + # replica count across metrics, so this keeps scaling working even + # when the detector/adapter path is down. + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 120 + scaleDown: + stabilizationWindowSeconds: 600 diff --git a/examples/k8s/keda-scaledobject.yaml b/examples/k8s/keda-scaledobject.yaml new file mode 100644 index 0000000..4d0cc85 --- /dev/null +++ b/examples/k8s/keda-scaledobject.yaml @@ -0,0 +1,62 @@ +# KEDA ScaledObject — scale a worker Deployment on anomalous backlog depth. +# +# This is the *good* shape for anomaly-driven scaling: react when the +# queue is anomalously DEEP (anomaly_density on a queue-depth group), not +# to how anomalous some latency metric looks. The guard-rails below are +# the point of the example — copy them, don't strip them. +# +# Prerequisites: +# * promanomaly detecting a "queue_depths" group (see +# examples/configs/*.yaml). +# * KEDA installed (it talks to the Prometheus/VictoriaMetrics HTTP API +# directly via the `prometheus` scaler — no metrics adapter needed for +# this path). +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: queue-workers + namespace: workloads +spec: + scaleTargetRef: + name: queue-workers # the Deployment to scale + minReplicaCount: 3 # GUARD-RAIL: never scale below a safe floor + maxReplicaCount: 30 # GUARD-RAIL: hard ceiling caps blast radius + # GUARD-RAIL: cooldown + stabilization avoid flapping on a single + # anomalous tick. Anomaly signal is sharp; react to sustained state. + cooldownPeriod: 300 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleUp: + stabilizationWindowSeconds: 120 + scaleDown: + stabilizationWindowSeconds: 600 # scale down slowly + triggers: + # Primary: scale on the *density* of anomalous backlog series, gated + # to only count when the anomaly is high-confidence AND sustained. + # `anomaly_density` is bounded-cardinality (a fraction 0-1), so this + # query is cheap and safe. + - type: prometheus + metadata: + serverAddress: http://victoriametrics.monitoring.svc:8428 + # Fire scaling only when density is meaningful (>0.2 of the group + # anomalous). The `and on` gates on severity-density (which already + # folds in confidence and duration) so a low-severity or momentary + # blip does not move replicas. + query: | + max( + anomaly_density{group="queue_depths"} + and on (group) (anomaly_severity_density{group="queue_depths"} > 0.5) + ) + threshold: "0.2" + activationThreshold: "0.1" + # GUARD-RAIL: reactive fallback. A plain backlog-length trigger keeps + # the workload safe if the detector or TSDB is unavailable, so the + # anomaly-driven trigger is never the *only* thing standing between + # you and an unbounded queue. KEDA scales to the max across triggers. + - type: prometheus + metadata: + serverAddress: http://victoriametrics.monitoring.svc:8428 + query: sum(queue_depth{queue="orders"}) + threshold: "1000" # target backlog per replica