From bfc6a45e1b9f1dd783f022e3033c1eca1c9b9e7e Mon Sep 17 00:00:00 2001 From: Werner Dijkerman Date: Fri, 29 May 2026 20:26:46 +0200 Subject: [PATCH] Ecosystem and integrations --- docs/README.md | 8 +- docs/canary-analysis.md | 76 ++++ docs/inputs/regressors.md | 95 +++++ docs/integrations/grafana-annotations.md | 56 +++ docs/reference/config-schema.md | 2 +- docs/triggers.md | 89 +++++ examples/configs/business-kpis.yaml | 19 + .../configs/calendars/release-freezes.yaml | 28 ++ forecaster/src/promforecast/annotations.py | 153 +++++++ forecaster/src/promforecast/calendar_file.py | 219 ++++++++++ forecaster/src/promforecast/canary.py | 166 ++++++++ forecaster/src/promforecast/config.py | 243 +++++++++++- forecaster/src/promforecast/exporter.py | 136 +++++++ forecaster/src/promforecast/main.py | 67 +++- forecaster/src/promforecast/regressors.py | 178 ++++++++- .../src/promforecast/runner/_scheduler.py | 373 ++++++++++++++++++ forecaster/src/promforecast/runner/_state.py | 5 + forecaster/src/promforecast/signing.py | 104 +++++ forecaster/src/promforecast/source.py | 57 +++ forecaster/src/promforecast/triggers.py | 231 +++++++++++ forecaster/src/promforecast/webhook.py | 254 ++++++++++++ forecaster/tests/test_annotations.py | 156 ++++++++ forecaster/tests/test_calendar_file.py | 128 ++++++ forecaster/tests/test_canary.py | 90 +++++ forecaster/tests/test_config_v19.py | 329 +++++++++++++++ forecaster/tests/test_regressors.py | 161 ++++++++ forecaster/tests/test_runner_canary.py | 121 ++++++ .../tests/test_runner_integrations_reload.py | 134 +++++++ forecaster/tests/test_signing.py | 81 ++++ forecaster/tests/test_triggers.py | 287 ++++++++++++++ forecaster/tests/test_webhook.py | 223 +++++++++++ 31 files changed, 4241 insertions(+), 28 deletions(-) create mode 100644 docs/canary-analysis.md create mode 100644 docs/integrations/grafana-annotations.md create mode 100644 docs/triggers.md create mode 100644 examples/configs/calendars/release-freezes.yaml create mode 100644 forecaster/src/promforecast/annotations.py create mode 100644 forecaster/src/promforecast/calendar_file.py create mode 100644 forecaster/src/promforecast/canary.py create mode 100644 forecaster/src/promforecast/signing.py create mode 100644 forecaster/src/promforecast/triggers.py create mode 100644 forecaster/src/promforecast/webhook.py create mode 100644 forecaster/tests/test_annotations.py create mode 100644 forecaster/tests/test_calendar_file.py create mode 100644 forecaster/tests/test_canary.py create mode 100644 forecaster/tests/test_config_v19.py create mode 100644 forecaster/tests/test_runner_canary.py create mode 100644 forecaster/tests/test_runner_integrations_reload.py create mode 100644 forecaster/tests/test_signing.py create mode 100644 forecaster/tests/test_triggers.py create mode 100644 forecaster/tests/test_webhook.py diff --git a/docs/README.md b/docs/README.md index 589b6ef..9400c01 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,7 +22,7 @@ Everything that shapes the data flowing into the forecasting model. - [Change-points](inputs/change-points.md) — input-side break detection and lookback adaptation - [Seasonality](inputs/seasonality.md) — periodicity detection - [Cold-start](inputs/cold-start.md) — sparse-history behaviour and peer-sibling borrowing -- [Regressors](inputs/regressors.md) — calendar, custom PromQL, and holiday regressors, plus conditional forecasts (baseline given the current regressor state) +- [Regressors](inputs/regressors.md) — calendar, custom PromQL, holiday, webhook (external pull), and calendar-file regressors, plus conditional forecasts (baseline given the current regressor state) and HMAC-signed webhooks - [Series discovery](inputs/discovery.md) — template-driven auto-fan-out over label values ## Outputs @@ -43,6 +43,12 @@ metrics, calibration, explainability, and ad-hoc what-if endpoints. - [Canary forecasts](outputs/canary.md) - [Predictive autoscaling](predictive-autoscaling.md) — Kubernetes custom-metrics adapter that exposes forecast metrics to HorizontalPodAutoscaler for scaling on the projected load curve +## Integrations + +- [Canary deploy forecast comparison](canary-analysis.md) — fit the baseline pods, project what the canary should look like, and emit `_canary_forecast_deviation` for Argo Rollouts / Flagger AnalysisRun gates +- [Forecast-driven webhook triggers](triggers.md) — push structured forecast events (predicted value, band, model, drift) to a webhook when a PromQL condition transitions true; optional HMAC signing +- [Grafana annotations](integrations/grafana-annotations.md) — post significant forecast events (drift, change-point, deploy-risk, predicted breach, deviation) as Grafana annotations on the dashboards you already watch + ## Operations Running the forecaster in production — diagnosis, capacity safeguards, diff --git a/docs/canary-analysis.md b/docs/canary-analysis.md new file mode 100644 index 0000000..a4e769b --- /dev/null +++ b/docs/canary-analysis.md @@ -0,0 +1,76 @@ +# Canary deploy forecast comparison + +When you run a canary deploy (a small slice of traffic to a new version), you want to know whether the canary is behaving like the stable fleet *would have* under the same conditions. + +promforecast can model that: it fits the **baseline** pods’ history, projects what the canary *should* look like, and surfaces the divergence between the canary’s actuals and that baseline-predicted forecast as a clear, alertable signal. + +This turns canary analysis into a data-driven comparison against a quality forecast, rather than manual eyeballing or hand-rolled thresholds. + +> **Note:** this is distinct from [canary / weighted sampling](outputs/canary.md), which fits only a fraction of a group’s series to save CPU. + +## Configuring + +Add a `canary:` block to a query. The selectors partition the already-fetched series into canary and baseline populations: + +```yaml +groups: + - name: checkout + queries: + - id: http_requests + promql: sum by (version, pod) (rate(http_requests_total[5m])) + canary: + enabled: true + canary_selector: '{version="canary"}' + baseline_selector: '{version!="canary"}' +``` + +- `canary_selector` / `baseline_selector` are Prometheus-style label matchers (`=` and `!=` only). Regex (`=~`) is rejected at load time. +- A series matching neither selector is ignored; matching both is a config error. + +## Emitted metrics + +For each canary series, the forecaster emits (in the same shape as existing deviation metrics): + +``` +_canary_forecast_deviation{deployment, replica, version, model, level, ...} +_canary_forecast_deviation_outside_band{...} +``` + +- `_canary_forecast_deviation` — signed normalised distance of the canary actual from the centre of the baseline-predicted band: `(actual - yhat) / (yhat_upper - yhat_lower)`. +- `_canary_forecast_deviation_outside_band` — `1` when the canary actual is outside the baseline band, else `0`. + +The labels preserve the canary series’ own labels plus the `model` that produced the baseline forecast and the confidence `level`. + +## Use with rollout controllers + +The emission shape is directly compatible with Argo Rollouts and Flagger. + +**Argo Rollouts** (`AnalysisTemplate`): + +```yaml +metrics: + - name: canary-divergence + successCondition: result[0] <= 0.5 + provider: + prometheus: + address: http://prometheus.monitoring:9090 + query: max(http_requests_canary_forecast_deviation{version="canary"}) +``` + +**Flagger** (`MetricTemplate`): + +```yaml +spec: + provider: + type: prometheus + address: http://prometheus.monitoring:9090 + query: max(http_requests_canary_forecast_deviation_outside_band{version="canary"}) +``` + +Gate the rollout on the deviation staying inside the band (or the ratio staying below an operator-chosen magnitude). + +## Trade-off: baseline history required + +Canary analysis depends on the baseline pods having stable history to fit on. For a brand-new service with no baseline, fall back to [cold-start sibling borrowing](inputs/cold-start.md) until the stable fleet has enough history. + +If either population is empty or the baseline is too short to fit, no comparison is emitted that tick — the normal forecast path is unaffected. diff --git a/docs/inputs/regressors.md b/docs/inputs/regressors.md index 1f3767f..42eb5d4 100644 --- a/docs/inputs/regressors.md +++ b/docs/inputs/regressors.md @@ -78,6 +78,98 @@ If the lookback window contains no holidays, falls back to a single binary colum Supported countries/regions are whatever the installed `holidays>=0.50` package provides (≥150 countries + most major subdivisions). +## Webhook regressors + +For a signal that does not live in Prometheus and is not on the built-in calendar list — active incidents from PagerDuty, deploys-in-flight from a CI/CD API, a custom internal "promotion-day" calendar — the `webhook` type lets the forecaster pull it in at fit time without standing up a Prometheus exporter or modifying the forecaster. + +```yaml +regressors: + - id: active_incidents + type: webhook + url: https://pagerduty-bridge.internal/incident-count + timeout: 5s # default + cache_ttl: 1h # default — reuse responses per (url, query, range) + headers: # optional static auth + Authorization: Bearer ${PAGERDUTY_TOKEN} + required: false # default +``` + +The forecaster POSTs to `url` once per primary query (not per series) with a JSON body: + +```json +{"query_id": "", "regressor_id": "active_incidents", "start": "", "end": ""} +``` + +and expects a JSON time series back: + +```json +{"timestamps": [1735689600, 1735693200], "values": [2, 1]} +``` + +Timestamps may be epoch seconds or ISO-8601 strings. The result is aligned to the primary series with the same nearest-asof join as a custom PromQL regressor, and cached per `(url, query_id, lookback)` for `cache_ttl` so a query returning many series — or a short refresh interval — doesn't hammer the external service. + +The forecaster stays stateless: the webhook owns whatever state it queries. Webhook failures (timeout, non-2xx, unparseable body) flow through the same graceful-degradation path as any other regressor — the forecast is fit *without* the regressor and `forecast_regressor_failures_total{…, reason="webhook_failure"}` increments. Set `required: true` to skip the series instead. + +`headers` values support `${ENV_VAR}` expansion against the process environment (injected from a Kubernetes Secret via `secretKeyRef`), so `Authorization: Bearer ${PAGERDUTY_TOKEN}` is resolved at request time rather than sent literally — keep the token out of the config. An unset variable expands to an empty string. + +> The shared webhook HTTP client is created when *any* query configures a `webhook` regressor. A config reload that adds the first webhook regressor wires the client on the fly, so no restart is required. + +### Example integrations + +**PagerDuty incident lookup** — a small bridge service queries the PagerDuty API for the count of active incidents in the requested window and returns it as a step series. Spikes in incidents become an exogenous input the model can attribute load changes to. + +**CI/CD deploy lookup** — a bridge to your CI/CD API (GitHub Actions, Argo, Jenkins) returns 1 on timestamps where a deploy completed. This is the dynamic sibling of the static `calendar_file` regressor below: use the webhook when "did a deploy happen?" is only answerable at query time, the calendar file when it's a known fixed schedule. + +### HMAC-verified responses + +When the webhook bridge signs its response body, add a `signing` block and the forecaster verifies it before accepting the data: + +```yaml +regressors: + - id: active_incidents + type: webhook + url: https://pagerduty-bridge.internal/incident-count + signing: + algorithm: hmac-sha256 # only value today + secret_ref: ${WEBHOOK_SECRET} +``` + +The forecaster recomputes `HMAC-SHA256(secret, response_body)` and compares it constant-time against the `X-Promforecast-Signature: sha256=` header the bridge sets. A missing or mismatched signature is treated as a regressor failure with `reason="signature_invalid"` (same graceful-degradation path). `secret_ref` resolves a `${ENV_VAR}` reference against the process environment (injected from a Kubernetes Secret via `secretKeyRef`) or accepts a literal for local testing; the same `signing` block is available on [webhook triggers](../triggers.md) for the outbound direction. + +## Calendar-file regressors + +For a static internal calendar — release freezes, fiscal-quarter ends, marketing-campaign launches — the `calendar_file` type reads a YAML or iCal file mounted into the pod, without needing a webhook just to read a file. + +```yaml +regressors: + - id: release_freeze + type: calendar_file + path: /etc/promforecast/calendars/release-freezes.yaml + expand: false # default — single binary column +``` + +The file format is YAML (preferred, git-friendly) with an `entries:` list: + +```yaml +# calendars/release-freezes.yaml +entries: + - start: 2026-01-01 + end: 2026-01-05 + name: new-year-freeze + - start: 2026-12-20 + end: 2026-12-31 + name: holiday-freeze +``` + +or iCal (`.ics`, for tooling compatibility) — each `VEVENT` becomes one window from its `DTSTART` / `DTEND` / `SUMMARY`. + +The emission shape matches the `holidays` regressor: + +- `expand: false` (default) emits a single binary column `release_freeze` (1.0 on any day inside any window, 0.0 otherwise). +- `expand: true` emits one column per distinct entry `name` (`release_freeze__new_year_freeze`, …). + +The calendar file is re-read automatically when its mtime changes, so a ConfigMap update to the mounted file is picked up on the next fit — the same effect as the main config's configmap-watch reload path, without a process restart. A missing or malformed file degrades gracefully with `reason="calendar_file_error"`. + ## Failure handling | Failure type | Reason label | `required: true` behaviour | @@ -88,6 +180,9 @@ Supported countries/regions are whatever the installed `holidays>=0.50` package | No datasource (dry-run / test) | `no_source` | Entire series skipped | | `holidays` package missing | `invalid_type` | Series skipped | | Unknown country / region | `invalid_type` | Series skipped | +| Webhook timeout / error / bad body | `webhook_failure` | Entire series skipped | +| Webhook HMAC verification failed | `signature_invalid` | Entire series skipped | +| Calendar file missing / malformed | `calendar_file_error` | Series skipped | Optional regressors (default) only increment the failure counter; the forecast still emits without that feature. diff --git a/docs/integrations/grafana-annotations.md b/docs/integrations/grafana-annotations.md new file mode 100644 index 0000000..0b23d96 --- /dev/null +++ b/docs/integrations/grafana-annotations.md @@ -0,0 +1,56 @@ +# Grafana annotations for forecast events + +If you live in Grafana, you want significant forecast events to appear *on the graph* — a vertical marker when a change-point is detected, drift threshold crossed, or a threshold breach is predicted. This lets you correlate forecasts with operational events without manually overlaying timelines. + +When enabled, the forecaster POSTs to Grafana’s annotations API on every enabled event that newly fires. Works with both Grafana OSS and Grafana Cloud. + +## Configuring + +Top-level `annotations:` block: + +```yaml +annotations: + enabled: true + grafana_url: https://grafana.example.com + api_token_ref: ${GRAFANA_API_TOKEN} # resolved from environment + events: + - drift + - change_point + - deploy_risk + - threshold_breach_predicted + - deviation_outside_band + timeout: 5s # default +``` + +- `grafana_url` is the base URL; posts go to `/api/annotations`. +- `api_token_ref` resolves a `${ENV_VAR}` (Grafana service-account token, typically from a Kubernetes Secret). +- `events` selects which families post annotations. Empty list = none. + +## Events + +| `event_type` | Fires when… | +|-------------------------------|-------------| +| `drift` | Drift score crosses alert threshold | +| `change_point` | Preprocessing detects a change-point | +| `deploy_risk` | Deploy-anchored risk score is active | +| `threshold_breach_predicted` | A `*_forecast_will_breach` sample predicts a breach | +| `deviation_outside_band` | Actual lands outside its forecast band | + +Each annotation carries a millisecond timestamp, human-readable text, and tags including `promforecast`, the `event_type`, and `group:`/`id:` scoping tags. + +## Rising-edge deduplication + +Sustained conditions post **once** on the rising edge, not every refresh. When the condition clears and re-fires, a new annotation is posted. This keeps dashboards readable. + +## Delivery telemetry + +``` +forecast_annotation_posts_total{event_type, outcome} # outcome = success | http_error | timeout +``` + +Delivery failures never block fits — they are logged and the run continues. + +## Operational notes + +- Annotations are built at process start. Changes to `enabled`, `events`, or `grafana_url` require a restart. +- Rising-edge state is in-process. After restart (or HA leader failover), still-active events re-post once — acceptable for annotations that mark moments rather than continuous state. diff --git a/docs/reference/config-schema.md b/docs/reference/config-schema.md index 0da2a57..348590e 100644 --- a/docs/reference/config-schema.md +++ b/docs/reference/config-schema.md @@ -48,7 +48,7 @@ No `helm upgrade` needed; the rolling reload picks up the change via the ConfigM ## Stability commitments (from `v1` onward) -- Top-level keys (`datasource`, `server`, `safety`, `defaults`, `sink`, `highAvailability`, `query_cache`, `telemetry`, `groups`) are part of the stable contract — no removals or renames. +- Top-level keys (`datasource`, `server`, `safety`, `defaults`, `sink`, `highAvailability`, `query_cache`, `telemetry`, `groups`, `triggers`, `annotations`) are part of the stable contract — no removals or renames. - Field defaults may change in minor releases **only** when the new default is safer. - Behavioural defaults that operators rely on (e.g. `defaults.accuracy.evaluate: true`) are pinned. - New optional fields are additive and never bump the `apiVersion`. diff --git a/docs/triggers.md b/docs/triggers.md new file mode 100644 index 0000000..b292d5e --- /dev/null +++ b/docs/triggers.md @@ -0,0 +1,89 @@ +# Forecast-driven webhook triggers + +Triggers let the forecaster **push** a structured event to a webhook when a PromQL condition over forecast metrics becomes true — e.g. auto-archive cleanup when a disk-fill ETA drops below a day, a Slack post when error-budget burn accelerates, or a CI job when drift is detected. + +**How this differs from alternatives**: +- vs. **Alertmanager / PrometheusRule** — the payload carries full forecast context (matched series, values, model identity, band/drift labels), not just “an alert fired”. +- vs. **[webhook regressors](inputs/regressors.md#webhook-regressors)** — the direction is reversed: regressors *pull* signals in; triggers *push* events out. + +## Configuring + +Top-level `triggers:` list: + +```yaml +triggers: + - name: disk-fill-imminent + condition: 'node_filesystem_avail_bytes_forecast{horizon="24h"} < 0' + webhook_url: https://hooks.slack.com/services/XXX/YYY/ZZZ + repeat_interval: 1h # default + timeout: 5s # default + headers: # optional static auth + Authorization: Bearer ${SLACK_TOKEN} + payload_template: | + {"text": "Disk fill predicted within 24h (value $value)"} +``` + +- `condition` is a PromQL expression evaluated after group runs (re-evaluations that land within a few seconds of each other — e.g. several groups finishing one refresh cycle together — are coalesced into a single instant query, so trigger evaluation doesn't scale with group count). +- The trigger fires on false → true transitions. Sustained-true conditions re-fire at most once per `repeat_interval`. + +## Payload + +With no `payload_template`, the forecaster POSTs: + +```json +{ + "name": "disk-fill-imminent", + "condition": "...", + "fired_at": "2026-05-29T12:00:00+00:00", + "value": -1.2e9, + "labels": {"instance": "node-7", "horizon": "24h", "model": "AutoARIMA"}, + "samples": [{"labels": {...}, "value": -1.2e9}] +} +``` + +`value` / `labels` come from the first matched series; `samples` carries the full matched vector. + +### Templating + +`payload_template` supports `$`-style placeholders: + +```yaml +payload_template: '{"text": "$name fired: value=$value"}' +``` + +Available variables: `$name`, `$condition`, `$value`, `$fired_at`, `$samples_json`. + +## Authentication + +Static headers use the `headers:` block. Header values support `${ENV_VAR}` expansion against the process environment (injected from a Kubernetes Secret via `secretKeyRef`), so `Authorization: Bearer ${SLACK_TOKEN}` is resolved at delivery time rather than sent literally. An unset variable expands to an empty string. + +For end-to-end verification, add a signing block: + +```yaml +signing: + algorithm: hmac-sha256 + secret_ref: ${TRIGGER_SIGNING_SECRET} +``` + +The forecaster adds `X-Promforecast-Signature: sha256=`. Receivers recompute the HMAC over the raw body. + +## Delivery telemetry + +``` +forecast_trigger_fires_total{name, outcome} # outcome = success | http_error | timeout +``` + +Delivery failures never block fits or other triggers. + +## Example receivers + +**Slack** — point `webhook_url` at a Slack incoming-webhook and use a simple `payload_template`. + +**Generic JSON receiver** — leave `payload_template` empty and verify the signature if configured. + +## Operational notes + +- Triggers are built at process start. Config changes require a restart. +- Conditions are evaluated after each group run. +- One-cycle lag is possible (TSDB ingestion). +- Evaluation or delivery errors are logged and never block the fit pipeline. diff --git a/examples/configs/business-kpis.yaml b/examples/configs/business-kpis.yaml index 48cd10f..101ebaf 100644 --- a/examples/configs/business-kpis.yaml +++ b/examples/configs/business-kpis.yaml @@ -41,6 +41,25 @@ groups: type: calendar - id: is_weekend type: calendar + # Optional webhook regressor: pull "deploys in flight" from a + # CI/CD bridge that isn't in Prometheus. Optional by default, so + # if the bridge is down the forecast still runs (the failure is + # counted in forecast_regressor_failures_total). The signing + # block is optional — drop it if the bridge doesn't sign. + - id: deploys_in_flight + type: webhook + url: https://cicd-bridge.internal/deploys-in-flight + cache_ttl: 15m + headers: + Authorization: Bearer ${CICD_BRIDGE_TOKEN} + signing: + algorithm: hmac-sha256 + secret_ref: ${WEBHOOK_SIGNING_SECRET} + # Static internal calendar mounted into the pod: 1 during a + # release freeze, 0 otherwise. See examples/configs/calendars/. + - id: release_freeze + type: calendar_file + path: /etc/promforecast/calendars/release-freezes.yaml - id: http_5xx_per_second promql: | diff --git a/examples/configs/calendars/release-freezes.yaml b/examples/configs/calendars/release-freezes.yaml new file mode 100644 index 0000000..12c07d7 --- /dev/null +++ b/examples/configs/calendars/release-freezes.yaml @@ -0,0 +1,28 @@ +# Example static calendar for a ``type: calendar_file`` regressor. +# +# Mount this file into the forecaster pod (e.g. via a ConfigMap) at the +# path referenced by the regressor's ``path:`` field, then reference it: +# +# regressors: +# - id: release_freeze +# type: calendar_file +# path: /etc/promforecast/calendars/release-freezes.yaml +# expand: false # single binary column; true = one column per name +# +# Each entry is an inclusive [start, end] date window with a name. With +# ``expand: false`` the regressor is 1 on any day inside any window; with +# ``expand: true`` it emits one column per distinct name. The file is +# re-read automatically when its mtime changes — no restart needed. +entries: + - start: 2026-01-01 + end: 2026-01-05 + name: new-year-freeze + - start: 2026-06-30 + end: 2026-06-30 + name: q2-financial-close + - start: 2026-11-25 + end: 2026-11-29 + name: thanksgiving-freeze + - start: 2026-12-20 + end: 2026-12-31 + name: holiday-freeze diff --git a/forecaster/src/promforecast/annotations.py b/forecaster/src/promforecast/annotations.py new file mode 100644 index 0000000..0180c83 --- /dev/null +++ b/forecaster/src/promforecast/annotations.py @@ -0,0 +1,153 @@ +"""Grafana-annotations sink for significant forecast events. + +When enabled, the forecaster posts a Grafana annotation each time an enabled +event *newly* fires — a change-point detected, a drift threshold crossed, a +deploy-risk elevation, a predicted threshold breach, or an actual landing +outside its band. Operators correlate forecast events with the dashboards +they already watch, without manually overlaying timelines. + +The runner builds a list of :class:`AnnotationEvent` from each group run and +hands it to :meth:`GrafanaAnnotationSink.post_events`. The sink filters to +the configured event families, suppresses repeats with a rising-edge dedup +(so a sustained "outside band" posts once, not every refresh), and POSTs to +Grafana's ``/api/annotations`` endpoint (compatible with both Grafana OSS and +Grafana Cloud). Delivery never blocks a fit: a failure bumps +``forecast_annotation_posts_total{outcome=http_error|timeout}`` and moves on. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import httpx +import structlog + +from . import signing +from .config import AnnotationsConfig + +if TYPE_CHECKING: + from .exporter import Exporter + +logger = structlog.get_logger(__name__) + + +@dataclass(frozen=True) +class AnnotationEvent: + """A single significant forecast event destined for Grafana. + + ``event_type`` is one of the configured families; ``dedup_key`` makes + the (type, series) pair unique so the rising-edge dedup can suppress + repeats while a condition stays active; ``tags`` and ``text`` are what + Grafana renders. + """ + + event_type: str + dedup_key: str + tags: list[str] + text: str + + +class GrafanaAnnotationSink: + """Posts forecast events to Grafana's annotations API. + + ``client`` is injectable for tests; ``now_ms`` is injectable so the + posted timestamp is deterministic under test. + """ + + def __init__( + self, + config: AnnotationsConfig, + exporter: Exporter, + *, + client: httpx.AsyncClient | None = None, + now_ms: Callable[[], int] = lambda: int(time.time() * 1000), + ) -> None: + self._config = config + self._exporter = exporter + self._enabled_events = set(config.events) + self._url = config.grafana_url.rstrip("/") + "/api/annotations" + if client is None: + self._client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=4, max_keepalive_connections=2), + ) + self._owned_client = True + else: + self._client = client + self._owned_client = False + self._now_ms = now_ms + # Keys currently firing -> the scope that posted them. Used to post + # only the rising edge. The value records *which* scope (group) owns + # a key so a per-group call only retires its own keys and can't clear + # another group's still-active annotations (the runner calls + # ``post_events`` once per group, each with only that group's events + # plus the global deploy-risk events). + self._active: dict[tuple[str, str], str] = {} + + async def aclose(self) -> None: + if self._owned_client: + await self._client.aclose() + + async def post_events(self, events: list[AnnotationEvent], *, scope: str = "") -> None: + """Post the rising edge of each enabled event family. + + ``events`` is the set of events firing this run *for ``scope``* (the + runner passes the group name so concurrent groups don't interfere). + A key that is freshly firing posts an annotation; a key owned by this + scope that fired last run but isn't in ``events`` now is cleared so it + can fire again later. Keys owned by *other* scopes are left untouched + — without this, a per-group call would clear sibling groups' active + keys and re-post every refresh, defeating the rising-edge dedup. Keys + for disabled event families are ignored entirely (they never enter + the active set, so toggling a family on doesn't retroactively post). + """ + firing = { + (e.event_type, e.dedup_key): e for e in events if e.event_type in self._enabled_events + } + # Retire only keys this scope owns that are no longer firing (and + # whose family is still enabled). A key owned by another scope, or + # whose family was disabled, stays put. + for key, owner in list(self._active.items()): + if owner == scope and key not in firing and key[0] in self._enabled_events: + del self._active[key] + for key, event in firing.items(): + if key in self._active: + continue # already posted on a prior run; suppress the repeat + self._active[key] = scope + await self._post_one(event) + + async def _post_one(self, event: AnnotationEvent) -> None: + ts = self._now_ms() + payload = {"time": ts, "tags": event.tags, "text": event.text} + headers = {"Content-Type": "application/json"} + if self._config.api_token_ref.strip(): + token = signing.resolve_secret_ref(self._config.api_token_ref) + headers["Authorization"] = f"Bearer {token}" + outcome = await self._deliver(event, payload, headers) + self._exporter.increment_annotation_post(event.event_type, outcome) + logger.info("annotation_posted", event_type=event.event_type, outcome=outcome) + + async def _deliver( + self, event: AnnotationEvent, payload: dict[str, object], headers: dict[str, str] + ) -> str: + try: + response = await self._client.post( + self._url, + json=payload, + headers=headers, + timeout=self._config.timeout.total_seconds(), + ) + except (TimeoutError, httpx.TimeoutException): + logger.warning("annotation_timeout", event_type=event.event_type) + return "timeout" + except httpx.HTTPError as exc: + logger.warning("annotation_error", event_type=event.event_type, error=str(exc)) + return "http_error" + if response.status_code >= 400: # noqa: PLR2004 + logger.warning( + "annotation_status", event_type=event.event_type, status=response.status_code + ) + return "http_error" + return "success" diff --git a/forecaster/src/promforecast/calendar_file.py b/forecaster/src/promforecast/calendar_file.py new file mode 100644 index 0000000..738d37c --- /dev/null +++ b/forecaster/src/promforecast/calendar_file.py @@ -0,0 +1,219 @@ +"""Static internal-calendar regressor source. + +Reads a YAML or iCal file mounted into the pod and turns it into a set of +named date windows (release freezes, fiscal-quarter ends, marketing-campaign +launches). The :mod:`promforecast.regressors` module then materialises those +windows into a regressor column with the same binary / one-hot shape as the +``holidays`` regressor: + +* ``expand: false`` (default) — a single binary column that is ``1`` on any + day inside any window and ``0`` otherwise. +* ``expand: true`` — one column per distinct entry ``name``. + +Two file formats are supported, chosen by extension: + +* ``.yaml`` / ``.yml`` (preferred, git-friendly) — a mapping with an + ``entries:`` list, each entry ``{start, end, name}``. A bare top-level + list is also accepted. +* ``.ics`` (for tooling compatibility) — minimal iCalendar: every ``VEVENT`` + contributes one window from ``DTSTART`` / ``DTEND`` / ``SUMMARY``. + +Parsed calendars are cached per path and re-read when the file's mtime +changes, so a ConfigMap update to the mounted calendar is picked up on the +next fit without a process restart — the same effect as the main config's +configmap-watch reload path. +""" + +from __future__ import annotations + +import datetime as dt +import os +from dataclasses import dataclass +from pathlib import Path + +import yaml + + +class CalendarFileError(ValueError): + """Raised when a calendar file is missing, unreadable, or malformed.""" + + +@dataclass(frozen=True) +class CalendarEntry: + """A single named, inclusive ``[start, end]`` date window.""" + + start: dt.date + end: dt.date + name: str + + def contains(self, day: dt.date) -> bool: + return self.start <= day <= self.end + + +# Per-path cache: path -> (mtime, parsed entries). Re-read on mtime change so +# a ConfigMap update to the mounted file is picked up without a restart. +_CACHE: dict[str, tuple[float, list[CalendarEntry]]] = {} + + +def get_calendar(path: str) -> list[CalendarEntry]: + """Return the parsed calendar for ``path``, re-reading on mtime change.""" + try: + mtime = os.path.getmtime(path) + except OSError as exc: + raise CalendarFileError(f"calendar file not readable: {path} ({exc})") from exc + cached = _CACHE.get(path) + if cached is not None and cached[0] == mtime: + return cached[1] + entries = load_calendar(path) + _CACHE[path] = (mtime, entries) + return entries + + +def load_calendar(path: str) -> list[CalendarEntry]: + """Parse a calendar file (YAML or iCal) into :class:`CalendarEntry` list. + + Dispatches on the file extension. Always reads from disk — callers that + want mtime-based caching should use :func:`get_calendar`. + """ + p = Path(path) + suffix = p.suffix.lower() + try: + text = p.read_text() + except OSError as exc: + raise CalendarFileError(f"calendar file not readable: {path} ({exc})") from exc + if suffix == ".ics": + return _parse_ical(text, path=path) + return _parse_yaml(text, path=path) + + +def _parse_yaml(text: str, *, path: str) -> list[CalendarEntry]: + try: + doc = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise CalendarFileError(f"calendar file {path} is not valid YAML: {exc}") from exc + if isinstance(doc, dict): + raw_entries = doc.get("entries") + elif isinstance(doc, list): + raw_entries = doc + else: + raise CalendarFileError( + f"calendar file {path} must be a mapping with 'entries:' or a top-level list" + ) + if not isinstance(raw_entries, list): + raise CalendarFileError(f"calendar file {path}: 'entries' must be a list") + entries: list[CalendarEntry] = [] + for index, item in enumerate(raw_entries): + if not isinstance(item, dict): + raise CalendarFileError(f"calendar file {path}: entry {index} must be a mapping") + entries.append(_entry_from_mapping(item, path=path, index=index)) + return entries + + +def _entry_from_mapping(item: dict[object, object], *, path: str, index: int) -> CalendarEntry: + name = item.get("name") + if not isinstance(name, str) or not name.strip(): + raise CalendarFileError(f"calendar file {path}: entry {index} requires a non-empty 'name'") + start = _coerce_date(item.get("start"), path=path, index=index, field="start") + end = _coerce_date(item.get("end"), path=path, index=index, field="end") + if end < start: + raise CalendarFileError( + f"calendar file {path}: entry {index} ({name!r}) has end before start" + ) + return CalendarEntry(start=start, end=end, name=name.strip()) + + +def _coerce_date(value: object, *, path: str, index: int, field: str) -> dt.date: + if isinstance(value, dt.datetime): + return value.date() + if isinstance(value, dt.date): + return value + if isinstance(value, str): + try: + return dt.date.fromisoformat(value.strip()[:10]) + except ValueError as exc: + raise CalendarFileError( + f"calendar file {path}: entry {index} field {field!r} is not an ISO date: {value!r}" + ) from exc + raise CalendarFileError( + f"calendar file {path}: entry {index} requires a date in field {field!r}" + ) + + +def _parse_ical(text: str, *, path: str) -> list[CalendarEntry]: + """Minimal iCalendar parser: one window per VEVENT. + + Handles the common shape mainstream calendar tools export: line-folded + ``BEGIN:VEVENT`` / ``END:VEVENT`` blocks with ``DTSTART``, ``DTEND``, + and ``SUMMARY`` properties. Property parameters (``;VALUE=DATE``, + ``;TZID=...``) are tolerated and ignored — only the date portion of the + value is used. An ``.ics`` without an explicit ``DTEND`` treats the + event as a single-day window. + + Per RFC 5545, ``DTEND`` for an all-day (``VALUE=DATE``) event is + *exclusive* — it names the day after the last day of the event — so a + date-only ``DTEND`` is rolled back one day to recover the inclusive + ``[start, end]`` window this module works in. A date-time ``DTEND`` + (with a ``T`` component) is treated as inclusive of its date. + """ + entries: list[CalendarEntry] = [] + in_event = False + start: dt.date | None = None + end: dt.date | None = None + end_is_date_only = False + summary: str | None = None + for raw_line in _unfold_ical(text): + line = raw_line.strip() + upper = line.upper() + if upper == "BEGIN:VEVENT": + in_event, start, end, end_is_date_only, summary = True, None, None, False, None + continue + if upper == "END:VEVENT": + if start is not None: + name = summary.strip() if summary and summary.strip() else "event" + resolved_end = start if end is None else end + # All-day DTEND is exclusive; pull it back to the last + # included day (never below ``start``). + if end is not None and end_is_date_only and resolved_end > start: + resolved_end = resolved_end - dt.timedelta(days=1) + entries.append(CalendarEntry(start=start, end=resolved_end, name=name)) + in_event = False + continue + if not in_event or ":" not in line: + continue + prop, _, value = line.partition(":") + key = prop.split(";", 1)[0].upper() + if key == "DTSTART": + start = _ical_date(value, path=path) + elif key == "DTEND": + end = _ical_date(value, path=path) + end_is_date_only = "T" not in value.strip() + elif key == "SUMMARY": + summary = value + return entries + + +def _unfold_ical(text: str) -> list[str]: + """Reverse RFC 5545 line folding (continuation lines start with space/tab).""" + lines: list[str] = [] + for line in text.splitlines(): + if line[:1] in (" ", "\t") and lines: + lines[-1] += line[1:] + else: + lines.append(line) + return lines + + +def _ical_date(value: str, *, path: str) -> dt.date: + token = value.strip() + # Date-time values look like 20260101T090000Z; date-only like 20260101. + digits = token.split("T", 1)[0] + try: + return dt.datetime.strptime(digits, "%Y%m%d").date() + except ValueError: + # Some producers emit ISO-with-dashes; fall back to fromisoformat. + try: + return dt.date.fromisoformat(token[:10]) + except ValueError as exc: + raise CalendarFileError( + f"calendar file {path}: unparseable iCal date {value!r}" + ) from exc diff --git a/forecaster/src/promforecast/canary.py b/forecaster/src/promforecast/canary.py new file mode 100644 index 0000000..035c925 --- /dev/null +++ b/forecaster/src/promforecast/canary.py @@ -0,0 +1,166 @@ +"""Canary-vs-baseline forecast comparison. + +Splits a query's fetched series into a *baseline* population (stable pods) +and a *canary* population (the new version) using Prometheus-style label +matchers, builds an aggregate baseline series, and lets the runner project +what the canary *should* look like. Each canary series' most recent actual +is then compared to the baseline-predicted forecast band, emitting +``_canary_forecast_deviation`` in the same shape as the existing +deviation metrics. + +Only ``=`` and ``!=`` matchers are supported: the selectors partition an +already-fetched result set rather than being pushed down to the TSDB. That +keeps the feature decoupled from PromQL parsing and matches how Argo +Rollouts / Flagger express their canary vs stable label filters. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .source import SeriesFrame + + +class CanarySelectorError(ValueError): + """Raised when a canary selector cannot be parsed.""" + + +@dataclass(frozen=True) +class LabelMatcher: + key: str + op: str # "=" or "!=" + value: str + + def matches(self, labels: dict[str, str]) -> bool: + actual = labels.get(self.key, "") + if self.op == "=": + return actual == self.value + return actual != self.value + + +# ``key="value"`` or ``key!="value"`` — double-quoted value, the only form +# Prometheus selectors use for equality matchers. +_MATCHER_RE = re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*(=|!=)\s*"((?:[^"\\]|\\.)*)"\s*$') + + +def parse_selector(selector: str) -> list[LabelMatcher]: + """Parse ``{k="v", k!="v"}`` into a list of :class:`LabelMatcher`. + + The surrounding braces are optional. An empty selector matches + everything (returns no matchers). Raises :class:`CanarySelectorError` + on an unsupported matcher (regex ``=~`` / ``!~`` or malformed token). + """ + text = selector.strip() + if text.startswith("{") and text.endswith("}"): + text = text[1:-1] + text = text.strip() + if not text: + return [] + matchers: list[LabelMatcher] = [] + for part in _split_top_level(text): + token = part.strip() + if not token: + continue + if "=~" in token or "!~" in token: + raise CanarySelectorError( + f"regex matchers are not supported in canary selectors: {token!r}" + ) + m = _MATCHER_RE.match(token) + if m is None: + raise CanarySelectorError(f"unparseable canary selector matcher: {token!r}") + key, op, value = m.group(1), m.group(2), m.group(3) + matchers.append(LabelMatcher(key=key, op=op, value=value.replace('\\"', '"'))) + return matchers + + +def _split_top_level(text: str) -> list[str]: + """Split on commas that are not inside a quoted value.""" + parts: list[str] = [] + buf: list[str] = [] + in_quote = False + escaped = False + for ch in text: + if escaped: + buf.append(ch) + escaped = False + continue + if ch == "\\": + buf.append(ch) + escaped = True + continue + if ch == '"': + in_quote = not in_quote + buf.append(ch) + continue + if ch == "," and not in_quote: + parts.append("".join(buf)) + buf = [] + continue + buf.append(ch) + parts.append("".join(buf)) + return parts + + +def matches_all(labels: dict[str, str], matchers: list[LabelMatcher]) -> bool: + return all(m.matches(labels) for m in matchers) + + +def split_frames( + frames: list[SeriesFrame], + *, + canary_selector: str, + baseline_selector: str, +) -> tuple[list[SeriesFrame], list[SeriesFrame]]: + """Partition ``frames`` into (canary, baseline) by their label selectors. + + A series may match neither selector (dropped) but not both — overlapping + selectors are a config error, surfaced as :class:`CanarySelectorError`. + """ + canary_matchers = parse_selector(canary_selector) + baseline_matchers = parse_selector(baseline_selector) + canary: list[SeriesFrame] = [] + baseline: list[SeriesFrame] = [] + for frame in frames: + in_canary = matches_all(frame.labels, canary_matchers) + in_baseline = matches_all(frame.labels, baseline_matchers) + if in_canary and in_baseline: + raise CanarySelectorError( + f"series {frame.labels!r} matches both canary and baseline selectors; " + "the selectors must be mutually exclusive" + ) + if in_canary: + canary.append(frame) + elif in_baseline: + baseline.append(frame) + return canary, baseline + + +def aggregate_baseline(frames: list[SeriesFrame]) -> tuple[list[datetime], list[float]]: + """Mean-across-baseline-series, grouped + sorted by timestamp. + + Returns ``([], [])`` when no usable points exist so the caller can skip + the fit gracefully. + """ + sums: dict[datetime, float] = {} + counts: dict[datetime, int] = {} + for frame in frames: + for ts, val in zip(frame.timestamps, frame.values, strict=True): + sums[ts] = sums.get(ts, 0.0) + float(val) + counts[ts] = counts.get(ts, 0) + 1 + if not sums: + return [], [] + ordered = sorted(sums) + values = [sums[ts] / counts[ts] for ts in ordered] + return ordered, values + + +def latest_actual(frame: SeriesFrame) -> float | None: + """The most recent (by timestamp) value of a canary series, or ``None``.""" + if not frame.values: + return None + pairs = sorted(zip(frame.timestamps, frame.values, strict=True), key=lambda p: p[0]) + return float(pairs[-1][1]) diff --git a/forecaster/src/promforecast/config.py b/forecaster/src/promforecast/config.py index 05f7a9e..2909483 100644 --- a/forecaster/src/promforecast/config.py +++ b/forecaster/src/promforecast/config.py @@ -26,7 +26,7 @@ # Bounded set of regressor "types" the schema accepts. Stays small on purpose: # every new type ships its own validator and runtime path, so a Literal here # prevents typos from silently degrading to ``custom``. -REGRESSOR_TYPES = ("custom", "calendar", "holidays") +REGRESSOR_TYPES = ("custom", "calendar", "holidays", "webhook", "calendar_file") # Versioned schema. ``apiVersion`` mirrors the Kubernetes convention so a # config file declares which schema it conforms to and the loader can route @@ -334,10 +334,33 @@ class DefaultsConfig(BaseModel): emission: EmissionConfig = EmissionConfig() +class SigningConfig(BaseModel): + """HMAC payload signing for an outbound or inbound webhook. + + The de-facto standard for webhook authentication: the sender computes + an HMAC over the raw request body and ships it in a header; the + receiver recomputes and compares. Both the webhook *regressor* (which + verifies the response it pulls in) and the webhook *trigger* (which + signs the request it pushes out) accept this block. + + ``secret_ref`` is resolved through the same ``${ENV_VAR}`` indirection + the chart uses for datasource credentials — set it to ``${NAME}`` to + read environment variable ``NAME`` (injected from a Kubernetes Secret + via ``secretKeyRef``), or to a literal value for local testing. An + empty resolved secret is rejected at load time so a missing Secret + can't silently disable signing. + """ + + model_config = ConfigDict(extra="forbid") + + algorithm: Literal["hmac-sha256"] = "hmac-sha256" + secret_ref: str + + class RegressorConfig(BaseModel): """Exogenous regressor wired into a query. - Three flavours: + Flavours: * ``type: calendar`` — built-in deterministic features (``hour_of_day``, ``day_of_week``, ``is_weekend``). ``id`` doubles as @@ -349,6 +372,18 @@ class RegressorConfig(BaseModel): (e.g. US states); ``expand: true`` emits one regressor column per holiday name (so the model can learn that Black Friday differs from Thanksgiving) instead of a single ``is_holiday`` binary column. + * ``type: webhook`` — a signal that lives outside Prometheus (active + incidents from PagerDuty, deploys-in-flight from a CI/CD API). The + forecaster POSTs to ``url`` at fit time with the query id and the + timestamp range and expects a JSON time series back + (``{"timestamps": [...], "values": [...]}``). Results are cached per + URL+range for ``cache_ttl``. ``signing`` opts into HMAC verification + of the response. + * ``type: calendar_file`` — a static internal calendar (release + freezes, fiscal-quarter ends) read from a YAML or iCal file mounted + into the pod at ``path``. ``expand`` matches the ``holidays`` shape: + ``false`` (default) emits a single binary column, ``true`` emits one + column per calendar-entry name. ``required`` flips the failure mode from graceful (skip the regressor, keep forecasting) to fatal-per-series (the whole series is skipped if @@ -360,7 +395,7 @@ class RegressorConfig(BaseModel): id: str promql: str | None = None - type: Literal["custom", "calendar", "holidays"] = "custom" + type: Literal["custom", "calendar", "holidays", "webhook", "calendar_file"] = "custom" required: bool = False # Holiday-regressor parameters. Only consulted when ``type == "holidays"``. # ``country`` is the ISO-3166-1 alpha-2 country code accepted by the @@ -372,10 +407,28 @@ class RegressorConfig(BaseModel): # date before the holiday lookup; defaults to ``UTC`` which matches the # original behaviour but is wrong for localised workloads (a US holiday # otherwise flips at UTC-midnight rather than US-Eastern midnight). + # + # ``expand`` is shared with ``type: calendar_file`` (same binary vs + # one-hot semantics). country: str | None = None regions: list[str] = Field(default_factory=list) expand: bool = False timezone: str = "UTC" + # Webhook-regressor parameters. Only consulted when ``type == "webhook"``. + # ``url`` is the endpoint POSTed at fit time; ``timeout`` bounds the + # request; ``cache_ttl`` controls how long a successful (url, range) + # response is reused to avoid hammering the external service; ``headers`` + # carry static auth (bearer tokens, shared secrets). ``signing`` opts + # into HMAC verification of the response body. + url: str | None = None + timeout: Duration = timedelta(seconds=5) + cache_ttl: Duration = timedelta(hours=1) + headers: dict[str, str] = Field(default_factory=dict) + signing: SigningConfig | None = None + # Calendar-file parameters. Only consulted when ``type == "calendar_file"``. + # ``path`` is the mounted YAML/iCal file; the runner reloads it on the + # same configmap-watch path as the main config. + path: str | None = None class DiscoveryVariable(BaseModel): @@ -743,6 +796,31 @@ class ColdStartConfig(BaseModel): max_quality_score: float = Field(default=0.5, ge=0.0, le=1.0) +class CanaryComparisonConfig(BaseModel): + """Canary-vs-baseline forecast comparison for a query. + + Distinct from the group-level :class:`CanaryConfig` (which *samples* a + fraction of series for cheaper fits). This block models a canary + deploy: the query's series are split by label selector into a + *baseline* population (the stable pods) and a *canary* population (the + new version). The forecaster fits the baseline, projects what the + canary *should* look like, and emits ``_canary_forecast_deviation`` + — how far the canary actuals are from the baseline-predicted forecast. + + ``canary_selector`` / ``baseline_selector`` are Prometheus-style label + matchers (``{version="canary"}``, ``{version!="canary"}``) applied to + the labels already attached to each returned series. Only ``=`` and + ``!=`` matchers are supported — the selectors partition an + already-fetched result set, they are not pushed down to the TSDB. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: bool = False + canary_selector: str = "" + baseline_selector: str = "" + + class QueryConfig(BaseModel): """A single PromQL query plus optional per-query overrides. @@ -804,6 +882,9 @@ class QueryConfig(BaseModel): # group can mix cheap-and-stable hourly metrics with expensive daily # capacity refits without forking the YAML into two groups. refresh_interval: Duration | None = None + # Per-query canary-vs-baseline comparison. Off by default; see + # :class:`CanaryComparisonConfig`. + canary: CanaryComparisonConfig = Field(default_factory=CanaryComparisonConfig) class CanaryConfig(BaseModel): @@ -1419,6 +1500,89 @@ class DeploySignalsConfig(BaseModel): max_active_deploys: int = Field(default=8, ge=1) +class TriggerConfig(BaseModel): + """A forecast-driven outbound webhook. + + Distinct from a PrometheusRule alert: the payload carries forecast + context (predicted value, confidence band, model identity, drift + score). Distinct from a webhook *regressor*: the direction is inverse + — the forecaster pushes out rather than pulls in. + + ``condition`` is a PromQL expression evaluated against the configured + datasource after every fit (so it can reference the forecast metrics + the sink wrote). The trigger POSTs to ``webhook_url`` when the + condition transitions from false to true; for sustained conditions it + re-fires no more often than ``repeat_interval``. ``payload_template`` + is an optional ``string.Template`` (``$var``-substitution) template + rendered with the event fields (``$name``, ``$condition``, ``$value``, + ``$fired_at``, ``$samples_json``) — ``$``-style rather than + ``str.format`` so a JSON template's literal ``{``/``}`` need no + escaping; an empty template ships the default JSON event. ``headers`` + carry static auth; ``signing`` opts into HMAC-signing the request + body. + """ + + model_config = ConfigDict(extra="forbid") + + name: str + condition: str + webhook_url: str + headers: dict[str, str] = Field(default_factory=dict) + payload_template: str = "" + repeat_interval: Duration = timedelta(hours=1) + timeout: Duration = timedelta(seconds=5) + signing: SigningConfig | None = None + + +# Bounded set of forecast events the Grafana-annotations sink can post. +# Each maps to a signal the runner already computes; keeping it a Literal +# stops a typo from silently disabling an annotation stream. +ANNOTATION_EVENT_TYPES = ( + "drift", + "change_point", + "deploy_risk", + "threshold_breach_predicted", + "deviation_outside_band", +) + + +class AnnotationsConfig(BaseModel): + """Grafana-annotations sink for significant forecast events. + + When enabled, the forecaster POSTs to Grafana's annotations API every + time an enabled event fires, so operators can correlate forecast + events with the dashboards they already watch. Compatible with both + Grafana OSS and Grafana Cloud (same API). + + ``api_token_ref`` is resolved through the same ``${ENV_VAR}`` + indirection as datasource credentials. ``events`` selects which event + families post annotations; an empty list (default) means none even + when ``enabled: true``. + + ``deploy_risk_min_score`` gates the ``deploy_risk`` event family: only + a deploy whose composite risk score is at or above this threshold posts + an annotation, so the stream reflects *elevated* risk rather than every + active rollout. Ignored unless ``deploy_risk`` is in ``events``. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: bool = False + grafana_url: str = "" + api_token_ref: str = "" + events: list[ + Literal[ + "drift", + "change_point", + "deploy_risk", + "threshold_breach_predicted", + "deviation_outside_band", + ] + ] = Field(default_factory=list) + timeout: Duration = timedelta(seconds=5) + deploy_risk_min_score: float = Field(default=0.5, ge=0.0, le=1.0) + + class Config(BaseModel): # ``populate_by_name=True`` lets in-Python construction use ``api_version`` # while YAML keeps the Kubernetes-idiomatic ``apiVersion`` spelling. @@ -1465,6 +1629,12 @@ class Config(BaseModel): # Operator-facing recommendations (right-sizing, ...). Off by # default; see :class:`RecommendationsConfig`. recommendations: RecommendationsConfig = Field(default_factory=RecommendationsConfig) + # Forecast-driven outbound webhooks. Off by default (empty list); + # see :class:`TriggerConfig`. + triggers: list[TriggerConfig] = Field(default_factory=list) + # Grafana-annotations sink for forecast events. Off by default; + # see :class:`AnnotationsConfig`. + annotations: AnnotationsConfig = Field(default_factory=AnnotationsConfig) def load(path: str | Path) -> Config: @@ -1502,13 +1672,60 @@ def load(path: str | Path) -> Config: for query in group.queries: validate_metric_name(query.id) _validate_query_overrides(group_name=group.name, query=query) + _validate_canary(group_name=group.name, query=query) for reg in query.regressors: _validate_regressor(group_name=group.name, query_id=query.id, reg=reg) _validate_deploy_signals(config) _validate_resource_sizing(config) + _validate_triggers(config) + _validate_annotations(config) return config +def _validate_canary(*, group_name: str, query: QueryConfig) -> None: + """A canary comparison needs both selectors when enabled.""" + if not query.canary.enabled: + return + label = f"groups[{group_name}].queries[{query.id}].canary" + if not query.canary.canary_selector.strip(): + raise ValueError(f"{label}: canary_selector is required when canary.enabled is true") + if not query.canary.baseline_selector.strip(): + raise ValueError(f"{label}: baseline_selector is required when canary.enabled is true") + + +def _validate_triggers(config: Config) -> None: + """Reject duplicate trigger names and empty condition/webhook_url. + + A duplicate ``name`` would collapse two triggers onto the same + ``forecast_trigger_fires_total{name}`` series and make the + repeat-interval bookkeeping ambiguous, so it fails fast at load. + """ + seen: set[str] = set() + for trigger in config.triggers: + label = f"triggers[{trigger.name}]" + if not trigger.name.strip(): + raise ValueError("triggers[]: name must be non-empty") + if trigger.name in seen: + raise ValueError(f"triggers: name {trigger.name!r} appears twice; names must be unique") + seen.add(trigger.name) + if not trigger.condition.strip(): + raise ValueError(f"{label}: condition must be a non-empty PromQL expression") + if not trigger.webhook_url.strip(): + raise ValueError(f"{label}: webhook_url must be non-empty") + + +def _validate_annotations(config: Config) -> None: + """A Grafana-annotations sink needs a URL when enabled.""" + ann = config.annotations + if not ann.enabled: + return + if not ann.grafana_url.strip(): + raise ValueError( + "annotations.enabled is true but annotations.grafana_url is empty; " + "set the Grafana base URL (e.g. https://grafana.example.com)" + ) + + def _validate_resource_sizing(config: Config) -> None: """Cross-reference every right-sizing workload against the live queries. @@ -2052,7 +2269,7 @@ def _validate_holiday_calendar(*, label: str, reg: RegressorConfig) -> None: ) from exc -def _validate_regressor(*, group_name: str, query_id: str, reg: RegressorConfig) -> None: +def _validate_regressor(*, group_name: str, query_id: str, reg: RegressorConfig) -> None: # noqa: PLR0912 """Per-regressor sanity checks. * Calendar regressors must use one of the supported feature names and @@ -2086,5 +2303,23 @@ def _validate_regressor(*, group_name: str, query_id: str, reg: RegressorConfig) # ``regions`` surfaces as a load-time error rather than as a # silent per-series ``invalid_type`` failure on every refresh. _validate_holiday_calendar(label=label, reg=reg) + elif reg.type == "webhook": + if not reg.url: + raise ValueError(f"{label}: webhook regressors require a non-empty url") + if reg.promql: + raise ValueError(f"{label}: webhook regressors must not set promql") + if reg.country or reg.regions: + raise ValueError(f"{label}: webhook regressors must not set country/regions") + if reg.path: + raise ValueError(f"{label}: webhook regressors must not set path") + elif reg.type == "calendar_file": + if not reg.path: + raise ValueError(f"{label}: calendar_file regressors require a non-empty path") + if reg.promql: + raise ValueError(f"{label}: calendar_file regressors must not set promql") + if reg.country or reg.regions: + raise ValueError(f"{label}: calendar_file regressors must not set country/regions") + if reg.url: + raise ValueError(f"{label}: calendar_file regressors must not set url") elif not reg.promql: raise ValueError(f"{label}: custom regressors require a non-empty promql") diff --git a/forecaster/src/promforecast/exporter.py b/forecaster/src/promforecast/exporter.py index 4d0617c..c035428 100644 --- a/forecaster/src/promforecast/exporter.py +++ b/forecaster/src/promforecast/exporter.py @@ -39,6 +39,7 @@ "AccuracyPoint", "BandCoveragePoint", "CalibrationOffsetPoint", + "CanaryDeviationPoint", "ComponentDecompositionPoint", "ConditionalCalibrationOffsetPoint", "ConditionalDeviationPoint", @@ -103,6 +104,26 @@ class DeviationPoint: outside_band: bool +@dataclass(frozen=True) +class CanaryDeviationPoint: + """A canary-vs-baseline deviation observation destined for /metrics. + + Rendered as the per-id family ``_canary_forecast_deviation`` (the + signed ratio) and ``_canary_forecast_deviation_outside_band`` (0/1) + — the same shape as the unconditional :class:`DeviationPoint`, but the + band is the *baseline*-predicted forecast and the observed value is the + *canary* series' most recent actual. ``metric`` is the query id (so the + family name carries the underlying signal); ``labels`` preserve the + canary series' own labels (``deployment``, ``replica``, ``version``, + ...) plus ``model`` and ``level``. + """ + + metric: str + labels: dict[str, str] + ratio: float + outside_band: bool + + @dataclass(frozen=True) class QualityPoint: """A composite forecast-quality observation destined for /metrics. @@ -505,6 +526,7 @@ class GroupSnapshot: points: list[ForecastPoint] = field(default_factory=list) accuracy_points: list[AccuracyPoint] = field(default_factory=list) deviation_points: list[DeviationPoint] = field(default_factory=list) + canary_deviation_points: list[CanaryDeviationPoint] = field(default_factory=list) quality_points: list[QualityPoint] = field(default_factory=list) ensemble_weights: list[EnsembleWeightPoint] = field(default_factory=list) contribution_points: list[ContributionPoint] = field(default_factory=list) @@ -714,6 +736,14 @@ def __init__(self) -> None: "error": 0, "invalid_body": 0, } + # Per-(trigger name, outcome) forecast-driven webhook fire counter. + # ``outcome`` is bounded to {success, http_error, timeout} so the + # label set stays small. Always emitted (even at zero) so a + # dashboard renders a series from the first configured trigger. + self._trigger_fires: dict[tuple[str, str], int] = {} + # Per-(event_type, outcome) Grafana-annotation post counter. + # ``outcome`` bounded to {success, http_error, timeout}. + self._annotation_posts: dict[tuple[str, str], int] = {} # Per-(group, query, model, reason) decomposition-skipped counter. # Bumped when ``emission.decompose: true`` is set on a query but # the chosen model does not decompose cleanly. The ``reason`` set @@ -929,6 +959,30 @@ def increment_what_if_run(self, outcome: str) -> None: with self._lock: self._what_if_runs[outcome] = self._what_if_runs.get(outcome, 0) + 1 + def increment_trigger_fire(self, name: str, outcome: str) -> None: + """Bump ``forecast_trigger_fires_total{name, outcome}``. + + ``outcome`` is one of ``success`` | ``http_error`` | ``timeout``; + an out-of-set value is collapsed to ``http_error`` so a future code + path can't blow the label cardinality. + """ + if outcome not in ("success", "http_error", "timeout"): + outcome = "http_error" + with self._lock: + key = (name, outcome) + self._trigger_fires[key] = self._trigger_fires.get(key, 0) + 1 + + def increment_annotation_post(self, event_type: str, outcome: str) -> None: + """Bump ``forecast_annotation_posts_total{event_type, outcome}``. + + ``outcome`` is one of ``success`` | ``http_error`` | ``timeout``. + """ + if outcome not in ("success", "http_error", "timeout"): + outcome = "http_error" + with self._lock: + key = (event_type, outcome) + self._annotation_posts[key] = self._annotation_posts.get(key, 0) + 1 + def record_discovery( self, *, @@ -1080,6 +1134,14 @@ def _what_if_runs_state(self) -> dict[str, int]: with self._lock: return dict(self._what_if_runs) + def _trigger_fires_state(self) -> dict[tuple[str, str], int]: + with self._lock: + return dict(self._trigger_fires) + + def _annotation_posts_state(self) -> dict[tuple[str, str], int]: + with self._lock: + return dict(self._annotation_posts) + def _discovery_state( self, ) -> tuple[dict[tuple[str, str], int], dict[tuple[str, str, str, str], int]]: @@ -1444,6 +1506,8 @@ def collect(self) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: # noqa: leader_state = self._exporter._leader_state() cache_state = self._exporter._query_cache_state() what_if_runs_state = self._exporter._what_if_runs_state() + trigger_fires_state = self._exporter._trigger_fires_state() + annotation_posts_state = self._exporter._annotation_posts_state() discovery_state = self._exporter._discovery_state() season_length_state = self._exporter._season_length_state() decomposition_skipped_state = self._exporter._decomposition_skipped_state() @@ -1460,6 +1524,7 @@ def collect(self) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: # noqa: yield from _component_families(groups) yield from _accuracy_families(groups) yield from _deviation_families(groups) + yield from _canary_deviation_families(groups) yield from _quality_families(groups) yield from _ensemble_families(groups) yield from _contribution_families(groups) @@ -1488,6 +1553,8 @@ def collect(self) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: # noqa: yield from _leader_families(leader_state) yield from _query_cache_families(cache_state) yield from _what_if_runs_families(what_if_runs_state) + yield from _trigger_fires_families(trigger_fires_state) + yield from _annotation_posts_families(annotation_posts_state) yield from _discovery_families(discovery_state) yield from _season_length_families(season_length_state) yield from _decomposition_skipped_families(decomposition_skipped_state) @@ -1626,6 +1693,37 @@ def _deviation_families( ) +def _canary_deviation_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render the ``_canary_forecast_deviation*`` family, one per id. + + Mirrors :func:`_deviation_families` (ratio + outside_band) but the + metric name is per-id so the canary signal sits alongside the + underlying forecast in Grafana, and the emission shape matches what + Argo Rollouts / Flagger AnalysisRun templates query. + """ + by_metric: dict[str, list[CanaryDeviationPoint]] = {} + for snap in groups: + for point in snap.canary_deviation_points: + by_metric.setdefault(point.metric, []).append(point) + for metric_name, points in sorted(by_metric.items()): + yield from _emit_gauge_family( + points, + f"{metric_name}_canary_forecast_deviation", + "Signed deviation of the canary actual from the baseline-predicted " + "forecast band, as (actual - yhat) / (yhat_upper - yhat_lower).", + value_fn=lambda p: p.ratio, + skip_fn=lambda p: not math.isfinite(p.ratio), + ) + yield from _emit_gauge_family( + points, + f"{metric_name}_canary_forecast_deviation_outside_band", + "1 if the canary actual is outside the baseline-predicted forecast band, else 0.", + value_fn=lambda p: 1.0 if p.outside_band else 0.0, + ) + + def _quality_families( groups: list[GroupSnapshot], ) -> Iterator[GaugeMetricFamily]: @@ -2333,6 +2431,44 @@ def _what_if_runs_families( yield counter +def _trigger_fires_families( + state: dict[tuple[str, str], int], +) -> Iterator[CounterMetricFamily]: + """Render ``forecast_trigger_fires_total{name, outcome}``. + + Bumped each time a forecast-driven webhook trigger delivers (or fails + to deliver) an event. ``outcome`` is bounded to + {success, http_error, timeout}. Emitted only once at least one trigger + has fired — there's no fixed name set to seed a zero baseline from. + """ + counter = CounterMetricFamily( + "forecast_trigger_fires", + "Forecast-driven webhook trigger delivery count, by trigger name and outcome.", + labels=["name", "outcome"], + ) + for (name, outcome), count in sorted(state.items()): + counter.add_metric([name, outcome], float(count)) + yield counter + + +def _annotation_posts_families( + state: dict[tuple[str, str], int], +) -> Iterator[CounterMetricFamily]: + """Render ``forecast_annotation_posts_total{event_type, outcome}``. + + Bumped each time the Grafana-annotations sink posts (or fails to post) + an annotation. ``outcome`` is bounded to {success, http_error, timeout}. + """ + counter = CounterMetricFamily( + "forecast_annotation_posts", + "Grafana-annotation post count, by event type and outcome.", + labels=["event_type", "outcome"], + ) + for (event_type, outcome), count in sorted(state.items()): + counter.add_metric([event_type, outcome], float(count)) + yield counter + + def _discovery_families( state: tuple[dict[tuple[str, str], int], dict[tuple[str, str, str, str], int]], ) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: diff --git a/forecaster/src/promforecast/main.py b/forecaster/src/promforecast/main.py index a4d3dd0..4f8320f 100644 --- a/forecaster/src/promforecast/main.py +++ b/forecaster/src/promforecast/main.py @@ -19,6 +19,7 @@ from fastapi.responses import JSONResponse, PlainTextResponse from . import adapter as adapter_module +from . import annotations as annotations_module from . import backfill as backfill_module from . import config as config_module from . import diagnose as diagnose_module @@ -26,9 +27,11 @@ from . import inspect as inspect_module from . import preview as preview_module from . import telemetry as telemetry_module +from . import triggers as triggers_module from . import tuner as tuner_module from . import validate as validate_module from . import warmup as warmup_module +from . import webhook as webhook_module from . import what_if as what_if_module from .cache import MemoryQueryCache, QueryCache, QueryCacheBackend, RedisQueryCache from .exporter import Exporter @@ -881,7 +884,19 @@ def run(config_path: Path, log_level: str) -> None: # group run. The disabled tracer is free, the real one carries SDK # init cost only when ``telemetry.enabled`` is true. tracer = telemetry_module.build_tracer(cfg.telemetry) - runner = Runner(config=cfg, source=source, exporter=exporter, sink=sink, tracer=tracer) + webhook_client = _build_webhook_client(cfg) + trigger_engine = _build_trigger_engine(cfg, source, exporter) + annotation_sink = _build_annotation_sink(cfg, exporter) + runner = Runner( + config=cfg, + source=source, + exporter=exporter, + sink=sink, + tracer=tracer, + webhook_client=webhook_client, + trigger_engine=trigger_engine, + annotation_sink=annotation_sink, + ) if snapshot_cache is not None: # Wire the leader's per-group snapshot hook so a successful run # publishes the rendered /metrics payload to Redis. Followers @@ -1494,6 +1509,56 @@ def _build_sink(cfg: config_module.Config) -> ForecastSink | None: ) +def _build_webhook_client( + cfg: config_module.Config, +) -> webhook_module.WebhookRegressorClient | None: + """Construct a webhook-regressor client iff any query configures one. + + The common case (no ``type: webhook`` regressor anywhere) returns + ``None`` so a deployment that never uses the feature pays no idle + connection pool. A config reload that *adds* the first webhook regressor + wires the client on the fly (see :meth:`Runner._rebuild_integrations`), + so no restart is required. + """ + has_webhook = any( + reg.type == "webhook" + for group in cfg.groups + for query in group.queries + for reg in query.regressors + ) + if not has_webhook: + return None + logger.info("webhook_regressor_client_enabled") + return webhook_module.WebhookRegressorClient() + + +def _build_trigger_engine( + cfg: config_module.Config, + source: PromSource, + exporter: Exporter, +) -> triggers_module.TriggerEngine | None: + """Construct the forecast-driven trigger engine iff triggers are configured.""" + if not cfg.triggers: + return None + logger.info("triggers_enabled", count=len(cfg.triggers)) + return triggers_module.TriggerEngine(triggers=cfg.triggers, source=source, exporter=exporter) + + +def _build_annotation_sink( + cfg: config_module.Config, + exporter: Exporter, +) -> annotations_module.GrafanaAnnotationSink | None: + """Construct the Grafana-annotations sink iff it is enabled.""" + if not cfg.annotations.enabled: + return None + logger.info( + "grafana_annotations_enabled", + grafana_url=cfg.annotations.grafana_url, + events=list(cfg.annotations.events), + ) + return annotations_module.GrafanaAnnotationSink(cfg.annotations, exporter) + + def _parse_listen(listen: str) -> tuple[str, int]: """Accept ``:9091`` or ``host:9091``.""" if listen.startswith(":"): diff --git a/forecaster/src/promforecast/regressors.py b/forecaster/src/promforecast/regressors.py index 3752518..3170294 100644 --- a/forecaster/src/promforecast/regressors.py +++ b/forecaster/src/promforecast/regressors.py @@ -45,6 +45,7 @@ import structlog from .config import CALENDAR_FEATURE_TYPES, RegressorConfig +from .webhook import WebhookError # Imported lazily inside ``_apply_holidays`` so the ``holidays`` package # is loaded only when a config actually configures a holiday regressor. @@ -55,6 +56,7 @@ import pandas as pd from .source import PromSource, SeriesFrame + from .webhook import WebhookRegressorClient logger = structlog.get_logger(__name__) @@ -76,9 +78,10 @@ class RegressorFailure: """Per-regressor failure record returned from ``resolve``. ``reason`` is a bounded token (``query_error`` | ``query_timeout`` | - ``empty_result`` | ``invalid_type``) that the runner uses as a label on - ``forecast_regressor_failures_total``. We keep the cardinality small so - the counter doesn't explode on a runaway regressor. + ``empty_result`` | ``invalid_type`` | ``no_source`` | ``webhook_failure`` + | ``signature_invalid`` | ``calendar_file_error``) that the runner uses + as a label on ``forecast_regressor_failures_total``. We keep the + cardinality small so the counter doesn't explode on a runaway regressor. """ regressor_id: str @@ -143,23 +146,71 @@ async def prefetch_promql( query_end: datetime, step_seconds: int, query_timeout_seconds: float, + query_id: str = "", + webhook_client: WebhookRegressorClient | None = None, ) -> PromQLPrefetch: - """Fan out every custom regressor's PromQL **once per primary query**. - - Calendar regressors are deferred to per-series alignment — they're - pure functions of the timestamps and don't benefit from caching. - Required-regressor failures raise :class:`RegressorError` so the - caller (the runner's ``_run_query``) can short-circuit the entire - query: every series produced by that PromQL would have been skipped - anyway, and emitting N copies of the same failure into the per-series - counter would just be noise. + """Fan out every PromQL + webhook regressor **once per primary query**. + + Calendar / calendar-file regressors are deferred to per-series alignment + — they're pure functions of the timestamps (plus a static file) and don't + benefit from this network-fetch phase. Webhook regressors *do* belong + here: a query returning 100 series must POST to the external endpoint + once, not 100 times (the webhook client also caches per URL+range to + cover repeated refresh ticks). + + Required-regressor failures raise :class:`RegressorError` so the caller + (the runner's ``_run_query``) can short-circuit the entire query: every + series produced by that PromQL would have been skipped anyway, and + emitting N copies of the same failure into the per-series counter would + just be noise. """ fetched: dict[str, list[SeriesFrame]] = {} failures: list[RegressorFailure] = [] promql_regs = [r for r in regressors if r.type == "custom"] - if not promql_regs: + webhook_regs = [r for r in regressors if r.type == "webhook"] + if not promql_regs and not webhook_regs: return PromQLPrefetch(fetched=fetched, failures=failures) + if promql_regs: + await _prefetch_promql_regs( + promql_regs=promql_regs, + source=source, + query_start=query_start, + query_end=query_end, + step_seconds=step_seconds, + query_timeout_seconds=query_timeout_seconds, + fetched=fetched, + failures=failures, + ) + for reg in webhook_regs: + await _prefetch_one_webhook( + reg=reg, + query_id=query_id, + query_start=query_start, + query_end=query_end, + webhook_client=webhook_client, + fetched=fetched, + failures=failures, + ) + # Reduce each regressor's frames to a sorted (ds, v) frame *once* per + # primary query. Per-series alignment becomes a cheap merge_asof against + # this pre-aggregated frame instead of re-running groupby+sort N times, + # where N is the primary query's series count. + aggregated = _aggregate_fetched(fetched) + return PromQLPrefetch(fetched=fetched, failures=failures, aggregated=aggregated) + + +async def _prefetch_promql_regs( + *, + promql_regs: list[RegressorConfig], + source: PromSource | None, + query_start: datetime, + query_end: datetime, + step_seconds: int, + query_timeout_seconds: float, + fetched: dict[str, list[SeriesFrame]], + failures: list[RegressorFailure], +) -> None: if source is None: # No datasource attached (test/dry-run). Treat every PromQL # regressor as "no_source" so optional ones degrade gracefully and @@ -173,7 +224,7 @@ async def prefetch_promql( if reg.required: raise RegressorError(no_source.reason, no_source.message) failures.append(no_source) - return PromQLPrefetch(fetched=fetched, failures=failures) + return raw_results = await asyncio.gather( *( @@ -198,12 +249,46 @@ async def prefetch_promql( continue assert isinstance(outcome, list) fetched[reg.id] = outcome - # Reduce each regressor's frames to a sorted (ds, v) frame *once* per - # primary query. Per-series alignment becomes a cheap merge_asof against - # this pre-aggregated frame instead of re-running groupby+sort N times, - # where N is the primary query's series count. - aggregated = _aggregate_fetched(fetched) - return PromQLPrefetch(fetched=fetched, failures=failures, aggregated=aggregated) + + +async def _prefetch_one_webhook( + *, + reg: RegressorConfig, + query_id: str, + query_start: datetime, + query_end: datetime, + webhook_client: WebhookRegressorClient | None, + fetched: dict[str, list[SeriesFrame]], + failures: list[RegressorFailure], +) -> None: + """Fetch one webhook regressor, recording graceful / fatal failures. + + Mirrors the PromQL path: an optional regressor that fails degrades + gracefully (recorded in ``failures``), a ``required: true`` one raises + :class:`RegressorError` to short-circuit the whole query. + """ + if webhook_client is None: + # No client wired (offline tests / dry-run). Same shape as the + # PromQL ``no_source`` path. + no_source = RegressorFailure( + regressor_id=reg.id, + reason="no_source", + message="webhook client is not available for regressor resolution", + ) + if reg.required: + raise RegressorError(no_source.reason, no_source.message) + failures.append(no_source) + return + try: + frames = await webhook_client.fetch( + reg=reg, query_id=query_id, start=query_start, end=query_end + ) + except WebhookError as exc: + if reg.required: + raise RegressorError(exc.reason, str(exc)) from exc + failures.append(RegressorFailure(regressor_id=reg.id, reason=exc.reason, message=str(exc))) + return + fetched[reg.id] = frames def _aggregate_fetched( @@ -245,7 +330,7 @@ def _aggregate_frames(frames: list[SeriesFrame]) -> pd.DataFrame | None: return src_grouped.sort_values(by="ds").reset_index(drop=True) -def align_for_series( +def align_for_series( # noqa: PLR0912 *, regressors: list[RegressorConfig], prefetch: PromQLPrefetch, @@ -285,6 +370,16 @@ def align_for_series( raise RegressorError(hol_failure.reason, hol_failure.message) failures.append(hol_failure) continue + if reg.type == "calendar_file": + cf_failure = _apply_calendar_file(reg, out) + if cf_failure is not None: + if reg.required: + raise RegressorError(cf_failure.reason, cf_failure.message) + failures.append(cf_failure) + continue + # ``custom`` and ``webhook`` both land here: the prefetch phase + # populated ``aggregated`` for them, so per-series alignment is the + # same nearest-asof merge. aggregated = _resolve_aggregated(prefetch, reg.id) if aggregated is None: # Pre-fetch failed for this optional regressor — already counted @@ -395,6 +490,47 @@ def _apply_holidays(reg: RegressorConfig, out: pd.DataFrame) -> RegressorFailure return None +def _apply_calendar_file(reg: RegressorConfig, out: pd.DataFrame) -> RegressorFailure | None: + """Materialise calendar-file regressor column(s) or return a failure record. + + Same binary / one-hot emission shape as :func:`_apply_holidays`: + + * ``expand: false`` (default) — a single binary column named ```` + that is ``1`` on any day inside any configured window and ``0`` + otherwise. + * ``expand: true`` — one column per distinct entry ``name``, named + ``__``. + + The file is parsed lazily (and re-read on mtime change) by + :mod:`promforecast.calendar_file`. A missing / malformed file surfaces + as a graceful per-regressor failure with reason ``calendar_file_error``; + dates are resolved in UTC to match how source timestamps arrive. + """ + from . import calendar_file as cf # noqa: PLC0415 + + try: + entries = cf.get_calendar(str(reg.path)) + except cf.CalendarFileError as exc: + return RegressorFailure(regressor_id=reg.id, reason="calendar_file_error", message=str(exc)) + + ds = out["ds"] + ds_utc = ds.dt.tz_localize("UTC") if ds.dt.tz is None else ds.dt.tz_convert("UTC") + dates = ds_utc.dt.date + if reg.expand: + names = sorted({e.name for e in entries}) + for name in names: + windows = [e for e in entries if e.name == name] + col = f"{reg.id}__{_slugify_holiday(name)}" + out[col] = [1.0 if any(w.contains(d) for w in windows) else 0.0 for d in dates] + if not names: + # No entries at all — emit the binary column so the frame always + # carries an explicit ``0`` column rather than silently vanishing. + out[reg.id] = [0.0] * len(dates) + else: + out[reg.id] = [1.0 if any(e.contains(d) for e in entries) else 0.0 for d in dates] + return None + + def _build_holiday_calendar(reg: RegressorConfig) -> Any: """Resolve the ``holidays.country_holidays`` calendar for ``reg``. diff --git a/forecaster/src/promforecast/runner/_scheduler.py b/forecaster/src/promforecast/runner/_scheduler.py index f66bc97..f9cd3be 100644 --- a/forecaster/src/promforecast/runner/_scheduler.py +++ b/forecaster/src/promforecast/runner/_scheduler.py @@ -24,7 +24,9 @@ import structlog +from .. import annotations as annotations_mod from .. import band_coverage as band_coverage_mod +from .. import canary as canary_mod from .. import capacity as capacity_mod from .. import cold_start as cold_start_mod from .. import conformal as conformal_mod @@ -69,6 +71,7 @@ ) from ..exporter import ( BandCoveragePoint, + CanaryDeviationPoint, DeployRiskPoint, DriftPoint, Exporter, @@ -127,8 +130,11 @@ if TYPE_CHECKING: import pandas as pd + from ..annotations import GrafanaAnnotationSink from ..config import Config from ..selector import SelectionResult + from ..triggers import TriggerEngine + from ..webhook import WebhookRegressorClient logger = structlog.get_logger(__name__) @@ -365,12 +371,25 @@ def __init__( executor: ThreadPoolExecutor | None = None, sink: ForecastSink | None = None, tracer: telemetry_mod.Tracer | None = None, + webhook_client: WebhookRegressorClient | None = None, + trigger_engine: TriggerEngine | None = None, + annotation_sink: GrafanaAnnotationSink | None = None, ) -> None: self._config = config self._source = source self._exporter = exporter self._executor = executor or ThreadPoolExecutor(max_workers=4) self._sink = sink + # Optional client for ``type: webhook`` regressors. ``None`` when no + # webhook regressor is configured (the common case) — the prefetch + # path then degrades any webhook regressor gracefully. + self._webhook_client = webhook_client + # Optional forecast-driven webhook triggers and Grafana-annotations + # sink. Both ``None`` unless the operator configured them; both are + # evaluated after each group run (post-snapshot, post-sink) so a + # delivery hiccup never delays the standard exposition. + self._trigger_engine = trigger_engine + self._annotation_sink = annotation_sink # Drift cache: persists across runs in the same process so # successive fits can be compared. HA followers don't see the # cache; they read the leader's rendered drift gauges via the @@ -638,11 +657,83 @@ async def apply_config(self, new_config: Config) -> ReloadResult: self._what_if_throttle.update_min_interval( new_config.safety.fleet_what_if_min_interval.total_seconds(), ) + # Rebuild the v1.9 outbound/inbound integrations (webhook + # regressor client, forecast-driven trigger engine, Grafana + # annotations sink) so a hot reload that adds, changes, or + # removes ``triggers:`` / ``annotations:`` / webhook regressors + # takes effect without a restart — the loops are quiesced here, + # so swapping them is race-free. + await self._rebuild_integrations(old_config=old) for group in new_config.groups: self._spawn_group_loop(group) return ReloadResult(added=added, removed=removed, changed=changed) + async def _rebuild_integrations(self, *, old_config: Config) -> None: + """Reconcile webhook client / trigger engine / annotation sink to config. + + Called from :meth:`apply_config` after the config swap (loops already + drained). Each integration is only rebuilt when its slice of the + config actually changed, so an unrelated reload preserves the trigger + rising-edge state and the annotation rising-edge dedup set rather than + re-firing everything that is currently active. + """ + new = self._config + # Webhook client: needed iff some query still configures a webhook + # regressor. The client is config-agnostic (each ``fetch`` reads the + # live ``reg``), so keep an existing one rather than churning its + # connection pool + response cache; only create / tear down on the + # needed-ness boundary. + needs_webhook = any( + reg.type == "webhook" + for group in new.groups + for query in group.queries + for reg in query.regressors + ) + if needs_webhook and self._webhook_client is None: + from ..webhook import WebhookRegressorClient # noqa: PLC0415 + + self._webhook_client = WebhookRegressorClient() + logger.info("webhook_regressor_client_enabled_on_reload") + elif not needs_webhook and self._webhook_client is not None: + stale_client = self._webhook_client + self._webhook_client = None + with contextlib.suppress(Exception): + await stale_client.aclose() + logger.info("webhook_regressor_client_disabled_on_reload") + + # Trigger engine: rebuild only when the trigger list changed. + if new.triggers != old_config.triggers: + stale_engine = self._trigger_engine + if new.triggers: + from ..triggers import TriggerEngine # noqa: PLC0415 + + self._trigger_engine = TriggerEngine( + triggers=new.triggers, source=self._source, exporter=self._exporter + ) + logger.info("triggers_reloaded", count=len(new.triggers)) + else: + self._trigger_engine = None + logger.info("triggers_disabled_on_reload") + if stale_engine is not None: + with contextlib.suppress(Exception): + await stale_engine.aclose() + + # Annotation sink: rebuild only when the annotations block changed. + if new.annotations != old_config.annotations: + stale_sink = self._annotation_sink + if new.annotations.enabled: + from ..annotations import GrafanaAnnotationSink # noqa: PLC0415 + + self._annotation_sink = GrafanaAnnotationSink(new.annotations, self._exporter) + logger.info("grafana_annotations_reloaded", events=list(new.annotations.events)) + else: + self._annotation_sink = None + logger.info("grafana_annotations_disabled_on_reload") + if stale_sink is not None: + with contextlib.suppress(Exception): + await stale_sink.aclose() + @staticmethod def _drain_grace_seconds(config: Config) -> float: """How long to wait for an in-flight run to finish before hard-cancelling. @@ -944,6 +1035,7 @@ async def run_group( # noqa: PLR0912, PLR0915 points=ctx.points, accuracy_points=ctx.accuracy_points, deviation_points=ctx.deviation_points, + canary_deviation_points=ctx.canary_deviation_points, quality_points=ctx.quality_points, ensemble_weights=ctx.ensemble_weight_points, contribution_points=ctx.contribution_points, @@ -1077,6 +1169,26 @@ def _record_sink_retry(reason: str) -> None: except Exception: log.exception("deploy_risk_refresh_failed") + # Grafana annotations + forecast-driven webhook triggers run + # last, after the snapshot, sink, and deploy-risk refresh, so + # (a) the standard exposition is never delayed by an external + # POST and (b) trigger conditions can read the freshest metrics + # (sink) and annotation events the freshest deploy-risk scores. + # Both swallow their own delivery errors; this wrapper guards + # against an unexpected defect crashing the group run. + if self._annotation_sink is not None: + try: + await self._annotation_sink.post_events( + self._build_annotation_events(group, ctx), scope=group.name + ) + except Exception: + log.exception("annotation_post_failed") + if self._trigger_engine is not None: + try: + await self._trigger_engine.evaluate() + except Exception: + log.exception("trigger_evaluate_failed") + # Recompute the service-level health composite across all # current group snapshots. Cheap (a single pass over the # already-on-exporter quality_points) and idempotent — every @@ -1480,6 +1592,14 @@ async def _do_run_query( # noqa: PLR0912, PLR0915 if frames is None: return + # Canary-vs-baseline deviation reads the *full* fetched population + # (it needs both the canary and the baseline series), so it runs + # before the group-level canary *sampling* and the overflow caps + # shrink the set. It emits its own ``_canary_forecast_deviation`` + # family and does not affect the normal forecast path below. + if query.canary.enabled: + await self._emit_canary_deviation(query, group, frames, ctx, qlog) + # Canary sampling runs *before* the overflow caps so the operator's # opt-in population shrink happens first, and the per-query/global # caps then bound the sampled population. The inverse order — caps @@ -1663,6 +1783,8 @@ async def _prefetch_query_regressors(self, query: QueryConfig) -> regressors_mod query_end=end, step_seconds=step_seconds, query_timeout_seconds=self._config.safety.query_timeout.total_seconds(), + query_id=query.id, + webhook_client=self._webhook_client, ) def _merge_preprocess_stats( @@ -3772,6 +3894,248 @@ def _record_fit_outcome(self, *, group: str, model: str, success: bool) -> None: """ self._exporter.record_fit_outcome(group=group, model=model, success=success) + def _build_annotation_events( + self, group: GroupConfig, ctx: _RunContext + ) -> list[annotations_mod.AnnotationEvent]: + """Translate this run's significant events into annotation events. + + Reads the per-group run context (drift alerts, change points, + predicted breaches, outside-band deviations) plus the global + deploy-risk scores. The sink itself filters by configured event + family and dedups repeats, so this builder emits liberally for + every *currently firing* event. + """ + events: list[annotations_mod.AnnotationEvent] = [] + g = group.name + for query_id, count in ctx.drift_alerts.items(): + if count > 0: + events.append( + annotations_mod.AnnotationEvent( + event_type="drift", + dedup_key=f"{g}:{query_id}", + tags=["promforecast", "drift", f"group:{g}", f"id:{query_id}"], + text=f"Forecast drift threshold crossed for {query_id} in group {g}", + ) + ) + for query_id, count in ctx.change_points.items(): + if count > 0: + events.append( + annotations_mod.AnnotationEvent( + event_type="change_point", + dedup_key=f"{g}:{query_id}", + tags=["promforecast", "change_point", f"group:{g}", f"id:{query_id}"], + text=f"Change-point detected in {query_id} (group {g})", + ) + ) + for point in ctx.will_breach_points: + if point.value >= 1.0: + qid = _query_id_from_metric(point.metric) + name = point.labels.get("name", "") + events.append( + annotations_mod.AnnotationEvent( + event_type="threshold_breach_predicted", + dedup_key=f"{g}:{qid}:{name}:{_series_key_from_labels(point.labels)}", + tags=[ + "promforecast", + "threshold_breach_predicted", + f"group:{g}", + f"id:{qid}", + ], + text=( + f"Predicted threshold breach for {qid} (threshold {name!r}, group {g})" + ), + ) + ) + for dev in ctx.deviation_points: + if dev.outside_band: + qid = dev.labels.get("id", "") + events.append( + annotations_mod.AnnotationEvent( + event_type="deviation_outside_band", + dedup_key=f"{g}:{qid}:{_series_key_from_labels(dev.labels)}", + tags=["promforecast", "deviation_outside_band", f"group:{g}", f"id:{qid}"], + text=f"Actual outside forecast band for {qid} (group {g})", + ) + ) + min_score = self._config.annotations.deploy_risk_min_score + for risk in self._exporter._deploy_risk_state(): + # Only an *elevated* deploy posts an annotation — a rollout sitting + # at a low risk score is steady-state, not a moment worth marking. + if risk.score < min_score: + continue + service = risk.labels.get("service", "") + deployment_id = risk.labels.get("deployment_id", "") + events.append( + annotations_mod.AnnotationEvent( + event_type="deploy_risk", + dedup_key=f"{service}:{deployment_id}", + tags=["promforecast", "deploy_risk", f"service:{service}"], + text=( + f"Deploy risk score {risk.score:.2f} for {service} " + f"(deployment {deployment_id})" + ), + ) + ) + return events + + async def _emit_canary_deviation( + self, + query: QueryConfig, + group: GroupConfig, + frames: list[SeriesFrame], + ctx: _RunContext, + qlog: structlog.stdlib.BoundLogger, + ) -> None: + """Fit the baseline population and score each canary series against it. + + Splits ``frames`` into a canary and a baseline population by the + configured selectors, fits the first configured model on the baseline + aggregate, backtests one horizon to obtain a trailing forecast band, + and emits ``_canary_forecast_deviation`` for each canary series' + most recent actual versus that band. Failures degrade gracefully — + an unfittable baseline or a missing population simply emits nothing. + """ + import pandas as pd # noqa: PLC0415 + + d = self._config.defaults + try: + canary_frames, baseline_frames = canary_mod.split_frames( + frames, + canary_selector=query.canary.canary_selector, + baseline_selector=query.canary.baseline_selector, + ) + except canary_mod.CanarySelectorError as exc: + qlog.warning("canary_selector_invalid", error=str(exc)) + ctx.failures["canary_selector"] = ctx.failures.get("canary_selector", 0) + 1 + return + if not canary_frames or not baseline_frames: + qlog.info( + "canary_population_empty", + canary=len(canary_frames), + baseline=len(baseline_frames), + ) + return + + timestamps, values = canary_mod.aggregate_baseline(baseline_frames) + eff_step = effective_step(query, d) + horizon_steps = max(1, int(d.horizon.total_seconds() // eff_step.total_seconds())) + eval_steps = min(horizon_steps, len(values) - 1) + if len(values) < max(d.min_points, eval_steps + 2): + qlog.info("canary_baseline_too_short", points=len(values)) + return + + df = pd.DataFrame( + { + "unique_id": ["baseline"] * len(values), + "ds": pd.to_datetime(timestamps), + "y": values, + } + ) + freq = _pandas_freq(eff_step) + season_length = effective_season_length(query, d, step=eff_step) + levels = list(d.confidence_levels) + spec = d.models[0] + summary = await self._backtest_baseline( + df=df, + spec=spec, + eval_steps=eval_steps, + freq=freq, + levels=levels, + season_length=season_length, + group_name=group.name, + query_id=query.id, + ) + if summary is None: + qlog.info("canary_baseline_unfittable") + return + _last_actual, last_yhat, last_bands = summary + + points: list[CanaryDeviationPoint] = [] + for frame in canary_frames: + actual = canary_mod.latest_actual(frame) + if actual is None: + continue + for level in levels: + band = last_bands.get(level) + if band is None: + continue + dev = deviation_mod.compute(actual, last_yhat, band.lower, band.upper) + # ``id`` mirrors the v0.2 deviation shape and lets the + # partial-tick inheritance registry scope these rows by query. + labels = { + **frame.labels, + "id": query.id, + "model": spec.name, + "level": str(level), + } + points.append( + CanaryDeviationPoint( + metric=query.id, + labels=labels, + ratio=dev.ratio, + outside_band=dev.outside_band, + ) + ) + ctx.canary_deviation_points.extend(points) + qlog.info( + "canary_deviation_emitted", + canary_series=len(canary_frames), + baseline_series=len(baseline_frames), + points=len(points), + ) + + async def _backtest_baseline( + self, + *, + df: pd.DataFrame, + spec: ModelSpec, + eval_steps: int, + freq: str, + levels: list[int], + season_length: int, + group_name: str, + query_id: str, + ) -> tuple[float, float, dict[int, Any]] | None: + """Run one baseline backtest in the executor; return its trailing band. + + Returns ``(last_actual, last_yhat, bands)`` exactly as + ``_summarise_backtest`` does, or ``None`` when the fit fails or the + frames are degenerate. + """ + loop = asyncio.get_running_loop() + timeout = self._config.safety.fit_timeout.total_seconds() + model = model_registry.build(spec.name, season_length=season_length, params=spec.params) + try: + with self._tracer.span( + "promforecast.fit", + model=spec.name, + kind="canary_baseline", + horizon=eval_steps, + freq=freq, + season_length=season_length, + ): + async with self._fits_semaphore: + outcome = await asyncio.wait_for( + loop.run_in_executor( + self._executor, + _run_backtest, + model, + df, + eval_steps, + freq, + levels, + None, + ), + timeout=timeout, + ) + except Exception: + logger.exception("canary_baseline_backtest_failed", group=group_name, query=query_id) + return None + if outcome is None: + return None + test, predictions = outcome + return _pkg._summarise_backtest(predictions=predictions, test=test, levels=levels) + async def _run_backtests( self, *, @@ -4273,3 +4637,12 @@ async def stop(self) -> None: if self._sink is not None: with contextlib.suppress(Exception): await self._sink.aclose() + if self._webhook_client is not None: + with contextlib.suppress(Exception): + await self._webhook_client.aclose() + if self._trigger_engine is not None: + with contextlib.suppress(Exception): + await self._trigger_engine.aclose() + if self._annotation_sink is not None: + with contextlib.suppress(Exception): + await self._annotation_sink.aclose() diff --git a/forecaster/src/promforecast/runner/_state.py b/forecaster/src/promforecast/runner/_state.py index 303242a..3573f98 100644 --- a/forecaster/src/promforecast/runner/_state.py +++ b/forecaster/src/promforecast/runner/_state.py @@ -21,6 +21,7 @@ from ..exporter import ( AccuracyPoint, CalibrationOffsetPoint, + CanaryDeviationPoint, ComponentDecompositionPoint, ConditionalCalibrationOffsetPoint, ConditionalDeviationPoint, @@ -235,6 +236,7 @@ class _RunContext: points: list[ForecastPoint] = field(default_factory=list) accuracy_points: list[AccuracyPoint] = field(default_factory=list) deviation_points: list[DeviationPoint] = field(default_factory=list) + canary_deviation_points: list[CanaryDeviationPoint] = field(default_factory=list) quality_points: list[QualityPoint] = field(default_factory=list) ensemble_weight_points: list[EnsembleWeightPoint] = field(default_factory=list) contribution_points: list[ContributionPoint] = field(default_factory=list) @@ -406,6 +408,9 @@ def _query_id_of(point: Any, by: _KeyBy) -> str: # Lists keyed by the point's ``id`` label. _InheritanceSpec("accuracy_points", "accuracy_points", "label"), _InheritanceSpec("deviation_points", "deviation_points", "label"), + # Canary deviation points carry the ``id`` label (added by the runner) + # so the label extractor scopes them per query. + _InheritanceSpec("canary_deviation_points", "canary_deviation_points", "label"), _InheritanceSpec("quality_points", "quality_points", "label"), _InheritanceSpec("ensemble_weights", "ensemble_weight_points", "label"), _InheritanceSpec("calibration_offsets", "calibration_offsets", "label"), diff --git a/forecaster/src/promforecast/signing.py b/forecaster/src/promforecast/signing.py new file mode 100644 index 0000000..073c5fa --- /dev/null +++ b/forecaster/src/promforecast/signing.py @@ -0,0 +1,104 @@ +"""HMAC signing and secret-reference resolution for webhooks. + +Shared by the webhook *regressor* (which verifies the HMAC on a response it +pulls in) and the webhook *trigger* (which signs a request it pushes out). +The signature is computed over the raw request/response body with +HMAC-SHA256 and carried in the ``X-Promforecast-Signature: sha256=`` +header — the de-facto convention GitHub, Stripe, and most webhook +producers use. + +Secrets are never written into the config literally in production: a field +like ``secret_ref: ${WEBHOOK_SECRET}`` is resolved against the process +environment (the chart injects the variable from a Kubernetes Secret via +``secretKeyRef``). A literal value is accepted too, for local testing. +""" + +from __future__ import annotations + +import hashlib +import hmac +import os +import re +from collections.abc import Mapping + +#: The header outbound triggers set and inbound regressors verify. +SIGNATURE_HEADER = "X-Promforecast-Signature" + +#: ``${NAME}`` references an environment variable; anything else is a literal. +_ENV_REF_RE = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$") + +#: ``${NAME}`` anywhere inside a string (used for header-value expansion, +#: where the reference is typically embedded — e.g. ``Bearer ${TOKEN}``). +_ENV_SUBST_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +def expand_env_refs(value: str) -> str: + """Substitute every ``${NAME}`` in ``value`` with environment variable ``NAME``. + + Unlike :func:`resolve_secret_ref` (which matches a *whole-value* + ``${NAME}`` and rejects an empty result), this expands references + *embedded* in a larger string — the shape webhook/trigger ``headers`` + use (``Authorization: Bearer ${TOKEN}``). An unset variable expands to + the empty string rather than raising: a header is delivered best-effort, + and a failed auth degrades through the same graceful path as any other + webhook failure. A value with no ``${...}`` is returned unchanged. + """ + return _ENV_SUBST_RE.sub(lambda m: os.environ.get(m.group(1), ""), value) + + +def expand_header_env_refs(headers: Mapping[str, str]) -> dict[str, str]: + """Expand ``${ENV_VAR}`` references in every header *value*. + + Header *names* are left verbatim. Lets operators inject secrets into + webhook/trigger headers from the environment (Kubernetes Secret → + env var) the same way ``secret_ref`` works for signing, instead of + baking a literal token into the config. + """ + return {name: expand_env_refs(value) for name, value in headers.items()} + + +class SecretResolutionError(ValueError): + """Raised when a ``${ENV_VAR}`` reference resolves to an empty value. + + A missing or empty secret must fail loudly rather than silently + disabling signing — an unsigned payload that the operator believed was + signed is worse than a hard error at startup. + """ + + +def resolve_secret_ref(ref: str) -> str: + """Resolve a ``${ENV_VAR}`` reference (or pass a literal through). + + Mirrors the indirection the chart already uses for datasource + credentials: a value of the form ``${NAME}`` reads environment variable + ``NAME``; any other value is returned verbatim. An empty resolved + secret raises :class:`SecretResolutionError`. + """ + match = _ENV_REF_RE.match(ref.strip()) + value = ref if match is None else os.environ.get(match.group(1), "") + if not value: + raise SecretResolutionError( + f"secret reference {ref!r} resolved to an empty value; " + "set the environment variable (injected from a Kubernetes Secret) " + "or provide a literal secret" + ) + return value + + +def compute_signature(secret: str, body: bytes) -> str: + """Return ``sha256=`` HMAC-SHA256 of ``body`` keyed by ``secret``.""" + digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + return f"sha256={digest}" + + +def verify_signature(secret: str, body: bytes, header_value: str | None) -> bool: + """Constant-time check that ``header_value`` matches ``body``'s signature. + + Returns ``False`` for a missing or malformed header so the caller can + treat it identically to a mismatch (both surface as a regressor + ``signature_invalid`` failure). + """ + if not header_value: + return False + expected = compute_signature(secret, body) + return hmac.compare_digest(expected, header_value.strip()) diff --git a/forecaster/src/promforecast/source.py b/forecaster/src/promforecast/source.py index cc01154..72162d5 100644 --- a/forecaster/src/promforecast/source.py +++ b/forecaster/src/promforecast/source.py @@ -43,6 +43,14 @@ class SeriesFrame: values: list[float] +@dataclass(frozen=True) +class InstantSample: + """A single sample returned by an instant PromQL query.""" + + labels: dict[str, str] + value: float + + class PromSource: """PromQL range query client with retry and timeout. @@ -128,6 +136,19 @@ async def query_range( await self._cache.set(cache_key, frames) return frames + async def query_instant(self, promql: str) -> list[InstantSample]: + """Run an instant PromQL query and return one sample per series. + + Used by the forecast-driven trigger engine to evaluate a condition + against the metrics already in the TSDB (including forecast metrics + the sink wrote). Scalar results collapse to a single unlabelled + sample; vector results map one sample per series. + """ + endpoint = f"{self._url}/api/v1/query" + async with self._semaphore: + payload = await self._request(endpoint, {"query": promql}) + return _parse_instant(payload) + async def _request(self, endpoint: str, params: dict[str, str]) -> dict[str, Any]: async for attempt in AsyncRetrying( stop=stop_after_attempt(3), @@ -180,6 +201,42 @@ def _is_retryable(exc: BaseException) -> bool: return False +def _parse_instant(payload: dict[str, Any]) -> list[InstantSample]: + """Parse an instant-query response (vector or scalar) into samples. + + Non-finite values are dropped so a NaN sample can't masquerade as a + truthy condition downstream. + """ + data = payload.get("data") or {} + result_type = data.get("resultType") + samples: list[InstantSample] = [] + if result_type == "scalar": + ts_val = data.get("result") or [None, None] + value = _coerce_float(ts_val[1] if len(ts_val) == 2 else None) # noqa: PLR2004 + if value is not None: + samples.append(InstantSample(labels={}, value=value)) + return samples + if result_type != "vector": + raise QueryError(f"unexpected resultType for instant query: {result_type!r}") + for series in data.get("result", []): + labels = dict(series.get("metric", {})) + point = series.get("value") or [] + if len(point) != 2: # noqa: PLR2004 + continue + value = _coerce_float(point[1]) + if value is not None: + samples.append(InstantSample(labels=labels, value=value)) + return samples + + +def _coerce_float(raw: Any) -> float | None: + try: + v = float(raw) + except (TypeError, ValueError): + return None + return None if math.isnan(v) else v + + def _parse_matrix(payload: dict[str, Any]) -> list[SeriesFrame]: data = payload.get("data") or {} if data.get("resultType") != "matrix": diff --git a/forecaster/src/promforecast/triggers.py b/forecaster/src/promforecast/triggers.py new file mode 100644 index 0000000..b5684d9 --- /dev/null +++ b/forecaster/src/promforecast/triggers.py @@ -0,0 +1,231 @@ +"""Forecast-driven webhook triggers. + +Evaluates each configured trigger's PromQL ``condition`` against the +datasource after every group run and POSTs a structured forecast event to +the trigger's ``webhook_url`` when the condition transitions from false to +true. For sustained conditions, re-fires are throttled to at most one per +``repeat_interval``. + +Different from a PrometheusRule alert: the payload carries forecast context +(the condition's matched series and their values — which naturally include +the forecast value, the model identity, and any band/drift labels the +operator's PromQL selects). Different from a webhook *regressor*: the +direction is inverse — the forecaster pushes out rather than pulls in. + +Delivery never blocks or fails a fit: a condition-evaluation error is logged +and skipped, and a delivery failure bumps +``forecast_trigger_fires_total{outcome=http_error|timeout}`` and moves on. +Outbound payloads can be HMAC-signed (the ``X-Promforecast-Signature`` +header) when the trigger sets a ``signing`` block. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime +from string import Template +from typing import TYPE_CHECKING + +import httpx +import structlog + +from . import signing +from .config import TriggerConfig + +if TYPE_CHECKING: + from .exporter import Exporter + from .source import InstantSample, PromSource + +logger = structlog.get_logger(__name__) + + +#: Default coalescing window. The runner evaluates triggers from each +#: group's post-run hook, so with G groups and T triggers a naive pass +#: issues T*G instant queries per refresh cycle — and in the common case +#: (all groups sharing ``server.refresh_interval``) those completions land +#: in one burst against an unchanged metric state. Collapsing re-evaluations +#: that fall within this window cuts the burst to one query per trigger +#: while staying far below any realistic refresh interval (minutes-to-hours), +#: so rising-edge detection latency is unaffected in practice. +_DEFAULT_COALESCE_SECONDS = 5.0 + + +@dataclass +class _TriggerState: + was_true: bool = False + last_fired: float = field(default=float("-inf")) + # Monotonic time of the last *evaluation* (query issued), used to + # coalesce the per-group-hook burst. Distinct from ``last_fired``, + # which tracks the last *delivery* for repeat-interval throttling. + last_evaluated: float = field(default=float("-inf")) + + +def condition_is_true(samples: list[InstantSample]) -> bool: + """A condition is true when it matches at least one non-zero sample. + + Mirrors how Prometheus alerting treats a vector: an empty result is + "inactive", and a sample's truthiness is its value being non-zero + (NaN samples were already dropped by the instant-query parser). + """ + return any(s.value != 0.0 for s in samples) + + +class TriggerEngine: + """Evaluates triggers and delivers their webhooks. + + ``client`` is injectable for tests (an ``httpx.AsyncClient`` over a + ``MockTransport``); ``clock`` and ``now_iso`` are injectable so + repeat-interval and timestamp assertions don't depend on wall-clock. + """ + + def __init__( + self, + *, + triggers: list[TriggerConfig], + source: PromSource, + exporter: Exporter, + client: httpx.AsyncClient | None = None, + clock: Callable[[], float] = time.monotonic, + now_iso: Callable[[], str] | None = None, + coalesce_seconds: float = _DEFAULT_COALESCE_SECONDS, + ) -> None: + self._triggers = triggers + self._source = source + self._exporter = exporter + self._coalesce_seconds = max(0.0, coalesce_seconds) + if client is None: + self._client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=4, max_keepalive_connections=2), + ) + self._owned_client = True + else: + self._client = client + self._owned_client = False + self._clock = clock + self._now_iso = now_iso or (lambda: datetime.now(tz=UTC).isoformat()) + self._states: dict[str, _TriggerState] = {t.name: _TriggerState() for t in triggers} + + async def aclose(self) -> None: + if self._owned_client: + await self._client.aclose() + + async def evaluate(self) -> None: + """Evaluate every configured trigger once. + + Each trigger is isolated: an unexpected error in one (e.g. a + signing ``secret_ref`` that resolves to an empty value) is logged + and does not prevent the remaining triggers from being evaluated. + """ + for trigger in self._triggers: + try: + await self._evaluate_one(trigger) + except Exception: + logger.exception("trigger_evaluate_failed", name=trigger.name) + + async def _evaluate_one(self, trigger: TriggerConfig) -> None: + state = self._states.setdefault(trigger.name, _TriggerState()) + now = self._clock() + # Coalesce the per-group-hook burst: if this trigger was evaluated + # within the window, the metric state can't meaningfully have moved, + # so skip the redundant instant query. State is untouched (no edge + # decision is made), so the next non-coalesced evaluation sees fresh + # data — this only defers, never drops. + if now - state.last_evaluated < self._coalesce_seconds: + return + state.last_evaluated = now + try: + samples = await self._source.query_instant(trigger.condition) + except Exception as exc: + # A bad condition or a TSDB blip must not block the fit loop — + # it's not a *delivery* outcome, so the counter stays untouched. + logger.warning("trigger_condition_failed", name=trigger.name, error=str(exc)) + return + is_true = condition_is_true(samples) + repeat = trigger.repeat_interval.total_seconds() + rising_edge = is_true and not state.was_true + sustained = is_true and state.was_true and (now - state.last_fired) >= repeat + state.was_true = is_true + if not (rising_edge or sustained): + return + state.last_fired = now + await self._fire(trigger, samples) + + async def _fire(self, trigger: TriggerConfig, samples: list[InstantSample]) -> None: + body = self._render_payload(trigger, samples) + headers = { + "Content-Type": "application/json", + **signing.expand_header_env_refs(trigger.headers), + } + if trigger.signing is not None: + secret = signing.resolve_secret_ref(trigger.signing.secret_ref) + headers[signing.SIGNATURE_HEADER] = signing.compute_signature(secret, body) + outcome = await self._deliver(trigger, body, headers) + self._exporter.increment_trigger_fire(trigger.name, outcome) + logger.info("trigger_fired", name=trigger.name, outcome=outcome, samples=len(samples)) + + async def _deliver(self, trigger: TriggerConfig, body: bytes, headers: dict[str, str]) -> str: + try: + response = await self._client.post( + trigger.webhook_url, + content=body, + headers=headers, + timeout=trigger.timeout.total_seconds(), + ) + except (TimeoutError, httpx.TimeoutException): + logger.warning("trigger_delivery_timeout", name=trigger.name) + return "timeout" + except httpx.HTTPError as exc: + logger.warning("trigger_delivery_error", name=trigger.name, error=str(exc)) + return "http_error" + if response.status_code >= 400: # noqa: PLR2004 + logger.warning( + "trigger_delivery_status", + name=trigger.name, + status=response.status_code, + ) + return "http_error" + return "success" + + def _render_payload(self, trigger: TriggerConfig, samples: list[InstantSample]) -> bytes: + event = _build_event(trigger=trigger, samples=samples, fired_at=self._now_iso()) + if not trigger.payload_template.strip(): + return json.dumps(event).encode("utf-8") + # ``string.Template`` ($var) substitution — chosen over str.format so + # a JSON template's literal ``{`` / ``}`` don't need escaping. + context = { + "name": event["name"], + "condition": event["condition"], + "value": event["value"], + "fired_at": event["fired_at"], + "samples_json": json.dumps(event["samples"]), + } + rendered = Template(trigger.payload_template).safe_substitute(context) + return rendered.encode("utf-8") + + +def _build_event( + *, + trigger: TriggerConfig, + samples: list[InstantSample], + fired_at: str, +) -> dict[str, object]: + """Assemble the default JSON event for a fired trigger. + + ``value`` / ``labels`` reflect the first matched series so a simple + Slack template can interpolate a single number; ``samples`` carries the + full matched vector (each series' labels — which include the model + identity and any band/drift discriminators the condition selected — plus + its value) so richer receivers have the full forecast context. + """ + first = samples[0] if samples else None + return { + "name": trigger.name, + "condition": trigger.condition, + "fired_at": fired_at, + "value": first.value if first is not None else None, + "labels": dict(first.labels) if first is not None else {}, + "samples": [{"labels": dict(s.labels), "value": s.value} for s in samples], + } diff --git a/forecaster/src/promforecast/webhook.py b/forecaster/src/promforecast/webhook.py new file mode 100644 index 0000000..90b1923 --- /dev/null +++ b/forecaster/src/promforecast/webhook.py @@ -0,0 +1,254 @@ +"""Webhook regressor client. + +Pulls a regressor signal that does not live in Prometheus and is not on the +built-in calendar list — active incidents from PagerDuty, deploys-in-flight +from a CI/CD API, a custom internal "promotion-day" calendar — by POSTing to +an operator-configured URL at fit time and parsing a JSON time series from +the response. + +The request body carries the query id and the timestamp range; the response +is ``{"timestamps": [...], "values": [...]}``. Successful responses are +cached per (url, query_id, lookback) for ``cache_ttl`` so a primary query +returning many series — or a short refresh interval — doesn't hammer the +external service. The forecaster stays stateless: the webhook owns whatever +state it queries. + +Failures are bounded to two reason tokens so the +``forecast_regressor_failures_total`` label set stays small: + +* ``webhook_failure`` — timeout, transport error, non-2xx status, or an + unparseable / mis-shaped body. +* ``signature_invalid`` — HMAC verification was requested but the response + carried a missing or mismatched signature. + +Both flow through the existing per-regressor graceful-degradation path: an +optional webhook regressor that fails is dropped (the forecast still runs); +a ``required: true`` one skips the series. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime + +import httpx +import structlog + +from . import signing +from .config import RegressorConfig +from .source import SeriesFrame + +logger = structlog.get_logger(__name__) + + +class WebhookError(RuntimeError): + """Raised when a webhook regressor cannot be resolved. + + ``reason`` is a bounded token (``webhook_failure`` | ``signature_invalid``) + suitable for use as a Prometheus counter label. + """ + + def __init__(self, reason: str, message: str) -> None: + super().__init__(message) + self.reason = reason + + +@dataclass +class _CacheEntry: + frames: list[SeriesFrame] + expires_at: float + + +class WebhookRegressorClient: + """Fetches + caches webhook-regressor time series. + + ``client`` may be injected (tests pass an ``httpx.AsyncClient`` backed by + a ``MockTransport``); when omitted a pooled async client is created and + owned by this instance. ``clock`` is injectable so cache-expiry tests + don't sleep. + """ + + def __init__( + self, + client: httpx.AsyncClient | None = None, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if client is None: + self._client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=4, max_keepalive_connections=2), + ) + self._owned_client = True + else: + self._client = client + self._owned_client = False + self._clock = clock + self._cache: dict[tuple[str, str, int], _CacheEntry] = {} + + async def aclose(self) -> None: + if self._owned_client: + await self._client.aclose() + + async def fetch( + self, + *, + reg: RegressorConfig, + query_id: str, + start: datetime, + end: datetime, + ) -> list[SeriesFrame]: + """Return one :class:`SeriesFrame` for the webhook regressor ``reg``. + + Reads from the per-(url, query_id, lookback) cache when a fresh entry + exists; otherwise POSTs to the configured URL, verifies the optional + HMAC signature, parses the JSON time series, caches it, and returns + it. Raises :class:`WebhookError` on any failure. + """ + url = str(reg.url) + lookback = max(1, int((end - start).total_seconds())) + cache_key = (url, query_id, lookback) + entry = self._cache.get(cache_key) + now = self._clock() + if entry is not None and entry.expires_at > now: + return entry.frames + + frames = await self._request(reg=reg, url=url, query_id=query_id, start=start, end=end) + ttl = max(1, int(reg.cache_ttl.total_seconds())) + self._cache[cache_key] = _CacheEntry(frames=frames, expires_at=now + ttl) + return frames + + async def _request( + self, + *, + reg: RegressorConfig, + url: str, + query_id: str, + start: datetime, + end: datetime, + ) -> list[SeriesFrame]: + body = { + "query_id": query_id, + "regressor_id": reg.id, + "start": start.isoformat(), + "end": end.isoformat(), + } + try: + response = await self._client.post( + url, + json=body, + headers=signing.expand_header_env_refs(reg.headers), + timeout=reg.timeout.total_seconds(), + ) + except (TimeoutError, httpx.TimeoutException) as exc: + raise WebhookError("webhook_failure", f"webhook regressor timed out: {url}") from exc + except httpx.HTTPError as exc: + raise WebhookError( + "webhook_failure", f"webhook regressor transport error: {exc}" + ) from exc + if response.status_code >= 400: # noqa: PLR2004 + raise WebhookError( + "webhook_failure", + f"webhook regressor returned {response.status_code}: {response.text[:200]}", + ) + if reg.signing is not None: + self._verify( + reg=reg, raw=response.content, header=response.headers.get(signing.SIGNATURE_HEADER) + ) + return [_parse_series(reg_id=reg.id, payload=_decode_json(response))] + + @staticmethod + def _verify(*, reg: RegressorConfig, raw: bytes, header: str | None) -> None: + assert reg.signing is not None # caller-guarded + try: + secret = signing.resolve_secret_ref(reg.signing.secret_ref) + except signing.SecretResolutionError as exc: + # A missing/empty signing secret is a config error, but we still + # route it through the regressor's graceful-degradation path + # (rather than crashing the whole query) so one mis-set Secret + # doesn't take down every series of the query. + raise WebhookError( + "signature_invalid", + f"webhook regressor {reg.id!r} signing secret could not be resolved: {exc}", + ) from exc + if not signing.verify_signature(secret, raw, header): + raise WebhookError( + "signature_invalid", + f"webhook regressor {reg.id!r} response failed HMAC verification", + ) + + +def _decode_json(response: httpx.Response) -> dict[str, object]: + try: + payload = response.json() + except ValueError as exc: + raise WebhookError("webhook_failure", f"webhook response was not JSON: {exc}") from exc + if not isinstance(payload, dict): + raise WebhookError("webhook_failure", "webhook response was not a JSON object") + return payload + + +def _parse_series(*, reg_id: str, payload: dict[str, object]) -> SeriesFrame: + """Build a :class:`SeriesFrame` from ``{"timestamps": [...], "values": [...]}``. + + Timestamps may be epoch seconds (int/float) or ISO-8601 strings. NaN / + unparseable points are dropped pairwise so a single bad sample doesn't + poison the whole series. + """ + raw_ts = payload.get("timestamps") + raw_vals = payload.get("values") + if not isinstance(raw_ts, list) or not isinstance(raw_vals, list): + raise WebhookError( + "webhook_failure", + "webhook response must contain 'timestamps' and 'values' arrays", + ) + if len(raw_ts) != len(raw_vals): + raise WebhookError( + "webhook_failure", + f"webhook 'timestamps' ({len(raw_ts)}) and 'values' ({len(raw_vals)}) " + "have different lengths", + ) + timestamps: list[datetime] = [] + values: list[float] = [] + for ts, val in zip(raw_ts, raw_vals, strict=True): + parsed_ts = _parse_timestamp(ts) + parsed_val = _parse_value(val) + if parsed_ts is None or parsed_val is None: + continue + timestamps.append(parsed_ts) + values.append(parsed_val) + if not values: + raise WebhookError( + "webhook_failure", "webhook response carried no usable (timestamp, value) pairs" + ) + return SeriesFrame(labels={"regressor_id": reg_id}, timestamps=timestamps, values=values) + + +def _parse_timestamp(ts: object) -> datetime | None: + from datetime import UTC # noqa: PLC0415 + + if isinstance(ts, bool): + return None + if isinstance(ts, (int, float)): + try: + return datetime.fromtimestamp(float(ts), tz=UTC) + except (OverflowError, OSError, ValueError): + return None + if isinstance(ts, str): + try: + parsed = datetime.fromisoformat(ts) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC) + return None + + +def _parse_value(val: object) -> float | None: + import math # noqa: PLC0415 + + if isinstance(val, bool): + return None + if isinstance(val, (int, float)): + f = float(val) + return None if math.isnan(f) else f + return None diff --git a/forecaster/tests/test_annotations.py b/forecaster/tests/test_annotations.py new file mode 100644 index 0000000..2dcba63 --- /dev/null +++ b/forecaster/tests/test_annotations.py @@ -0,0 +1,156 @@ +"""Tests for the Grafana-annotations sink.""" + +from __future__ import annotations + +import json + +import httpx +import pytest + +from promforecast.annotations import AnnotationEvent, GrafanaAnnotationSink +from promforecast.config import AnnotationsConfig +from promforecast.exporter import Exporter + + +def _sink(config: AnnotationsConfig, exporter: Exporter, handler: object) -> GrafanaAnnotationSink: + http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) # type: ignore[arg-type] + return GrafanaAnnotationSink(config, exporter, client=http, now_ms=lambda: 1_700_000_000_000) + + +def _config(events: list[str], **kwargs: object) -> AnnotationsConfig: + base: dict[str, object] = { + "enabled": True, + "grafana_url": "https://grafana.example.com", + "events": events, + } + base.update(kwargs) + return AnnotationsConfig(**base) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_posts_enabled_event() -> None: + posted: list[dict[str, object]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/annotations" + posted.append(json.loads(request.content)) + return httpx.Response(200) + + exporter = Exporter() + sink = _sink(_config(["drift"]), exporter, handler) + event = AnnotationEvent( + event_type="drift", dedup_key="g:m", tags=["promforecast", "drift"], text="drift!" + ) + try: + await sink.post_events([event]) + finally: + await sink.aclose() + assert len(posted) == 1 + assert posted[0]["text"] == "drift!" + assert posted[0]["time"] == 1_700_000_000_000 + body = exporter.render().decode() + assert 'forecast_annotation_posts_total{event_type="drift",outcome="success"} 1.0' in body + + +@pytest.mark.asyncio +async def test_disabled_event_not_posted() -> None: + posts = {"n": 0} + + def handler(_: httpx.Request) -> httpx.Response: + posts["n"] += 1 + return httpx.Response(200) + + exporter = Exporter() + sink = _sink(_config(["drift"]), exporter, handler) + event = AnnotationEvent(event_type="deploy_risk", dedup_key="svc:1", tags=[], text="x") + try: + await sink.post_events([event]) + finally: + await sink.aclose() + assert posts["n"] == 0 + + +@pytest.mark.asyncio +async def test_rising_edge_dedup() -> None: + posts = {"n": 0} + + def handler(_: httpx.Request) -> httpx.Response: + posts["n"] += 1 + return httpx.Response(200) + + exporter = Exporter() + sink = _sink(_config(["deviation_outside_band"]), exporter, handler) + event = AnnotationEvent( + event_type="deviation_outside_band", dedup_key="g:m:s", tags=[], text="outside" + ) + try: + await sink.post_events([event]) # rising edge + await sink.post_events([event]) # sustained -> suppressed + assert posts["n"] == 1 + await sink.post_events([]) # clears the active key + await sink.post_events([event]) # re-fires + assert posts["n"] == 2 + finally: + await sink.aclose() + + +@pytest.mark.asyncio +async def test_rising_edge_dedup_is_scoped_per_group() -> None: + # The runner calls post_events once per group, each with only that + # group's events. A sustained event in group "a" must not be re-posted + # just because group "b" posts its own (different) event in between. + posts: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + posts.append(json.loads(request.content)["text"]) + return httpx.Response(200) + + exporter = Exporter() + sink = _sink(_config(["drift"]), exporter, handler) + ev_a = AnnotationEvent(event_type="drift", dedup_key="a:m", tags=[], text="a-drift") + ev_b = AnnotationEvent(event_type="drift", dedup_key="b:m", tags=[], text="b-drift") + try: + # Cycle 1: both groups fire their rising edge. + await sink.post_events([ev_a], scope="a") + await sink.post_events([ev_b], scope="b") + # Cycle 2: both still firing — both must be suppressed, not re-posted. + await sink.post_events([ev_a], scope="a") + await sink.post_events([ev_b], scope="b") + finally: + await sink.aclose() + assert posts == ["a-drift", "b-drift"] + + +@pytest.mark.asyncio +async def test_auth_token_header(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GRAFANA_TOKEN", "glsa_abc") + captured: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["auth"] = request.headers.get("Authorization", "") + return httpx.Response(200) + + exporter = Exporter() + sink = _sink(_config(["drift"], api_token_ref="${GRAFANA_TOKEN}"), exporter, handler) + event = AnnotationEvent(event_type="drift", dedup_key="g:m", tags=[], text="d") + try: + await sink.post_events([event]) + finally: + await sink.aclose() + assert captured["auth"] == "Bearer glsa_abc" + + +@pytest.mark.asyncio +async def test_http_error_counts_outcome() -> None: + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response(503) + + exporter = Exporter() + sink = _sink(_config(["drift"]), exporter, handler) + event = AnnotationEvent(event_type="drift", dedup_key="g:m", tags=[], text="d") + try: + await sink.post_events([event]) + finally: + await sink.aclose() + body = exporter.render().decode() + assert 'forecast_annotation_posts_total{event_type="drift",outcome="http_error"} 1.0' in body diff --git a/forecaster/tests/test_calendar_file.py b/forecaster/tests/test_calendar_file.py new file mode 100644 index 0000000..ca2f28c --- /dev/null +++ b/forecaster/tests/test_calendar_file.py @@ -0,0 +1,128 @@ +"""Tests for the static calendar-file regressor source (YAML + iCal).""" + +from __future__ import annotations + +import datetime as dt +from pathlib import Path + +import pytest + +from promforecast import calendar_file + + +def _write(tmp_path: Path, name: str, body: str) -> str: + path = tmp_path / name + path.write_text(body) + return str(path) + + +def test_parse_yaml_entries(tmp_path: Path) -> None: + path = _write( + tmp_path, + "freezes.yaml", + """ +entries: + - start: 2026-01-01 + end: 2026-01-05 + name: new-year-freeze + - start: 2026-07-01 + end: 2026-07-01 + name: q3-close +""", + ) + entries = calendar_file.load_calendar(path) + assert len(entries) == 2 + assert entries[0].name == "new-year-freeze" + assert entries[0].contains(dt.date(2026, 1, 3)) + assert not entries[0].contains(dt.date(2026, 1, 6)) + assert entries[1].start == entries[1].end == dt.date(2026, 7, 1) + + +def test_parse_yaml_bare_list(tmp_path: Path) -> None: + path = _write( + tmp_path, + "list.yaml", + """ +- start: 2026-02-01 + end: 2026-02-02 + name: campaign +""", + ) + entries = calendar_file.load_calendar(path) + assert entries[0].name == "campaign" + + +def test_parse_yaml_rejects_end_before_start(tmp_path: Path) -> None: + path = _write( + tmp_path, + "bad.yaml", + """ +entries: + - start: 2026-02-05 + end: 2026-02-01 + name: backwards +""", + ) + with pytest.raises(calendar_file.CalendarFileError): + calendar_file.load_calendar(path) + + +def test_parse_yaml_rejects_missing_name(tmp_path: Path) -> None: + path = _write( + tmp_path, + "noname.yaml", + """ +entries: + - start: 2026-02-05 + end: 2026-02-06 +""", + ) + with pytest.raises(calendar_file.CalendarFileError): + calendar_file.load_calendar(path) + + +def test_parse_ical(tmp_path: Path) -> None: + path = _write( + tmp_path, + "cal.ics", + """BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART;VALUE=DATE:20260101 +DTEND;VALUE=DATE:20260105 +SUMMARY:New Year Freeze +END:VEVENT +BEGIN:VEVENT +DTSTART:20260701T090000Z +SUMMARY:Q3 Close +END:VEVENT +END:VCALENDAR +""", + ) + entries = calendar_file.load_calendar(path) + assert len(entries) == 2 + assert entries[0].name == "New Year Freeze" + assert entries[0].contains(dt.date(2026, 1, 3)) + # All-day DTEND is exclusive (RFC 5545): 20260105 means "through Jan 4". + assert entries[0].end == dt.date(2026, 1, 4) + assert not entries[0].contains(dt.date(2026, 1, 5)) + # No DTEND -> single-day window. + assert entries[1].start == entries[1].end == dt.date(2026, 7, 1) + + +def test_missing_file_raises() -> None: + with pytest.raises(calendar_file.CalendarFileError): + calendar_file.get_calendar("/nonexistent/path/to/calendar.yaml") + + +def test_get_calendar_caches_and_reparses_on_mtime_change(tmp_path: Path) -> None: + path = tmp_path / "c.yaml" + path.write_text("entries:\n - start: 2026-01-01\n end: 2026-01-02\n name: a\n") + first = calendar_file.get_calendar(str(path)) + assert [e.name for e in first] == ["a"] + # Rewrite with a different mtime so the cache is invalidated. + import os # noqa: PLC0415 + + path.write_text("entries:\n - start: 2026-01-01\n end: 2026-01-02\n name: b\n") + os.utime(path, (1_000_000_000, 1_000_000_000)) + second = calendar_file.get_calendar(str(path)) + assert [e.name for e in second] == ["b"] diff --git a/forecaster/tests/test_canary.py b/forecaster/tests/test_canary.py new file mode 100644 index 0000000..28f59cd --- /dev/null +++ b/forecaster/tests/test_canary.py @@ -0,0 +1,90 @@ +"""Tests for canary selector parsing, frame splitting, and aggregation.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from promforecast import canary +from promforecast.source import SeriesFrame + + +def _frame(labels: dict[str, str], values: list[float]) -> SeriesFrame: + base = datetime(2026, 1, 1, tzinfo=UTC) + ts = [base + timedelta(hours=i) for i in range(len(values))] + return SeriesFrame(labels=labels, timestamps=ts, values=values) + + +def test_parse_selector_equality_and_inequality() -> None: + matchers = canary.parse_selector('{version="canary", tier!="db"}') + assert len(matchers) == 2 + assert matchers[0].key == "version" + assert matchers[0].op == "=" + assert matchers[1].op == "!=" + + +def test_parse_selector_without_braces() -> None: + matchers = canary.parse_selector('version="canary"') + assert matchers[0].value == "canary" + + +def test_parse_selector_empty_matches_everything() -> None: + assert canary.parse_selector("") == [] + assert canary.parse_selector("{}") == [] + + +def test_parse_selector_rejects_regex() -> None: + with pytest.raises(canary.CanarySelectorError): + canary.parse_selector('{version=~"can.*"}') + + +def test_matcher_matches() -> None: + matchers = canary.parse_selector('{version="canary"}') + assert canary.matches_all({"version": "canary"}, matchers) + assert not canary.matches_all({"version": "stable"}, matchers) + + +def test_split_frames_partitions_populations() -> None: + frames = [ + _frame({"version": "canary", "pod": "a"}, [1.0]), + _frame({"version": "stable", "pod": "b"}, [2.0]), + _frame({"version": "stable", "pod": "c"}, [3.0]), + ] + canary_frames, baseline_frames = canary.split_frames( + frames, + canary_selector='{version="canary"}', + baseline_selector='{version!="canary"}', + ) + assert [f.labels["pod"] for f in canary_frames] == ["a"] + assert [f.labels["pod"] for f in baseline_frames] == ["b", "c"] + + +def test_split_frames_rejects_overlapping_selectors() -> None: + frames = [_frame({"version": "canary"}, [1.0])] + with pytest.raises(canary.CanarySelectorError): + canary.split_frames( + frames, + canary_selector='{version="canary"}', + baseline_selector='{version="canary"}', + ) + + +def test_aggregate_baseline_means_across_series() -> None: + frames = [ + _frame({"pod": "b"}, [2.0, 4.0]), + _frame({"pod": "c"}, [4.0, 8.0]), + ] + ts, values = canary.aggregate_baseline(frames) + assert len(ts) == 2 + assert values == [3.0, 6.0] + + +def test_aggregate_baseline_empty() -> None: + assert canary.aggregate_baseline([]) == ([], []) + + +def test_latest_actual() -> None: + frame = _frame({"pod": "a"}, [1.0, 2.0, 5.0]) + assert canary.latest_actual(frame) == 5.0 + assert canary.latest_actual(_frame({"pod": "x"}, [])) is None diff --git a/forecaster/tests/test_config_v19.py b/forecaster/tests/test_config_v19.py new file mode 100644 index 0000000..ca14a52 --- /dev/null +++ b/forecaster/tests/test_config_v19.py @@ -0,0 +1,329 @@ +"""Config-schema tests for the ecosystem/integration features. + +Covers the webhook + calendar_file regressor types, the per-query canary +comparison block, the top-level triggers and annotations blocks, and the +shared signing block. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import promforecast.config as config_module + +_HEADER = """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +""" + + +def _write(tmp_path: Path, body: str) -> Path: + path = tmp_path / "config.yaml" + path.write_text(_HEADER + body) + return path + + +# -- webhook regressor ------------------------------------------------------ + + +def test_webhook_regressor_loads(tmp_path: Path) -> None: + cfg = config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up + regressors: + - id: incidents + type: webhook + url: https://pagerduty.example/incidents + timeout: 5s + cache_ttl: 1h + headers: + Authorization: Bearer abc + signing: + algorithm: hmac-sha256 + secret_ref: ${HOOK_SECRET} +""", + ) + ) + reg = cfg.groups[0].queries[0].regressors[0] + assert reg.type == "webhook" + assert reg.url == "https://pagerduty.example/incidents" + assert reg.signing is not None + assert reg.signing.secret_ref == "${HOOK_SECRET}" + + +def test_webhook_regressor_requires_url(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="webhook regressors require a non-empty url"): + config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up + regressors: + - id: incidents + type: webhook +""", + ) + ) + + +def test_webhook_regressor_rejects_promql(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="webhook regressors must not set promql"): + config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up + regressors: + - id: incidents + type: webhook + url: https://x/y + promql: up +""", + ) + ) + + +# -- calendar_file regressor ------------------------------------------------ + + +def test_calendar_file_regressor_loads(tmp_path: Path) -> None: + cfg = config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up + regressors: + - id: release_freeze + type: calendar_file + path: /etc/promforecast/calendars/release-freezes.yaml + expand: true +""", + ) + ) + reg = cfg.groups[0].queries[0].regressors[0] + assert reg.type == "calendar_file" + assert reg.path.endswith("release-freezes.yaml") + assert reg.expand is True + + +def test_calendar_file_regressor_requires_path(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="calendar_file regressors require a non-empty path"): + config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up + regressors: + - id: cal + type: calendar_file +""", + ) + ) + + +# -- canary comparison ------------------------------------------------------ + + +def test_canary_comparison_loads(tmp_path: Path) -> None: + cfg = config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: http_requests_total + canary: + enabled: true + canary_selector: '{version="canary"}' + baseline_selector: '{version!="canary"}' +""", + ) + ) + canary = cfg.groups[0].queries[0].canary + assert canary.enabled is True + assert canary.canary_selector == '{version="canary"}' + + +def test_canary_comparison_requires_selectors(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="canary_selector is required"): + config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up + canary: + enabled: true + baseline_selector: '{version!="canary"}' +""", + ) + ) + + +# -- triggers --------------------------------------------------------------- + + +def test_triggers_load(tmp_path: Path) -> None: + cfg = config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up +triggers: + - name: disk-fill-soon + condition: 'node_filesystem_avail_bytes_forecast{horizon="24h"} < 0' + webhook_url: https://hooks.slack.example/abc + repeat_interval: 1h + payload_template: '{"text": "disk fill predicted"}' + signing: + secret_ref: ${TRIGGER_SECRET} +""", + ) + ) + assert cfg.triggers[0].name == "disk-fill-soon" + assert cfg.triggers[0].signing is not None + + +def test_triggers_reject_duplicate_names(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="appears twice"): + config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up +triggers: + - name: dup + condition: up + webhook_url: https://x/y + - name: dup + condition: down + webhook_url: https://x/z +""", + ) + ) + + +# -- annotations ------------------------------------------------------------ + + +def test_annotations_load(tmp_path: Path) -> None: + cfg = config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up +annotations: + enabled: true + grafana_url: https://grafana.example.com + api_token_ref: ${GRAFANA_TOKEN} + events: [drift, deploy_risk, deviation_outside_band] +""", + ) + ) + assert cfg.annotations.enabled is True + assert "drift" in cfg.annotations.events + # Defaults to the "elevated" gate for deploy-risk annotations. + assert cfg.annotations.deploy_risk_min_score == 0.5 + + +def test_annotations_deploy_risk_min_score_override(tmp_path: Path) -> None: + cfg = config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up +annotations: + enabled: true + grafana_url: https://grafana.example.com + events: [deploy_risk] + deploy_risk_min_score: 0.8 +""", + ) + ) + assert cfg.annotations.deploy_risk_min_score == 0.8 + + +def test_annotations_deploy_risk_min_score_out_of_range(tmp_path: Path) -> None: + with pytest.raises(ValueError, match=r"deploy_risk_min_score"): + config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up +annotations: + enabled: true + grafana_url: https://grafana.example.com + events: [deploy_risk] + deploy_risk_min_score: 1.5 +""", + ) + ) + + +def test_annotations_enabled_requires_url(tmp_path: Path) -> None: + with pytest.raises(ValueError, match=r"annotations\.grafana_url is empty"): + config_module.load( + _write( + tmp_path, + """ +groups: + - name: g1 + queries: + - id: m + promql: up +annotations: + enabled: true + events: [drift] +""", + ) + ) diff --git a/forecaster/tests/test_regressors.py b/forecaster/tests/test_regressors.py index ced1afd..f0aa171 100644 --- a/forecaster/tests/test_regressors.py +++ b/forecaster/tests/test_regressors.py @@ -14,6 +14,7 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta +from pathlib import Path import pytest @@ -25,6 +26,7 @@ prefetch_promql, ) from promforecast.source import PromSource, SeriesFrame +from promforecast.webhook import WebhookError class _FakeSource(PromSource): @@ -214,3 +216,162 @@ def test_align_mixes_calendar_and_promql() -> None: assert out.frame is not None assert list(out.frame["hour_of_day"]) == [12.0, 13.0] assert list(out.frame["deploys"]) == [10.0, 20.0] + + +# -- webhook regressors (prefetch wiring) ---------------------------------- + + +class _FakeWebhookClient: + """Returns canned frames (or raises) without touching the network.""" + + def __init__(self, result: object) -> None: + self._result = result + self.calls = 0 + + async def fetch( + self, + *, + reg: RegressorConfig, + query_id: str, + start: datetime, + end: datetime, + ) -> list[SeriesFrame]: + self.calls += 1 + if isinstance(self._result, Exception): + raise self._result + return self._result # type: ignore[return-value] + + async def aclose(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_prefetch_webhook_success_is_aligned() -> None: + timestamps = _hourly_timestamps(2) + frame = SeriesFrame( + labels={"regressor_id": "incidents"}, timestamps=timestamps, values=[1.0, 2.0] + ) + client = _FakeWebhookClient([frame]) + reg = RegressorConfig(id="incidents", type="webhook", url="https://hook/x") + pf = await prefetch_promql( + regressors=[reg], + source=None, + query_start=timestamps[0], + query_end=timestamps[-1], + step_seconds=3600, + query_timeout_seconds=5.0, + query_id="cpu", + webhook_client=client, # type: ignore[arg-type] + ) + assert client.calls == 1 + out = align_for_series(regressors=[reg], prefetch=pf, timestamps=timestamps) + assert out.frame is not None + assert list(out.frame["incidents"]) == [1.0, 2.0] + + +@pytest.mark.asyncio +async def test_prefetch_webhook_optional_failure_degrades() -> None: + client = _FakeWebhookClient(WebhookError("webhook_failure", "boom")) + reg = RegressorConfig(id="incidents", type="webhook", url="https://hook/x") + pf = await prefetch_promql( + regressors=[reg], + source=None, + query_start=datetime.now(tz=UTC), + query_end=datetime.now(tz=UTC), + step_seconds=3600, + query_timeout_seconds=5.0, + query_id="cpu", + webhook_client=client, # type: ignore[arg-type] + ) + assert pf.fetched == {} + assert [f.reason for f in pf.failures] == ["webhook_failure"] + + +@pytest.mark.asyncio +async def test_prefetch_webhook_required_failure_raises() -> None: + client = _FakeWebhookClient(WebhookError("signature_invalid", "bad sig")) + reg = RegressorConfig(id="incidents", type="webhook", url="https://hook/x", required=True) + with pytest.raises(RegressorError) as excinfo: + await prefetch_promql( + regressors=[reg], + source=None, + query_start=datetime.now(tz=UTC), + query_end=datetime.now(tz=UTC), + step_seconds=3600, + query_timeout_seconds=5.0, + query_id="cpu", + webhook_client=client, # type: ignore[arg-type] + ) + assert excinfo.value.reason == "signature_invalid" + + +@pytest.mark.asyncio +async def test_prefetch_webhook_no_client_degrades() -> None: + reg = RegressorConfig(id="incidents", type="webhook", url="https://hook/x") + pf = await prefetch_promql( + regressors=[reg], + source=None, + query_start=datetime.now(tz=UTC), + query_end=datetime.now(tz=UTC), + step_seconds=3600, + query_timeout_seconds=5.0, + query_id="cpu", + webhook_client=None, + ) + assert [f.reason for f in pf.failures] == ["no_source"] + + +# -- calendar_file regressors (per-series alignment) ----------------------- + + +def _write_calendar(tmp_path: object, body: str) -> str: + path = Path(str(tmp_path)) / "freezes.yaml" + path.write_text(body) + return str(path) + + +def test_align_calendar_file_binary(tmp_path: object) -> None: + path = _write_calendar( + tmp_path, + "entries:\n - start: 2024-06-03\n end: 2024-06-03\n name: freeze\n", + ) + # _hourly_timestamps starts 2024-06-03 12:00 UTC; 2 points same day. + timestamps = _hourly_timestamps(2) + reg = RegressorConfig(id="release_freeze", type="calendar_file", path=path) + out = align_for_series( + regressors=[reg], + prefetch=PromQLPrefetch(fetched={}, failures=[]), + timestamps=timestamps, + ) + assert out.frame is not None + assert list(out.frame["release_freeze"]) == [1.0, 1.0] + + +def test_align_calendar_file_expand_one_hot(tmp_path: object) -> None: + path = _write_calendar( + tmp_path, + "entries:\n" + " - start: 2024-06-03\n end: 2024-06-03\n name: freeze-a\n" + " - start: 2030-01-01\n end: 2030-01-01\n name: freeze-b\n", + ) + timestamps = _hourly_timestamps(1) + reg = RegressorConfig(id="cal", type="calendar_file", path=path, expand=True) + out = align_for_series( + regressors=[reg], + prefetch=PromQLPrefetch(fetched={}, failures=[]), + timestamps=timestamps, + ) + assert out.frame is not None + assert list(out.frame["cal__freeze_a"]) == [1.0] + assert list(out.frame["cal__freeze_b"]) == [0.0] + + +def test_align_calendar_file_missing_file_degrades() -> None: + reg = RegressorConfig(id="cal", type="calendar_file", path="/nope/missing.yaml") + out = align_for_series( + regressors=[reg], + prefetch=PromQLPrefetch(fetched={}, failures=[]), + timestamps=_hourly_timestamps(2), + ) + assert out.frame is None + assert [f.reason for f in out.failures] == ["calendar_file_error"] diff --git a/forecaster/tests/test_runner_canary.py b/forecaster/tests/test_runner_canary.py new file mode 100644 index 0000000..32cd417 --- /dev/null +++ b/forecaster/tests/test_runner_canary.py @@ -0,0 +1,121 @@ +"""Integration test for the canary-vs-baseline deviation emission. + +Uses the real SeasonalNaive model on a small baseline so the runner path +(split → baseline fit → per-canary deviation → exporter family) is exercised +end to end without mocking the fit. +""" + +from __future__ import annotations + +import math +from datetime import UTC, datetime, timedelta + +import pytest + +from promforecast.config import ( + CanaryComparisonConfig, + Config, + DatasourceConfig, + DefaultsConfig, + GroupConfig, + QueryConfig, + SafetyConfig, + ServerConfig, +) +from promforecast.exporter import Exporter +from promforecast.runner import Runner +from promforecast.source import PromSource, SeriesFrame + + +class _FakeSource(PromSource): + def __init__(self, by_query: dict[str, list[SeriesFrame]]) -> None: + self._by_query = by_query + + async def query_range( + self, promql: str, start: datetime, end: datetime, step_seconds: int + ) -> list[SeriesFrame]: + return self._by_query.get(promql, []) + + async def aclose(self) -> None: + return None + + +def _series(labels: dict[str, str], values: list[float]) -> SeriesFrame: + base = datetime(2026, 1, 1, tzinfo=UTC) + ts = [base + timedelta(hours=i) for i in range(len(values))] + return SeriesFrame(labels=labels, timestamps=ts, values=values) + + +def _canary_config() -> Config: + return Config( + datasource=DatasourceConfig(url="http://x"), + server=ServerConfig(), + safety=SafetyConfig(max_series_per_query=500, max_total_series=0), + defaults=DefaultsConfig( + min_points=10, + models=["SeasonalNaive"], + step=timedelta(hours=1), + horizon=timedelta(hours=3), + season_length=12, + confidence_levels=[80], + ), + groups=[ + GroupConfig( + name="g", + queries=[ + QueryConfig( + id="http_requests", + promql="http_requests_total", + canary=CanaryComparisonConfig( + enabled=True, + canary_selector='{version="canary"}', + baseline_selector='{version!="canary"}', + ), + ) + ], + ) + ], + ) + + +@pytest.mark.asyncio +async def test_run_group_emits_canary_deviation() -> None: + n = 40 + baseline_a = _series({"version": "stable", "pod": "a"}, [10.0 + (i % 12) for i in range(n)]) + baseline_b = _series({"version": "stable", "pod": "b"}, [12.0 + (i % 12) for i in range(n)]) + # Canary pod running far hotter than the baseline band predicts. + canary = _series({"version": "canary", "pod": "c"}, [200.0 + (i % 12) for i in range(n)]) + source = _FakeSource({"http_requests_total": [baseline_a, baseline_b, canary]}) + exporter = Exporter() + runner = Runner(config=_canary_config(), source=source, exporter=exporter) + + await runner.run_group(_canary_config().groups[0]) + + body = exporter.render().decode() + assert "http_requests_canary_forecast_deviation" in body + assert 'pod="c"' in body + # The hot canary should land outside the baseline band. + assert "http_requests_canary_forecast_deviation_outside_band" in body + outside = [ + line + for line in body.splitlines() + if line.startswith("http_requests_canary_forecast_deviation_outside_band") + and not line.startswith("#") + ] + assert outside, "expected an outside_band sample" + assert any(math.isclose(float(line.rsplit(" ", 1)[1]), 1.0) for line in outside) + + +@pytest.mark.asyncio +async def test_canary_skipped_when_population_missing() -> None: + n = 40 + # Only baseline series — no canary population. + baseline = _series({"version": "stable", "pod": "a"}, [10.0 + (i % 12) for i in range(n)]) + source = _FakeSource({"http_requests_total": [baseline]}) + exporter = Exporter() + runner = Runner(config=_canary_config(), source=source, exporter=exporter) + + await runner.run_group(_canary_config().groups[0]) + + body = exporter.render().decode() + assert "http_requests_canary_forecast_deviation" not in body diff --git a/forecaster/tests/test_runner_integrations_reload.py b/forecaster/tests/test_runner_integrations_reload.py new file mode 100644 index 0000000..da27769 --- /dev/null +++ b/forecaster/tests/test_runner_integrations_reload.py @@ -0,0 +1,134 @@ +"""Hot-reload coverage for the v1.9 outbound/inbound integrations. + +A config reload that adds, changes, or removes ``triggers:`` / +``annotations:`` / webhook regressors must reconcile the live trigger +engine, annotation sink, and webhook client without a process restart. +The fit path is irrelevant here, so ``run_group`` is stubbed to a no-op to +keep the respawned group loops inert. +""" + +from __future__ import annotations + +import pytest + +from promforecast.annotations import GrafanaAnnotationSink +from promforecast.config import ( + AnnotationsConfig, + Config, + DatasourceConfig, + DefaultsConfig, + GroupConfig, + QueryConfig, + RegressorConfig, + SafetyConfig, + ServerConfig, + TriggerConfig, +) +from promforecast.exporter import Exporter +from promforecast.runner import Runner +from promforecast.source import PromSource + + +class _FakeSource(PromSource): + def __init__(self) -> None: + pass + + async def aclose(self) -> None: + return None + + +def _config( + *, + triggers: list[TriggerConfig] | None = None, + annotations: AnnotationsConfig | None = None, + webhook: bool = False, +) -> Config: + regressors = ( + [RegressorConfig(id="wh", type="webhook", url="https://bridge/x")] if webhook else [] + ) + return Config( + datasource=DatasourceConfig(url="http://x"), + server=ServerConfig(), + safety=SafetyConfig(), + defaults=DefaultsConfig(models=["AutoARIMA"]), + groups=[ + GroupConfig( + name="g", + queries=[QueryConfig(id="m", promql="up", regressors=regressors)], + ) + ], + triggers=triggers or [], + annotations=annotations or AnnotationsConfig(), + ) + + +@pytest.mark.asyncio +async def test_reload_wires_and_tears_down_integrations(monkeypatch: pytest.MonkeyPatch) -> None: + async def _noop_run_group(self: Runner, group: object, queries: object = None) -> None: + return None + + monkeypatch.setattr(Runner, "run_group", _noop_run_group) + + runner = Runner(config=_config(), source=_FakeSource(), exporter=Exporter()) + # Nothing configured at startup. + assert runner._trigger_engine is None + assert runner._annotation_sink is None + assert runner._webhook_client is None + + try: + # Reload adds all three. + await runner.apply_config( + _config( + triggers=[ + TriggerConfig(name="t", condition="up == 1", webhook_url="https://hook/x") + ], + annotations=AnnotationsConfig( + enabled=True, grafana_url="https://grafana", events=["drift"] + ), + webhook=True, + ) + ) + assert runner._trigger_engine is not None + assert runner._annotation_sink is not None + assert runner._webhook_client is not None + + # Reload removes all three again. + await runner.apply_config(_config()) + assert runner._trigger_engine is None + assert runner._annotation_sink is None + assert runner._webhook_client is None + finally: + await runner.stop() + + +@pytest.mark.asyncio +async def test_reload_preserves_annotation_dedup_when_unchanged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # An unrelated reload (a group changes, annotations block identical) must + # NOT rebuild the annotation sink, or the rising-edge dedup state resets + # and every active annotation re-posts. + async def _noop_run_group(self: Runner, group: object, queries: object = None) -> None: + return None + + monkeypatch.setattr(Runner, "run_group", _noop_run_group) + + ann = AnnotationsConfig(enabled=True, grafana_url="https://grafana", events=["drift"]) + exporter = Exporter() + # The sink is built and injected by main.py at startup; mirror that here. + runner = Runner( + config=_config(annotations=ann), + source=_FakeSource(), + exporter=exporter, + annotation_sink=GrafanaAnnotationSink(ann, exporter), + ) + sink_before = runner._annotation_sink + assert sink_before is not None + try: + # Change only the group's query id; annotations block is identical. + changed = _config(annotations=ann) + changed.groups[0].queries[0] = QueryConfig(id="m2", promql="up") + await runner.apply_config(changed) + assert runner._annotation_sink is sink_before # same instance, state intact + finally: + await runner.stop() diff --git a/forecaster/tests/test_signing.py b/forecaster/tests/test_signing.py new file mode 100644 index 0000000..ea37b22 --- /dev/null +++ b/forecaster/tests/test_signing.py @@ -0,0 +1,81 @@ +"""Tests for HMAC signing and secret-reference resolution.""" + +from __future__ import annotations + +import hashlib +import hmac + +import pytest + +from promforecast import signing + + +def test_compute_signature_matches_stdlib_hmac() -> None: + secret = "topsecret" + body = b'{"hello": "world"}' + expected = "sha256=" + hmac.new(secret.encode(), body, hashlib.sha256).hexdigest() + assert signing.compute_signature(secret, body) == expected + + +def test_verify_signature_round_trip() -> None: + secret = "topsecret" + body = b"payload" + sig = signing.compute_signature(secret, body) + assert signing.verify_signature(secret, body, sig) + + +def test_verify_signature_rejects_tampered_body() -> None: + secret = "topsecret" + sig = signing.compute_signature(secret, b"original") + assert not signing.verify_signature(secret, b"tampered", sig) + + +def test_verify_signature_rejects_wrong_secret() -> None: + body = b"payload" + sig = signing.compute_signature("right", body) + assert not signing.verify_signature("wrong", body, sig) + + +def test_verify_signature_missing_header_is_false() -> None: + assert not signing.verify_signature("secret", b"payload", None) + assert not signing.verify_signature("secret", b"payload", "") + + +def test_resolve_secret_ref_literal_passthrough() -> None: + assert signing.resolve_secret_ref("literal-secret") == "literal-secret" + + +def test_resolve_secret_ref_env_expansion(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PF_TEST_SECRET", "from-env") + assert signing.resolve_secret_ref("${PF_TEST_SECRET}") == "from-env" + + +def test_resolve_secret_ref_empty_env_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PF_MISSING_SECRET", raising=False) + with pytest.raises(signing.SecretResolutionError): + signing.resolve_secret_ref("${PF_MISSING_SECRET}") + + +def test_resolve_secret_ref_empty_literal_raises() -> None: + with pytest.raises(signing.SecretResolutionError): + signing.resolve_secret_ref("") + + +def test_expand_env_refs_embedded(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PF_TEST_TOKEN", "abc123") + assert signing.expand_env_refs("Bearer ${PF_TEST_TOKEN}") == "Bearer abc123" + + +def test_expand_env_refs_missing_is_empty(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PF_MISSING_TOKEN", raising=False) + assert signing.expand_env_refs("Bearer ${PF_MISSING_TOKEN}") == "Bearer " + + +def test_expand_env_refs_no_ref_passthrough() -> None: + assert signing.expand_env_refs("plain-value") == "plain-value" + + +def test_expand_header_env_refs(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PF_TEST_TOKEN", "xyz") + out = signing.expand_header_env_refs({"Authorization": "Bearer ${PF_TEST_TOKEN}", "X": "lit"}) + assert out == {"Authorization": "Bearer xyz", "X": "lit"} diff --git a/forecaster/tests/test_triggers.py b/forecaster/tests/test_triggers.py new file mode 100644 index 0000000..fd63ff1 --- /dev/null +++ b/forecaster/tests/test_triggers.py @@ -0,0 +1,287 @@ +"""Tests for the forecast-driven webhook trigger engine.""" + +from __future__ import annotations + +import json +from datetime import timedelta + +import httpx +import pytest + +from promforecast import signing +from promforecast.config import SigningConfig, TriggerConfig +from promforecast.exporter import Exporter +from promforecast.source import InstantSample, PromSource +from promforecast.triggers import TriggerEngine, condition_is_true + + +class _FakeSource(PromSource): + """Returns a scripted instant-query result per call.""" + + def __init__(self, results: list[list[InstantSample]]) -> None: + self._results = results + self._i = 0 + + async def query_instant(self, promql: str) -> list[InstantSample]: + result = self._results[min(self._i, len(self._results) - 1)] + self._i += 1 + return result + + async def aclose(self) -> None: + return None + + +def _engine( + source: PromSource, + exporter: Exporter, + handler: object, + *, + triggers: list[TriggerConfig], + clock: object | None = None, +) -> TriggerEngine: + http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) # type: ignore[arg-type] + kwargs: dict[str, object] = {"now_iso": lambda: "2026-01-01T00:00:00+00:00"} + if clock is not None: + kwargs["clock"] = clock + return TriggerEngine( + triggers=triggers, + source=source, + exporter=exporter, + client=http, + **kwargs, # type: ignore[arg-type] + ) + + +def _trigger(**kwargs: object) -> TriggerConfig: + base: dict[str, object] = { + "name": "t1", + "condition": "up == 1", + "webhook_url": "https://hook/x", + } + base.update(kwargs) + return TriggerConfig(**base) # type: ignore[arg-type] + + +def test_condition_is_true() -> None: + assert condition_is_true([InstantSample(labels={}, value=1.0)]) + assert not condition_is_true([InstantSample(labels={}, value=0.0)]) + assert not condition_is_true([]) + + +@pytest.mark.asyncio +async def test_trigger_fires_on_rising_edge() -> None: + posted: list[dict[str, object]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + posted.append(json.loads(request.content)) + return httpx.Response(200) + + source = _FakeSource([[InstantSample(labels={"model": "AutoARIMA"}, value=1.0)]]) + exporter = Exporter() + engine = _engine(source, exporter, handler, triggers=[_trigger()]) + try: + await engine.evaluate() + finally: + await engine.aclose() + assert len(posted) == 1 + assert posted[0]["name"] == "t1" + assert posted[0]["value"] == 1.0 + body = exporter.render().decode() + assert 'forecast_trigger_fires_total{name="t1",outcome="success"} 1.0' in body + + +@pytest.mark.asyncio +async def test_trigger_does_not_refire_while_sustained() -> None: + posts = {"n": 0} + + def handler(_: httpx.Request) -> httpx.Response: + posts["n"] += 1 + return httpx.Response(200) + + # Condition stays true on every evaluation. + source = _FakeSource([[InstantSample(labels={}, value=1.0)]]) + exporter = Exporter() + now = {"t": 0.0} + engine = _engine( + source, + exporter, + handler, + triggers=[_trigger(repeat_interval=timedelta(hours=1))], + clock=lambda: now["t"], + ) + try: + await engine.evaluate() # rising edge -> fire + now["t"] += 60 # still within repeat_interval + await engine.evaluate() # suppressed + assert posts["n"] == 1 + now["t"] += 3600 # past repeat_interval + await engine.evaluate() # re-fires + assert posts["n"] == 2 + finally: + await engine.aclose() + + +@pytest.mark.asyncio +async def test_trigger_coalesces_per_group_burst() -> None: + # The runner evaluates triggers once per group run, so a refresh cycle + # with several groups calls evaluate() back-to-back. Those calls land + # within the coalescing window and must collapse to a single instant + # query + delivery rather than one per group. + queries = {"n": 0} + + class _CountingSource(PromSource): + def __init__(self) -> None: + pass + + async def query_instant(self, promql: str) -> list[InstantSample]: + queries["n"] += 1 + return [InstantSample(labels={}, value=1.0)] + + async def aclose(self) -> None: + return None + + posts = {"n": 0} + + def handler(_: httpx.Request) -> httpx.Response: + posts["n"] += 1 + return httpx.Response(200) + + exporter = Exporter() + now = {"t": 100.0} + http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + engine = TriggerEngine( + triggers=[_trigger(repeat_interval=timedelta(hours=1))], + source=_CountingSource(), + exporter=exporter, + client=http, + clock=lambda: now["t"], + now_iso=lambda: "2026-01-01T00:00:00+00:00", + coalesce_seconds=5.0, + ) + try: + # Three group runs in the same instant -> one query, one fire. + await engine.evaluate() + await engine.evaluate() + await engine.evaluate() + assert queries["n"] == 1 + assert posts["n"] == 1 + # Past the window the trigger is re-evaluated (query issued again), + # but the sustained condition is still inside repeat_interval -> no + # second delivery. + now["t"] += 10.0 + await engine.evaluate() + assert queries["n"] == 2 + assert posts["n"] == 1 + finally: + await engine.aclose() + + +@pytest.mark.asyncio +async def test_trigger_signs_payload(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TRIGGER_SECRET", "shh") + captured: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["sig"] = request.headers.get(signing.SIGNATURE_HEADER, "") + captured["body"] = request.content.decode() + return httpx.Response(200) + + source = _FakeSource([[InstantSample(labels={}, value=1.0)]]) + exporter = Exporter() + trigger = _trigger(signing=SigningConfig(secret_ref="${TRIGGER_SECRET}")) + engine = _engine(source, exporter, handler, triggers=[trigger]) + try: + await engine.evaluate() + finally: + await engine.aclose() + expected = signing.compute_signature("shh", captured["body"].encode()) + assert captured["sig"] == expected + + +@pytest.mark.asyncio +async def test_trigger_payload_template_renders() -> None: + posted: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + posted.append(request.content.decode()) + return httpx.Response(200) + + source = _FakeSource([[InstantSample(labels={}, value=42.0)]]) + exporter = Exporter() + trigger = _trigger(payload_template='{"text": "value is $value for $name"}') + engine = _engine(source, exporter, handler, triggers=[trigger]) + try: + await engine.evaluate() + finally: + await engine.aclose() + assert posted[0] == '{"text": "value is 42.0 for t1"}' + + +@pytest.mark.asyncio +async def test_trigger_http_error_counts_outcome() -> None: + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response(500) + + source = _FakeSource([[InstantSample(labels={}, value=1.0)]]) + exporter = Exporter() + engine = _engine(source, exporter, handler, triggers=[_trigger()]) + try: + await engine.evaluate() + finally: + await engine.aclose() + body = exporter.render().decode() + assert 'forecast_trigger_fires_total{name="t1",outcome="http_error"} 1.0' in body + + +@pytest.mark.asyncio +async def test_trigger_condition_failure_does_not_fire() -> None: + class _BadSource(PromSource): + def __init__(self) -> None: + pass + + async def query_instant(self, promql: str) -> list[InstantSample]: + raise RuntimeError("tsdb down") + + async def aclose(self) -> None: + return None + + posts = {"n": 0} + + def handler(_: httpx.Request) -> httpx.Response: + posts["n"] += 1 + return httpx.Response(200) + + exporter = Exporter() + engine = _engine(_BadSource(), exporter, handler, triggers=[_trigger()]) + try: + await engine.evaluate() + finally: + await engine.aclose() + assert posts["n"] == 0 + + +@pytest.mark.asyncio +async def test_one_bad_trigger_does_not_block_others(monkeypatch: pytest.MonkeyPatch) -> None: + # First trigger has an unresolvable signing secret (raises mid-fire); + # the second must still be evaluated and delivered. + monkeypatch.delenv("PF_MISSING_TRIGGER_SECRET", raising=False) + posted: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + posted.append(str(request.url)) + return httpx.Response(200) + + source = _FakeSource([[InstantSample(labels={}, value=1.0)]]) + exporter = Exporter() + bad = _trigger( + name="bad", + webhook_url="https://hook/bad", + signing=SigningConfig(secret_ref="${PF_MISSING_TRIGGER_SECRET}"), + ) + good = _trigger(name="good", webhook_url="https://hook/good") + engine = _engine(source, exporter, handler, triggers=[bad, good]) + try: + await engine.evaluate() + finally: + await engine.aclose() + assert "https://hook/good" in posted diff --git a/forecaster/tests/test_webhook.py b/forecaster/tests/test_webhook.py new file mode 100644 index 0000000..7a54e7a --- /dev/null +++ b/forecaster/tests/test_webhook.py @@ -0,0 +1,223 @@ +"""Tests for the webhook regressor client. + +Covers JSON parsing, per-(url, range) caching, HMAC verification, and the +bounded failure reasons that drive ``forecast_regressor_failures_total``. +The end-to-end prefetch wiring (graceful degradation, required short-circuit) +is covered in ``test_regressors.py``. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import httpx +import pytest + +from promforecast import signing +from promforecast.config import RegressorConfig, SigningConfig +from promforecast.webhook import WebhookError, WebhookRegressorClient + + +def _client(handler: object, clock: object | None = None) -> WebhookRegressorClient: + transport = httpx.MockTransport(handler) # type: ignore[arg-type] + http = httpx.AsyncClient(transport=transport) + if clock is None: + return WebhookRegressorClient(client=http) + return WebhookRegressorClient(client=http, clock=clock) # type: ignore[arg-type] + + +def _reg(**kwargs: object) -> RegressorConfig: + base: dict[str, object] = {"id": "incidents", "type": "webhook", "url": "https://hook/x"} + base.update(kwargs) + return RegressorConfig(**base) # type: ignore[arg-type] + + +_START = datetime(2026, 1, 1, tzinfo=UTC) +_END = datetime(2026, 1, 2, tzinfo=UTC) + + +@pytest.mark.asyncio +async def test_fetch_parses_epoch_timestamps() -> None: + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"timestamps": [_START.timestamp(), _END.timestamp()], "values": [1.0, 3.0]}, + ) + + client = _client(handler) + try: + frames = await client.fetch(reg=_reg(), query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert len(frames) == 1 + assert frames[0].values == [1.0, 3.0] + assert frames[0].timestamps[0] == _START + + +@pytest.mark.asyncio +async def test_fetch_expands_env_refs_in_headers(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PF_BRIDGE_TOKEN", "secret-token") + captured: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["auth"] = request.headers.get("Authorization", "") + return httpx.Response(200, json={"timestamps": [_START.timestamp()], "values": [1.0]}) + + client = _client(handler) + reg = _reg(headers={"Authorization": "Bearer ${PF_BRIDGE_TOKEN}"}) + try: + await client.fetch(reg=reg, query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert captured["auth"] == "Bearer secret-token" + + +@pytest.mark.asyncio +async def test_fetch_parses_iso_timestamps_and_drops_nan() -> None: + def handler(_: httpx.Request) -> httpx.Response: + # Raw body so we can include a JSON ``NaN`` token (the standard JSON + # encoder rejects it, but Python's decoder accepts it on parse). + body = ( + f'{{"timestamps": ["{_START.isoformat()}", "{_END.isoformat()}"], ' + '"values": [2.0, NaN]}' + ) + return httpx.Response(200, content=body, headers={"Content-Type": "application/json"}) + + client = _client(handler) + try: + frames = await client.fetch(reg=_reg(), query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert frames[0].values == [2.0] + + +@pytest.mark.asyncio +async def test_fetch_request_body_carries_query_id_and_range() -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + import json # noqa: PLC0415 + + seen.update(json.loads(request.content)) + return httpx.Response(200, json={"timestamps": [_START.timestamp()], "values": [1.0]}) + + client = _client(handler) + try: + await client.fetch(reg=_reg(), query_id="cpu_busy", start=_START, end=_END) + finally: + await client.aclose() + assert seen["query_id"] == "cpu_busy" + assert seen["regressor_id"] == "incidents" + assert seen["start"] == _START.isoformat() + + +@pytest.mark.asyncio +async def test_fetch_caches_within_ttl() -> None: + calls = {"n": 0} + + def handler(_: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(200, json={"timestamps": [_START.timestamp()], "values": [1.0]}) + + now = {"t": 1000.0} + client = _client(handler, clock=lambda: now["t"]) + reg = _reg(cache_ttl=timedelta(hours=1)) + try: + await client.fetch(reg=reg, query_id="cpu", start=_START, end=_END) + now["t"] += 60 # still inside the 1h TTL + await client.fetch(reg=reg, query_id="cpu", start=_START, end=_END) + assert calls["n"] == 1 + now["t"] += 3600 # past TTL + await client.fetch(reg=reg, query_id="cpu", start=_START, end=_END) + assert calls["n"] == 2 + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_fetch_http_error_raises_webhook_failure() -> None: + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response(503, text="overloaded") + + client = _client(handler) + try: + with pytest.raises(WebhookError) as exc: + await client.fetch(reg=_reg(), query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert exc.value.reason == "webhook_failure" + + +@pytest.mark.asyncio +async def test_fetch_bad_shape_raises_webhook_failure() -> None: + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"not": "a timeseries"}) + + client = _client(handler) + try: + with pytest.raises(WebhookError) as exc: + await client.fetch(reg=_reg(), query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert exc.value.reason == "webhook_failure" + + +@pytest.mark.asyncio +async def test_fetch_valid_signature_accepted(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PF_HOOK_SECRET", "s3cret") + body = b'{"timestamps": [1735689600.0], "values": [1.0]}' + sig = signing.compute_signature("s3cret", body) + + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=body, headers={signing.SIGNATURE_HEADER: sig}) + + reg = _reg(signing=SigningConfig(secret_ref="${PF_HOOK_SECRET}")) + client = _client(handler) + try: + frames = await client.fetch(reg=reg, query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert frames[0].values == [1.0] + + +@pytest.mark.asyncio +async def test_fetch_invalid_signature_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PF_HOOK_SECRET", "s3cret") + + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"timestamps": [_START.timestamp()], "values": [1.0]}, + headers={signing.SIGNATURE_HEADER: "sha256=deadbeef"}, + ) + + reg = _reg(signing=SigningConfig(secret_ref="${PF_HOOK_SECRET}")) + client = _client(handler) + try: + with pytest.raises(WebhookError) as exc: + await client.fetch(reg=reg, query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert exc.value.reason == "signature_invalid" + + +@pytest.mark.asyncio +async def test_fetch_unresolvable_secret_degrades_gracefully( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A missing signing secret is a config error, but it must route through + # the graceful-degradation path (WebhookError) rather than crashing the + # whole query with a SecretResolutionError. + monkeypatch.delenv("PF_MISSING_HOOK_SECRET", raising=False) + + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"timestamps": [_START.timestamp()], "values": [1.0]}) + + reg = _reg(signing=SigningConfig(secret_ref="${PF_MISSING_HOOK_SECRET}")) + client = _client(handler) + try: + with pytest.raises(WebhookError) as exc: + await client.fetch(reg=reg, query_id="cpu", start=_START, end=_END) + finally: + await client.aclose() + assert exc.value.reason == "signature_invalid"