Skip to content

Various small improvements related to performance #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dj-wasabi merged 3 commits into from
May 30, 2026
Merged

Various small improvements related to performance #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a5c283f
Usage of multiple backends
dj-wasabi May 30, 2026
00c11aa
Various small improvements everywhere
dj-wasabi May 30, 2026
f49d9eb
fix lint issue
dj-wasabi May 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,10 +145,6 @@ dev-up-load: ## Start the full demo stack: minimum + http-demo + loadgen + Grafa
dev-down: ## Stop the local dev stack (any profile).
docker compose -f docker-compose.dev.yml --profile load down -v

.PHONY: load-test
load-test: ## Synthetic high-cardinality run for safety-control validation.
cd $(DETECTOR) && $(UV) run python -m promanomaly.scripts.load_test

# ---------- Aggregate ----------

.PHONY: ci
Expand Down
42 changes: 41 additions & 1 deletion charts/promanomaly/templates/configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ data:
/etc/promanomaly-datasource-auth and referenced by file, so they
never land in this ConfigMap. Expected Secret keys per type:
bearer -> token; basic -> password (+ auth.username here);
mtls -> ca.crt / tls.crt / tls.key.
mtls -> ca.crt / tls.crt / tls.key;
oauth2 -> client_secret; azure -> client_secret.
*/}}
{{- if eq .Values.datasource.auth.type "bearer" }}
token_file: /etc/promanomaly-datasource-auth/token
Expand All @@ -38,6 +39,45 @@ data:
key_file: /etc/promanomaly-datasource-auth/tls.key
{{- end }}
{{- end }}
{{- if eq .Values.datasource.auth.type "sigv4" }}
sigv4:
region: {{ .Values.datasource.auth.sigv4.region | quote }}
{{- with .Values.datasource.auth.sigv4.profile }}
profile: {{ . | quote }}
{{- end }}
{{- else if eq .Values.datasource.auth.type "gcp" }}
gcp:
{{- with .Values.datasource.auth.gcp.credentials_file }}
credentials_file: {{ . | quote }}
{{- end }}
{{- else if eq .Values.datasource.auth.type "azure" }}
azure:
{{- with .Values.datasource.auth.azure.client_id }}
client_id: {{ . | quote }}
{{- end }}
{{- with .Values.datasource.auth.azure.tenant_id }}
tenant_id: {{ . | quote }}
{{- end }}
{{- if .Values.datasource.auth.azure.existingSecret }}
client_secret_file: /etc/promanomaly-datasource-auth/client_secret
{{- end }}
{{- else if eq .Values.datasource.auth.type "oauth2" }}
oauth2:
token_url: {{ .Values.datasource.auth.oauth2.token_url | quote }}
client_id: {{ .Values.datasource.auth.oauth2.client_id | quote }}
{{- if .Values.datasource.auth.oauth2.existingSecret }}
client_secret_file: /etc/promanomaly-datasource-auth/client_secret
{{- end }}
{{- with .Values.datasource.auth.oauth2.scopes }}
scopes:
{{ toYaml . | indent 12 }}
{{- end }}
{{- end }}
{{- if .Values.datasource.tenant.id }}
tenant:
id: {{ .Values.datasource.tenant.id | quote }}
header: {{ .Values.datasource.tenant.header | quote }}
{{- end }}
server:
listen: {{ .Values.server.listen | quote }}
refresh_interval: {{ .Values.server.refresh_interval | quote }}
Expand Down
77 changes: 75 additions & 2 deletions charts/promanomaly/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,46 @@
"auth": {
"type": "object",
"properties": {
"type": {"type": "string", "enum": ["none", "bearer", "basic", "mtls"]},
"type": {"type": "string", "enum": ["none", "bearer", "basic", "mtls", "sigv4", "gcp", "azure", "oauth2"]},
"username": {"type": "string"},
"existingSecret": {"type": "string"}
"existingSecret": {"type": "string"},
"sigv4": {
"type": "object",
"properties": {
"region": {"type": "string"},
"profile": {"type": "string"}
}
},
"gcp": {
"type": "object",
"properties": {
"credentials_file": {"type": "string"}
}
},
"azure": {
"type": "object",
"properties": {
"client_id": {"type": "string"},
"tenant_id": {"type": "string"},
"existingSecret": {"type": "string"}
}
},
"oauth2": {
"type": "object",
"properties": {
"token_url": {"type": "string"},
"client_id": {"type": "string"},
"existingSecret": {"type": "string"},
"scopes": {"type": "array", "items": {"type": "string"}}
}
}
}
},
"tenant": {
"type": "object",
"properties": {
"id": {"type": "string"},
"header": {"type": "string"}
}
}
}
Expand All @@ -55,6 +92,7 @@
"properties": {
"listen": {"type": "string"},
"refresh_interval": {"type": "string"},
"ready_endpoint": {"type": "boolean"},
"expose_warmup_endpoint": {"type": "boolean"},
"selftest": {
"type": "object",
Expand All @@ -63,6 +101,20 @@
"detector": {"type": "string"},
"threshold": {"type": "number", "exclusiveMinimum": 0}
}
},
"reload": {
"type": "object",
"properties": {
"enabled": {"type": "boolean"},
"watch_configmap": {"type": "boolean"},
"auth": {
"type": "object",
"properties": {
"type": {"type": "string", "enum": ["none", "bearer", "mtls"]},
"existingSecret": {"type": "string"}
}
}
}
}
}
},
Expand All @@ -72,8 +124,11 @@
"max_series_per_query": {"type": "integer", "minimum": 1},
"max_total_series": {"type": "integer", "minimum": 1},
"series_overflow": {"type": "string", "enum": ["drop_lowest_priority", "reject", "sample"]},
"detect_timeout": {"type": "string"},
"query_timeout": {"type": "string"},
"on_source_failure": {"type": "string", "enum": ["serve_stale", "drop_scores", "fail_ready"]},
"fail_ready_after": {"type": "integer", "minimum": 1},
"max_stratified_cache_entries": {"type": "integer", "minimum": 1},
"query_cache": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -102,9 +157,27 @@
"defaults": {
"type": "object",
"properties": {
"window": {"type": "string"},
"step": {"type": "string"},
"min_points": {"type": "integer", "minimum": 1},
"warmup_policy": {
"type": "string",
"enum": ["emit_warming_up", "suppress", "emit_with_flag"]
},
"min_abs_delta": {"type": "number", "minimum": 0},
"min_relative_delta": {"type": "number", "minimum": 0},
"emit_baseline": {"type": "boolean"},
"emit_change_points": {"type": "boolean"},
"emit_baseline_stability": {"type": "boolean"},
"emit_duration": {"type": "boolean"},
"emit_severity": {"type": "boolean"},
"emit_anomaly_type": {"type": "boolean"},
"density_by": {"type": "array", "items": {"type": "string"}},
"alert_thresholds": {
"type": "object",
"properties": {
"score": {"type": "number", "exclusiveMinimum": 0}
}
}
}
},
Expand Down
30 changes: 29 additions & 1 deletion charts/promanomaly/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,43 @@ datasource:
url: http://victoria-metrics-single-server:8428/
timeout: 10s
auth:
type: none # none | bearer | basic | mtls
type: none # none | bearer | basic | mtls | sigv4 | gcp | azure | oauth2
username: "" # basic-auth username (not sensitive; the password
# comes from existingSecret)
# Secret carrying the credential, mounted as files and referenced by
# path so it never lands in the ConfigMap. Expected keys per type:
# bearer -> token
# basic -> password (username above)
# mtls -> ca.crt, tls.crt, tls.key
# oauth2 -> client_secret
existingSecret: ""
# AWS SigV4 signing for Amazon Managed Service for Prometheus.
# Only consulted when type: sigv4.
sigv4:
region: ""
profile: "" # optional AWS CLI profile name
# Google Cloud auth for Google Managed Prometheus.
# Only consulted when type: gcp.
gcp:
credentials_file: "" # path to service-account JSON key; empty = ADC
# Azure auth for Azure Monitor managed Prometheus.
# Only consulted when type: azure.
azure:
client_id: ""
tenant_id: ""
existingSecret: "" # Secret with a `client_secret` key
# Generic OAuth2 client-credentials grant.
# Only consulted when type: oauth2.
oauth2:
token_url: ""
client_id: ""
existingSecret: "" # Secret with a `client_secret` key
scopes: []
# Multi-tenant datasource support (Mimir / Cortex). Sets a tenant
# header on every PromQL request. Overridable per group.
tenant:
id: "" # tenant ID (e.g. "team-a"); empty = disabled
header: X-Scope-OrgID

# Reload endpoint protection. The shipped NetworkPolicy denies ingress
# to /-/reload and the debug endpoints by default; auth is defence in
Expand Down
20 changes: 20 additions & 0 deletions detector/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,23 @@ seasonal = [
matrixprofile = [
"stumpy>=1.12.0",
]
# AWS SigV4 request signing for Amazon Managed Service for Prometheus.
# Uses the standard AWS credential chain. Install with
# ``pip install promanomaly[aws]`` when pointing at an AMP endpoint.
aws = [
"botocore>=1.35.0",
]
# Google Cloud auth for Google Managed Prometheus / Cloud Monitoring.
# Uses Application Default Credentials or Workload Identity. Install
# with ``pip install promanomaly[gcp]``.
gcp = [
"google-auth>=2.35.0",
]
# Azure auth for Azure Monitor managed Prometheus. Uses
# DefaultAzureCredential. Install with ``pip install promanomaly[azure]``.
azure = [
"azure-identity>=1.19.0",
]
dev = [
"pytest>=8.3.0",
"pytest-asyncio>=0.24.0",
Expand Down Expand Up @@ -168,6 +185,9 @@ module = [
"statsmodels.*",
"stumpy.*",
"scipy.*",
"botocore.*",
"google.*",
"azure.*",
]
ignore_missing_imports = true

Expand Down
73 changes: 41 additions & 32 deletions detector/src/promanomaly/anomalies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,52 +237,61 @@ def collect_blast_radius(store: SnapshotStore) -> list[BlastRadiusRow]:
# breach is read from the same series that is firing.
durations: dict[tuple[tuple[str, str], ...], float] = {}

# Deferred outside-threshold samples: we need durations (populated in
# the same pass) before computing max breach, so these are buffered
# and replayed after the single pass over all snapshots.
deferred_outside: list[tuple[dict[str, str], float]] = []

# Single pass over all snapshots — collects durations, rollup gauges,
# warming-up markers, and outside-threshold samples in one traversal
# instead of three separate loops.
for snap in store.all_snapshots():
for sample in snap.samples:
labels = dict(sample.labels)
grp = labels.get("group", "")
metric = sample.metric
if metric == "anomaly_duration_seconds":
labels = dict(sample.labels)
durations[_detector_key(labels)] = sample.value
elif metric == "anomaly_active_series" and set(labels) == {"group"}:
group_active[grp] = sample.value
elif metric == "anomaly_density" and set(labels) == {"group"}:
group_density[grp] = sample.value

for snap in store.all_snapshots():
for sample in snap.samples:
labels = dict(sample.labels)
if sample.metric == "anomaly_warming_up" and sample.value >= 1.0:
elif metric == "anomaly_active_series":
labels = dict(sample.labels)
if set(labels) == {"group"}:
group_active[labels.get("group", "")] = sample.value
elif metric == "anomaly_density":
labels = dict(sample.labels)
if set(labels) == {"group"}:
group_density[labels.get("group", "")] = sample.value
elif metric == "anomaly_warming_up" and sample.value >= 1.0:
labels = dict(sample.labels)
grp = labels.get("group", "")
ident = _series_identity(labels)
group_warming.setdefault(grp, set()).add(ident)
axis = labels.get("cohort_label")
if axis is not None:
key = cohort_key_for(labels, axis)
cohort_warming.setdefault((grp, axis, key), set()).add(ident)

for snap in store.all_snapshots():
for sample in snap.samples:
if sample.metric != "anomaly_outside_threshold":
continue
labels = dict(sample.labels)
grp = labels.get("group", "")
ident = _series_identity(labels)
group_total.setdefault(grp, set()).add(ident)
firing_now = sample.value >= 1.0
elif metric == "anomaly_outside_threshold":
labels = dict(sample.labels)
deferred_outside.append((labels, sample.value))

# Replay the buffered outside-threshold samples now that durations
# are fully populated.
for labels, value in deferred_outside:
grp = labels.get("group", "")
ident = _series_identity(labels)
group_total.setdefault(grp, set()).add(ident)
firing_now = value >= 1.0
if firing_now:
group_firing.setdefault(grp, set()).add(ident)
breach = durations.get(_detector_key(labels), 0.0)
group_max_duration[grp] = max(group_max_duration.get(grp, 0.0), breach)
axis = labels.get("cohort_label")
if axis is not None:
key = cohort_key_for(labels, axis)
cid = (grp, axis, key)
cohort_total.setdefault(cid, set()).add(ident)
if firing_now:
group_firing.setdefault(grp, set()).add(ident)
cohort_firing.setdefault(cid, set()).add(ident)
breach = durations.get(_detector_key(labels), 0.0)
group_max_duration[grp] = max(group_max_duration.get(grp, 0.0), breach)
axis = labels.get("cohort_label")
if axis is not None:
key = cohort_key_for(labels, axis)
cid = (grp, axis, key)
cohort_total.setdefault(cid, set()).add(ident)
if firing_now:
cohort_firing.setdefault(cid, set()).add(ident)
breach = durations.get(_detector_key(labels), 0.0)
cohort_max_duration[cid] = max(cohort_max_duration.get(cid, 0.0), breach)
cohort_max_duration[cid] = max(cohort_max_duration.get(cid, 0.0), breach)

rows: list[BlastRadiusRow] = []

Expand Down
Loading
Loading