Fix profiler overhead, canary baseline rollback, pool leak detection, WebAuthn downgrade protection

backend/src/canary.rs

@@ -40,13 +40,25 @@ pub struct CanaryStage {
 pub struct MetricThresholds {
     pub max_error_rate: f64,
     pub max_latency_p99_ms: f64,
+    /// Maximum amount the canary cohort's error rate may exceed the
+    /// baseline (non-canary) cohort's error rate before an automatic
+    /// rollback is triggered, independent of `max_error_rate`. E.g. `0.01`
+    /// means the canary is rolled back once it's more than 1 percentage
+    /// point worse than baseline.
+    #[serde(default = "default_error_rate_margin")]
+    pub max_error_rate_margin: f64,
+}
+
+fn default_error_rate_margin() -> f64 {
+    0.01
 }
 
 impl Default for MetricThresholds {
     fn default() -> Self {
         Self {
             max_error_rate: 0.02,
             max_latency_p99_ms: 500.0,
+            max_error_rate_margin: default_error_rate_margin(),
         }
     }
 }
@@ -85,6 +97,9 @@ pub struct CanaryDeployment {
     pub thresholds: MetricThresholds,
     pub status: CanaryStatus,
     pub last_metrics: Option<CanaryMetrics>,
+    /// Most recently reported error rate for the baseline (non-canary)
+    /// cohort, used for continuous canary-vs-baseline comparison.
+    pub last_baseline_error_rate: Option<f64>,
     pub history: Vec<CanaryEvent>,
     pub started_at: DateTime<Utc>,
     pub stage_started_at: DateTime<Utc>,
@@ -122,6 +137,11 @@ pub struct StartCanaryRequest {
 #[derive(Debug, Deserialize)]
 pub struct EvaluateCanaryRequest {
     pub metrics: CanaryMetrics,
+    /// Current error rate observed in the baseline (non-canary) cohort, for
+    /// continuous canary-vs-baseline comparison. If omitted, the previously
+    /// reported baseline error rate (if any) continues to be used.
+    #[serde(default)]
+    pub baseline_error_rate: Option<f64>,
 }
 
 /// Request body for `POST /deployments/canary/:id/rollback`.
@@ -188,6 +208,7 @@ pub async fn start_canary_deployment(
         thresholds: body.thresholds.unwrap_or_default(),
         status: CanaryStatus::InProgress,
         last_metrics: None,
+        last_baseline_error_rate: None,
         history: vec![],
         started_at: now,
         stage_started_at: now,
@@ -229,20 +250,43 @@ pub async fn evaluate_canary(
     }
 
     deployment.last_metrics = Some(body.metrics.clone());
+    if let Some(baseline_error_rate) = body.baseline_error_rate {
+        deployment.last_baseline_error_rate = Some(baseline_error_rate);
+    }
 
-    let breached = body.metrics.error_rate > deployment.thresholds.max_error_rate
+    let absolute_breach = body.metrics.error_rate > deployment.thresholds.max_error_rate
         || body.metrics.latency_p99_ms > deployment.thresholds.max_latency_p99_ms;
 
-    if breached {
+    // Continuous canary-vs-baseline comparison: roll back if the canary's
+    // error rate has drifted more than `max_error_rate_margin` above the
+    // baseline cohort's error rate, even if it hasn't breached the absolute
+    // `max_error_rate` threshold on its own.
+    let baseline_breach = deployment.last_baseline_error_rate.is_some_and(|baseline| {
+        body.metrics.error_rate > baseline + deployment.thresholds.max_error_rate_margin
+    });
+
+    if absolute_breach || baseline_breach {
         deployment.status = CanaryStatus::RolledBack;
-        deployment.push_event(format!(
-            "automated rollback: error_rate={:.4} latency_p99_ms={:.1} breached thresholds",
-            body.metrics.error_rate, body.metrics.latency_p99_ms
-        ));
+        let reason = if baseline_breach {
+            format!(
+                "automated rollback: canary error_rate={:.4} exceeded baseline error_rate={:.4} by more than the configured margin ({:.4})",
+                body.metrics.error_rate,
+                deployment.last_baseline_error_rate.unwrap_or(0.0),
+                deployment.thresholds.max_error_rate_margin
+            )
+        } else {
+            format!(
+                "automated rollback: error_rate={:.4} latency_p99_ms={:.1} breached thresholds",
+                body.metrics.error_rate, body.metrics.latency_p99_ms
+            )
+        };
+        deployment.push_event(reason);
         tracing::error!(
             deployment_id = %id,
             error_rate = body.metrics.error_rate,
             latency_p99_ms = body.metrics.latency_p99_ms,
+            baseline_error_rate = deployment.last_baseline_error_rate,
+            baseline_breach,
             "canary rolled back automatically due to metric breach"
         );
         return Ok(Json(deployment.clone()));
@@ -285,3 +329,114 @@ pub async fn rollback_canary(
 
     Ok(Json(deployment.clone()))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    async fn start_deployment(state: &Arc<CanaryState>) -> CanaryDeployment {
+        let (_, Json(deployment)) = start_canary_deployment(
+            State(Arc::clone(state)),
+            Json(StartCanaryRequest {
+                service: "vault-api".into(),
+                version: "v1.0.0".into(),
+                stages: Some(vec![CanaryStage {
+                    traffic_percent: 5,
+                    min_duration_minutes: 60,
+                }]),
+                thresholds: None,
+            }),
+        )
+        .await
+        .unwrap();
+        deployment
+    }
+
+    #[tokio::test]
+    async fn rollback_triggers_when_canary_exceeds_baseline_margin() {
+        let state = Arc::new(CanaryState::new());
+        let deployment = start_deployment(&state).await;
+
+        let (_, Json(result)) = evaluate_canary(
+            State(Arc::clone(&state)),
+            Path(deployment.id.clone()),
+            Json(EvaluateCanaryRequest {
+                metrics: CanaryMetrics {
+                    error_rate: 0.015,
+                    latency_p99_ms: 100.0,
+                },
+                baseline_error_rate: Some(0.001),
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.status, CanaryStatus::RolledBack);
+    }
+
+    #[tokio::test]
+    async fn rollback_does_not_trigger_within_baseline_margin() {
+        let state = Arc::new(CanaryState::new());
+        let deployment = start_deployment(&state).await;
+
+        let (_, Json(result)) = evaluate_canary(
+            State(Arc::clone(&state)),
+            Path(deployment.id.clone()),
+            Json(EvaluateCanaryRequest {
+                metrics: CanaryMetrics {
+                    error_rate: 0.005,
+                    latency_p99_ms: 100.0,
+                },
+                baseline_error_rate: Some(0.001),
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.status, CanaryStatus::InProgress);
+    }
+
+    #[tokio::test]
+    async fn rollback_still_triggers_on_absolute_threshold_without_baseline() {
+        let state = Arc::new(CanaryState::new());
+        let deployment = start_deployment(&state).await;
+
+        let (_, Json(result)) = evaluate_canary(
+            State(Arc::clone(&state)),
+            Path(deployment.id.clone()),
+            Json(EvaluateCanaryRequest {
+                metrics: CanaryMetrics {
+                    error_rate: 0.05,
+                    latency_p99_ms: 100.0,
+                },
+                baseline_error_rate: None,
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.status, CanaryStatus::RolledBack);
+    }
+
+    #[tokio::test]
+    async fn no_rollback_when_healthy_and_no_baseline_reported() {
+        let state = Arc::new(CanaryState::new());
+        let deployment = start_deployment(&state).await;
+
+        let (_, Json(result)) = evaluate_canary(
+            State(Arc::clone(&state)),
+            Path(deployment.id.clone()),
+            Json(EvaluateCanaryRequest {
+                metrics: CanaryMetrics {
+                    error_rate: 0.001,
+                    latency_p99_ms: 100.0,
+                },
+                baseline_error_rate: None,
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.status, CanaryStatus::InProgress);
+    }
+}

backend/src/oncall.rs

@@ -159,6 +159,51 @@ impl Default for OnCallState {
     }
 }
 
+/// An ad-hoc alert raised against a schedule's primary escalation contacts,
+/// outside the normal `trigger_escalation` HTTP flow — used by other
+/// subsystems (e.g. connection-pool leak detection) that need to page
+/// on-call without going through a pre-defined escalation level.
+#[derive(Debug, Clone)]
+pub struct AlertRecord {
+    pub schedule_id: String,
+    pub source: String,
+    pub message: String,
+    pub contacts_notified: Vec<String>,
+}
+
+/// Raise an alert against `schedule_id`'s primary (level-1) escalation
+/// contacts and log it. Returns `None` if the schedule doesn't exist, in
+/// which case the alert is only logged, not attributed to any contacts.
+pub fn raise_alert(
+    state: &OnCallState,
+    schedule_id: &str,
+    source: &str,
+    message: &str,
+) -> Option<AlertRecord> {
+    let contacts = {
+        let store = state.store.lock().unwrap();
+        store
+            .get(schedule_id)
+            .and_then(|schedule| schedule.escalation_policy.levels.first())
+            .map(|level| level.contacts.clone())
+    };
+
+    tracing::error!(
+        schedule_id = %schedule_id,
+        source = %source,
+        message = %message,
+        contacts = ?contacts,
+        "alert raised"
+    );
+
+    contacts.map(|contacts_notified| AlertRecord {
+        schedule_id: schedule_id.to_string(),
+        source: source.to_string(),
+        message: message.to_string(),
+        contacts_notified,
+    })
+}
+
 /// Build a round-robin rotation of `shift_count` shifts across
 /// `participants`, each `rotation_hours` long, starting now.
 fn build_rotation(
@@ -309,3 +354,50 @@ pub async fn trigger_escalation(
         reason: body.reason,
     }))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn schedule_with_contacts(contacts: Vec<String>) -> OnCallSchedule {
+        OnCallSchedule {
+            id: "sched-1".into(),
+            name: "Backend On-Call".into(),
+            rotation_hours: 24,
+            shifts: vec![],
+            escalation_policy: EscalationPolicy {
+                levels: vec![EscalationLevel {
+                    level: 1,
+                    delay_minutes: 5,
+                    contacts,
+                }],
+            },
+            handoffs: vec![],
+            created_at: Utc::now(),
+        }
+    }
+
+    #[test]
+    fn raise_alert_notifies_primary_escalation_contacts() {
+        let state = OnCallState::new();
+        let schedule = schedule_with_contacts(vec!["oncall@example.com".into()]);
+        state
+            .store
+            .lock()
+            .unwrap()
+            .insert(schedule.id.clone(), schedule.clone());
+
+        let alert = raise_alert(&state, &schedule.id, "pool_optimizer", "leak detected")
+            .expect("schedule exists");
+
+        assert_eq!(alert.contacts_notified, vec!["oncall@example.com".to_string()]);
+        assert_eq!(alert.source, "pool_optimizer");
+    }
+
+    #[test]
+    fn raise_alert_on_unknown_schedule_returns_none() {
+        let state = OnCallState::new();
+        let alert = raise_alert(&state, "does-not-exist", "pool_optimizer", "leak detected");
+        assert!(alert.is_none());
+    }
+}