feat: add global p95 latency guardrail

README.md

@@ -18,9 +18,11 @@ This tool makes those problems visible before release.
 
 - Scores each test case using expected + forbidden phrases
 - Calculates pass rate, average latency, and average cost
+- Calculates pass rate, average latency, and p95 latency (tail behavior)
 - Fails if quality drops below your threshold
 - Optionally compares against a baseline report and blocks regressions
 - Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast
+- Optionally enforces a global p95 latency limit to catch long-tail slow responses
 - Optionally records run summaries and detects sustained pass-rate drift across releases
 - Supports per-case latency/cost limits to catch outliers hidden by averages
 - Enforces telemetry presence when global average latency/cost limits are configured
@@ -74,6 +76,7 @@ global:
   minimum_pass_rate: 0.8
   allowed_regression: 0.02
   max_avg_latency_ms: 1500
+  max_p95_latency_ms: 2000
   max_avg_cost_usd: 0.03
   max_avg_latency_regression_pct: 0.15
   max_avg_cost_regression_pct: 0.10
@@ -92,6 +95,8 @@ cases:
 
 When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run.
 
+When `max_p95_latency_ms` is configured globally, the gate fails if p95 latency is above the threshold or latency telemetry is missing.
+
 When `--baseline` is provided, you can also set `max_avg_latency_regression_pct` and/or `max_avg_cost_regression_pct` to fail if average latency or cost regresses beyond the allowed percentage increase vs baseline.
 
 ## Repo layout

examples/spec.yaml

@@ -2,6 +2,7 @@ global:
   minimum_pass_rate: 0.66
   allowed_regression: 0.03
   max_avg_latency_ms: 1500
+  max_p95_latency_ms: 1800
   max_avg_cost_usd: 0.03
   max_avg_latency_regression_pct: 0.15
   max_avg_cost_regression_pct: 0.1

src/agent_release_gate/evaluator.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import math
 from pathlib import Path
 from typing import Optional, Union
 
@@ -31,6 +32,15 @@ def normalize(text: str) -> str:
     return " ".join(text.lower().split())
 
 
+def percentile(values: list[float], pct: float) -> Optional[float]:
+    if not values:
+        return None
+
+    sorted_values = sorted(values)
+    rank = max(0, math.ceil(pct * len(sorted_values)) - 1)
+    return float(sorted_values[rank])
+
+
 def score_case(case: GateCase, result: CaseResult) -> ScoredCase:
     response = normalize(result.response)
     notes: list[str] = []
@@ -127,6 +137,7 @@ def evaluate(
     latencies = [r.latency_ms for r in results if r.latency_ms is not None]
     costs = [r.cost_usd for r in results if r.cost_usd is not None]
     avg_latency = (sum(latencies) / len(latencies)) if latencies else None
+    p95_latency = percentile([float(v) for v in latencies], 0.95)
     avg_cost = (sum(costs) / len(costs)) if costs else None
 
     reasons: list[str] = []
@@ -150,6 +161,18 @@ def evaluate(
                 f"Average latency {avg_latency:.1f}ms exceeds limit {spec.max_avg_latency_ms}ms"
             )
 
+    if spec.max_p95_latency_ms is not None:
+        if p95_latency is None:
+            gate_passed = False
+            reasons.append(
+                "P95 latency limit is configured, but no latency telemetry was provided"
+            )
+        elif p95_latency > spec.max_p95_latency_ms:
+            gate_passed = False
+            reasons.append(
+                f"P95 latency {p95_latency:.1f}ms exceeds limit {spec.max_p95_latency_ms}ms"
+            )
+
     if spec.max_avg_cost_usd is not None:
         if avg_cost is None:
             gate_passed = False
@@ -231,6 +254,7 @@ def evaluate(
         avg_cost_usd=round(avg_cost, 6) if avg_cost is not None else None,
         gate_passed=gate_passed,
         gate_reasons=reasons,
+        p95_latency_ms=round(p95_latency, 2) if p95_latency is not None else None,
     )
 
     return GateReport(summary=summary, cases=scored)
@@ -244,6 +268,7 @@ def to_dict(report: GateReport) -> dict:
             "pass_rate": report.summary.pass_rate,
             "avg_latency_ms": report.summary.avg_latency_ms,
             "avg_cost_usd": report.summary.avg_cost_usd,
+            "p95_latency_ms": report.summary.p95_latency_ms,
             "gate_passed": report.summary.gate_passed,
             "gate_reasons": report.summary.gate_reasons,
         },
@@ -264,6 +289,8 @@ def to_markdown(report: GateReport) -> str:
     ]
     if s.avg_latency_ms is not None:
         lines.append(f"- Avg latency: {s.avg_latency_ms:.2f}ms")
+    if s.p95_latency_ms is not None:
+        lines.append(f"- P95 latency: {s.p95_latency_ms:.2f}ms")
     if s.avg_cost_usd is not None:
         lines.append(f"- Avg cost: ${s.avg_cost_usd:.6f}")
 

src/agent_release_gate/models.py

@@ -20,6 +20,7 @@ class GateSpec:
     minimum_pass_rate: float = 0.8
     allowed_regression: float = 0.02
     max_avg_latency_ms: Optional[int] = None
+    max_p95_latency_ms: Optional[int] = None
     max_avg_cost_usd: Optional[float] = None
     max_avg_latency_regression_pct: Optional[float] = None
     max_avg_cost_regression_pct: Optional[float] = None
@@ -51,6 +52,7 @@ class GateSummary:
     avg_cost_usd: Optional[float]
     gate_passed: bool
     gate_reasons: list[str]
+    p95_latency_ms: Optional[float] = None
 
 
 @dataclass
@@ -95,6 +97,11 @@ def parse_spec(data: dict[str, Any]) -> GateSpec:
             if global_cfg.get("max_avg_latency_ms") is not None
             else None
         ),
+        max_p95_latency_ms=(
+            int(global_cfg["max_p95_latency_ms"])
+            if global_cfg.get("max_p95_latency_ms") is not None
+            else None
+        ),
         max_avg_cost_usd=(
             float(global_cfg["max_avg_cost_usd"])
             if global_cfg.get("max_avg_cost_usd") is not None

tests/test_evaluator.py

@@ -39,6 +39,7 @@ def test_evaluate_happy_path(tmp_path: Path):
     report = evaluate(spec, results)
     assert report.summary.gate_passed is True
     assert report.summary.pass_rate == 1.0
+    assert report.summary.p95_latency_ms == 800.0
     assert report.cases[0].passed is True
 
 
@@ -178,6 +179,104 @@ def test_global_latency_limit_requires_telemetry_when_configured(tmp_path: Path)
     )
 
 
+def test_global_p95_latency_limit_blocks_tail_latency(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_p95_latency_ms: 1000
+cases:
+  - id: case_1
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_2
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_3
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_4
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_5
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_6
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_7
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_8
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_9
+    expected_all: ["ok"]
+    min_score: 0.7
+  - id: case_10
+    expected_all: ["ok"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        """
+{
+  "cases": [
+    {"id": "case_1", "response": "ok", "latency_ms": 500},
+    {"id": "case_2", "response": "ok", "latency_ms": 500},
+    {"id": "case_3", "response": "ok", "latency_ms": 500},
+    {"id": "case_4", "response": "ok", "latency_ms": 500},
+    {"id": "case_5", "response": "ok", "latency_ms": 500},
+    {"id": "case_6", "response": "ok", "latency_ms": 500},
+    {"id": "case_7", "response": "ok", "latency_ms": 500},
+    {"id": "case_8", "response": "ok", "latency_ms": 500},
+    {"id": "case_9", "response": "ok", "latency_ms": 500},
+    {"id": "case_10", "response": "ok", "latency_ms": 2500}
+  ]
+}
+""".strip(),
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert report.summary.p95_latency_ms == 2500.0
+    assert any("P95 latency 2500.0ms exceeds limit 1000ms" in r for r in report.summary.gate_reasons)
+
+
+def test_global_p95_latency_limit_requires_telemetry_when_configured(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_p95_latency_ms: 1000
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed"}]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert any(
+        "P95 latency limit is configured, but no latency telemetry was provided" in r
+        for r in report.summary.gate_reasons
+    )
+
+
 def test_global_cost_limit_requires_telemetry_when_configured(tmp_path: Path):
     spec = tmp_path / "spec.yaml"
     spec.write_text(