From 01d91bc18b421fc0d1b0ccfa5766824ef024f306 Mon Sep 17 00:00:00 2001 From: Brandon Date: Tue, 21 Apr 2026 21:10:50 -0400 Subject: [PATCH] feat: enforce global p95 latency release gate --- README.md | 5 ++ examples/spec.yaml | 1 + src/agent_release_gate/evaluator.py | 27 ++++++++ src/agent_release_gate/models.py | 7 ++ tests/test_evaluator.py | 99 +++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+) diff --git a/README.md b/README.md index e54b54a..b85433f 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,11 @@ This tool makes those problems visible before release. - Scores each test case using expected + forbidden phrases - Calculates pass rate, average latency, and average cost +- Calculates pass rate, average latency, and p95 latency (tail behavior) - Fails if quality drops below your threshold - Optionally compares against a baseline report and blocks regressions - Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast +- Optionally enforces a global p95 latency limit to catch long-tail slow responses - Optionally records run summaries and detects sustained pass-rate drift across releases - Supports per-case latency/cost limits to catch outliers hidden by averages - Enforces telemetry presence when global average latency/cost limits are configured @@ -74,6 +76,7 @@ global: minimum_pass_rate: 0.8 allowed_regression: 0.02 max_avg_latency_ms: 1500 + max_p95_latency_ms: 2000 max_avg_cost_usd: 0.03 max_avg_latency_regression_pct: 0.15 max_avg_cost_regression_pct: 0.10 @@ -92,6 +95,8 @@ cases: When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run. +When `max_p95_latency_ms` is configured globally, the gate fails if p95 latency is above the threshold or latency telemetry is missing. + When `--baseline` is provided, you can also set `max_avg_latency_regression_pct` and/or `max_avg_cost_regression_pct` to fail if average latency or cost regresses beyond the allowed percentage increase vs baseline. ## Repo layout diff --git a/examples/spec.yaml b/examples/spec.yaml index 5804da9..f769c5b 100644 --- a/examples/spec.yaml +++ b/examples/spec.yaml @@ -2,6 +2,7 @@ global: minimum_pass_rate: 0.66 allowed_regression: 0.03 max_avg_latency_ms: 1500 + max_p95_latency_ms: 1800 max_avg_cost_usd: 0.03 max_avg_latency_regression_pct: 0.15 max_avg_cost_regression_pct: 0.1 diff --git a/src/agent_release_gate/evaluator.py b/src/agent_release_gate/evaluator.py index afc3062..7dc7c0f 100644 --- a/src/agent_release_gate/evaluator.py +++ b/src/agent_release_gate/evaluator.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math from pathlib import Path from typing import Optional, Union @@ -31,6 +32,15 @@ def normalize(text: str) -> str: return " ".join(text.lower().split()) +def percentile(values: list[float], pct: float) -> Optional[float]: + if not values: + return None + + sorted_values = sorted(values) + rank = max(0, math.ceil(pct * len(sorted_values)) - 1) + return float(sorted_values[rank]) + + def score_case(case: GateCase, result: CaseResult) -> ScoredCase: response = normalize(result.response) notes: list[str] = [] @@ -127,6 +137,7 @@ def evaluate( latencies = [r.latency_ms for r in results if r.latency_ms is not None] costs = [r.cost_usd for r in results if r.cost_usd is not None] avg_latency = (sum(latencies) / len(latencies)) if latencies else None + p95_latency = percentile([float(v) for v in latencies], 0.95) avg_cost = (sum(costs) / len(costs)) if costs else None reasons: list[str] = [] @@ -150,6 +161,18 @@ def evaluate( f"Average latency {avg_latency:.1f}ms exceeds limit {spec.max_avg_latency_ms}ms" ) + if spec.max_p95_latency_ms is not None: + if p95_latency is None: + gate_passed = False + reasons.append( + "P95 latency limit is configured, but no latency telemetry was provided" + ) + elif p95_latency > spec.max_p95_latency_ms: + gate_passed = False + reasons.append( + f"P95 latency {p95_latency:.1f}ms exceeds limit {spec.max_p95_latency_ms}ms" + ) + if spec.max_avg_cost_usd is not None: if avg_cost is None: gate_passed = False @@ -231,6 +254,7 @@ def evaluate( avg_cost_usd=round(avg_cost, 6) if avg_cost is not None else None, gate_passed=gate_passed, gate_reasons=reasons, + p95_latency_ms=round(p95_latency, 2) if p95_latency is not None else None, ) return GateReport(summary=summary, cases=scored) @@ -244,6 +268,7 @@ def to_dict(report: GateReport) -> dict: "pass_rate": report.summary.pass_rate, "avg_latency_ms": report.summary.avg_latency_ms, "avg_cost_usd": report.summary.avg_cost_usd, + "p95_latency_ms": report.summary.p95_latency_ms, "gate_passed": report.summary.gate_passed, "gate_reasons": report.summary.gate_reasons, }, @@ -264,6 +289,8 @@ def to_markdown(report: GateReport) -> str: ] if s.avg_latency_ms is not None: lines.append(f"- Avg latency: {s.avg_latency_ms:.2f}ms") + if s.p95_latency_ms is not None: + lines.append(f"- P95 latency: {s.p95_latency_ms:.2f}ms") if s.avg_cost_usd is not None: lines.append(f"- Avg cost: ${s.avg_cost_usd:.6f}") diff --git a/src/agent_release_gate/models.py b/src/agent_release_gate/models.py index 4084c92..a203e5c 100644 --- a/src/agent_release_gate/models.py +++ b/src/agent_release_gate/models.py @@ -20,6 +20,7 @@ class GateSpec: minimum_pass_rate: float = 0.8 allowed_regression: float = 0.02 max_avg_latency_ms: Optional[int] = None + max_p95_latency_ms: Optional[int] = None max_avg_cost_usd: Optional[float] = None max_avg_latency_regression_pct: Optional[float] = None max_avg_cost_regression_pct: Optional[float] = None @@ -51,6 +52,7 @@ class GateSummary: avg_cost_usd: Optional[float] gate_passed: bool gate_reasons: list[str] + p95_latency_ms: Optional[float] = None @dataclass @@ -95,6 +97,11 @@ def parse_spec(data: dict[str, Any]) -> GateSpec: if global_cfg.get("max_avg_latency_ms") is not None else None ), + max_p95_latency_ms=( + int(global_cfg["max_p95_latency_ms"]) + if global_cfg.get("max_p95_latency_ms") is not None + else None + ), max_avg_cost_usd=( float(global_cfg["max_avg_cost_usd"]) if global_cfg.get("max_avg_cost_usd") is not None diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index fc04e40..099ec23 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -39,6 +39,7 @@ def test_evaluate_happy_path(tmp_path: Path): report = evaluate(spec, results) assert report.summary.gate_passed is True assert report.summary.pass_rate == 1.0 + assert report.summary.p95_latency_ms == 800.0 assert report.cases[0].passed is True @@ -178,6 +179,104 @@ def test_global_latency_limit_requires_telemetry_when_configured(tmp_path: Path) ) +def test_global_p95_latency_limit_blocks_tail_latency(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + """ +global: + minimum_pass_rate: 1.0 + max_p95_latency_ms: 1000 +cases: + - id: case_1 + expected_all: ["ok"] + min_score: 0.7 + - id: case_2 + expected_all: ["ok"] + min_score: 0.7 + - id: case_3 + expected_all: ["ok"] + min_score: 0.7 + - id: case_4 + expected_all: ["ok"] + min_score: 0.7 + - id: case_5 + expected_all: ["ok"] + min_score: 0.7 + - id: case_6 + expected_all: ["ok"] + min_score: 0.7 + - id: case_7 + expected_all: ["ok"] + min_score: 0.7 + - id: case_8 + expected_all: ["ok"] + min_score: 0.7 + - id: case_9 + expected_all: ["ok"] + min_score: 0.7 + - id: case_10 + expected_all: ["ok"] + min_score: 0.7 +""".strip(), + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + """ +{ + "cases": [ + {"id": "case_1", "response": "ok", "latency_ms": 500}, + {"id": "case_2", "response": "ok", "latency_ms": 500}, + {"id": "case_3", "response": "ok", "latency_ms": 500}, + {"id": "case_4", "response": "ok", "latency_ms": 500}, + {"id": "case_5", "response": "ok", "latency_ms": 500}, + {"id": "case_6", "response": "ok", "latency_ms": 500}, + {"id": "case_7", "response": "ok", "latency_ms": 500}, + {"id": "case_8", "response": "ok", "latency_ms": 500}, + {"id": "case_9", "response": "ok", "latency_ms": 500}, + {"id": "case_10", "response": "ok", "latency_ms": 2500} + ] +} +""".strip(), + encoding="utf-8", + ) + + report = evaluate(spec, results) + assert report.summary.gate_passed is False + assert report.summary.p95_latency_ms == 2500.0 + assert any("P95 latency 2500.0ms exceeds limit 1000ms" in r for r in report.summary.gate_reasons) + + +def test_global_p95_latency_limit_requires_telemetry_when_configured(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + """ +global: + minimum_pass_rate: 1.0 + max_p95_latency_ms: 1000 +cases: + - id: case_1 + expected_all: ["refund"] + min_score: 0.7 +""".strip(), + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + '{"cases":[{"id":"case_1","response":"refund confirmed"}]}', + encoding="utf-8", + ) + + report = evaluate(spec, results) + assert report.summary.gate_passed is False + assert any( + "P95 latency limit is configured, but no latency telemetry was provided" in r + for r in report.summary.gate_reasons + ) + + def test_global_cost_limit_requires_telemetry_when_configured(tmp_path: Path): spec = tmp_path / "spec.yaml" spec.write_text(