feat: add per-case latency/cost guardrails

README.md

@@ -20,6 +20,7 @@ This tool makes those problems visible before release.
 - Calculates pass rate, average latency, and average cost
 - Fails if quality drops below your threshold
 - Optionally compares against a baseline report and blocks regressions
+- Supports per-case latency/cost limits to catch outliers hidden by averages
 - Outputs both JSON (for machines) and Markdown (for humans)
 
 ## Install
@@ -60,8 +61,12 @@ cases:
     expected_any: ["order", "transaction"]
     forbidden: ["cannot help", "policy not found"]
     min_score: 0.72
+    max_latency_ms: 1200
+    max_cost_usd: 0.02
 ```
 
+`max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit.
+
 ## Repo layout
 
 - `src/agent_release_gate/`: scoring + gate logic

examples/spec.yaml

@@ -10,6 +10,8 @@ cases:
     expected_any: ["order", "transaction"]
     forbidden: ["cannot help", "policy not found"]
     min_score: 0.72
+    max_latency_ms: 1200
+    max_cost_usd: 0.02
 
   - id: address_change
     expected_all: ["address", "update"]

src/agent_release_gate/evaluator.py

@@ -70,6 +70,26 @@ def score_case(case: GateCase, result: CaseResult) -> ScoredCase:
     score = max(0.0, min(score, 1.0))
     passed = score >= case.min_score and forbidden_hits == 0
 
+    if case.max_latency_ms is not None:
+        if result.latency_ms is None:
+            passed = False
+            notes.append("Missing latency_ms for case with max_latency_ms set")
+        elif result.latency_ms > case.max_latency_ms:
+            passed = False
+            notes.append(
+                f"Latency {result.latency_ms}ms exceeds case max {case.max_latency_ms}ms"
+            )
+
+    if case.max_cost_usd is not None:
+        if result.cost_usd is None:
+            passed = False
+            notes.append("Missing cost_usd for case with max_cost_usd set")
+        elif result.cost_usd > case.max_cost_usd:
+            passed = False
+            notes.append(
+                f"Cost ${result.cost_usd:.6f} exceeds case max ${case.max_cost_usd:.6f}"
+            )
+
     if passed and not notes:
         notes.append("Looks good")
 

src/agent_release_gate/models.py

@@ -11,6 +11,8 @@ class GateCase:
     expected_any: list[str] = field(default_factory=list)
     forbidden: list[str] = field(default_factory=list)
     min_score: float = 0.7
+    max_latency_ms: Optional[int] = None
+    max_cost_usd: Optional[float] = None
 
 
 @dataclass
@@ -73,6 +75,12 @@ def parse_spec(data: dict[str, Any]) -> GateSpec:
             expected_any=_as_list(c.get("expected_any")),
             forbidden=_as_list(c.get("forbidden")),
             min_score=float(c.get("min_score", 0.7)),
+            max_latency_ms=(
+                int(c["max_latency_ms"]) if c.get("max_latency_ms") is not None else None
+            ),
+            max_cost_usd=(
+                float(c["max_cost_usd"]) if c.get("max_cost_usd") is not None else None
+            ),
         )
         for c in raw_cases
     ]

tests/test_evaluator.py

@@ -69,3 +69,81 @@ def test_evaluate_regression(tmp_path: Path):
     report = evaluate(spec, results, baseline)
     assert report.summary.gate_passed is False
     assert any("Regression detected" in r for r in report.summary.gate_reasons)
+
+
+def test_case_limits_fail_on_latency_and_cost_outliers(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+    max_latency_ms: 1200
+    max_cost_usd: 0.02
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        """
+{
+  "cases": [
+    {
+      "id": "case_1",
+      "response": "refund confirmed",
+      "latency_ms": 1900,
+      "cost_usd": 0.031
+    }
+  ]
+}
+""".strip(),
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert report.cases[0].passed is False
+    assert any("Latency 1900ms exceeds case max 1200ms" in n for n in report.cases[0].notes)
+    assert any("Cost $0.031000 exceeds case max $0.020000" in n for n in report.cases[0].notes)
+
+
+def test_case_limits_require_telemetry_when_configured(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+    max_latency_ms: 1200
+    max_cost_usd: 0.02
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        """
+{
+  "cases": [
+    {
+      "id": "case_1",
+      "response": "refund confirmed"
+    }
+  ]
+}
+""".strip(),
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert report.cases[0].passed is False
+    assert "Missing latency_ms for case with max_latency_ms set" in report.cases[0].notes
+    assert "Missing cost_usd for case with max_cost_usd set" in report.cases[0].notes