feat: add baseline latency/cost regression caps to release gate

README.md

@@ -20,6 +20,7 @@ This tool makes those problems visible before release.
 - Calculates pass rate, average latency, and average cost
 - Fails if quality drops below your threshold
 - Optionally compares against a baseline report and blocks regressions
+- Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast
 - Supports per-case latency/cost limits to catch outliers hidden by averages
 - Enforces telemetry presence when global average latency/cost limits are configured
 - Outputs both JSON (for machines) and Markdown (for humans)
@@ -55,6 +56,8 @@ global:
   allowed_regression: 0.02
   max_avg_latency_ms: 1500
   max_avg_cost_usd: 0.03
+  max_avg_latency_regression_pct: 0.15
+  max_avg_cost_regression_pct: 0.10
 
 cases:
   - id: refund_status
@@ -70,6 +73,8 @@ cases:
 
 When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run.
 
+When `--baseline` is provided, you can also set `max_avg_latency_regression_pct` and/or `max_avg_cost_regression_pct` to fail if average latency or cost regresses beyond the allowed percentage increase vs baseline.
+
 ## Repo layout
 
 - `src/agent_release_gate/`: scoring + gate logic

examples/spec.yaml

@@ -3,6 +3,8 @@ global:
   allowed_regression: 0.03
   max_avg_latency_ms: 1500
   max_avg_cost_usd: 0.03
+  max_avg_latency_regression_pct: 0.15
+  max_avg_cost_regression_pct: 0.1
 
 cases:
   - id: refund_status

src/agent_release_gate/evaluator.py

@@ -164,13 +164,62 @@ def evaluate(
 
     if baseline_path:
         baseline = load_json(baseline_path)
-        baseline_pass_rate = float(baseline.get("summary", {}).get("pass_rate", 0.0))
+        baseline_summary = baseline.get("summary", {}) or {}
+        baseline_pass_rate = float(baseline_summary.get("pass_rate", 0.0))
         if pass_rate < (baseline_pass_rate - spec.allowed_regression):
             gate_passed = False
             reasons.append(
                 f"Regression detected: current {pass_rate:.2%}, baseline {baseline_pass_rate:.2%}, allowed drop {spec.allowed_regression:.2%}"
             )
 
+        if spec.max_avg_latency_regression_pct is not None:
+            baseline_latency = baseline_summary.get("avg_latency_ms")
+            if baseline_latency is None or avg_latency is None:
+                gate_passed = False
+                reasons.append(
+                    "Latency regression limit is configured, but baseline or current average latency telemetry is missing"
+                )
+            else:
+                baseline_latency_f = float(baseline_latency)
+                if baseline_latency_f <= 0:
+                    gate_passed = False
+                    reasons.append(
+                        "Latency regression limit is configured, but baseline avg_latency_ms must be > 0"
+                    )
+                else:
+                    latency_increase_pct = (avg_latency - baseline_latency_f) / baseline_latency_f
+                    if latency_increase_pct > spec.max_avg_latency_regression_pct:
+                        gate_passed = False
+                        reasons.append(
+                            "Latency regression detected: "
+                            f"current {avg_latency:.1f}ms vs baseline {baseline_latency_f:.1f}ms "
+                            f"({latency_increase_pct:.2%} increase, allowed {spec.max_avg_latency_regression_pct:.2%})"
+                        )
+
+        if spec.max_avg_cost_regression_pct is not None:
+            baseline_cost = baseline_summary.get("avg_cost_usd")
+            if baseline_cost is None or avg_cost is None:
+                gate_passed = False
+                reasons.append(
+                    "Cost regression limit is configured, but baseline or current average cost telemetry is missing"
+                )
+            else:
+                baseline_cost_f = float(baseline_cost)
+                if baseline_cost_f <= 0:
+                    gate_passed = False
+                    reasons.append(
+                        "Cost regression limit is configured, but baseline avg_cost_usd must be > 0"
+                    )
+                else:
+                    cost_increase_pct = (avg_cost - baseline_cost_f) / baseline_cost_f
+                    if cost_increase_pct > spec.max_avg_cost_regression_pct:
+                        gate_passed = False
+                        reasons.append(
+                            "Cost regression detected: "
+                            f"current ${avg_cost:.6f} vs baseline ${baseline_cost_f:.6f} "
+                            f"({cost_increase_pct:.2%} increase, allowed {spec.max_avg_cost_regression_pct:.2%})"
+                        )
+
     if gate_passed:
         reasons.append("Gate passed")
 

src/agent_release_gate/models.py

@@ -21,6 +21,8 @@ class GateSpec:
     allowed_regression: float = 0.02
     max_avg_latency_ms: Optional[int] = None
     max_avg_cost_usd: Optional[float] = None
+    max_avg_latency_regression_pct: Optional[float] = None
+    max_avg_cost_regression_pct: Optional[float] = None
     cases: list[GateCase] = field(default_factory=list)
 
 
@@ -98,6 +100,16 @@ def parse_spec(data: dict[str, Any]) -> GateSpec:
             if global_cfg.get("max_avg_cost_usd") is not None
             else None
         ),
+        max_avg_latency_regression_pct=(
+            float(global_cfg["max_avg_latency_regression_pct"])
+            if global_cfg.get("max_avg_latency_regression_pct") is not None
+            else None
+        ),
+        max_avg_cost_regression_pct=(
+            float(global_cfg["max_avg_cost_regression_pct"])
+            if global_cfg.get("max_avg_cost_regression_pct") is not None
+            else None
+        ),
         cases=cases,
     )
 

tests/test_evaluator.py

@@ -205,3 +205,101 @@ def test_global_cost_limit_requires_telemetry_when_configured(tmp_path: Path):
         "Average cost limit is configured, but no cost telemetry was provided" in r
         for r in report.summary.gate_reasons
     )
+
+
+def test_baseline_latency_regression_limit_blocks_slowdown(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_avg_latency_regression_pct: 0.10
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed","latency_ms":1300}]}',
+        encoding="utf-8",
+    )
+
+    baseline = tmp_path / "baseline.json"
+    baseline.write_text(
+        '{"summary":{"pass_rate":1.0,"avg_latency_ms":1000},"cases":[]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results, baseline)
+    assert report.summary.gate_passed is False
+    assert any("Latency regression detected" in r for r in report.summary.gate_reasons)
+
+
+def test_baseline_cost_regression_limit_blocks_spend_drift(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_avg_cost_regression_pct: 0.20
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed","cost_usd":0.018}]}',
+        encoding="utf-8",
+    )
+
+    baseline = tmp_path / "baseline.json"
+    baseline.write_text(
+        '{"summary":{"pass_rate":1.0,"avg_cost_usd":0.01},"cases":[]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results, baseline)
+    assert report.summary.gate_passed is False
+    assert any("Cost regression detected" in r for r in report.summary.gate_reasons)
+
+
+def test_baseline_regression_limits_require_current_and_baseline_telemetry(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_avg_latency_regression_pct: 0.15
+  max_avg_cost_regression_pct: 0.15
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed"}]}',
+        encoding="utf-8",
+    )
+
+    baseline = tmp_path / "baseline.json"
+    baseline.write_text(
+        '{"summary":{"pass_rate":1.0},"cases":[]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results, baseline)
+    assert report.summary.gate_passed is False
+    assert any("Latency regression limit is configured" in r for r in report.summary.gate_reasons)
+    assert any("Cost regression limit is configured" in r for r in report.summary.gate_reasons)