From a85975c460237a752ca6dbdfdb0c9697059d7eb0 Mon Sep 17 00:00:00 2001 From: Brandon Date: Thu, 2 Apr 2026 21:09:22 -0400 Subject: [PATCH] feat: gate baseline latency and cost drift regressions --- README.md | 5 ++ examples/spec.yaml | 2 + src/agent_release_gate/evaluator.py | 51 ++++++++++++++- src/agent_release_gate/models.py | 12 ++++ tests/test_evaluator.py | 98 +++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b71ba0e..fb0af4e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ This tool makes those problems visible before release. - Calculates pass rate, average latency, and average cost - Fails if quality drops below your threshold - Optionally compares against a baseline report and blocks regressions +- Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast - Supports per-case latency/cost limits to catch outliers hidden by averages - Enforces telemetry presence when global average latency/cost limits are configured - Outputs both JSON (for machines) and Markdown (for humans) @@ -55,6 +56,8 @@ global: allowed_regression: 0.02 max_avg_latency_ms: 1500 max_avg_cost_usd: 0.03 + max_avg_latency_regression_pct: 0.15 + max_avg_cost_regression_pct: 0.10 cases: - id: refund_status @@ -70,6 +73,8 @@ cases: When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run. +When `--baseline` is provided, you can also set `max_avg_latency_regression_pct` and/or `max_avg_cost_regression_pct` to fail if average latency or cost regresses beyond the allowed percentage increase vs baseline. + ## Repo layout - `src/agent_release_gate/`: scoring + gate logic diff --git a/examples/spec.yaml b/examples/spec.yaml index 28fc585..5804da9 100644 --- a/examples/spec.yaml +++ b/examples/spec.yaml @@ -3,6 +3,8 @@ global: allowed_regression: 0.03 max_avg_latency_ms: 1500 max_avg_cost_usd: 0.03 + max_avg_latency_regression_pct: 0.15 + max_avg_cost_regression_pct: 0.1 cases: - id: refund_status diff --git a/src/agent_release_gate/evaluator.py b/src/agent_release_gate/evaluator.py index e35d2f2..afc3062 100644 --- a/src/agent_release_gate/evaluator.py +++ b/src/agent_release_gate/evaluator.py @@ -164,13 +164,62 @@ def evaluate( if baseline_path: baseline = load_json(baseline_path) - baseline_pass_rate = float(baseline.get("summary", {}).get("pass_rate", 0.0)) + baseline_summary = baseline.get("summary", {}) or {} + baseline_pass_rate = float(baseline_summary.get("pass_rate", 0.0)) if pass_rate < (baseline_pass_rate - spec.allowed_regression): gate_passed = False reasons.append( f"Regression detected: current {pass_rate:.2%}, baseline {baseline_pass_rate:.2%}, allowed drop {spec.allowed_regression:.2%}" ) + if spec.max_avg_latency_regression_pct is not None: + baseline_latency = baseline_summary.get("avg_latency_ms") + if baseline_latency is None or avg_latency is None: + gate_passed = False + reasons.append( + "Latency regression limit is configured, but baseline or current average latency telemetry is missing" + ) + else: + baseline_latency_f = float(baseline_latency) + if baseline_latency_f <= 0: + gate_passed = False + reasons.append( + "Latency regression limit is configured, but baseline avg_latency_ms must be > 0" + ) + else: + latency_increase_pct = (avg_latency - baseline_latency_f) / baseline_latency_f + if latency_increase_pct > spec.max_avg_latency_regression_pct: + gate_passed = False + reasons.append( + "Latency regression detected: " + f"current {avg_latency:.1f}ms vs baseline {baseline_latency_f:.1f}ms " + f"({latency_increase_pct:.2%} increase, allowed {spec.max_avg_latency_regression_pct:.2%})" + ) + + if spec.max_avg_cost_regression_pct is not None: + baseline_cost = baseline_summary.get("avg_cost_usd") + if baseline_cost is None or avg_cost is None: + gate_passed = False + reasons.append( + "Cost regression limit is configured, but baseline or current average cost telemetry is missing" + ) + else: + baseline_cost_f = float(baseline_cost) + if baseline_cost_f <= 0: + gate_passed = False + reasons.append( + "Cost regression limit is configured, but baseline avg_cost_usd must be > 0" + ) + else: + cost_increase_pct = (avg_cost - baseline_cost_f) / baseline_cost_f + if cost_increase_pct > spec.max_avg_cost_regression_pct: + gate_passed = False + reasons.append( + "Cost regression detected: " + f"current ${avg_cost:.6f} vs baseline ${baseline_cost_f:.6f} " + f"({cost_increase_pct:.2%} increase, allowed {spec.max_avg_cost_regression_pct:.2%})" + ) + if gate_passed: reasons.append("Gate passed") diff --git a/src/agent_release_gate/models.py b/src/agent_release_gate/models.py index 65ef088..4084c92 100644 --- a/src/agent_release_gate/models.py +++ b/src/agent_release_gate/models.py @@ -21,6 +21,8 @@ class GateSpec: allowed_regression: float = 0.02 max_avg_latency_ms: Optional[int] = None max_avg_cost_usd: Optional[float] = None + max_avg_latency_regression_pct: Optional[float] = None + max_avg_cost_regression_pct: Optional[float] = None cases: list[GateCase] = field(default_factory=list) @@ -98,6 +100,16 @@ def parse_spec(data: dict[str, Any]) -> GateSpec: if global_cfg.get("max_avg_cost_usd") is not None else None ), + max_avg_latency_regression_pct=( + float(global_cfg["max_avg_latency_regression_pct"]) + if global_cfg.get("max_avg_latency_regression_pct") is not None + else None + ), + max_avg_cost_regression_pct=( + float(global_cfg["max_avg_cost_regression_pct"]) + if global_cfg.get("max_avg_cost_regression_pct") is not None + else None + ), cases=cases, ) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index d39de83..fc04e40 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -205,3 +205,101 @@ def test_global_cost_limit_requires_telemetry_when_configured(tmp_path: Path): "Average cost limit is configured, but no cost telemetry was provided" in r for r in report.summary.gate_reasons ) + + +def test_baseline_latency_regression_limit_blocks_slowdown(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + """ +global: + minimum_pass_rate: 1.0 + max_avg_latency_regression_pct: 0.10 +cases: + - id: case_1 + expected_all: ["refund"] + min_score: 0.7 +""".strip(), + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + '{"cases":[{"id":"case_1","response":"refund confirmed","latency_ms":1300}]}', + encoding="utf-8", + ) + + baseline = tmp_path / "baseline.json" + baseline.write_text( + '{"summary":{"pass_rate":1.0,"avg_latency_ms":1000},"cases":[]}', + encoding="utf-8", + ) + + report = evaluate(spec, results, baseline) + assert report.summary.gate_passed is False + assert any("Latency regression detected" in r for r in report.summary.gate_reasons) + + +def test_baseline_cost_regression_limit_blocks_spend_drift(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + """ +global: + minimum_pass_rate: 1.0 + max_avg_cost_regression_pct: 0.20 +cases: + - id: case_1 + expected_all: ["refund"] + min_score: 0.7 +""".strip(), + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + '{"cases":[{"id":"case_1","response":"refund confirmed","cost_usd":0.018}]}', + encoding="utf-8", + ) + + baseline = tmp_path / "baseline.json" + baseline.write_text( + '{"summary":{"pass_rate":1.0,"avg_cost_usd":0.01},"cases":[]}', + encoding="utf-8", + ) + + report = evaluate(spec, results, baseline) + assert report.summary.gate_passed is False + assert any("Cost regression detected" in r for r in report.summary.gate_reasons) + + +def test_baseline_regression_limits_require_current_and_baseline_telemetry(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + """ +global: + minimum_pass_rate: 1.0 + max_avg_latency_regression_pct: 0.15 + max_avg_cost_regression_pct: 0.15 +cases: + - id: case_1 + expected_all: ["refund"] + min_score: 0.7 +""".strip(), + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + '{"cases":[{"id":"case_1","response":"refund confirmed"}]}', + encoding="utf-8", + ) + + baseline = tmp_path / "baseline.json" + baseline.write_text( + '{"summary":{"pass_rate":1.0},"cases":[]}', + encoding="utf-8", + ) + + report = evaluate(spec, results, baseline) + assert report.summary.gate_passed is False + assert any("Latency regression limit is configured" in r for r in report.summary.gate_reasons) + assert any("Cost regression limit is configured" in r for r in report.summary.gate_reasons)