From f16b25b0fd93b469521146895fb23116b5710521 Mon Sep 17 00:00:00 2001 From: Brandon Date: Fri, 20 Mar 2026 21:10:54 -0400 Subject: [PATCH] feat: add per-case latency and cost guardrails --- README.md | 5 ++ examples/spec.yaml | 2 + src/agent_release_gate/evaluator.py | 20 ++++++++ src/agent_release_gate/models.py | 8 +++ tests/test_evaluator.py | 78 +++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+) diff --git a/README.md b/README.md index 81df136..3840cc9 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ This tool makes those problems visible before release. - Calculates pass rate, average latency, and average cost - Fails if quality drops below your threshold - Optionally compares against a baseline report and blocks regressions +- Supports per-case latency/cost limits to catch outliers hidden by averages - Outputs both JSON (for machines) and Markdown (for humans) ## Install @@ -60,8 +61,12 @@ cases: expected_any: ["order", "transaction"] forbidden: ["cannot help", "policy not found"] min_score: 0.72 + max_latency_ms: 1200 + max_cost_usd: 0.02 ``` +`max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit. + ## Repo layout - `src/agent_release_gate/`: scoring + gate logic diff --git a/examples/spec.yaml b/examples/spec.yaml index 3740365..28fc585 100644 --- a/examples/spec.yaml +++ b/examples/spec.yaml @@ -10,6 +10,8 @@ cases: expected_any: ["order", "transaction"] forbidden: ["cannot help", "policy not found"] min_score: 0.72 + max_latency_ms: 1200 + max_cost_usd: 0.02 - id: address_change expected_all: ["address", "update"] diff --git a/src/agent_release_gate/evaluator.py b/src/agent_release_gate/evaluator.py index 4c996fd..4c0155b 100644 --- a/src/agent_release_gate/evaluator.py +++ b/src/agent_release_gate/evaluator.py @@ -70,6 +70,26 @@ def score_case(case: GateCase, result: CaseResult) -> ScoredCase: score = max(0.0, min(score, 1.0)) passed = score >= case.min_score and forbidden_hits == 0 + if case.max_latency_ms is not None: + if result.latency_ms is None: + passed = False + notes.append("Missing latency_ms for case with max_latency_ms set") + elif result.latency_ms > case.max_latency_ms: + passed = False + notes.append( + f"Latency {result.latency_ms}ms exceeds case max {case.max_latency_ms}ms" + ) + + if case.max_cost_usd is not None: + if result.cost_usd is None: + passed = False + notes.append("Missing cost_usd for case with max_cost_usd set") + elif result.cost_usd > case.max_cost_usd: + passed = False + notes.append( + f"Cost ${result.cost_usd:.6f} exceeds case max ${case.max_cost_usd:.6f}" + ) + if passed and not notes: notes.append("Looks good") diff --git a/src/agent_release_gate/models.py b/src/agent_release_gate/models.py index a57500c..65ef088 100644 --- a/src/agent_release_gate/models.py +++ b/src/agent_release_gate/models.py @@ -11,6 +11,8 @@ class GateCase: expected_any: list[str] = field(default_factory=list) forbidden: list[str] = field(default_factory=list) min_score: float = 0.7 + max_latency_ms: Optional[int] = None + max_cost_usd: Optional[float] = None @dataclass @@ -73,6 +75,12 @@ def parse_spec(data: dict[str, Any]) -> GateSpec: expected_any=_as_list(c.get("expected_any")), forbidden=_as_list(c.get("forbidden")), min_score=float(c.get("min_score", 0.7)), + max_latency_ms=( + int(c["max_latency_ms"]) if c.get("max_latency_ms") is not None else None + ), + max_cost_usd=( + float(c["max_cost_usd"]) if c.get("max_cost_usd") is not None else None + ), ) for c in raw_cases ] diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index dfc30ba..86e5cfa 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -69,3 +69,81 @@ def test_evaluate_regression(tmp_path: Path): report = evaluate(spec, results, baseline) assert report.summary.gate_passed is False assert any("Regression detected" in r for r in report.summary.gate_reasons) + + +def test_case_limits_fail_on_latency_and_cost_outliers(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + """ +global: + minimum_pass_rate: 1.0 +cases: + - id: case_1 + expected_all: ["refund"] + min_score: 0.7 + max_latency_ms: 1200 + max_cost_usd: 0.02 +""".strip(), + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + """ +{ + "cases": [ + { + "id": "case_1", + "response": "refund confirmed", + "latency_ms": 1900, + "cost_usd": 0.031 + } + ] +} +""".strip(), + encoding="utf-8", + ) + + report = evaluate(spec, results) + assert report.summary.gate_passed is False + assert report.cases[0].passed is False + assert any("Latency 1900ms exceeds case max 1200ms" in n for n in report.cases[0].notes) + assert any("Cost $0.031000 exceeds case max $0.020000" in n for n in report.cases[0].notes) + + +def test_case_limits_require_telemetry_when_configured(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + """ +global: + minimum_pass_rate: 1.0 +cases: + - id: case_1 + expected_all: ["refund"] + min_score: 0.7 + max_latency_ms: 1200 + max_cost_usd: 0.02 +""".strip(), + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + """ +{ + "cases": [ + { + "id": "case_1", + "response": "refund confirmed" + } + ] +} +""".strip(), + encoding="utf-8", + ) + + report = evaluate(spec, results) + assert report.summary.gate_passed is False + assert report.cases[0].passed is False + assert "Missing latency_ms for case with max_latency_ms set" in report.cases[0].notes + assert "Missing cost_usd for case with max_cost_usd set" in report.cases[0].notes