From 190bf4b8d2b163cae445394ec7ebd58c8957d555 Mon Sep 17 00:00:00 2001 From: Brandon Date: Wed, 22 Jul 2026 21:09:18 -0400 Subject: [PATCH] feat: gate required evaluation cases --- README.md | 4 ++ examples/spec.yaml | 1 + src/agent_release_gate.egg-info/PKG-INFO | 72 +++++++++++++++++++-- src/agent_release_gate.egg-info/SOURCES.txt | 4 +- src/agent_release_gate/evaluator.py | 20 +++++- src/agent_release_gate/models.py | 4 ++ tests/test_evaluator.py | 46 +++++++++++++ 7 files changed, 144 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4ebda23..b9c2427 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ This tool makes those problems visible before release. - Scores each test case using expected + forbidden phrases - Calculates pass rate, average latency, p95 latency, and average cost - Fails if quality drops below your threshold +- Lets you mark critical cases as required so a must-pass workflow can block release even when aggregate pass rate looks healthy - Groups results by optional case tags so failure clusters are obvious in JSON and Markdown reports - Optionally compares against a baseline report and blocks regressions - Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast @@ -102,6 +103,7 @@ global: cases: - id: refund_status tags: ["billing", "support"] + required: true expected_all: ["refund", "3-5 business days"] expected_any: ["order", "transaction"] forbidden: ["cannot help", "policy not found"] @@ -110,6 +112,8 @@ cases: max_cost_usd: 0.02 ``` +`required` is optional per case. Set it to `true` for launch-blocking journeys, compliance checks, or executive-demo flows that cannot fail just because the aggregate pass rate still clears the bar. + `max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit. `tags` is optional per case. Use it to group failures by workflow, surface area, or owner-adjacent bucket (for example `["onboarding", "activation"]`). Reports include a tag summary so PMs and engineers can see where regressions cluster instead of reading every case one by one. diff --git a/examples/spec.yaml b/examples/spec.yaml index b4811d4..b22dded 100644 --- a/examples/spec.yaml +++ b/examples/spec.yaml @@ -10,6 +10,7 @@ global: cases: - id: refund_status tags: ["billing", "support"] + required: true expected_all: ["refund", "3-5 business days"] expected_any: ["order", "transaction"] forbidden: ["cannot help", "policy not found"] diff --git a/src/agent_release_gate.egg-info/PKG-INFO b/src/agent_release_gate.egg-info/PKG-INFO index 8ff9501..1212500 100644 --- a/src/agent_release_gate.egg-info/PKG-INFO +++ b/src/agent_release_gate.egg-info/PKG-INFO @@ -25,9 +25,16 @@ This tool makes those problems visible before release. ## What it does - Scores each test case using expected + forbidden phrases -- Calculates pass rate, average latency, and average cost +- Calculates pass rate, average latency, p95 latency, and average cost - Fails if quality drops below your threshold +- Lets you mark critical cases as required so a must-pass workflow can block release even when aggregate pass rate looks healthy +- Groups results by optional case tags so failure clusters are obvious in JSON and Markdown reports - Optionally compares against a baseline report and blocks regressions +- Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast +- Optionally enforces a global p95 latency limit to catch long-tail slow responses +- Optionally records run summaries and detects cross-run pass-rate, latency, p95 latency, and cost drift across releases +- Supports per-case latency/cost limits to catch outliers hidden by averages +- Enforces telemetry presence when global average latency/cost limits are configured - Outputs both JSON (for machines) and Markdown (for humans) ## Install @@ -53,6 +60,42 @@ The command exits with: Perfect for CI pipelines. +## Track trend drift across runs + +Single-baseline checks catch point-in-time regressions. Trend analysis catches slow degradation over many runs. + +```bash +# Record this run in a history folder +argate evaluate \ + --spec examples/spec.yaml \ + --results examples/results.json \ + --record-history .gate-history + +# Analyze the latest 10 runs and print trend JSON +argate trend --history .gate-history --window 10 + +# CI mode: fail when pass-rate trend is declining +argate trend --history .gate-history --fail-on-regression + +# Optional CI mode: fail when average latency, p95 latency, or cost is trending up +argate trend --history .gate-history --fail-on-latency-regression +argate trend --history .gate-history --fail-on-p95-regression +argate trend --history .gate-history --fail-on-cost-regression + +# Fail on any tracked regression across pass rate, latency, p95 latency, or cost +argate trend --history .gate-history --fail-on-any-regression +``` + +`argate evaluate --record-history ...` stores pass rate, average latency, p95 latency, and average cost in each history file when that telemetry is available in the evaluation report. + +`argate trend` keeps the existing pass-rate output and now also includes slope, direction, and regression booleans for: + +- `avg_latency_ms` +- `p95_latency_ms` +- `avg_cost_usd` + +Older history files that predate these metrics are still supported. When there are fewer than three runs with a given metric inside the analysis window, that metric reports `insufficient_data` instead of forcing a regression. + ## Spec format ```yaml @@ -60,22 +103,41 @@ global: minimum_pass_rate: 0.8 allowed_regression: 0.02 max_avg_latency_ms: 1500 + max_p95_latency_ms: 2000 max_avg_cost_usd: 0.03 + max_avg_latency_regression_pct: 0.15 + max_avg_cost_regression_pct: 0.10 cases: - id: refund_status + tags: ["billing", "support"] + required: true expected_all: ["refund", "3-5 business days"] expected_any: ["order", "transaction"] forbidden: ["cannot help", "policy not found"] min_score: 0.72 + max_latency_ms: 1200 + max_cost_usd: 0.02 ``` +`required` is optional per case. Set it to `true` for launch-blocking journeys, compliance checks, or executive-demo flows that cannot fail just because the aggregate pass rate still clears the bar. + +`max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit. + +`tags` is optional per case. Use it to group failures by workflow, surface area, or owner-adjacent bucket (for example `["onboarding", "activation"]`). Reports include a tag summary so PMs and engineers can see where regressions cluster instead of reading every case one by one. + +When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run. + +When `max_p95_latency_ms` is configured globally, the gate fails if p95 latency is above the threshold or latency telemetry is missing. + +When `--baseline` is provided, you can also set `max_avg_latency_regression_pct` and/or `max_avg_cost_regression_pct` to fail if average latency or cost regresses beyond the allowed percentage increase vs baseline. + ## Repo layout -- `src/agent_release_gate/` — scoring + gate logic -- `examples/` — runnable demo spec and results -- `tests/` — unit tests -- `.github/workflows/ci.yml` — CI example +- `src/agent_release_gate/`: scoring + gate logic +- `examples/`: runnable demo spec and results +- `tests/`: unit tests +- `.github/workflows/ci.yml`: CI example ## Development diff --git a/src/agent_release_gate.egg-info/SOURCES.txt b/src/agent_release_gate.egg-info/SOURCES.txt index 5309a58..8b218be 100644 --- a/src/agent_release_gate.egg-info/SOURCES.txt +++ b/src/agent_release_gate.egg-info/SOURCES.txt @@ -3,6 +3,7 @@ pyproject.toml src/agent_release_gate/__init__.py src/agent_release_gate/cli.py src/agent_release_gate/evaluator.py +src/agent_release_gate/history.py src/agent_release_gate/models.py src/agent_release_gate.egg-info/PKG-INFO src/agent_release_gate.egg-info/SOURCES.txt @@ -11,4 +12,5 @@ src/agent_release_gate.egg-info/entry_points.txt src/agent_release_gate.egg-info/requires.txt src/agent_release_gate.egg-info/top_level.txt tests/test_cli.py -tests/test_evaluator.py \ No newline at end of file +tests/test_evaluator.py +tests/test_history.py \ No newline at end of file diff --git a/src/agent_release_gate/evaluator.py b/src/agent_release_gate/evaluator.py index 6e6522b..afa8ac3 100644 --- a/src/agent_release_gate/evaluator.py +++ b/src/agent_release_gate/evaluator.py @@ -110,6 +110,7 @@ def score_case(case: GateCase, result: CaseResult) -> ScoredCase: passed=passed, notes=notes, tags=list(case.tags), + required=case.required, ) @@ -164,6 +165,7 @@ def evaluate( passed=False, notes=["Missing case result"], tags=list(case.tags), + required=case.required, ) ) continue @@ -188,6 +190,11 @@ def evaluate( f"Pass rate {pass_rate:.2%} is below minimum {spec.minimum_pass_rate:.2%}" ) + required_case_failures = [case.id for case in scored if case.required and not case.passed] + if required_case_failures: + gate_passed = False + reasons.append(f"Required cases failed: {', '.join(required_case_failures)}") + if spec.max_avg_latency_ms is not None: if avg_latency is None: gate_passed = False @@ -296,6 +303,7 @@ def evaluate( gate_passed=gate_passed, gate_reasons=reasons, p95_latency_ms=round(p95_latency, 2) if p95_latency is not None else None, + required_case_failures=required_case_failures, tag_summaries=tag_summaries, ) @@ -313,6 +321,7 @@ def to_dict(report: GateReport) -> dict: "p95_latency_ms": report.summary.p95_latency_ms, "gate_passed": report.summary.gate_passed, "gate_reasons": report.summary.gate_reasons, + "required_case_failures": report.summary.required_case_failures, "tag_summaries": [ { "tag": tag.tag, @@ -325,7 +334,14 @@ def to_dict(report: GateReport) -> dict: ], }, "cases": [ - {"id": c.id, "score": c.score, "passed": c.passed, "notes": c.notes, "tags": c.tags} + { + "id": c.id, + "score": c.score, + "passed": c.passed, + "notes": c.notes, + "tags": c.tags, + "required": c.required, + } for c in report.cases ], } @@ -370,6 +386,8 @@ def to_markdown(report: GateReport) -> str: for c in report.cases: lines.append(f"### {c.id} {'✅' if c.passed else '❌'}") lines.append(f"- Score: {c.score:.2f}") + if c.required: + lines.append("- Required: yes") if c.tags: lines.append(f"- Tags: {', '.join(c.tags)}") for note in c.notes: diff --git a/src/agent_release_gate/models.py b/src/agent_release_gate/models.py index 4d2f66b..84697de 100644 --- a/src/agent_release_gate/models.py +++ b/src/agent_release_gate/models.py @@ -11,6 +11,7 @@ class GateCase: expected_any: list[str] = field(default_factory=list) forbidden: list[str] = field(default_factory=list) tags: list[str] = field(default_factory=list) + required: bool = False min_score: float = 0.7 max_latency_ms: Optional[int] = None max_cost_usd: Optional[float] = None @@ -43,6 +44,7 @@ class ScoredCase: passed: bool notes: list[str] tags: list[str] = field(default_factory=list) + required: bool = False @dataclass @@ -64,6 +66,7 @@ class GateSummary: gate_passed: bool gate_reasons: list[str] p95_latency_ms: Optional[float] = None + required_case_failures: list[str] = field(default_factory=list) tag_summaries: list[TagSummary] = field(default_factory=list) @@ -91,6 +94,7 @@ def parse_spec(data: dict[str, Any]) -> GateSpec: expected_any=_as_list(c.get("expected_any")), forbidden=_as_list(c.get("forbidden")), tags=_as_list(c.get("tags")), + required=bool(c.get("required", False)), min_score=float(c.get("min_score", 0.7)), max_latency_ms=( int(c["max_latency_ms"]) if c.get("max_latency_ms") is not None else None diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index cc634b7..d243abc 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -453,6 +453,48 @@ def test_evaluate_rolls_up_tag_summaries(tmp_path: Path): assert tag_summaries["safety"].failing_case_ids == [] +def test_required_case_failure_blocks_release_even_when_pass_rate_passes(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + "\n".join( + [ + "global:", + " minimum_pass_rate: 0.5", + "cases:", + " - id: checkout_receipt", + ' expected_all: ["receipt"]', + " required: true", + " - id: account_summary", + ' expected_all: ["summary"]', + ] + ) + + "\n", + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + json.dumps( + { + "cases": [ + {"id": "checkout_receipt", "response": "order queued"}, + {"id": "account_summary", "response": "summary ready"}, + ] + } + ), + encoding="utf-8", + ) + + report = evaluate(spec, results) + + assert report.summary.pass_rate == 0.5 + assert report.summary.gate_passed is False + assert report.summary.required_case_failures == ["checkout_receipt"] + assert any("Required cases failed: checkout_receipt" in r for r in report.summary.gate_reasons) + assert report.cases[0].required is True + assert report.cases[1].required is False + + def test_report_exports_include_tag_context(tmp_path: Path): spec = tmp_path / "spec.yaml" spec.write_text( @@ -463,6 +505,7 @@ def test_report_exports_include_tag_context(tmp_path: Path): "cases:", " - id: onboarding_answer", ' expected_all: ["setup"]', + " required: true", ' tags: ["onboarding"]', " - id: onboarding_followup", ' expected_all: ["next steps"]', @@ -491,7 +534,10 @@ def test_report_exports_include_tag_context(tmp_path: Path): markdown = to_markdown(report) assert payload["summary"]["tag_summaries"][0]["tag"] == "onboarding" + assert payload["summary"]["required_case_failures"] == [] + assert payload["cases"][0]["required"] is True assert payload["cases"][1]["tags"] == ["onboarding", "support"] assert "## Tag summary" in markdown assert "| onboarding | 50.00% | 1/2 | onboarding_followup |" in markdown + assert "- Required: yes" in markdown assert "- Tags: onboarding, support" in markdown