From aaff3ef7d3d623b88a14530250e38b84029ffba9 Mon Sep 17 00:00:00 2001 From: Brandon Date: Tue, 19 May 2026 10:49:09 -0400 Subject: [PATCH] feat: add case tag rollups to gate reports --- README.md | 4 ++ examples/spec.yaml | 3 + src/agent_release_gate/evaluator.py | 74 +++++++++++++++++++++- src/agent_release_gate/models.py | 13 ++++ tests/test_evaluator.py | 95 ++++++++++++++++++++++++++++- 5 files changed, 186 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a6ae3f5..4ebda23 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ This tool makes those problems visible before release. - Scores each test case using expected + forbidden phrases - Calculates pass rate, average latency, p95 latency, and average cost - Fails if quality drops below your threshold +- Groups results by optional case tags so failure clusters are obvious in JSON and Markdown reports - Optionally compares against a baseline report and blocks regressions - Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast - Optionally enforces a global p95 latency limit to catch long-tail slow responses @@ -100,6 +101,7 @@ global: cases: - id: refund_status + tags: ["billing", "support"] expected_all: ["refund", "3-5 business days"] expected_any: ["order", "transaction"] forbidden: ["cannot help", "policy not found"] @@ -110,6 +112,8 @@ cases: `max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit. +`tags` is optional per case. Use it to group failures by workflow, surface area, or owner-adjacent bucket (for example `["onboarding", "activation"]`). Reports include a tag summary so PMs and engineers can see where regressions cluster instead of reading every case one by one. + When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run. When `max_p95_latency_ms` is configured globally, the gate fails if p95 latency is above the threshold or latency telemetry is missing. diff --git a/examples/spec.yaml b/examples/spec.yaml index f769c5b..b4811d4 100644 --- a/examples/spec.yaml +++ b/examples/spec.yaml @@ -9,6 +9,7 @@ global: cases: - id: refund_status + tags: ["billing", "support"] expected_all: ["refund", "3-5 business days"] expected_any: ["order", "transaction"] forbidden: ["cannot help", "policy not found"] @@ -17,12 +18,14 @@ cases: max_cost_usd: 0.02 - id: address_change + tags: ["account", "support"] expected_all: ["address", "update"] expected_any: ["shipping", "profile"] forbidden: ["not possible"] min_score: 0.7 - id: cancel_order + tags: ["billing", "order-management"] expected_all: ["cancel", "order"] expected_any: ["window", "before shipment"] forbidden: ["no idea"] diff --git a/src/agent_release_gate/evaluator.py b/src/agent_release_gate/evaluator.py index 7dc7c0f..6e6522b 100644 --- a/src/agent_release_gate/evaluator.py +++ b/src/agent_release_gate/evaluator.py @@ -13,6 +13,7 @@ GateReport, GateSummary, ScoredCase, + TagSummary, parse_results, parse_spec, ) @@ -103,7 +104,44 @@ def score_case(case: GateCase, result: CaseResult) -> ScoredCase: if passed and not notes: notes.append("Looks good") - return ScoredCase(id=case.id, score=round(score, 4), passed=passed, notes=notes) + return ScoredCase( + id=case.id, + score=round(score, 4), + passed=passed, + notes=notes, + tags=list(case.tags), + ) + + +def build_tag_summaries(spec_cases: list[GateCase], scored_cases: list[ScoredCase]) -> list[TagSummary]: + scored_by_id = {case.id: case for case in scored_cases} + tag_buckets: dict[str, list[ScoredCase]] = {} + + for spec_case in spec_cases: + scored_case = scored_by_id.get(spec_case.id) + if scored_case is None: + continue + + for tag in dict.fromkeys(spec_case.tags): + tag_buckets.setdefault(tag, []).append(scored_case) + + summaries: list[TagSummary] = [] + for tag in sorted(tag_buckets): + cases = tag_buckets[tag] + total_cases = len(cases) + passed_cases = sum(1 for case in cases if case.passed) + failing_case_ids = [case.id for case in cases if not case.passed] + summaries.append( + TagSummary( + tag=tag, + total_cases=total_cases, + passed_cases=passed_cases, + pass_rate=round((passed_cases / total_cases) if total_cases else 0.0, 4), + failing_case_ids=failing_case_ids, + ) + ) + + return summaries def evaluate( @@ -125,6 +163,7 @@ def evaluate( score=0.0, passed=False, notes=["Missing case result"], + tags=list(case.tags), ) ) continue @@ -246,6 +285,8 @@ def evaluate( if gate_passed: reasons.append("Gate passed") + tag_summaries = build_tag_summaries(spec.cases, scored) + summary = GateSummary( total_cases=total, passed_cases=passed, @@ -255,6 +296,7 @@ def evaluate( gate_passed=gate_passed, gate_reasons=reasons, p95_latency_ms=round(p95_latency, 2) if p95_latency is not None else None, + tag_summaries=tag_summaries, ) return GateReport(summary=summary, cases=scored) @@ -271,9 +313,19 @@ def to_dict(report: GateReport) -> dict: "p95_latency_ms": report.summary.p95_latency_ms, "gate_passed": report.summary.gate_passed, "gate_reasons": report.summary.gate_reasons, + "tag_summaries": [ + { + "tag": tag.tag, + "total_cases": tag.total_cases, + "passed_cases": tag.passed_cases, + "pass_rate": tag.pass_rate, + "failing_case_ids": tag.failing_case_ids, + } + for tag in report.summary.tag_summaries + ], }, "cases": [ - {"id": c.id, "score": c.score, "passed": c.passed, "notes": c.notes} + {"id": c.id, "score": c.score, "passed": c.passed, "notes": c.notes, "tags": c.tags} for c in report.cases ], } @@ -298,10 +350,28 @@ def to_markdown(report: GateReport) -> str: for reason in s.gate_reasons: lines.append(f"- {reason}") + if s.tag_summaries: + lines.extend( + [ + "", + "## Tag summary", + "", + "| Tag | Pass rate | Passed / Total | Failing cases |", + "|---|---:|---:|---|", + ] + ) + for tag in s.tag_summaries: + failing_cases = ", ".join(tag.failing_case_ids) if tag.failing_case_ids else "-" + lines.append( + f"| {tag.tag} | {tag.pass_rate:.2%} | {tag.passed_cases}/{tag.total_cases} | {failing_cases} |" + ) + lines.extend(["", "## Case details", ""]) for c in report.cases: lines.append(f"### {c.id} {'✅' if c.passed else '❌'}") lines.append(f"- Score: {c.score:.2f}") + if c.tags: + lines.append(f"- Tags: {', '.join(c.tags)}") for note in c.notes: lines.append(f"- {note}") lines.append("") diff --git a/src/agent_release_gate/models.py b/src/agent_release_gate/models.py index a203e5c..4d2f66b 100644 --- a/src/agent_release_gate/models.py +++ b/src/agent_release_gate/models.py @@ -10,6 +10,7 @@ class GateCase: expected_all: list[str] = field(default_factory=list) expected_any: list[str] = field(default_factory=list) forbidden: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) min_score: float = 0.7 max_latency_ms: Optional[int] = None max_cost_usd: Optional[float] = None @@ -41,6 +42,16 @@ class ScoredCase: score: float passed: bool notes: list[str] + tags: list[str] = field(default_factory=list) + + +@dataclass +class TagSummary: + tag: str + total_cases: int + passed_cases: int + pass_rate: float + failing_case_ids: list[str] = field(default_factory=list) @dataclass @@ -53,6 +64,7 @@ class GateSummary: gate_passed: bool gate_reasons: list[str] p95_latency_ms: Optional[float] = None + tag_summaries: list[TagSummary] = field(default_factory=list) @dataclass @@ -78,6 +90,7 @@ def parse_spec(data: dict[str, Any]) -> GateSpec: expected_all=_as_list(c.get("expected_all")), expected_any=_as_list(c.get("expected_any")), forbidden=_as_list(c.get("forbidden")), + tags=_as_list(c.get("tags")), min_score=float(c.get("min_score", 0.7)), max_latency_ms=( int(c["max_latency_ms"]) if c.get("max_latency_ms") is not None else None diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 099ec23..cc634b7 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -1,6 +1,7 @@ +import json from pathlib import Path -from agent_release_gate.evaluator import evaluate +from agent_release_gate.evaluator import evaluate, to_dict, to_markdown def test_evaluate_happy_path(tmp_path: Path): @@ -402,3 +403,95 @@ def test_baseline_regression_limits_require_current_and_baseline_telemetry(tmp_p assert report.summary.gate_passed is False assert any("Latency regression limit is configured" in r for r in report.summary.gate_reasons) assert any("Cost regression limit is configured" in r for r in report.summary.gate_reasons) + + +def test_evaluate_rolls_up_tag_summaries(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + "\n".join( + [ + "global:", + " minimum_pass_rate: 0.5", + "cases:", + " - id: billing_refund", + ' expected_all: ["refund"]', + ' tags: ["billing", "support"]', + " - id: billing_status", + ' expected_all: ["invoice"]', + ' tags: ["billing"]', + " - id: safety_escalation", + ' expected_all: ["escalate"]', + ' tags: ["safety", "support"]', + ] + ) + + "\n", + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + json.dumps( + { + "cases": [ + {"id": "billing_refund", "response": "refund approved"}, + {"id": "billing_status", "response": "status unknown"}, + {"id": "safety_escalation", "response": "please escalate this issue"}, + ] + } + ), + encoding="utf-8", + ) + + report = evaluate(spec, results) + tag_summaries = {item.tag: item for item in report.summary.tag_summaries} + + assert report.cases[0].tags == ["billing", "support"] + assert tag_summaries["billing"].total_cases == 2 + assert tag_summaries["billing"].passed_cases == 1 + assert tag_summaries["billing"].failing_case_ids == ["billing_status"] + assert tag_summaries["support"].pass_rate == 1.0 + assert tag_summaries["safety"].failing_case_ids == [] + + +def test_report_exports_include_tag_context(tmp_path: Path): + spec = tmp_path / "spec.yaml" + spec.write_text( + "\n".join( + [ + "global:", + " minimum_pass_rate: 0.5", + "cases:", + " - id: onboarding_answer", + ' expected_all: ["setup"]', + ' tags: ["onboarding"]', + " - id: onboarding_followup", + ' expected_all: ["next steps"]', + ' tags: ["onboarding", "support"]', + ] + ) + + "\n", + encoding="utf-8", + ) + + results = tmp_path / "results.json" + results.write_text( + json.dumps( + { + "cases": [ + {"id": "onboarding_answer", "response": "setup guide is here"}, + {"id": "onboarding_followup", "response": "no answer available"}, + ] + } + ), + encoding="utf-8", + ) + + report = evaluate(spec, results) + payload = to_dict(report) + markdown = to_markdown(report) + + assert payload["summary"]["tag_summaries"][0]["tag"] == "onboarding" + assert payload["cases"][1]["tags"] == ["onboarding", "support"] + assert "## Tag summary" in markdown + assert "| onboarding | 50.00% | 1/2 | onboarding_followup |" in markdown + assert "- Tags: onboarding, support" in markdown