feat: add case tag rollups to gate reports

README.md

@@ -19,6 +19,7 @@ This tool makes those problems visible before release.
 - Scores each test case using expected + forbidden phrases
 - Calculates pass rate, average latency, p95 latency, and average cost
 - Fails if quality drops below your threshold
+- Groups results by optional case tags so failure clusters are obvious in JSON and Markdown reports
 - Optionally compares against a baseline report and blocks regressions
 - Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast
 - Optionally enforces a global p95 latency limit to catch long-tail slow responses
@@ -100,6 +101,7 @@ global:
 
 cases:
   - id: refund_status
+    tags: ["billing", "support"]
     expected_all: ["refund", "3-5 business days"]
     expected_any: ["order", "transaction"]
     forbidden: ["cannot help", "policy not found"]
@@ -110,6 +112,8 @@ cases:
 
 `max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit.
 
+`tags` is optional per case. Use it to group failures by workflow, surface area, or owner-adjacent bucket (for example `["onboarding", "activation"]`). Reports include a tag summary so PMs and engineers can see where regressions cluster instead of reading every case one by one.
+
 When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run.
 
 When `max_p95_latency_ms` is configured globally, the gate fails if p95 latency is above the threshold or latency telemetry is missing.

examples/spec.yaml

@@ -9,6 +9,7 @@ global:
 
 cases:
   - id: refund_status
+    tags: ["billing", "support"]
     expected_all: ["refund", "3-5 business days"]
     expected_any: ["order", "transaction"]
     forbidden: ["cannot help", "policy not found"]
@@ -17,12 +18,14 @@ cases:
     max_cost_usd: 0.02
 
   - id: address_change
+    tags: ["account", "support"]
     expected_all: ["address", "update"]
     expected_any: ["shipping", "profile"]
     forbidden: ["not possible"]
     min_score: 0.7
 
   - id: cancel_order
+    tags: ["billing", "order-management"]
     expected_all: ["cancel", "order"]
     expected_any: ["window", "before shipment"]
     forbidden: ["no idea"]

src/agent_release_gate/evaluator.py

@@ -13,6 +13,7 @@
     GateReport,
     GateSummary,
     ScoredCase,
+    TagSummary,
     parse_results,
     parse_spec,
 )
@@ -103,7 +104,44 @@ def score_case(case: GateCase, result: CaseResult) -> ScoredCase:
     if passed and not notes:
         notes.append("Looks good")
 
-    return ScoredCase(id=case.id, score=round(score, 4), passed=passed, notes=notes)
+    return ScoredCase(
+        id=case.id,
+        score=round(score, 4),
+        passed=passed,
+        notes=notes,
+        tags=list(case.tags),
+    )
+
+
+def build_tag_summaries(spec_cases: list[GateCase], scored_cases: list[ScoredCase]) -> list[TagSummary]:
+    scored_by_id = {case.id: case for case in scored_cases}
+    tag_buckets: dict[str, list[ScoredCase]] = {}
+
+    for spec_case in spec_cases:
+        scored_case = scored_by_id.get(spec_case.id)
+        if scored_case is None:
+            continue
+
+        for tag in dict.fromkeys(spec_case.tags):
+            tag_buckets.setdefault(tag, []).append(scored_case)
+
+    summaries: list[TagSummary] = []
+    for tag in sorted(tag_buckets):
+        cases = tag_buckets[tag]
+        total_cases = len(cases)
+        passed_cases = sum(1 for case in cases if case.passed)
+        failing_case_ids = [case.id for case in cases if not case.passed]
+        summaries.append(
+            TagSummary(
+                tag=tag,
+                total_cases=total_cases,
+                passed_cases=passed_cases,
+                pass_rate=round((passed_cases / total_cases) if total_cases else 0.0, 4),
+                failing_case_ids=failing_case_ids,
+            )
+        )
+
+    return summaries
 
 
 def evaluate(
@@ -125,6 +163,7 @@ def evaluate(
                     score=0.0,
                     passed=False,
                     notes=["Missing case result"],
+                    tags=list(case.tags),
                 )
             )
             continue
@@ -246,6 +285,8 @@ def evaluate(
     if gate_passed:
         reasons.append("Gate passed")
 
+    tag_summaries = build_tag_summaries(spec.cases, scored)
+
     summary = GateSummary(
         total_cases=total,
         passed_cases=passed,
@@ -255,6 +296,7 @@ def evaluate(
         gate_passed=gate_passed,
         gate_reasons=reasons,
         p95_latency_ms=round(p95_latency, 2) if p95_latency is not None else None,
+        tag_summaries=tag_summaries,
     )
 
     return GateReport(summary=summary, cases=scored)
@@ -271,9 +313,19 @@ def to_dict(report: GateReport) -> dict:
             "p95_latency_ms": report.summary.p95_latency_ms,
             "gate_passed": report.summary.gate_passed,
             "gate_reasons": report.summary.gate_reasons,
+            "tag_summaries": [
+                {
+                    "tag": tag.tag,
+                    "total_cases": tag.total_cases,
+                    "passed_cases": tag.passed_cases,
+                    "pass_rate": tag.pass_rate,
+                    "failing_case_ids": tag.failing_case_ids,
+                }
+                for tag in report.summary.tag_summaries
+            ],
         },
         "cases": [
-            {"id": c.id, "score": c.score, "passed": c.passed, "notes": c.notes}
+            {"id": c.id, "score": c.score, "passed": c.passed, "notes": c.notes, "tags": c.tags}
             for c in report.cases
         ],
     }
@@ -298,10 +350,28 @@ def to_markdown(report: GateReport) -> str:
     for reason in s.gate_reasons:
         lines.append(f"- {reason}")
 
+    if s.tag_summaries:
+        lines.extend(
+            [
+                "",
+                "## Tag summary",
+                "",
+                "| Tag | Pass rate | Passed / Total | Failing cases |",
+                "|---|---:|---:|---|",
+            ]
+        )
+        for tag in s.tag_summaries:
+            failing_cases = ", ".join(tag.failing_case_ids) if tag.failing_case_ids else "-"
+            lines.append(
+                f"| {tag.tag} | {tag.pass_rate:.2%} | {tag.passed_cases}/{tag.total_cases} | {failing_cases} |"
+            )
+
     lines.extend(["", "## Case details", ""])
     for c in report.cases:
         lines.append(f"### {c.id} {'✅' if c.passed else '❌'}")
         lines.append(f"- Score: {c.score:.2f}")
+        if c.tags:
+            lines.append(f"- Tags: {', '.join(c.tags)}")
         for note in c.notes:
             lines.append(f"- {note}")
         lines.append("")

src/agent_release_gate/models.py

@@ -10,6 +10,7 @@ class GateCase:
     expected_all: list[str] = field(default_factory=list)
     expected_any: list[str] = field(default_factory=list)
     forbidden: list[str] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
     min_score: float = 0.7
     max_latency_ms: Optional[int] = None
     max_cost_usd: Optional[float] = None
@@ -41,6 +42,16 @@ class ScoredCase:
     score: float
     passed: bool
     notes: list[str]
+    tags: list[str] = field(default_factory=list)
+
+
+@dataclass
+class TagSummary:
+    tag: str
+    total_cases: int
+    passed_cases: int
+    pass_rate: float
+    failing_case_ids: list[str] = field(default_factory=list)
 
 
 @dataclass
@@ -53,6 +64,7 @@ class GateSummary:
     gate_passed: bool
     gate_reasons: list[str]
     p95_latency_ms: Optional[float] = None
+    tag_summaries: list[TagSummary] = field(default_factory=list)
 
 
 @dataclass
@@ -78,6 +90,7 @@ def parse_spec(data: dict[str, Any]) -> GateSpec:
             expected_all=_as_list(c.get("expected_all")),
             expected_any=_as_list(c.get("expected_any")),
             forbidden=_as_list(c.get("forbidden")),
+            tags=_as_list(c.get("tags")),
             min_score=float(c.get("min_score", 0.7)),
             max_latency_ms=(
                 int(c["max_latency_ms"]) if c.get("max_latency_ms") is not None else None

tests/test_evaluator.py

@@ -1,6 +1,7 @@
+import json
 from pathlib import Path
 
-from agent_release_gate.evaluator import evaluate
+from agent_release_gate.evaluator import evaluate, to_dict, to_markdown
 
 
 def test_evaluate_happy_path(tmp_path: Path):
@@ -402,3 +403,95 @@ def test_baseline_regression_limits_require_current_and_baseline_telemetry(tmp_p
     assert report.summary.gate_passed is False
     assert any("Latency regression limit is configured" in r for r in report.summary.gate_reasons)
     assert any("Cost regression limit is configured" in r for r in report.summary.gate_reasons)
+
+
+def test_evaluate_rolls_up_tag_summaries(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        "\n".join(
+            [
+                "global:",
+                "  minimum_pass_rate: 0.5",
+                "cases:",
+                "  - id: billing_refund",
+                '    expected_all: ["refund"]',
+                '    tags: ["billing", "support"]',
+                "  - id: billing_status",
+                '    expected_all: ["invoice"]',
+                '    tags: ["billing"]',
+                "  - id: safety_escalation",
+                '    expected_all: ["escalate"]',
+                '    tags: ["safety", "support"]',
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        json.dumps(
+            {
+                "cases": [
+                    {"id": "billing_refund", "response": "refund approved"},
+                    {"id": "billing_status", "response": "status unknown"},
+                    {"id": "safety_escalation", "response": "please escalate this issue"},
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    tag_summaries = {item.tag: item for item in report.summary.tag_summaries}
+
+    assert report.cases[0].tags == ["billing", "support"]
+    assert tag_summaries["billing"].total_cases == 2
+    assert tag_summaries["billing"].passed_cases == 1
+    assert tag_summaries["billing"].failing_case_ids == ["billing_status"]
+    assert tag_summaries["support"].pass_rate == 1.0
+    assert tag_summaries["safety"].failing_case_ids == []
+
+
+def test_report_exports_include_tag_context(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        "\n".join(
+            [
+                "global:",
+                "  minimum_pass_rate: 0.5",
+                "cases:",
+                "  - id: onboarding_answer",
+                '    expected_all: ["setup"]',
+                '    tags: ["onboarding"]',
+                "  - id: onboarding_followup",
+                '    expected_all: ["next steps"]',
+                '    tags: ["onboarding", "support"]',
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        json.dumps(
+            {
+                "cases": [
+                    {"id": "onboarding_answer", "response": "setup guide is here"},
+                    {"id": "onboarding_followup", "response": "no answer available"},
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    payload = to_dict(report)
+    markdown = to_markdown(report)
+
+    assert payload["summary"]["tag_summaries"][0]["tag"] == "onboarding"
+    assert payload["cases"][1]["tags"] == ["onboarding", "support"]
+    assert "## Tag summary" in markdown
+    assert "| onboarding | 50.00% | 1/2 | onboarding_followup |" in markdown
+    assert "- Tags: onboarding, support" in markdown