feat: fail when global telemetry limits cannot be enforced

README.md

@@ -21,6 +21,7 @@ This tool makes those problems visible before release.
 - Fails if quality drops below your threshold
 - Optionally compares against a baseline report and blocks regressions
 - Supports per-case latency/cost limits to catch outliers hidden by averages
+- Enforces telemetry presence when global average latency/cost limits are configured
 - Outputs both JSON (for machines) and Markdown (for humans)
 
 ## Install
@@ -67,6 +68,8 @@ cases:
 
 `max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit.
 
+When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run.
+
 ## Repo layout
 
 - `src/agent_release_gate/`: scoring + gate logic

src/agent_release_gate/evaluator.py

@@ -138,17 +138,29 @@ def evaluate(
             f"Pass rate {pass_rate:.2%} is below minimum {spec.minimum_pass_rate:.2%}"
         )
 
-    if spec.max_avg_latency_ms is not None and avg_latency is not None and avg_latency > spec.max_avg_latency_ms:
-        gate_passed = False
-        reasons.append(
-            f"Average latency {avg_latency:.1f}ms exceeds limit {spec.max_avg_latency_ms}ms"
-        )
+    if spec.max_avg_latency_ms is not None:
+        if avg_latency is None:
+            gate_passed = False
+            reasons.append(
+                "Average latency limit is configured, but no latency telemetry was provided"
+            )
+        elif avg_latency > spec.max_avg_latency_ms:
+            gate_passed = False
+            reasons.append(
+                f"Average latency {avg_latency:.1f}ms exceeds limit {spec.max_avg_latency_ms}ms"
+            )
 
-    if spec.max_avg_cost_usd is not None and avg_cost is not None and avg_cost > spec.max_avg_cost_usd:
-        gate_passed = False
-        reasons.append(
-            f"Average cost ${avg_cost:.4f} exceeds limit ${spec.max_avg_cost_usd:.4f}"
-        )
+    if spec.max_avg_cost_usd is not None:
+        if avg_cost is None:
+            gate_passed = False
+            reasons.append(
+                "Average cost limit is configured, but no cost telemetry was provided"
+            )
+        elif avg_cost > spec.max_avg_cost_usd:
+            gate_passed = False
+            reasons.append(
+                f"Average cost ${avg_cost:.4f} exceeds limit ${spec.max_avg_cost_usd:.4f}"
+            )
 
     if baseline_path:
         baseline = load_json(baseline_path)

tests/test_evaluator.py

@@ -147,3 +147,61 @@ def test_case_limits_require_telemetry_when_configured(tmp_path: Path):
     assert report.cases[0].passed is False
     assert "Missing latency_ms for case with max_latency_ms set" in report.cases[0].notes
     assert "Missing cost_usd for case with max_cost_usd set" in report.cases[0].notes
+
+
+def test_global_latency_limit_requires_telemetry_when_configured(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_avg_latency_ms: 1000
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed"}]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert any(
+        "Average latency limit is configured, but no latency telemetry was provided" in r
+        for r in report.summary.gate_reasons
+    )
+
+
+def test_global_cost_limit_requires_telemetry_when_configured(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_avg_cost_usd: 0.01
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed"}]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert any(
+        "Average cost limit is configured, but no cost telemetry was provided" in r
+        for r in report.summary.gate_reasons
+    )