From 43339b098cd0b6c2cc5a3c715e0e98fdf75a2985 Mon Sep 17 00:00:00 2001
From: Brandon <bwise@orbit.bwise.local>
Date: Tue, 24 Mar 2026 21:11:15 -0400
Subject: [PATCH] feat: fail gate when global telemetry limits lack metrics

---
 README.md                           |  3 ++
 src/agent_release_gate/evaluator.py | 32 +++++++++++-----
 tests/test_evaluator.py             | 58 +++++++++++++++++++++++++++++
 3 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 3840cc9..b71ba0e 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,7 @@ This tool makes those problems visible before release.
 - Fails if quality drops below your threshold
 - Optionally compares against a baseline report and blocks regressions
 - Supports per-case latency/cost limits to catch outliers hidden by averages
+- Enforces telemetry presence when global average latency/cost limits are configured
 - Outputs both JSON (for machines) and Markdown (for humans)
 
 ## Install
@@ -67,6 +68,8 @@ cases:
 
 `max_latency_ms` and `max_cost_usd` are optional per-case guardrails. If set, that case fails when telemetry is missing or exceeds the limit.
 
+When `max_avg_latency_ms` or `max_avg_cost_usd` is configured globally, the gate also fails if the corresponding telemetry is missing across the run.
+
 ## Repo layout
 
 - `src/agent_release_gate/`: scoring + gate logic
diff --git a/src/agent_release_gate/evaluator.py b/src/agent_release_gate/evaluator.py
index 4c0155b..e35d2f2 100644
--- a/src/agent_release_gate/evaluator.py
+++ b/src/agent_release_gate/evaluator.py
@@ -138,17 +138,29 @@ def evaluate(
             f"Pass rate {pass_rate:.2%} is below minimum {spec.minimum_pass_rate:.2%}"
         )
 
-    if spec.max_avg_latency_ms is not None and avg_latency is not None and avg_latency > spec.max_avg_latency_ms:
-        gate_passed = False
-        reasons.append(
-            f"Average latency {avg_latency:.1f}ms exceeds limit {spec.max_avg_latency_ms}ms"
-        )
+    if spec.max_avg_latency_ms is not None:
+        if avg_latency is None:
+            gate_passed = False
+            reasons.append(
+                "Average latency limit is configured, but no latency telemetry was provided"
+            )
+        elif avg_latency > spec.max_avg_latency_ms:
+            gate_passed = False
+            reasons.append(
+                f"Average latency {avg_latency:.1f}ms exceeds limit {spec.max_avg_latency_ms}ms"
+            )
 
-    if spec.max_avg_cost_usd is not None and avg_cost is not None and avg_cost > spec.max_avg_cost_usd:
-        gate_passed = False
-        reasons.append(
-            f"Average cost ${avg_cost:.4f} exceeds limit ${spec.max_avg_cost_usd:.4f}"
-        )
+    if spec.max_avg_cost_usd is not None:
+        if avg_cost is None:
+            gate_passed = False
+            reasons.append(
+                "Average cost limit is configured, but no cost telemetry was provided"
+            )
+        elif avg_cost > spec.max_avg_cost_usd:
+            gate_passed = False
+            reasons.append(
+                f"Average cost ${avg_cost:.4f} exceeds limit ${spec.max_avg_cost_usd:.4f}"
+            )
 
     if baseline_path:
         baseline = load_json(baseline_path)
diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py
index 86e5cfa..d39de83 100644
--- a/tests/test_evaluator.py
+++ b/tests/test_evaluator.py
@@ -147,3 +147,61 @@ def test_case_limits_require_telemetry_when_configured(tmp_path: Path):
     assert report.cases[0].passed is False
     assert "Missing latency_ms for case with max_latency_ms set" in report.cases[0].notes
     assert "Missing cost_usd for case with max_cost_usd set" in report.cases[0].notes
+
+
+def test_global_latency_limit_requires_telemetry_when_configured(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_avg_latency_ms: 1000
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed"}]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert any(
+        "Average latency limit is configured, but no latency telemetry was provided" in r
+        for r in report.summary.gate_reasons
+    )
+
+
+def test_global_cost_limit_requires_telemetry_when_configured(tmp_path: Path):
+    spec = tmp_path / "spec.yaml"
+    spec.write_text(
+        """
+global:
+  minimum_pass_rate: 1.0
+  max_avg_cost_usd: 0.01
+cases:
+  - id: case_1
+    expected_all: ["refund"]
+    min_score: 0.7
+""".strip(),
+        encoding="utf-8",
+    )
+
+    results = tmp_path / "results.json"
+    results.write_text(
+        '{"cases":[{"id":"case_1","response":"refund confirmed"}]}',
+        encoding="utf-8",
+    )
+
+    report = evaluate(spec, results)
+    assert report.summary.gate_passed is False
+    assert any(
+        "Average cost limit is configured, but no cost telemetry was provided" in r
+        for r in report.summary.gate_reasons
+    )