feat: add multi-metric trend release gates

README.md

@@ -17,13 +17,12 @@ This tool makes those problems visible before release.
 ## What it does
 
 - Scores each test case using expected + forbidden phrases
-- Calculates pass rate, average latency, and average cost
-- Calculates pass rate, average latency, and p95 latency (tail behavior)
+- Calculates pass rate, average latency, p95 latency, and average cost
 - Fails if quality drops below your threshold
 - Optionally compares against a baseline report and blocks regressions
 - Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast
 - Optionally enforces a global p95 latency limit to catch long-tail slow responses
-- Optionally records run summaries and detects sustained pass-rate drift across releases
+- Optionally records run summaries and detects cross-run pass-rate, latency, p95 latency, and cost drift across releases
 - Supports per-case latency/cost limits to catch outliers hidden by averages
 - Enforces telemetry presence when global average latency/cost limits are configured
 - Outputs both JSON (for machines) and Markdown (for humans)
@@ -67,8 +66,26 @@ argate trend --history .gate-history --window 10
 
 # CI mode: fail when pass-rate trend is declining
 argate trend --history .gate-history --fail-on-regression
+
+# Optional CI mode: fail when average latency, p95 latency, or cost is trending up
+argate trend --history .gate-history --fail-on-latency-regression
+argate trend --history .gate-history --fail-on-p95-regression
+argate trend --history .gate-history --fail-on-cost-regression
+
+# Fail on any tracked regression across pass rate, latency, p95 latency, or cost
+argate trend --history .gate-history --fail-on-any-regression
 ```
 
+`argate evaluate --record-history ...` stores pass rate, average latency, p95 latency, and average cost in each history file when that telemetry is available in the evaluation report.
+
+`argate trend` keeps the existing pass-rate output and now also includes slope, direction, and regression booleans for:
+
+- `avg_latency_ms`
+- `p95_latency_ms`
+- `avg_cost_usd`
+
+Older history files that predate these metrics are still supported. When there are fewer than three runs with a given metric inside the analysis window, that metric reports `insufficient_data` instead of forcing a regression.
+
 ## Spec format
 
 ```yaml

src/agent_release_gate/cli.py

@@ -25,14 +25,37 @@ def build_parser() -> argparse.ArgumentParser:
         help="Optional directory to append run summaries for cross-run trend analysis",
     )
 
-    trend_cmd = sub.add_parser("trend", help="Analyze pass-rate trend across historical run summaries")
+    trend_cmd = sub.add_parser(
+        "trend",
+        help="Analyze cross-run quality, latency, and cost trends across historical run summaries",
+    )
     trend_cmd.add_argument("--history", required=True, help="Directory containing historical run JSON")
     trend_cmd.add_argument("--window", type=int, default=10, help="Number of most recent runs to analyze")
     trend_cmd.add_argument("--output", help="Write trend report JSON to this path")
     trend_cmd.add_argument(
         "--fail-on-regression",
         action="store_true",
-        help="Exit non-zero when the trend direction is declining",
+        help="Exit non-zero when the pass-rate trend direction is declining",
+    )
+    trend_cmd.add_argument(
+        "--fail-on-latency-regression",
+        action="store_true",
+        help="Exit non-zero when the average latency trend direction is increasing",
+    )
+    trend_cmd.add_argument(
+        "--fail-on-p95-regression",
+        action="store_true",
+        help="Exit non-zero when the p95 latency trend direction is increasing",
+    )
+    trend_cmd.add_argument(
+        "--fail-on-cost-regression",
+        action="store_true",
+        help="Exit non-zero when the average cost trend direction is increasing",
+    )
+    trend_cmd.add_argument(
+        "--fail-on-any-regression",
+        action="store_true",
+        help="Exit non-zero when pass rate, average latency, p95 latency, or average cost regresses",
     )
 
     return parser
@@ -68,7 +91,19 @@ def main(argv: Optional[List[str]] = None) -> int:
         if args.output:
             Path(args.output).write_text(json.dumps(payload, indent=2), encoding="utf-8")
 
-        if args.fail_on_regression and trend_report.any_regression:
+        if args.fail_on_regression and trend_report.pass_rate_regression:
+            return 1
+
+        if args.fail_on_latency_regression and trend_report.avg_latency_regression:
+            return 1
+
+        if args.fail_on_p95_regression and trend_report.p95_latency_regression:
+            return 1
+
+        if args.fail_on_cost_regression and trend_report.avg_cost_regression:
+            return 1
+
+        if args.fail_on_any_regression and trend_report.any_trend_regression:
             return 1
 
         return 0

src/agent_release_gate/history.py

@@ -8,24 +8,48 @@
 
 from .models import GateReport
 
+MIN_TREND_POINTS = 3
+PASS_RATE_TREND_EPSILON = 0.001
+LATENCY_TREND_EPSILON_MS = 1.0
+COST_TREND_EPSILON_USD = 0.000001
+
 
 @dataclass
 class GateRunSummary:
     run_id: str
     timestamp: str
     pass_rate: float
     avg_latency_ms: Optional[float]
+    p95_latency_ms: Optional[float]
     avg_cost_usd: Optional[float]
     gate_passed: bool
 
 
+@dataclass
+class MetricTrend:
+    slope: Optional[float]
+    direction: str
+    regression: bool
+
+
 @dataclass
 class GateTrendReport:
     window: int
     runs: list[GateRunSummary]
     pass_rate_slope: float
     pass_rate_direction: str
+    pass_rate_regression: bool
+    avg_latency_ms_slope: Optional[float]
+    avg_latency_ms_direction: str
+    avg_latency_regression: bool
+    p95_latency_ms_slope: Optional[float]
+    p95_latency_ms_direction: str
+    p95_latency_regression: bool
+    avg_cost_usd_slope: Optional[float]
+    avg_cost_usd_direction: str
+    avg_cost_regression: bool
     any_regression: bool
+    any_trend_regression: bool
 
 
 def _timestamp_id() -> str:
@@ -43,6 +67,7 @@ def _run_payload(run: GateRunSummary) -> dict:
         "summary": {
             "pass_rate": run.pass_rate,
             "avg_latency_ms": run.avg_latency_ms,
+            "p95_latency_ms": run.p95_latency_ms,
             "avg_cost_usd": run.avg_cost_usd,
             "gate_passed": run.gate_passed,
         },
@@ -60,6 +85,11 @@ def _run_from_payload(data: dict, fallback_run_id: str) -> GateRunSummary:
             if summary.get("avg_latency_ms") is not None
             else None
         ),
+        p95_latency_ms=(
+            float(summary["p95_latency_ms"])
+            if summary.get("p95_latency_ms") is not None
+            else None
+        ),
         avg_cost_usd=(
             float(summary["avg_cost_usd"])
             if summary.get("avg_cost_usd") is not None
@@ -69,6 +99,14 @@ def _run_from_payload(data: dict, fallback_run_id: str) -> GateRunSummary:
     )
 
 
+def _looks_like_run_payload(data: dict) -> bool:
+    summary = data.get("summary")
+    if isinstance(summary, dict) and summary.get("pass_rate") is not None:
+        return True
+
+    return data.get("pass_rate") is not None
+
+
 def _sort_key(run: GateRunSummary) -> tuple[str, str]:
     return (run.timestamp, run.run_id)
 
@@ -88,12 +126,65 @@ def _ols_slope(values: list[float]) -> float:
     return numerator / denominator
 
 
+def _trend_direction(
+    slope: float,
+    *,
+    positive_direction: str,
+    negative_direction: str,
+    epsilon: float,
+) -> str:
+    if slope > epsilon:
+        return positive_direction
+    if slope < -epsilon:
+        return negative_direction
+    return "stable"
+
+
+def _available_metric_values(runs: list[GateRunSummary], field_name: str) -> list[float]:
+    values: list[float] = []
+    for run in runs:
+        value = getattr(run, field_name)
+        if value is not None:
+            values.append(float(value))
+    return values
+
+
+def _analyze_optional_metric(
+    values: list[float],
+    *,
+    epsilon: float,
+    positive_direction: str,
+    negative_direction: str,
+    regression_direction: str,
+) -> MetricTrend:
+    if len(values) < MIN_TREND_POINTS:
+        return MetricTrend(
+            slope=None,
+            direction="insufficient_data",
+            regression=False,
+        )
+
+    slope = _ols_slope(values)
+    direction = _trend_direction(
+        slope,
+        positive_direction=positive_direction,
+        negative_direction=negative_direction,
+        epsilon=epsilon,
+    )
+    return MetricTrend(
+        slope=round(slope, 6),
+        direction=direction,
+        regression=direction == regression_direction,
+    )
+
+
 def summary_from_report(report: GateReport, run_id: Optional[str] = None) -> GateRunSummary:
     return GateRunSummary(
         run_id=run_id or _timestamp_id(),
         timestamp=_iso_utc_now(),
         pass_rate=report.summary.pass_rate,
         avg_latency_ms=report.summary.avg_latency_ms,
+        p95_latency_ms=report.summary.p95_latency_ms,
         avg_cost_usd=report.summary.avg_cost_usd,
         gate_passed=report.summary.gate_passed,
     )
@@ -123,7 +214,14 @@ def load_recent_runs(history_dir: Union[str, Path], window: int = 10) -> list[Ga
 
     runs: list[GateRunSummary] = []
     for file_path in history_path.glob("*.json"):
-        data = json.loads(file_path.read_text(encoding="utf-8"))
+        try:
+            data = json.loads(file_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            continue
+
+        if not isinstance(data, dict) or not _looks_like_run_payload(data):
+            continue
+
         runs.append(_run_from_payload(data, fallback_run_id=file_path.stem))
 
     runs.sort(key=_sort_key)
@@ -134,21 +232,63 @@ def load_recent_runs(history_dir: Union[str, Path], window: int = 10) -> list[Ga
 
 def analyze_history(history_dir: Union[str, Path], window: int = 10) -> GateTrendReport:
     runs = load_recent_runs(history_dir=history_dir, window=window)
-    slope = _ols_slope([r.pass_rate for r in runs]) if len(runs) >= 3 else 0.0
-
-    if slope < -0.001:
-        direction = "declining"
-    elif slope > 0.001:
-        direction = "improving"
-    else:
-        direction = "stable"
+    pass_rate_trend = _analyze_optional_metric(
+        [r.pass_rate for r in runs],
+        epsilon=PASS_RATE_TREND_EPSILON,
+        positive_direction="improving",
+        negative_direction="declining",
+        regression_direction="declining",
+    )
+    avg_latency_trend = _analyze_optional_metric(
+        _available_metric_values(runs, "avg_latency_ms"),
+        epsilon=LATENCY_TREND_EPSILON_MS,
+        positive_direction="increasing",
+        negative_direction="decreasing",
+        regression_direction="increasing",
+    )
+    p95_latency_trend = _analyze_optional_metric(
+        _available_metric_values(runs, "p95_latency_ms"),
+        epsilon=LATENCY_TREND_EPSILON_MS,
+        positive_direction="increasing",
+        negative_direction="decreasing",
+        regression_direction="increasing",
+    )
+    avg_cost_trend = _analyze_optional_metric(
+        _available_metric_values(runs, "avg_cost_usd"),
+        epsilon=COST_TREND_EPSILON_USD,
+        positive_direction="increasing",
+        negative_direction="decreasing",
+        regression_direction="increasing",
+    )
+    pass_rate_regression = pass_rate_trend.regression
+    any_trend_regression = any(
+        (
+            pass_rate_regression,
+            avg_latency_trend.regression,
+            p95_latency_trend.regression,
+            avg_cost_trend.regression,
+        )
+    )
 
     return GateTrendReport(
         window=window,
         runs=runs,
-        pass_rate_slope=round(slope, 6),
-        pass_rate_direction=direction,
-        any_regression=direction == "declining",
+        pass_rate_slope=pass_rate_trend.slope or 0.0,
+        pass_rate_direction=pass_rate_trend.direction
+        if pass_rate_trend.slope is not None
+        else "stable",
+        pass_rate_regression=pass_rate_regression,
+        avg_latency_ms_slope=avg_latency_trend.slope,
+        avg_latency_ms_direction=avg_latency_trend.direction,
+        avg_latency_regression=avg_latency_trend.regression,
+        p95_latency_ms_slope=p95_latency_trend.slope,
+        p95_latency_ms_direction=p95_latency_trend.direction,
+        p95_latency_regression=p95_latency_trend.regression,
+        avg_cost_usd_slope=avg_cost_trend.slope,
+        avg_cost_usd_direction=avg_cost_trend.direction,
+        avg_cost_regression=avg_cost_trend.regression,
+        any_regression=pass_rate_regression,
+        any_trend_regression=any_trend_regression,
     )
 
 
@@ -161,12 +301,24 @@ def trend_to_dict(report: GateTrendReport) -> dict:
                 "timestamp": run.timestamp,
                 "pass_rate": run.pass_rate,
                 "avg_latency_ms": run.avg_latency_ms,
+                "p95_latency_ms": run.p95_latency_ms,
                 "avg_cost_usd": run.avg_cost_usd,
                 "gate_passed": run.gate_passed,
             }
             for run in report.runs
         ],
         "pass_rate_slope": report.pass_rate_slope,
         "pass_rate_direction": report.pass_rate_direction,
+        "pass_rate_regression": report.pass_rate_regression,
+        "avg_latency_ms_slope": report.avg_latency_ms_slope,
+        "avg_latency_ms_direction": report.avg_latency_ms_direction,
+        "avg_latency_regression": report.avg_latency_regression,
+        "p95_latency_ms_slope": report.p95_latency_ms_slope,
+        "p95_latency_ms_direction": report.p95_latency_ms_direction,
+        "p95_latency_regression": report.p95_latency_regression,
+        "avg_cost_usd_slope": report.avg_cost_usd_slope,
+        "avg_cost_usd_direction": report.avg_cost_usd_direction,
+        "avg_cost_regression": report.avg_cost_regression,
         "any_regression": report.any_regression,
+        "any_trend_regression": report.any_trend_regression,
     }