From 5022edfd48a0bb6f2190a772753ea8dd6d667df2 Mon Sep 17 00:00:00 2001 From: Brandon Date: Wed, 29 Apr 2026 21:25:17 -0400 Subject: [PATCH] feat: add multi-metric trend release gates --- README.md | 23 +++- src/agent_release_gate/cli.py | 41 +++++- src/agent_release_gate/history.py | 176 +++++++++++++++++++++++-- tests/test_cli.py | 207 +++++++++++++++++++++++++++--- tests/test_history.py | 163 ++++++++++++++++++++++- 5 files changed, 569 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index b85433f..a6ae3f5 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,12 @@ This tool makes those problems visible before release. ## What it does - Scores each test case using expected + forbidden phrases -- Calculates pass rate, average latency, and average cost -- Calculates pass rate, average latency, and p95 latency (tail behavior) +- Calculates pass rate, average latency, p95 latency, and average cost - Fails if quality drops below your threshold - Optionally compares against a baseline report and blocks regressions - Optionally enforces baseline latency/cost drift caps so slower or pricier runs fail fast - Optionally enforces a global p95 latency limit to catch long-tail slow responses -- Optionally records run summaries and detects sustained pass-rate drift across releases +- Optionally records run summaries and detects cross-run pass-rate, latency, p95 latency, and cost drift across releases - Supports per-case latency/cost limits to catch outliers hidden by averages - Enforces telemetry presence when global average latency/cost limits are configured - Outputs both JSON (for machines) and Markdown (for humans) @@ -67,8 +66,26 @@ argate trend --history .gate-history --window 10 # CI mode: fail when pass-rate trend is declining argate trend --history .gate-history --fail-on-regression + +# Optional CI mode: fail when average latency, p95 latency, or cost is trending up +argate trend --history .gate-history --fail-on-latency-regression +argate trend --history .gate-history --fail-on-p95-regression +argate trend --history .gate-history --fail-on-cost-regression + +# Fail on any tracked regression across pass rate, latency, p95 latency, or cost +argate trend --history .gate-history --fail-on-any-regression ``` +`argate evaluate --record-history ...` stores pass rate, average latency, p95 latency, and average cost in each history file when that telemetry is available in the evaluation report. + +`argate trend` keeps the existing pass-rate output and now also includes slope, direction, and regression booleans for: + +- `avg_latency_ms` +- `p95_latency_ms` +- `avg_cost_usd` + +Older history files that predate these metrics are still supported. When there are fewer than three runs with a given metric inside the analysis window, that metric reports `insufficient_data` instead of forcing a regression. + ## Spec format ```yaml diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index 6733680..0817bf5 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -25,14 +25,37 @@ def build_parser() -> argparse.ArgumentParser: help="Optional directory to append run summaries for cross-run trend analysis", ) - trend_cmd = sub.add_parser("trend", help="Analyze pass-rate trend across historical run summaries") + trend_cmd = sub.add_parser( + "trend", + help="Analyze cross-run quality, latency, and cost trends across historical run summaries", + ) trend_cmd.add_argument("--history", required=True, help="Directory containing historical run JSON") trend_cmd.add_argument("--window", type=int, default=10, help="Number of most recent runs to analyze") trend_cmd.add_argument("--output", help="Write trend report JSON to this path") trend_cmd.add_argument( "--fail-on-regression", action="store_true", - help="Exit non-zero when the trend direction is declining", + help="Exit non-zero when the pass-rate trend direction is declining", + ) + trend_cmd.add_argument( + "--fail-on-latency-regression", + action="store_true", + help="Exit non-zero when the average latency trend direction is increasing", + ) + trend_cmd.add_argument( + "--fail-on-p95-regression", + action="store_true", + help="Exit non-zero when the p95 latency trend direction is increasing", + ) + trend_cmd.add_argument( + "--fail-on-cost-regression", + action="store_true", + help="Exit non-zero when the average cost trend direction is increasing", + ) + trend_cmd.add_argument( + "--fail-on-any-regression", + action="store_true", + help="Exit non-zero when pass rate, average latency, p95 latency, or average cost regresses", ) return parser @@ -68,7 +91,19 @@ def main(argv: Optional[List[str]] = None) -> int: if args.output: Path(args.output).write_text(json.dumps(payload, indent=2), encoding="utf-8") - if args.fail_on_regression and trend_report.any_regression: + if args.fail_on_regression and trend_report.pass_rate_regression: + return 1 + + if args.fail_on_latency_regression and trend_report.avg_latency_regression: + return 1 + + if args.fail_on_p95_regression and trend_report.p95_latency_regression: + return 1 + + if args.fail_on_cost_regression and trend_report.avg_cost_regression: + return 1 + + if args.fail_on_any_regression and trend_report.any_trend_regression: return 1 return 0 diff --git a/src/agent_release_gate/history.py b/src/agent_release_gate/history.py index 90a5a7d..577c036 100644 --- a/src/agent_release_gate/history.py +++ b/src/agent_release_gate/history.py @@ -8,6 +8,11 @@ from .models import GateReport +MIN_TREND_POINTS = 3 +PASS_RATE_TREND_EPSILON = 0.001 +LATENCY_TREND_EPSILON_MS = 1.0 +COST_TREND_EPSILON_USD = 0.000001 + @dataclass class GateRunSummary: @@ -15,17 +20,36 @@ class GateRunSummary: timestamp: str pass_rate: float avg_latency_ms: Optional[float] + p95_latency_ms: Optional[float] avg_cost_usd: Optional[float] gate_passed: bool +@dataclass +class MetricTrend: + slope: Optional[float] + direction: str + regression: bool + + @dataclass class GateTrendReport: window: int runs: list[GateRunSummary] pass_rate_slope: float pass_rate_direction: str + pass_rate_regression: bool + avg_latency_ms_slope: Optional[float] + avg_latency_ms_direction: str + avg_latency_regression: bool + p95_latency_ms_slope: Optional[float] + p95_latency_ms_direction: str + p95_latency_regression: bool + avg_cost_usd_slope: Optional[float] + avg_cost_usd_direction: str + avg_cost_regression: bool any_regression: bool + any_trend_regression: bool def _timestamp_id() -> str: @@ -43,6 +67,7 @@ def _run_payload(run: GateRunSummary) -> dict: "summary": { "pass_rate": run.pass_rate, "avg_latency_ms": run.avg_latency_ms, + "p95_latency_ms": run.p95_latency_ms, "avg_cost_usd": run.avg_cost_usd, "gate_passed": run.gate_passed, }, @@ -60,6 +85,11 @@ def _run_from_payload(data: dict, fallback_run_id: str) -> GateRunSummary: if summary.get("avg_latency_ms") is not None else None ), + p95_latency_ms=( + float(summary["p95_latency_ms"]) + if summary.get("p95_latency_ms") is not None + else None + ), avg_cost_usd=( float(summary["avg_cost_usd"]) if summary.get("avg_cost_usd") is not None @@ -69,6 +99,14 @@ def _run_from_payload(data: dict, fallback_run_id: str) -> GateRunSummary: ) +def _looks_like_run_payload(data: dict) -> bool: + summary = data.get("summary") + if isinstance(summary, dict) and summary.get("pass_rate") is not None: + return True + + return data.get("pass_rate") is not None + + def _sort_key(run: GateRunSummary) -> tuple[str, str]: return (run.timestamp, run.run_id) @@ -88,12 +126,65 @@ def _ols_slope(values: list[float]) -> float: return numerator / denominator +def _trend_direction( + slope: float, + *, + positive_direction: str, + negative_direction: str, + epsilon: float, +) -> str: + if slope > epsilon: + return positive_direction + if slope < -epsilon: + return negative_direction + return "stable" + + +def _available_metric_values(runs: list[GateRunSummary], field_name: str) -> list[float]: + values: list[float] = [] + for run in runs: + value = getattr(run, field_name) + if value is not None: + values.append(float(value)) + return values + + +def _analyze_optional_metric( + values: list[float], + *, + epsilon: float, + positive_direction: str, + negative_direction: str, + regression_direction: str, +) -> MetricTrend: + if len(values) < MIN_TREND_POINTS: + return MetricTrend( + slope=None, + direction="insufficient_data", + regression=False, + ) + + slope = _ols_slope(values) + direction = _trend_direction( + slope, + positive_direction=positive_direction, + negative_direction=negative_direction, + epsilon=epsilon, + ) + return MetricTrend( + slope=round(slope, 6), + direction=direction, + regression=direction == regression_direction, + ) + + def summary_from_report(report: GateReport, run_id: Optional[str] = None) -> GateRunSummary: return GateRunSummary( run_id=run_id or _timestamp_id(), timestamp=_iso_utc_now(), pass_rate=report.summary.pass_rate, avg_latency_ms=report.summary.avg_latency_ms, + p95_latency_ms=report.summary.p95_latency_ms, avg_cost_usd=report.summary.avg_cost_usd, gate_passed=report.summary.gate_passed, ) @@ -123,7 +214,14 @@ def load_recent_runs(history_dir: Union[str, Path], window: int = 10) -> list[Ga runs: list[GateRunSummary] = [] for file_path in history_path.glob("*.json"): - data = json.loads(file_path.read_text(encoding="utf-8")) + try: + data = json.loads(file_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + + if not isinstance(data, dict) or not _looks_like_run_payload(data): + continue + runs.append(_run_from_payload(data, fallback_run_id=file_path.stem)) runs.sort(key=_sort_key) @@ -134,21 +232,63 @@ def load_recent_runs(history_dir: Union[str, Path], window: int = 10) -> list[Ga def analyze_history(history_dir: Union[str, Path], window: int = 10) -> GateTrendReport: runs = load_recent_runs(history_dir=history_dir, window=window) - slope = _ols_slope([r.pass_rate for r in runs]) if len(runs) >= 3 else 0.0 - - if slope < -0.001: - direction = "declining" - elif slope > 0.001: - direction = "improving" - else: - direction = "stable" + pass_rate_trend = _analyze_optional_metric( + [r.pass_rate for r in runs], + epsilon=PASS_RATE_TREND_EPSILON, + positive_direction="improving", + negative_direction="declining", + regression_direction="declining", + ) + avg_latency_trend = _analyze_optional_metric( + _available_metric_values(runs, "avg_latency_ms"), + epsilon=LATENCY_TREND_EPSILON_MS, + positive_direction="increasing", + negative_direction="decreasing", + regression_direction="increasing", + ) + p95_latency_trend = _analyze_optional_metric( + _available_metric_values(runs, "p95_latency_ms"), + epsilon=LATENCY_TREND_EPSILON_MS, + positive_direction="increasing", + negative_direction="decreasing", + regression_direction="increasing", + ) + avg_cost_trend = _analyze_optional_metric( + _available_metric_values(runs, "avg_cost_usd"), + epsilon=COST_TREND_EPSILON_USD, + positive_direction="increasing", + negative_direction="decreasing", + regression_direction="increasing", + ) + pass_rate_regression = pass_rate_trend.regression + any_trend_regression = any( + ( + pass_rate_regression, + avg_latency_trend.regression, + p95_latency_trend.regression, + avg_cost_trend.regression, + ) + ) return GateTrendReport( window=window, runs=runs, - pass_rate_slope=round(slope, 6), - pass_rate_direction=direction, - any_regression=direction == "declining", + pass_rate_slope=pass_rate_trend.slope or 0.0, + pass_rate_direction=pass_rate_trend.direction + if pass_rate_trend.slope is not None + else "stable", + pass_rate_regression=pass_rate_regression, + avg_latency_ms_slope=avg_latency_trend.slope, + avg_latency_ms_direction=avg_latency_trend.direction, + avg_latency_regression=avg_latency_trend.regression, + p95_latency_ms_slope=p95_latency_trend.slope, + p95_latency_ms_direction=p95_latency_trend.direction, + p95_latency_regression=p95_latency_trend.regression, + avg_cost_usd_slope=avg_cost_trend.slope, + avg_cost_usd_direction=avg_cost_trend.direction, + avg_cost_regression=avg_cost_trend.regression, + any_regression=pass_rate_regression, + any_trend_regression=any_trend_regression, ) @@ -161,6 +301,7 @@ def trend_to_dict(report: GateTrendReport) -> dict: "timestamp": run.timestamp, "pass_rate": run.pass_rate, "avg_latency_ms": run.avg_latency_ms, + "p95_latency_ms": run.p95_latency_ms, "avg_cost_usd": run.avg_cost_usd, "gate_passed": run.gate_passed, } @@ -168,5 +309,16 @@ def trend_to_dict(report: GateTrendReport) -> dict: ], "pass_rate_slope": report.pass_rate_slope, "pass_rate_direction": report.pass_rate_direction, + "pass_rate_regression": report.pass_rate_regression, + "avg_latency_ms_slope": report.avg_latency_ms_slope, + "avg_latency_ms_direction": report.avg_latency_ms_direction, + "avg_latency_regression": report.avg_latency_regression, + "p95_latency_ms_slope": report.p95_latency_ms_slope, + "p95_latency_ms_direction": report.p95_latency_ms_direction, + "p95_latency_regression": report.p95_latency_regression, + "avg_cost_usd_slope": report.avg_cost_usd_slope, + "avg_cost_usd_direction": report.avg_cost_usd_direction, + "avg_cost_regression": report.avg_cost_regression, "any_regression": report.any_regression, + "any_trend_regression": report.any_trend_regression, } diff --git a/tests/test_cli.py b/tests/test_cli.py index bd64e48..e153961 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,34 @@ import json from pathlib import Path +from typing import Optional from agent_release_gate.cli import build_parser, main +def _write_run( + history_dir: Path, + run_id: str, + timestamp: str, + pass_rate: float, + *, + avg_latency_ms: Optional[float] = 800.0, + p95_latency_ms: Optional[float] = 1200.0, + avg_cost_usd: Optional[float] = 0.01, +) -> None: + payload = { + "run_id": run_id, + "timestamp": timestamp, + "summary": { + "pass_rate": pass_rate, + "avg_latency_ms": avg_latency_ms, + "p95_latency_ms": p95_latency_ms, + "avg_cost_usd": avg_cost_usd, + "gate_passed": True, + }, + } + (history_dir / f"{run_id}.json").write_text(json.dumps(payload), encoding="utf-8") + + def test_parser_has_evaluate_command(): parser = build_parser() args = parser.parse_args(["evaluate", "--spec", "a.yaml", "--results", "b.json"]) @@ -39,6 +64,26 @@ def test_parser_has_trend_command(): assert args.window == 8 +def test_parser_trend_supports_metric_regression_flags(): + parser = build_parser() + args = parser.parse_args( + [ + "trend", + "--history", + ".gate-history", + "--fail-on-latency-regression", + "--fail-on-p95-regression", + "--fail-on-cost-regression", + "--fail-on-any-regression", + ] + ) + + assert args.fail_on_latency_regression is True + assert args.fail_on_p95_regression is True + assert args.fail_on_cost_regression is True + assert args.fail_on_any_regression is True + + def test_main_evaluate_records_history(tmp_path: Path): spec = tmp_path / "spec.yaml" spec.write_text( @@ -88,23 +133,151 @@ def test_main_trend_fail_on_regression(tmp_path: Path): history_dir = tmp_path / "history" history_dir.mkdir(parents=True) - runs = [ - ("run-1", "2026-04-01T00:00:00+00:00", 0.92), - ("run-2", "2026-04-02T00:00:00+00:00", 0.88), - ("run-3", "2026-04-03T00:00:00+00:00", 0.84), - ] - for run_id, timestamp, pass_rate in runs: - payload = { - "run_id": run_id, - "timestamp": timestamp, - "summary": { - "pass_rate": pass_rate, - "avg_latency_ms": 800.0, - "avg_cost_usd": 0.01, - "gate_passed": True, - }, - } - (history_dir / f"{run_id}.json").write_text(json.dumps(payload), encoding="utf-8") + _write_run(history_dir, "run-1", "2026-04-01T00:00:00+00:00", 0.92) + _write_run(history_dir, "run-2", "2026-04-02T00:00:00+00:00", 0.88) + _write_run(history_dir, "run-3", "2026-04-03T00:00:00+00:00", 0.84) code = main(["trend", "--history", str(history_dir), "--fail-on-regression"]) assert code == 1 + + +def test_main_trend_fail_on_regression_only_checks_pass_rate(tmp_path: Path): + history_dir = tmp_path / "history" + history_dir.mkdir(parents=True) + + _write_run( + history_dir, + "run-1", + "2026-04-01T00:00:00+00:00", + 0.90, + avg_latency_ms=700.0, + p95_latency_ms=1000.0, + avg_cost_usd=0.010, + ) + _write_run( + history_dir, + "run-2", + "2026-04-02T00:00:00+00:00", + 0.92, + avg_latency_ms=800.0, + p95_latency_ms=1200.0, + avg_cost_usd=0.011, + ) + _write_run( + history_dir, + "run-3", + "2026-04-03T00:00:00+00:00", + 0.94, + avg_latency_ms=900.0, + p95_latency_ms=1400.0, + avg_cost_usd=0.012, + ) + + assert main(["trend", "--history", str(history_dir), "--fail-on-regression"]) == 0 + assert main(["trend", "--history", str(history_dir), "--fail-on-latency-regression"]) == 1 + assert main(["trend", "--history", str(history_dir), "--fail-on-any-regression"]) == 1 + + +def test_main_trend_fail_on_p95_regression(tmp_path: Path): + history_dir = tmp_path / "history" + history_dir.mkdir(parents=True) + + _write_run( + history_dir, + "run-1", + "2026-04-01T00:00:00+00:00", + 0.95, + avg_latency_ms=800.0, + p95_latency_ms=1000.0, + avg_cost_usd=0.010, + ) + _write_run( + history_dir, + "run-2", + "2026-04-02T00:00:00+00:00", + 0.95, + avg_latency_ms=800.0, + p95_latency_ms=1200.0, + avg_cost_usd=0.010, + ) + _write_run( + history_dir, + "run-3", + "2026-04-03T00:00:00+00:00", + 0.95, + avg_latency_ms=800.0, + p95_latency_ms=1400.0, + avg_cost_usd=0.010, + ) + + assert main(["trend", "--history", str(history_dir), "--fail-on-p95-regression"]) == 1 + + +def test_main_trend_fail_on_cost_regression(tmp_path: Path): + history_dir = tmp_path / "history" + history_dir.mkdir(parents=True) + + _write_run( + history_dir, + "run-1", + "2026-04-01T00:00:00+00:00", + 0.95, + avg_latency_ms=800.0, + p95_latency_ms=1200.0, + avg_cost_usd=0.010, + ) + _write_run( + history_dir, + "run-2", + "2026-04-02T00:00:00+00:00", + 0.95, + avg_latency_ms=800.0, + p95_latency_ms=1200.0, + avg_cost_usd=0.011, + ) + _write_run( + history_dir, + "run-3", + "2026-04-03T00:00:00+00:00", + 0.95, + avg_latency_ms=800.0, + p95_latency_ms=1200.0, + avg_cost_usd=0.012, + ) + + assert main(["trend", "--history", str(history_dir), "--fail-on-cost-regression"]) == 1 + + +def test_main_trend_missing_optional_metric_history_does_not_trigger_metric_failure(tmp_path: Path): + history_dir = tmp_path / "history" + history_dir.mkdir(parents=True) + + _write_run( + history_dir, + "run-1", + "2026-04-01T00:00:00+00:00", + 0.90, + avg_latency_ms=800.0, + p95_latency_ms=None, + avg_cost_usd=0.010, + ) + _write_run( + history_dir, + "run-2", + "2026-04-02T00:00:00+00:00", + 0.91, + avg_latency_ms=810.0, + p95_latency_ms=None, + avg_cost_usd=0.010, + ) + _write_run( + history_dir, + "run-3", + "2026-04-03T00:00:00+00:00", + 0.92, + avg_latency_ms=820.0, + p95_latency_ms=None, + avg_cost_usd=0.010, + ) + + assert main(["trend", "--history", str(history_dir), "--fail-on-p95-regression"]) == 0 diff --git a/tests/test_history.py b/tests/test_history.py index 2c75b80..9cbc7c9 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -1,36 +1,56 @@ import json from pathlib import Path +from typing import Optional -from agent_release_gate.history import analyze_history, record_history +from agent_release_gate.history import analyze_history, record_history, trend_to_dict from agent_release_gate.models import GateReport, GateSummary, ScoredCase -def _report(pass_rate: float, gate_passed: bool = True) -> GateReport: +def _report( + pass_rate: float, + gate_passed: bool = True, + *, + avg_latency_ms: float = 850.0, + p95_latency_ms: float = 1100.0, + avg_cost_usd: float = 0.011, +) -> GateReport: return GateReport( summary=GateSummary( total_cases=10, passed_cases=int(pass_rate * 10), pass_rate=pass_rate, - avg_latency_ms=850.0, - avg_cost_usd=0.011, + avg_latency_ms=avg_latency_ms, + avg_cost_usd=avg_cost_usd, gate_passed=gate_passed, gate_reasons=["ok"], + p95_latency_ms=p95_latency_ms, ), cases=[ScoredCase(id="case_1", score=0.9, passed=gate_passed, notes=["ok"])], ) -def _write_run(history_dir: Path, run_id: str, timestamp: str, pass_rate: float) -> None: +def _write_run( + history_dir: Path, + run_id: str, + timestamp: str, + pass_rate: float, + *, + avg_latency_ms: Optional[float] = 800.0, + p95_latency_ms: Optional[float] = 1200.0, + avg_cost_usd: Optional[float] = 0.01, +) -> None: payload = { "run_id": run_id, "timestamp": timestamp, "summary": { "pass_rate": pass_rate, "avg_latency_ms": 800.0, - "avg_cost_usd": 0.01, "gate_passed": pass_rate >= 0.8, }, } + payload["summary"]["avg_latency_ms"] = avg_latency_ms + payload["summary"]["p95_latency_ms"] = p95_latency_ms + payload["summary"]["avg_cost_usd"] = avg_cost_usd (history_dir / f"{run_id}.json").write_text(json.dumps(payload), encoding="utf-8") @@ -42,6 +62,7 @@ def test_record_history_writes_run_summary(tmp_path: Path): payload = json.loads(path.read_text(encoding="utf-8")) assert payload["run_id"] == "run-001" assert payload["summary"]["pass_rate"] == 0.9 + assert payload["summary"]["p95_latency_ms"] == 1100.0 def test_analyze_history_detects_declining_trend(tmp_path: Path): @@ -57,7 +78,9 @@ def test_analyze_history_detects_declining_trend(tmp_path: Path): trend = analyze_history(history_dir=history_dir, window=10) assert trend.pass_rate_direction == "declining" + assert trend.pass_rate_regression is True assert trend.any_regression is True + assert trend.any_trend_regression is True assert trend.pass_rate_slope < 0 @@ -77,3 +100,131 @@ def test_analyze_history_window_uses_most_recent_runs(tmp_path: Path): assert len(trend.runs) == 3 assert trend.runs[0].run_id == "run-4" assert trend.pass_rate_direction == "improving" + + +def test_analyze_history_detects_latency_p95_and_cost_regressions(tmp_path: Path): + history_dir = tmp_path / "history" + history_dir.mkdir(parents=True) + + _write_run( + history_dir, + "run-1", + "2026-04-01T00:00:00+00:00", + 0.95, + avg_latency_ms=800.0, + p95_latency_ms=1200.0, + avg_cost_usd=0.010, + ) + _write_run( + history_dir, + "run-2", + "2026-04-02T00:00:00+00:00", + 0.96, + avg_latency_ms=850.0, + p95_latency_ms=1400.0, + avg_cost_usd=0.011, + ) + _write_run( + history_dir, + "run-3", + "2026-04-03T00:00:00+00:00", + 0.97, + avg_latency_ms=900.0, + p95_latency_ms=1600.0, + avg_cost_usd=0.012, + ) + + trend = analyze_history(history_dir=history_dir, window=10) + payload = trend_to_dict(trend) + + assert trend.pass_rate_direction == "improving" + assert trend.pass_rate_regression is False + assert trend.avg_latency_ms_direction == "increasing" + assert trend.avg_latency_regression is True + assert trend.avg_latency_ms_slope and trend.avg_latency_ms_slope > 0 + assert trend.p95_latency_ms_direction == "increasing" + assert trend.p95_latency_regression is True + assert trend.p95_latency_ms_slope and trend.p95_latency_ms_slope > 0 + assert trend.avg_cost_usd_direction == "increasing" + assert trend.avg_cost_regression is True + assert trend.avg_cost_usd_slope and trend.avg_cost_usd_slope > 0 + assert trend.any_regression is False + assert trend.any_trend_regression is True + assert payload["runs"][0]["p95_latency_ms"] == 1200.0 + assert payload["avg_latency_regression"] is True + assert payload["p95_latency_regression"] is True + assert payload["avg_cost_regression"] is True + assert payload["any_trend_regression"] is True + + +def test_analyze_history_tolerates_older_history_without_optional_metrics(tmp_path: Path): + history_dir = tmp_path / "history" + history_dir.mkdir(parents=True) + + legacy_payload = { + "run_id": "run-1", + "timestamp": "2026-04-01T00:00:00+00:00", + "summary": { + "pass_rate": 0.90, + "avg_latency_ms": 780.0, + "gate_passed": True, + }, + } + (history_dir / "run-1.json").write_text(json.dumps(legacy_payload), encoding="utf-8") + + _write_run( + history_dir, + "run-2", + "2026-04-02T00:00:00+00:00", + 0.91, + avg_latency_ms=800.0, + p95_latency_ms=1200.0, + avg_cost_usd=None, + ) + _write_run( + history_dir, + "run-3", + "2026-04-03T00:00:00+00:00", + 0.92, + avg_latency_ms=820.0, + p95_latency_ms=1300.0, + avg_cost_usd=None, + ) + _write_run( + history_dir, + "run-4", + "2026-04-04T00:00:00+00:00", + 0.93, + avg_latency_ms=840.0, + p95_latency_ms=1400.0, + avg_cost_usd=None, + ) + + trend = analyze_history(history_dir=history_dir, window=10) + + assert trend.p95_latency_ms_direction == "increasing" + assert trend.p95_latency_regression is True + assert trend.avg_cost_usd_slope is None + assert trend.avg_cost_usd_direction == "insufficient_data" + assert trend.avg_cost_regression is False + + +def test_analyze_history_ignores_non_run_json_files(tmp_path: Path): + history_dir = tmp_path / "history" + history_dir.mkdir(parents=True) + + _write_run(history_dir, "run-1", "2026-04-01T00:00:00+00:00", 0.91) + _write_run(history_dir, "run-2", "2026-04-02T00:00:00+00:00", 0.92) + _write_run(history_dir, "run-3", "2026-04-03T00:00:00+00:00", 0.93) + + (history_dir / "trend-report.json").write_text( + json.dumps({"window": 10, "runs": []}), + encoding="utf-8", + ) + (history_dir / "incomplete.json").write_text("", encoding="utf-8") + + trend = analyze_history(history_dir=history_dir, window=10) + + assert len(trend.runs) == 3 + assert [run.run_id for run in trend.runs] == ["run-1", "run-2", "run-3"] + assert trend.pass_rate_direction == "improving"