|
| 1 | +"""heart/checks/unit_test_timing.py — track library unit-test durations, flag |
| 2 | +slow-test regressions. |
| 3 | +
|
| 4 | +OFF-TICK by design. Running the library suites costs minutes (test_autofit is |
| 5 | +~1500 tests), which does not fit the <30s watch-loop budget (``docs/internals.md`` |
| 6 | +rule 3). Run on a slower cadence — a daily/weekly cron or on demand — and do |
| 7 | +**not** wire it into ``heart/tick.sh``: |
| 8 | +
|
| 9 | + python -m heart.checks.unit_test_timing |
| 10 | + HYGIENE_PYTHON=~/venv/PyAuto/bin/python python -m heart.checks.unit_test_timing |
| 11 | +
|
| 12 | +The suites run in a SUBPROCESS (``pytest --durations``) — Heart's own process |
| 13 | +never imports the test/science stack (rule 4) — and the runner is injected so |
| 14 | +the stdlib-only tests feed fixtures and never run pytest-in-pytest. Advisory: |
| 15 | +surfaced on the board but never a release gate (test speed is not release-blocking). |
| 16 | +
|
| 17 | +State: ~/.pyauto-heart/timings-unit/<slug>.json — rolling window per tracked test. |
| 18 | +Output: ~/.pyauto-heart/unit_test_timing.json — the latest regression summary. |
| 19 | +Classification (mirrors script_timing): ratio = latest / median(prior window); |
| 20 | +green <= yellow_factor (1.5), yellow <= red_factor (3.0), else red. |
| 21 | +""" |
| 22 | + |
| 23 | +from __future__ import annotations |
| 24 | + |
| 25 | +import json |
| 26 | +import os |
| 27 | +import re |
| 28 | +import statistics |
| 29 | +import subprocess |
| 30 | +import sys |
| 31 | +from pathlib import Path |
| 32 | +from typing import Any, Callable |
| 33 | + |
| 34 | +import yaml |
| 35 | + |
| 36 | +HEART_STATE_DIR = Path( |
| 37 | + os.environ.get("HEART_STATE_DIR") |
| 38 | + or Path.home() / ".pyauto-heart" |
| 39 | +) |
| 40 | +HEART_UNIT_TIMINGS_DIR = HEART_STATE_DIR / "timings-unit" |
| 41 | +HEART_HOME = Path(__file__).resolve().parents[2] |
| 42 | +CONFIG_PATH = HEART_HOME / "config" / "repos.yaml" |
| 43 | + |
| 44 | +# Each library's suite is discovered via its own pytest config (testpaths), so we |
| 45 | +# just invoke pytest from the repo root. Import-name maps to repo dir name. |
| 46 | +DEFAULT_REPOS = ["PyAutoConf", "PyAutoFit", "PyAutoArray", "PyAutoGalaxy", "PyAutoLens"] |
| 47 | + |
| 48 | +# A runner maps a repo checkout dir to {test_nodeid: call_seconds} for the |
| 49 | +# slowest tests, or None when the suite could not be run. Injected in tests. |
| 50 | +Runner = Callable[[Path], "dict[str, float] | None"] |
| 51 | + |
| 52 | +# pytest's "slowest durations" report lines: "1.23s call path::test_x" |
| 53 | +_DURATION_RE = re.compile(r"^\s*([0-9.]+)s\s+call\s+(\S+)\s*$") |
| 54 | + |
| 55 | + |
| 56 | +def load_thresholds() -> tuple[float, float, int, int]: |
| 57 | + """Return (yellow_factor, red_factor, baseline_window, top_n) from config.""" |
| 58 | + yellow, red, window, top_n = 1.5, 3.0, 7, 15 |
| 59 | + if CONFIG_PATH.is_file(): |
| 60 | + cfg = yaml.safe_load(CONFIG_PATH.read_text()) or {} |
| 61 | + t = cfg.get("thresholds", {}).get("unit_test_timing", {}) |
| 62 | + yellow = float(t.get("yellow_factor", yellow)) |
| 63 | + red = float(t.get("red_factor", red)) |
| 64 | + window = int(t.get("baseline_window", window)) |
| 65 | + top_n = int(t.get("top_n", top_n)) |
| 66 | + return yellow, red, window, top_n |
| 67 | + |
| 68 | + |
| 69 | +def parse_durations(stdout: str) -> dict[str, float]: |
| 70 | + """Extract {nodeid: call_seconds} from a pytest --durations report.""" |
| 71 | + out: dict[str, float] = {} |
| 72 | + for line in stdout.splitlines(): |
| 73 | + m = _DURATION_RE.match(line) |
| 74 | + if m: |
| 75 | + out[m.group(2)] = float(m.group(1)) |
| 76 | + return out |
| 77 | + |
| 78 | + |
| 79 | +def default_runner(python: str, timeout: float, top_n: int) -> Runner: |
| 80 | + """Run ``pytest --durations`` per repo in a subprocess; parse the slowest.""" |
| 81 | + env = { |
| 82 | + **os.environ, |
| 83 | + "NUMBA_CACHE_DIR": os.environ.get("NUMBA_CACHE_DIR", "/tmp/numba_cache"), |
| 84 | + "MPLCONFIGDIR": os.environ.get("MPLCONFIGDIR", "/tmp/matplotlib"), |
| 85 | + } |
| 86 | + |
| 87 | + def run_repo(repo_dir: Path) -> dict[str, float] | None: |
| 88 | + if not repo_dir.is_dir(): |
| 89 | + return None |
| 90 | + cmd = [python, "-m", "pytest", "--durations", str(top_n), |
| 91 | + "-q", "-p", "no:cacheprovider"] |
| 92 | + try: |
| 93 | + proc = subprocess.run( |
| 94 | + cmd, cwd=str(repo_dir), capture_output=True, text=True, |
| 95 | + timeout=timeout, env=env, |
| 96 | + ) |
| 97 | + except (subprocess.TimeoutExpired, FileNotFoundError): |
| 98 | + return None |
| 99 | + # A non-zero exit (test failures) still prints the durations report; parse |
| 100 | + # regardless. Only a total absence of parseable lines means "no data". |
| 101 | + parsed = parse_durations(proc.stdout) |
| 102 | + return parsed or None |
| 103 | + |
| 104 | + return run_repo |
| 105 | + |
| 106 | + |
| 107 | +def _slug(repo: str, nodeid: str) -> str: |
| 108 | + safe = re.sub(r"[^A-Za-z0-9]+", "_", f"{repo}__{nodeid}").strip("_") |
| 109 | + return f"{safe}.json" |
| 110 | + |
| 111 | + |
| 112 | +def update_history(slug: str, duration: float, window: int) -> list[float]: |
| 113 | + HEART_UNIT_TIMINGS_DIR.mkdir(parents=True, exist_ok=True) |
| 114 | + history_path = HEART_UNIT_TIMINGS_DIR / slug |
| 115 | + if history_path.is_file(): |
| 116 | + try: |
| 117 | + history = json.loads(history_path.read_text()) |
| 118 | + except json.JSONDecodeError: |
| 119 | + history = [] |
| 120 | + else: |
| 121 | + history = [] |
| 122 | + history.append(duration) |
| 123 | + history = history[-window:] |
| 124 | + history_path.write_text(json.dumps(history)) |
| 125 | + return history |
| 126 | + |
| 127 | + |
| 128 | +def classify(ratio: float, yellow: float, red: float) -> str: |
| 129 | + if ratio > red: |
| 130 | + return "red" |
| 131 | + if ratio > yellow: |
| 132 | + return "yellow" |
| 133 | + return "green" |
| 134 | + |
| 135 | + |
| 136 | +def run(runner: Runner | None = None, repos: list[str] | None = None) -> dict[str, Any]: |
| 137 | + """Run each repo's suite, update rolling per-test baselines, classify.""" |
| 138 | + yellow_factor, red_factor, window, top_n = load_thresholds() |
| 139 | + repos = repos if repos is not None else DEFAULT_REPOS |
| 140 | + python = os.environ.get("HYGIENE_PYTHON", "python3") |
| 141 | + timeout = float(os.environ.get("HEART_UNIT_TEST_TIMEOUT", "1800")) |
| 142 | + root = Path(os.environ.get("PYAUTO_ROOT", str(Path.home() / "Code" / "PyAutoLabs"))) |
| 143 | + runner = runner or default_runner(python, timeout, top_n) |
| 144 | + |
| 145 | + findings: dict[str, list[dict[str, Any]]] = {"red": [], "yellow": [], "green": []} |
| 146 | + repos_measured = 0 |
| 147 | + unavailable: list[str] = [] |
| 148 | + new_tests = 0 |
| 149 | + |
| 150 | + for repo in repos: |
| 151 | + durations = runner(root / repo) |
| 152 | + if durations is None: |
| 153 | + unavailable.append(repo) |
| 154 | + continue |
| 155 | + repos_measured += 1 |
| 156 | + for nodeid, duration in durations.items(): |
| 157 | + history = update_history(_slug(repo, nodeid), duration, window) |
| 158 | + if len(history) <= 1: |
| 159 | + new_tests += 1 |
| 160 | + continue |
| 161 | + baseline = statistics.median(history[:-1]) |
| 162 | + if baseline <= 0: |
| 163 | + continue |
| 164 | + ratio = duration / baseline |
| 165 | + findings[classify(ratio, yellow_factor, red_factor)].append({ |
| 166 | + "repo": repo, |
| 167 | + "test": nodeid, |
| 168 | + "latest_seconds": round(duration, 3), |
| 169 | + "baseline_seconds": round(baseline, 3), |
| 170 | + "ratio": round(ratio, 2), |
| 171 | + "samples": len(history) - 1, |
| 172 | + }) |
| 173 | + |
| 174 | + summary = { |
| 175 | + "python": python, |
| 176 | + "repos_measured": repos_measured, |
| 177 | + "repos_unavailable": unavailable, |
| 178 | + "new_tests_no_baseline": new_tests, |
| 179 | + "red_count": len(findings["red"]), |
| 180 | + "yellow_count": len(findings["yellow"]), |
| 181 | + "green_count": len(findings["green"]), |
| 182 | + "red": sorted(findings["red"], key=lambda x: -x["ratio"]), |
| 183 | + "yellow": sorted(findings["yellow"], key=lambda x: -x["ratio"]), |
| 184 | + } |
| 185 | + |
| 186 | + sys.path.insert(0, str(HEART_HOME)) |
| 187 | + from heart import state |
| 188 | + |
| 189 | + state.atomic_write_json(HEART_STATE_DIR / "unit_test_timing.json", summary) |
| 190 | + return summary |
| 191 | + |
| 192 | + |
| 193 | +def main(argv: list[str]) -> int: |
| 194 | + summary = run() |
| 195 | + |
| 196 | + from heart_color import c_ok, c_warn, c_fail, c_info, c_meta, glyph_ok, glyph_warn, glyph_fail |
| 197 | + |
| 198 | + if summary["red_count"]: |
| 199 | + glyph = glyph_fail() |
| 200 | + label = c_fail(f"{summary['red_count']} red") + " " + c_warn(f"{summary['yellow_count']} yellow") |
| 201 | + elif summary["yellow_count"]: |
| 202 | + glyph = glyph_warn() |
| 203 | + label = c_warn(f"{summary['yellow_count']} slower than baseline") |
| 204 | + elif not summary["repos_measured"]: |
| 205 | + glyph = glyph_warn() |
| 206 | + label = c_warn("no suite runnable (set HYGIENE_PYTHON / PYAUTO_ROOT)") |
| 207 | + else: |
| 208 | + glyph = glyph_ok() |
| 209 | + label = c_ok(f"{summary['green_count']} tracked tests within baseline") |
| 210 | + extra = c_meta(f" ({summary['new_tests_no_baseline']} new, no baseline)") |
| 211 | + print(f"{glyph} {c_info('unit_test_timing')} {label}{extra}") |
| 212 | + return 0 |
| 213 | + |
| 214 | + |
| 215 | +if __name__ == "__main__": |
| 216 | + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) |
| 217 | + sys.exit(main(sys.argv)) |
0 commit comments