Skip to content

Commit 82fc92d

Browse files
Jammy2211claude
authored andcommitted
test_run: caller-decided fetch_cloud + local/cloud agreement (#83 fix 1)
The server-first cloud verdict was unreachable from every real entrypoint: run() inferred fetch_cloud from `results_dir is None` and main() — the tick and the mobile path — always passed results_dir, leaving the leg on the local test_results/latest (stale since 2026-07-09) since the design merged. main() now passes fetch_cloud=True explicitly; bare run() defaults to no network. When both surfaces carry a verdict, green requires both (ready = local AND cloud), cloud_ready is always recorded, and a mismatch writes a disagreement field — closing the audit's local-green/cloud-red hole and its mirror. Verified against the real tick invocation: the sidecar now carries the actual latest workspace-validation run. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent fbcd972 commit 82fc92d

2 files changed

Lines changed: 80 additions & 9 deletions

File tree

heart/checks/test_run.py

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -190,10 +190,14 @@ def _from_per_job(results_dir: Path) -> dict[str, Any]:
190190

191191

192192
def run(results_dir: Path | None = None, fetch_cloud: bool | None = None) -> dict[str, Any]:
193-
default_path = results_dir is None
193+
# fetch_cloud is decided by the CALLER, never inferred: the old
194+
# `results_dir is None` inference meant main() — the tick and the mobile
195+
# path — always disabled the cloud fetch, leaving the leg on stale local
196+
# evidence while claiming to be server-first (PyAutoHeart#83 finding A).
197+
# Library/test callers default to no network.
194198
results_dir = results_dir or TEST_RESULTS_LATEST
195199
if fetch_cloud is None:
196-
fetch_cloud = default_path # only hit the network on the real tick path
200+
fetch_cloud = False
197201

198202
summary: dict[str, Any]
199203
report = _read_json(results_dir / "report.json")
@@ -208,18 +212,32 @@ def run(results_dir: Path | None = None, fetch_cloud: bool | None = None) -> dic
208212
report_path.stat().st_mtime, datetime.timezone.utc
209213
).isoformat()
210214

211-
# The server workspace-validation run is the authoritative continuous verdict
212-
# (server-first: MCP-supplied file, else `gh`); it sets ready/ts so a missing
213-
# local report.json no longer forces "unknown" when the cloud run is green.
214-
# The local report, when present, still supplies the count detail.
215+
# The server workspace-validation run (MCP-supplied file, else `gh`) sets
216+
# ready/ts so a missing local report.json no longer forces "unknown" when
217+
# the cloud run is green. The local report, when present, still supplies
218+
# the count detail. When BOTH surfaces carry a verdict they must agree to
219+
# be green — a disagreement is surfaced, never silently resolved in either
220+
# direction (PyAutoHeart#83 §4-§5: a fresh local pass must not green the
221+
# leg while the server surface fails, and vice versa).
215222
cloud = _server_verdict() if fetch_cloud else None
216223
if cloud is not None:
224+
local_ready = summary.get("ready") if summary else None
217225
if not summary:
218226
summary = {
219227
"passed": 0, "failed": 0, "skipped": 0, "timeout": 0,
220228
"per_project": {}, "parked_stale_count": 0, "parked_stale": [],
221229
}
222-
summary["ready"] = cloud["ready"]
230+
summary["cloud_ready"] = cloud["ready"]
231+
if local_ready is None:
232+
summary["ready"] = cloud["ready"]
233+
elif cloud["ready"] is None:
234+
summary["ready"] = local_ready # cloud run in progress: keep local
235+
else:
236+
summary["ready"] = bool(local_ready) and bool(cloud["ready"])
237+
if bool(local_ready) != bool(cloud["ready"]):
238+
summary["disagreement"] = (
239+
f"local ready={local_ready} vs cloud ready={cloud['ready']}"
240+
)
223241
summary["ts"] = cloud["ts"]
224242
summary["run_label"] = summary.get("run_label") or f"cloud#{cloud['run_id']}"
225243
summary["cloud_url"] = cloud["url"]
@@ -230,7 +248,9 @@ def run(results_dir: Path | None = None, fetch_cloud: bool | None = None) -> dic
230248

231249
def main(argv: list[str]) -> int:
232250
results_dir = Path(argv[1]) if len(argv) > 1 else TEST_RESULTS_LATEST
233-
summary = run(results_dir)
251+
# The tick/CLI is the real path: always consult the server verdict here,
252+
# explicitly — this is the entrypoint the old inference silently disabled.
253+
summary = run(results_dir, fetch_cloud=True)
234254

235255
sys.path.insert(0, str(HEART_HOME))
236256
from heart import state

tests/test_test_run.py

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,12 +168,63 @@ def test_run_writes_nothing_to_state_dir(tmp_path):
168168
assert after == before
169169

170170

171-
def test_main_persists_summary_to_state_dir(tmp_path):
171+
def test_main_persists_summary_to_state_dir(monkeypatch, tmp_path):
172172
"""The tick path (python -m heart.checks.test_run) must still persist."""
173173
import os
174174
from pathlib import Path
175+
monkeypatch.setattr(tr, "_server_verdict", lambda: None) # hermetic
175176
(tmp_path / "report.json").write_text(json.dumps({
176177
"ready": True, "run_label": "R1", "summary": {"passed": 1}}))
177178
assert tr.main(["test_run", str(tmp_path)]) == 0
178179
written = json.loads((Path(os.environ["HEART_STATE_DIR"]) / "test_run.json").read_text())
179180
assert written["ready"] is True and written["passed"] == 1
181+
182+
183+
# --- entrypoint wiring + disagreement (PyAutoHeart#83 finding A) ----------------
184+
185+
def test_main_consults_the_server_verdict(monkeypatch, tmp_path):
186+
"""The tick entrypoint must fetch the cloud verdict — the old inference
187+
(fetch_cloud from `results_dir is None`) silently disabled it forever."""
188+
import os
189+
from pathlib import Path
190+
monkeypatch.setattr(tr, "_server_verdict", lambda: {
191+
"ready": False, "ts": "2026-07-16T00:00:00Z", "run_id": 3, "url": "U"})
192+
(tmp_path / "report.json").write_text(json.dumps({
193+
"ready": True, "run_label": "local", "summary": {"passed": 5}}))
194+
assert tr.main(["test_run", str(tmp_path)]) == 0
195+
written = json.loads((Path(os.environ["HEART_STATE_DIR"]) / "test_run.json").read_text())
196+
assert written["source"] == "cloud"
197+
assert written["cloud_ready"] is False
198+
199+
200+
def test_local_and_cloud_must_agree_to_be_green(monkeypatch, tmp_path):
201+
# local False + cloud True → NOT green, disagreement surfaced (the mirror
202+
# of the local-green/cloud-red hole; neither side wins silently).
203+
(tmp_path / "report.json").write_text(json.dumps({
204+
"ready": False, "run_label": "local", "summary": {"passed": 1, "failed": 2}}))
205+
monkeypatch.setattr(tr, "_server_verdict", lambda: {
206+
"ready": True, "ts": "t", "run_id": 4, "url": "u"})
207+
out = tr.run(results_dir=tmp_path, fetch_cloud=True)
208+
assert out["ready"] is False
209+
assert "disagreement" in out
210+
assert out["cloud_ready"] is True
211+
212+
213+
def test_cloud_in_progress_keeps_local_verdict(monkeypatch, tmp_path):
214+
(tmp_path / "report.json").write_text(json.dumps({
215+
"ready": True, "run_label": "local", "summary": {"passed": 1}}))
216+
monkeypatch.setattr(tr, "_server_verdict", lambda: {
217+
"ready": None, "ts": "t", "run_id": 5, "url": "u"})
218+
out = tr.run(results_dir=tmp_path, fetch_cloud=True)
219+
assert out["ready"] is True
220+
assert out["cloud_ready"] is None
221+
assert "disagreement" not in out
222+
223+
224+
def test_run_without_explicit_fetch_never_touches_network(monkeypatch, tmp_path):
225+
calls = []
226+
monkeypatch.setattr(tr, "_server_verdict", lambda: calls.append(1))
227+
(tmp_path / "report.json").write_text(json.dumps({
228+
"ready": True, "summary": {"passed": 1}}))
229+
tr.run(results_dir=tmp_path)
230+
assert calls == []

0 commit comments

Comments
 (0)