diff --git a/scripts/aggregate_report.py b/scripts/aggregate_report.py index 219844c..326ec0b 100644 --- a/scripts/aggregate_report.py +++ b/scripts/aggregate_report.py @@ -17,6 +17,16 @@ HIGHER_IS_BETTER = {"R2_test", "R2_val"} +def _status_ready(path: Path) -> bool: + if not path.exists(): + return False + try: + status = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return False + return bool(status.get("end_time")) + + def _fig_to_base64(fig) -> str: buf = io.BytesIO() fig.savefig(buf, format="png", bbox_inches="tight") @@ -205,9 +215,11 @@ def main() -> None: print("[aggregator] waiting for scenario statuses", flush=True) deadline = time.time() + 60 * 60 while time.time() < deadline: - if all(p.exists() for p in status_paths): + if all(_status_ready(p) for p in status_paths): break time.sleep(5) + else: + print("[aggregator] timeout waiting for fresh scenario statuses; proceeding with available files", flush=True) out_dir = Path("results/final") out_dir.mkdir(parents=True, exist_ok=True) diff --git a/scripts/run_scenario.py b/scripts/run_scenario.py index b4ec7d2..3b5e954 100644 --- a/scripts/run_scenario.py +++ b/scripts/run_scenario.py @@ -67,6 +67,7 @@ def main() -> None: "split_metadata_path": str(sdir / "model" / "split_metadata.json"), } + (sdir / "status.json").write_text(json.dumps(status, indent=2), encoding="utf-8") try: print(f"[{scenario}] scenario started", flush=True) csv = _resolve_dataset_path(scenario, args.dataset) diff --git a/tests/test_aggregate_report_features.py b/tests/test_aggregate_report_features.py index c4cc769..642381d 100644 --- a/tests/test_aggregate_report_features.py +++ b/tests/test_aggregate_report_features.py @@ -24,6 +24,7 @@ def test_aggregate_report_handles_feature_and_new_metrics(tmp_path, monkeypatch) "preds_path": str(sdir / "preds.csv"), "dataset_path": "", "split_metadata": {"train_start_index": 0, "train_end_index": 5, "val_start_index": 6, "val_end_index": 7, "test_start_index": 8, "test_end_index": 9, "train_rows": 6, "val_rows": 2, "test_rows": 2, "train_pct": 0.6, "val_pct": 0.2, "test_pct": 0.2}, + "end_time": "2024-01-01T00:00:00Z", } ), encoding="utf-8", @@ -38,6 +39,7 @@ def test_aggregate_report_handles_feature_and_new_metrics(tmp_path, monkeypatch) "features": ["dl_cqi", "ul_sinr"], "epochs": 1, "split_metadata": {"train_start_index": 0, "train_end_index": 5, "val_start_index": 6, "val_end_index": 7, "test_start_index": 8, "test_end_index": 9, "train_rows": 6, "val_rows": 2, "test_rows": 2, "train_pct": 0.6, "val_pct": 0.2, "test_pct": 0.2}, + "end_time": "2024-01-01T00:00:00Z", } ), encoding="utf-8", diff --git a/tests/test_aggregate_waiting.py b/tests/test_aggregate_waiting.py new file mode 100644 index 0000000..07d6ab1 --- /dev/null +++ b/tests/test_aggregate_waiting.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import json + +from scripts.aggregate_report import _status_ready + + +def test_status_ready_requires_end_time(tmp_path) -> None: + p = tmp_path / "status.json" + p.write_text(json.dumps({"success": True}), encoding="utf-8") + assert _status_ready(p) is False + + p.write_text(json.dumps({"success": True, "end_time": "2024-01-01T00:00:00Z"}), encoding="utf-8") + assert _status_ready(p) is True