diff --git a/generate_data.py b/generate_data.py index a18a082..1566536 100644 --- a/generate_data.py +++ b/generate_data.py @@ -10,6 +10,7 @@ from oran_sim.config import FEATURE_ORDER from oran_sim.data import load_timeseries_from_kpm, write_json +from oran_sim.splitting import chronological_split def _build_target(df: pd.DataFrame, target: str, horizon_steps: int) -> pd.DataFrame: @@ -52,16 +53,6 @@ def _expand_to_exact_rows(df: pd.DataFrame, steps: int, seed: int) -> tuple[pd.D return expanded.iloc[:steps].copy(), logs -def _det_split(df: pd.DataFrame, seed: int) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - shuffled = df.sample(frac=1.0, random_state=seed).reset_index(drop=True) - n = len(shuffled) - n_train = int(n * 0.6) - n_val = int(n * 0.3) - n_test = n - n_train - n_val - train = shuffled.iloc[:n_train].copy() - val = shuffled.iloc[n_train : n_train + n_val].copy() - test = shuffled.iloc[n_train + n_val : n_train + n_val + n_test].copy() - return train, val, test def main() -> None: @@ -85,7 +76,7 @@ def main() -> None: base = base[keep_cols] exact_df, sampling_logs = _expand_to_exact_rows(base, args.steps, args.seed) - train_df, val_df, test_df = _det_split(exact_df, args.seed) + train_df, val_df, test_df, split_meta = chronological_split(exact_df, train_ratio=0.6, val_ratio=0.3, test_ratio=0.1) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) @@ -110,6 +101,7 @@ def main() -> None: "features_present": [c for c in FEATURE_ORDER if c in exact_df.columns], "target_definition": {"name": args.target, "horizon_steps": int(args.horizon_steps)}, "sampling_logs": sampling_logs, + "split_metadata": split_meta, } summary_path = Path("results/data/traffic_data_summary.json") write_json(summary_path, summary) diff --git a/oran_sim/splitting.py b/oran_sim/splitting.py new file mode 100644 index 0000000..3fdbadd --- /dev/null +++ b/oran_sim/splitting.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import pandas as pd + + +def _sorted(df: pd.DataFrame, time_col: str = "time_ms") -> pd.DataFrame: + if time_col in df.columns: + return df.sort_values(time_col).reset_index(drop=True) + return df.reset_index(drop=True) + + +def build_split_metadata( + train_df: pd.DataFrame, + val_df: pd.DataFrame, + test_df: pd.DataFrame, + *, + time_col: str = "time_ms", +) -> dict: + n_train, n_val, n_test = len(train_df), len(val_df), len(test_df) + total = max(n_train + n_val + n_test, 1) + + def _time_bounds(df: pd.DataFrame) -> tuple[float | None, float | None]: + if time_col not in df.columns or df.empty: + return None, None + return float(df[time_col].iloc[0]), float(df[time_col].iloc[-1]) + + tr_s, tr_e = _time_bounds(train_df) + va_s, va_e = _time_bounds(val_df) + te_s, te_e = _time_bounds(test_df) + + return { + "split_type": "chronological", + "train_start_index": 0, + "train_end_index": max(n_train - 1, -1), + "val_start_index": n_train, + "val_end_index": n_train + n_val - 1, + "test_start_index": n_train + n_val, + "test_end_index": n_train + n_val + n_test - 1, + "train_start": tr_s, + "train_end": tr_e, + "val_start": va_s, + "val_end": va_e, + "test_start": te_s, + "test_end": te_e, + "train_rows": n_train, + "val_rows": n_val, + "test_rows": n_test, + "train_pct": n_train / total, + "val_pct": n_val / total, + "test_pct": n_test / total, + } + + +def chronological_split( + df: pd.DataFrame, + train_ratio: float = 0.6, + val_ratio: float = 0.2, + test_ratio: float = 0.2, + *, + time_col: str = "time_ms", +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]: + if abs((train_ratio + val_ratio + test_ratio) - 1.0) > 1e-9: + raise ValueError("train_ratio + val_ratio + test_ratio must equal 1") + + ordered = _sorted(df, time_col=time_col) + n = len(ordered) + n_train = int(n * train_ratio) + n_val = int(n * val_ratio) + n_test = n - n_train - n_val + + train = ordered.iloc[:n_train].copy() + val = ordered.iloc[n_train : n_train + n_val].copy() + test = ordered.iloc[n_train + n_val : n_train + n_val + n_test].copy() + metadata = build_split_metadata(train, val, test, time_col=time_col) + return train, val, test, metadata + + +def sort_split(df: pd.DataFrame, time_col: str = "time_ms") -> pd.DataFrame: + return _sorted(df, time_col=time_col) diff --git a/scripts/aggregate_report.py b/scripts/aggregate_report.py index fa109e2..219844c 100644 --- a/scripts/aggregate_report.py +++ b/scripts/aggregate_report.py @@ -44,20 +44,91 @@ def _prepare_preds_for_plot(preds: pd.DataFrame) -> pd.DataFrame: return preds.sort_values("time_ms").reset_index(drop=True) return preds.reset_index(drop=True) -def _build_timeseries_chart(preds: pd.DataFrame, scenario: str) -> str: +def _build_timeseries_chart(preds: pd.DataFrame, scenario: str, split_meta: dict | None = None) -> str: sorted_preds = _prepare_preds_for_plot(preds) fig, ax = plt.subplots(figsize=(9, 3)) - x = sorted_preds["time_ms"] if "time_ms" in sorted_preds.columns else sorted_preds.index - ax.plot(x.values, sorted_preds["y_true"].values, label="y_true", linewidth=1.6) - ax.plot(x.values, sorted_preds["y_pred"].values, label="y_pred", linewidth=1.2) - ax.set_title(f"{scenario}: y_true/y_pred vs timestamp") - ax.set_xlabel("timestamp") - ax.legend(loc="best") + + if split_meta: + tr_s = split_meta.get("train_start_index", 0) + tr_e = split_meta.get("train_end_index", -1) + va_s = split_meta.get("val_start_index", 0) + va_e = split_meta.get("val_end_index", -1) + te_s = split_meta.get("test_start_index", 0) + te_e = split_meta.get("test_end_index", len(sorted_preds) - 1) + + full_x = np.arange(int(max(te_e + 1, len(sorted_preds)))) + y_true_full = np.full(full_x.shape, np.nan, dtype=float) + y_pred_full = np.full(full_x.shape, np.nan, dtype=float) + test_slice = slice(int(te_s), int(min(te_s + len(sorted_preds), len(full_x)))) + n_fill = test_slice.stop - test_slice.start + y_true_full[test_slice] = sorted_preds["y_true"].values[:n_fill] + y_pred_full[test_slice] = sorted_preds["y_pred"].values[:n_fill] + + ax.axvspan(tr_s, tr_e, alpha=0.15, color="#2ca02c", label="Train region") + ax.axvspan(va_s, va_e, alpha=0.15, color="#ff7f0e", label="Validation region") + ax.axvspan(te_s, te_e, alpha=0.15, color="#1f77b4", label="Test region") + ax.axvline(tr_e, color="#2ca02c", linestyle="--", linewidth=1.2) + ax.axvline(va_e, color="#ff7f0e", linestyle="--", linewidth=1.2) + + ax.plot(full_x, y_true_full, label="y_true (test window)", linewidth=1.4, color="black") + ax.plot(full_x, y_pred_full, label="y_pred (test window)", linewidth=1.1, color="red") + ax.set_xlabel("global row index") + else: + x = sorted_preds["time_ms"] if "time_ms" in sorted_preds.columns else sorted_preds.index + ax.plot(x.values, sorted_preds["y_true"].values, label="y_true", linewidth=1.6) + ax.plot(x.values, sorted_preds["y_pred"].values, label="y_pred", linewidth=1.2) + ax.set_xlabel("timestamp") + + ax.set_title(f"{scenario}: predictions with chronological split boundaries") + ax.legend(loc="best", fontsize=8) + img = _fig_to_base64(fig) + plt.close(fig) + return img + + +def _build_split_timeline(split_meta: dict, out_path: Path) -> str: + fig, ax = plt.subplots(figsize=(9, 1.8)) + tr = int(split_meta.get("train_rows", 0)) + va = int(split_meta.get("val_rows", 0)) + te = int(split_meta.get("test_rows", 0)) + total = max(tr + va + te, 1) + + left = 0 + ax.barh([0], [tr], left=left, color="#2ca02c", label="Train") + left += tr + ax.barh([0], [va], left=left, color="#ff7f0e", label="Validation") + left += va + ax.barh([0], [te], left=left, color="#1f77b4", label="Test") + + ax.set_xlim(0, total) + ax.set_yticks([]) + ax.set_xlabel("Row index timeline") + ax.set_title("| Train | Validation | Test | (chronological split)") + ax.legend(loc="upper center", ncol=3) + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=140, bbox_inches="tight") img = _fig_to_base64(fig) plt.close(fig) return img +def _split_table(split_meta: dict) -> pd.DataFrame: + rows = [] + for split_name, pfx in [("Train", "train"), ("Validation", "val"), ("Test", "test")]: + rows.append( + { + "Split": split_name, + "Start index": split_meta.get(f"{pfx}_start_index"), + "End index": split_meta.get(f"{pfx}_end_index"), + "Start time": split_meta.get(f"{pfx}_start"), + "End time": split_meta.get(f"{pfx}_end"), + "Rows": split_meta.get(f"{pfx}_rows"), + "Percentage": f"{100.0 * float(split_meta.get(f'{pfx}_pct', 0.0)):.1f}%", + } + ) + return pd.DataFrame(rows) + + def _build_benchmark_table(df: pd.DataFrame) -> pd.DataFrame: if df.empty: return pd.DataFrame() @@ -171,6 +242,7 @@ def main() -> None: "profile_note": status.get("profile_note"), "selected_features": ", ".join(status.get("selected_features", [])), "seq_len": status.get("seq_len"), + "split_metadata": status.get("split_metadata"), } metrics_path = Path(row["metrics_path"]) @@ -188,6 +260,7 @@ def main() -> None: row["num_features"] = len(cfg.get("features", [])) row["epochs"] = cfg.get("epochs", row.get("epochs")) row["seq_len"] = cfg.get("seq_len", row.get("seq_len")) + row["split_metadata"] = cfg.get("split_metadata", row.get("split_metadata")) else: row["model_type"] = None @@ -210,7 +283,7 @@ def main() -> None: row["mean_abs_error"] = float(preds["abs_error"].mean()) row["mean_abs_pct_error"] = float(preds["pct_error"].abs().mean()) - model_chart = _build_timeseries_chart(preds, scenario) + model_chart = _build_timeseries_chart(preds, scenario, row.get("split_metadata")) model_chart_sections.append(f"

{scenario}

") dpath_str = str(row.get("dataset_path", "")).strip() @@ -274,6 +347,17 @@ def main() -> None: comp_df = comp_df.merge(benchmark_df[["scenario", "benchmark_score"]], on="scenario", how="left") comp_df.to_csv(out_dir / "scenario_status.csv", index=False) + split_meta = None + for row in rows: + if row.get("split_metadata"): + split_meta = row["split_metadata"] + break + split_table_html = "

No split metadata available.

" + split_timeline_html = "

No split timeline available.

" + if split_meta: + split_table_html = _split_table(split_meta).to_html(index=False) + split_timeline_html = f"" + feature_importance_path = Path("results/feature_importance.json") feature_importance_html = "

Feature importance artifact not found.

" if not feature_importance_path.exists(): @@ -286,15 +370,6 @@ def main() -> None: if not feature_importance_df.empty: feature_importance_html = feature_importance_df.to_html(index=False) - - feature_importance_path = Path("results/feature_importance.json") - feature_importance_html = "

Feature importance artifact not found.

" - if feature_importance_path.exists(): - feature_importance_payload = json.loads(feature_importance_path.read_text(encoding="utf-8")) - feature_importance_df = pd.DataFrame(feature_importance_payload.get("feature_importance", [])) - if not feature_importance_df.empty: - feature_importance_html = feature_importance_df.to_html(index=False) - html = f"""

KPM Final Report

@@ -306,6 +381,11 @@ def main() -> None: {table_df.to_html(index=False)}

Benchmark Leaderboard

{benchmark_df.to_html(index=False) if not benchmark_df.empty else '

No benchmark-ready metrics available.

'} +

Chronological Time-Series Split

+

All experiments use contiguous chronological blocks only: | Train | Validation | Test |. No random shuffling is used in the official benchmark path.

+ {split_table_html} + {split_timeline_html} +

Temporal windows are built from past values only; validation/test windows do not use future labels. Feature importance is computed on the training split only.

Global Feature Importance (train-only Random Forest)

{feature_importance_html}

Model Predictions vs Ground Truth (timestamp axis)

diff --git a/scripts/run_scenario.py b/scripts/run_scenario.py index 707248d..b4ec7d2 100644 --- a/scripts/run_scenario.py +++ b/scripts/run_scenario.py @@ -64,6 +64,7 @@ def main() -> None: "epochs": epochs, "selected_features": [], "feature_importance_path": str(sdir / "model" / "feature_importance.json"), + "split_metadata_path": str(sdir / "model" / "split_metadata.json"), } try: @@ -99,6 +100,10 @@ def main() -> None: if features_path.exists(): status["selected_features"] = json.loads(features_path.read_text(encoding="utf-8")) + split_meta_path = sdir / "model" / "split_metadata.json" + if split_meta_path.exists(): + status["split_metadata"] = json.loads(split_meta_path.read_text(encoding="utf-8")) + cfg_path = sdir / "model" / "config.json" if cfg_path.exists(): cfg = json.loads(cfg_path.read_text(encoding="utf-8")) diff --git a/scripts/train.py b/scripts/train.py index 19e96b7..0988160 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -16,8 +16,9 @@ from oran_sim.config import FEATURE_ORDER, SCENARIOS from oran_sim.feature_selection import rank_features_by_importance, select_top_k_features, write_feature_importance_artifacts from oran_sim.model import build_model, get_model_metadata -from oran_sim.sequence_data import make_sequences, sort_by_time +from oran_sim.sequence_data import make_sequences from oran_sim.temporal import TemporalSpec, build_temporal_model, predict_temporal_model, train_temporal_model +from oran_sim.splitting import build_split_metadata, chronological_split, sort_split def _metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict: @@ -35,7 +36,7 @@ def _metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict: } -def _load_splits(csv: Path, seed: int, temporal: bool) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: +def _load_splits(csv: Path) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]: root = csv.parent stem = csv.stem train_path = root / f"{stem}_train.csv" @@ -43,22 +44,18 @@ def _load_splits(csv: Path, seed: int, temporal: bool) -> tuple[pd.DataFrame, pd test_path = root / f"{stem}_test.csv" if train_path.exists() and val_path.exists() and test_path.exists(): - train_df = pd.read_csv(train_path) - val_df = pd.read_csv(val_path) - test_df = pd.read_csv(test_path) + train_df = sort_split(pd.read_csv(train_path)) + val_df = sort_split(pd.read_csv(val_path)) + test_df = sort_split(pd.read_csv(test_path)) + split_meta = build_split_metadata(train_df, val_df, test_df) else: full = pd.read_csv(csv) - if temporal: - full = sort_by_time(full) - else: - full = full.sample(frac=1.0, random_state=seed).reset_index(drop=True) - n = len(full) - a, b = int(0.6 * n), int(0.9 * n) - train_df, val_df, test_df = full.iloc[:a].copy(), full.iloc[a:b].copy(), full.iloc[b:].copy() + train_df, val_df, test_df, split_meta = chronological_split(full, train_ratio=0.6, val_ratio=0.3, test_ratio=0.1) + train_df.to_csv(train_path, index=False) + val_df.to_csv(val_path, index=False) + test_df.to_csv(test_path, index=False) - if temporal: - return sort_by_time(train_df), sort_by_time(val_df), sort_by_time(test_df) - return train_df, val_df, test_df + return train_df, val_df, test_df, split_meta def _prepare_tabular_preprocessor(features: list[str]) -> ColumnTransformer: @@ -88,7 +85,7 @@ def main() -> None: is_temporal = bool(scenario_cfg and scenario_cfg.kind == "temporal") csv = Path(args.csv) - train_df, val_df, test_df = _load_splits(csv, args.seed, is_temporal) + train_df, val_df, test_df, split_meta = _load_splits(csv) candidate_features = [c for c in FEATURE_ORDER if c in train_df.columns] importance_df = rank_features_by_importance(train_df, candidate_features, random_state=args.seed) @@ -171,6 +168,7 @@ def encode(df: pd.DataFrame) -> pd.DataFrame: write_feature_importance_artifacts(importance_df, json_path=Path("results") / "feature_importance.json", csv_path=Path("results") / "feature_importance.csv") write_feature_importance_artifacts(importance_df, json_path=out_dir / "feature_importance.json", csv_path=out_dir / "feature_importance.csv") + (out_dir / "split_metadata.json").write_text(json.dumps(split_meta, indent=2), encoding="utf-8") torch.save(model.state_dict(), out_dir / "temporal_model.pt") joblib.dump(scaler, out_dir / "temporal_scaler.joblib") (out_dir / "temporal_spec.json").write_text(json.dumps(spec.__dict__, indent=2), encoding="utf-8") @@ -194,6 +192,7 @@ def encode(df: pd.DataFrame) -> pd.DataFrame: "requested_seq_len": requested_seq_len, "temporal": True, "input_shape": [int(seq_len), int(len(features))], + "split_metadata": split_meta, }, indent=2, ), @@ -239,6 +238,7 @@ def encode(df: pd.DataFrame) -> pd.DataFrame: out_dir.mkdir(parents=True, exist_ok=True) write_feature_importance_artifacts(importance_df, json_path=Path("results") / "feature_importance.json", csv_path=Path("results") / "feature_importance.csv") write_feature_importance_artifacts(importance_df, json_path=out_dir / "feature_importance.json", csv_path=out_dir / "feature_importance.csv") + (out_dir / "split_metadata.json").write_text(json.dumps(split_meta, indent=2), encoding="utf-8") joblib.dump(pipe, out_dir / "model.joblib") (out_dir / "features.json").write_text(json.dumps(features, indent=2), encoding="utf-8") (out_dir / "config.json").write_text( @@ -255,6 +255,7 @@ def encode(df: pd.DataFrame) -> pd.DataFrame: "feature_selection": "random_forest_importance_top_k", "epochs": epochs, "temporal": False, + "split_metadata": split_meta, }, indent=2, ), @@ -264,6 +265,7 @@ def encode(df: pd.DataFrame) -> pd.DataFrame: pd.DataFrame(epoch_rows).to_csv(out_dir / "epoch_metrics.csv", index=False) print(f"train_rows={len(train_df)} val_rows={len(val_df)} test_rows={len(test_df)}") + print(f"split_meta={split_meta}") if __name__ == "__main__": diff --git a/tests/test_aggregate_report_features.py b/tests/test_aggregate_report_features.py index ee1553b..c4cc769 100644 --- a/tests/test_aggregate_report_features.py +++ b/tests/test_aggregate_report_features.py @@ -23,6 +23,7 @@ def test_aggregate_report_handles_feature_and_new_metrics(tmp_path, monkeypatch) "metrics_path": str(model_dir / "metrics.json"), "preds_path": str(sdir / "preds.csv"), "dataset_path": "", + "split_metadata": {"train_start_index": 0, "train_end_index": 5, "val_start_index": 6, "val_end_index": 7, "test_start_index": 8, "test_end_index": 9, "train_rows": 6, "val_rows": 2, "test_rows": 2, "train_pct": 0.6, "val_pct": 0.2, "test_pct": 0.2}, } ), encoding="utf-8", @@ -36,6 +37,7 @@ def test_aggregate_report_handles_feature_and_new_metrics(tmp_path, monkeypatch) "profile_note": "note", "features": ["dl_cqi", "ul_sinr"], "epochs": 1, + "split_metadata": {"train_start_index": 0, "train_end_index": 5, "val_start_index": 6, "val_end_index": 7, "test_start_index": 8, "test_end_index": 9, "train_rows": 6, "val_rows": 2, "test_rows": 2, "train_pct": 0.6, "val_pct": 0.2, "test_pct": 0.2}, } ), encoding="utf-8", @@ -62,6 +64,8 @@ def test_aggregate_report_handles_feature_and_new_metrics(tmp_path, monkeypatch) assert "Global Feature Importance" in out assert "sMAPE" in out assert "wMAPE" in out + assert "Chronological Time-Series Split" in out + assert Path("results/final/split_timeline.png").exists() status_df = pd.read_csv("results/final/scenario_status.csv") assert pd.notna(status_df.loc[0, "benchmark_score"]) if "benchmark_score" in status_df.columns else True diff --git a/tests/test_feature_selection.py b/tests/test_feature_selection.py index a8f1224..bafa4c4 100644 --- a/tests/test_feature_selection.py +++ b/tests/test_feature_selection.py @@ -62,4 +62,6 @@ def test_feature_importance_file_generation_and_selected_features_in_status(tmp_ assert status.get("model_backend") == "Ridge" assert status.get("logical_profile") == "tabular_baseline" assert (sdir / "model" / "feature_importance.json").exists() + assert (sdir / "model" / "split_metadata.json").exists() + assert "split_metadata" in status assert Path("results/feature_importance.json").exists() diff --git a/tests/test_splitting.py b/tests/test_splitting.py new file mode 100644 index 0000000..807fe97 --- /dev/null +++ b/tests/test_splitting.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pandas as pd + +from oran_sim.splitting import chronological_split + + +def test_chronological_split_order_and_counts() -> None: + df = pd.DataFrame({"time_ms": [5, 1, 4, 2, 3], "target": [50, 10, 40, 20, 30]}) + train, val, test, meta = chronological_split(df, train_ratio=0.6, val_ratio=0.2, test_ratio=0.2) + + assert train["time_ms"].tolist() == [1, 2, 3] + assert val["time_ms"].tolist() == [4] + assert test["time_ms"].tolist() == [5] + assert meta["train_rows"] == 3 + assert meta["val_rows"] == 1 + assert meta["test_rows"] == 1 + + +def test_chronological_split_no_overlap() -> None: + df = pd.DataFrame({"time_ms": list(range(10)), "target": list(range(10))}) + train, val, test, meta = chronological_split(df) + assert train["time_ms"].max() < val["time_ms"].min() + assert val["time_ms"].max() < test["time_ms"].min() + assert meta["train_end_index"] < meta["val_start_index"] + assert meta["val_end_index"] < meta["test_start_index"]