python_pipeline.py

#!/usr/bin/env python3
"""
Bulletproof Python Pipeline for HOSER Evaluation

This script imports gene.py and evaluation.py directly to orchestrate
trajectory generation and evaluation. Designed to work with evaluation
directories created by setup_evaluation.py.

Usage:
    # From inside an evaluation directory created by setup_evaluation.py:
    cd hoser-evaluation-xyz-20241024_123456
    uv run python ../python_pipeline.py [OPTIONS]

    # Or with explicit paths:
    uv run python python_pipeline.py --eval-dir path/to/eval/dir [OPTIONS]

Options:
    --eval-dir DIR           Evaluation directory (default: current directory)
    --seed SEED              Random seed (default: from config)
    --models MODEL1,MODEL2   Models to run (default: auto-detect all)
    --od-source SOURCE      OD source: train or test (default: from config)
    --skip-gene             Skip generation (use existing trajectories)
    --skip-eval             Skip evaluation
    --force                 Force re-run even if results exist
    --cuda DEVICE           CUDA device (default: from config)
    --num-gene N            Number of trajectories (default: from config)
    --wandb-project PROJECT WandB project name (default: from config)
    --no-wandb              Disable WandB logging entirely
    --verbose               Enable verbose output
    --run-scenarios         Run scenario analysis after evaluation
    --scenarios-config PATH Path to scenarios config YAML
"""

import os
import sys
import argparse
import threading
import re
from functools import wraps
from pathlib import Path
from typing import List, Dict, Any, Optional, Callable, Set
import logging


class _SafeStreamHandler(logging.StreamHandler):
    def emit(self, record):
        try:
            super().emit(record)
        except (PermissionError, OSError, ValueError):
            # Some cluster environments can intermittently make stdout/stderr
            # unwritable (e.g., closed TTY, redirected stream). Logging should
            # never crash or spam internal tracebacks.
            return

    def flush(self):
        try:
            super().flush()
        except (PermissionError, OSError, ValueError):
            return


class _SafeFileHandler(logging.FileHandler):
    def emit(self, record):
        try:
            super().emit(record)
        except (PermissionError, OSError, ValueError):
            return

    def flush(self):
        try:
            super().flush()
        except (PermissionError, OSError, ValueError):
            return

# Detect if we're running from inside an eval directory or from project root
SCRIPT_DIR = Path(__file__).parent
CURRENT_DIR = Path.cwd()

# Add appropriate paths for imports based on location
if (CURRENT_DIR / "models").exists() and (CURRENT_DIR / "config").exists():
    # Running from inside an evaluation directory
    EVAL_DIR = CURRENT_DIR
    PROJECT_ROOT = SCRIPT_DIR  # python_pipeline.py is in project root
else:
    # Running from project root or elsewhere
    EVAL_DIR = None  # Will be set from args
    PROJECT_ROOT = SCRIPT_DIR

sys.path.insert(0, str(PROJECT_ROOT))

# Import the programmatic interfaces
from gene import generate_trajectories_programmatic  # noqa: E402
from evaluation import evaluate_trajectories_programmatic  # noqa: E402
from utils import set_seed  # noqa: E402
import yaml  # noqa: E402
import torch  # noqa: E402
import wandb  # noqa: E402
from tools.model_detection import extract_model_name  # noqa: E402

# Configure logging
# Some environments (shared filesystems, read-only eval dirs, restrictive symlinks)
# can make a log file unwritable. In that case, fall back to stderr logging only.
logging.raiseExceptions = False

_log_handlers: List[logging.Handler] = [_SafeStreamHandler()]
try:
    # Open immediately so unwritable paths are detected here (not during emit/flush).
    _log_handlers.append(
        _SafeFileHandler(Path.cwd() / "pipeline.log", mode="a", encoding="utf-8")
    )
except OSError:
    pass

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=_log_handlers,
)
logger = logging.getLogger(__name__)


# Phase Decorator Infrastructure
# Phase registry - auto-populated by decorator
PHASE_REGISTRY: Dict[str, Callable] = {}


def phase(name: str, critical: bool = False):
    """Decorator to register a pipeline phase

    Args:
        name: Phase identifier (used in CLI)
        critical: If True, pipeline stops on failure
    """

    def decorator(func):
        PHASE_REGISTRY[name] = {"func": func, "critical": critical, "name": name}

        @wraps(func)
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)

        return wrapper

    return decorator


class PipelineConfig:
    """Configuration for the evaluation pipeline"""

    def __init__(self, config_path: str = None, eval_dir: Path = None):
        # Store evaluation directory
        self.eval_dir = eval_dir

        # Default values
        self.wandb_project = "hoser-evaluation"
        self.dataset = "Beijing"
        self.cuda_device = 0
        self.num_gene = 100
        self.seed = 42
        self.models = []  # Auto-detect
        self.od_sources = ["train", "test"]

        # Optional dataset root override (can be absolute or relative to eval dir)
        # Used by eval workspaces that keep datasets outside ../data/{dataset}.
        self.data_dir: Optional[str] = None

        # NEW: Phase-based control (replaces skip_* flags)
        # Note: stored as a set for membership checks; execution order is defined
        # in `_phase_execution_order()`.
        self.phases: Set[str] = {
            "generation",
            "base_eval",
            "perturbation_correction",
            "paired_analysis",
            "cross_dataset",
            "road_network_translate",
            "abnormal",
            "abnormal_od_extract",
            "abnormal_od_generate",
            "abnormal_od_evaluate",
            "wang_abnormality",
            "lmtad_spatial_abnormality",
            "scenarios",
        }

        # When True, `phases` was set explicitly by the operator (e.g., via --only).
        # In that case we should NOT auto-add phases based on feature flags, so that
        # `--only X` really means only X.
        self.phases_explicit: bool = False

        # Other settings
        self.force = False
        self.enable_wandb = True
        self.verbose = False
        self.beam_width = 4
        self.beam_search = True  # Use beam search by default (set False for A* search)
        self.grid_size = 0.001
        self.edr_eps = 100.0
        self.background_sync = True  # Background WandB sync
        self.run_scenarios = False  # NEW: Run scenario analysis after eval
        self.scenarios_config = None  # NEW: Path to scenarios config
        self.cross_dataset_eval = None  # NEW: Path to cross-dataset for evaluation
        self.cross_dataset_name = (
            None  # NEW: Name of cross-dataset (e.g., BJUT_Beijing)
        )
        self.run_abnormal_detection = False  # NEW: Run abnormal trajectory detection
        self.abnormal_config = None  # NEW: Path to abnormal detection config
        self.run_wang_detection = (
            False  # NEW: Run Wang statistical abnormality detection
        )
        self.wang_config = None  # NEW: Path to Wang statistical detection config
        self.run_lmtad_spatial_detection = (
            False  # NEW: Run LM-TAD spatial abnormality detection
        )
        self.lmtad_spatial_config = None  # NEW: Path to LM-TAD spatial config file
        self.lmtad_source_eval_dir = None  # NEW: Path to LM-TAD source eval directory
        self.lmtad_num_trajectories_per_od = (
            None  # NEW: Override num trajectories per OD pair
        )
        self.lmtad_max_od_pairs = None  # NEW: Override max OD pairs to sample
        self.lmtad_max_duplicate_ratio = (
            0.1  # NEW: Duplicate threshold for LM-TAD validation
        )

        # Phase B: perturbation correction evaluation (optional)
        self.perturbation_source_csv: Optional[str] = None
        self.perturbation_od_source: str = "train"
        self.perturbation_max_entries: Optional[int] = None
        self.perturbation_seed: int = 0
        self.perturbation_use_astar: bool = True
        self.perturbation_lmtad_checkpoint: Optional[str] = None
        self.perturbation_lmtad_repo: Optional[str] = None
        self.perturbation_lmtad_batch_size: int = 128

        # Load from YAML if provided
        if config_path:
            self.load_from_yaml(config_path)

    def load_from_yaml(self, config_path: str):
        """Load configuration from YAML file"""
        with open(config_path, "r") as f:
            config_data = yaml.safe_load(f)

        # Update attributes from config
        for key, value in config_data.items():
            if key == "wandb":
                wandb_config = value
                self.wandb_project = wandb_config.get("project", self.wandb_project)
                self.enable_wandb = wandb_config.get("enable", self.enable_wandb)
                self.background_sync = wandb_config.get(
                    "background_sync", self.background_sync
                )
            elif key == "logging":
                logging_config = value
                self.verbose = logging_config.get("verbose", self.verbose)
            elif key == "phases":
                # Ensure phases is always a set (YAML may load as list)
                self.phases = set(value) if isinstance(value, (list, set)) else value
            elif hasattr(self, key):
                setattr(self, key, value)


class ModelDetector:
    """Auto-detect models in the models directory"""

    def __init__(self, models_dir: Path):
        self.models_dir = models_dir

    @staticmethod
    def _extract_model_type_from_filename(filename: str) -> str:
        """Extract a canonical model type from a checkpoint filename.

        Delegates to the centralized `tools.model_detection.extract_model_name` so
        naming conventions remain consistent across the codebase.
        """

        return extract_model_name(filename)

    def detect_models(self) -> List[str]:
        """Detect all available models and return unique model types"""
        if not self.models_dir.exists():
            raise FileNotFoundError(f"Models directory not found: {self.models_dir}")

        model_files = list(self.models_dir.glob("*.pth"))
        if not model_files:
            raise FileNotFoundError(f"No .pth files found in {self.models_dir}")

        model_types = []
        for model_file in model_files:
            model_type = self._extract_model_type_from_filename(model_file.name)
            model_types.append(model_type)

        # Remove duplicates and sort
        unique_types = sorted(list(set(model_types)))
        logger.info(f"Detected models: {unique_types}")
        return unique_types

    def find_model_file(self, model_type: str) -> Optional[Path]:
        """Find the actual model file for a given model type"""
        for model_file in self.models_dir.glob("*.pth"):
            extracted_type = self._extract_model_type_from_filename(model_file.name)

            if extracted_type == model_type:
                return model_file

        return None


class TrajectoryGenerator:
    """Generate trajectories using HOSER models"""

    def __init__(self, config: PipelineConfig):
        self.config = config

    @staticmethod
    def _infer_seed_from_model_type(model_type: str) -> Optional[int]:
        """Infer a numeric seed from a model type string.

        Examples:
            - "seed42" -> 42
            - "distilled_l0p001_seed43" -> 43

        Returns:
            Parsed seed integer if present, otherwise None.
        """

        match = re.search(r"seed(\d+)", model_type)
        if not match:
            return None

        try:
            return int(match.group(1))
        except ValueError:
            return None

    def generate_trajectories(
        self, model_path: Path, model_type: str, od_source: str
    ) -> tuple[Path, dict]:
        """Generate trajectories for a specific model and OD source

        Returns:
            Tuple of (output_path, performance_metrics)
        """
        logger.info(f"Generating trajectories: {model_type} ({od_source} OD)")

        model_seed = self._infer_seed_from_model_type(model_type) or self.config.seed

        # Use the programmatic interface (returns dict with output_file, num_generated, performance)
        result = generate_trajectories_programmatic(
            dataset=self.config.dataset,
            model_path=str(model_path),
            od_source=od_source,
            seed=model_seed,
            num_gene=self.config.num_gene,
            data_dir=self.config.data_dir,
            cuda_device=self.config.cuda_device,
            beam_search=self.config.beam_search,
            beam_width=self.config.beam_width,
            enable_wandb=False,  # We'll handle WandB separately
            wandb_project=None,
            wandb_run_name=None,
            wandb_tags=None,
            model_type=model_type,
        )

        output_path = Path(result["output_file"])
        performance = result.get("performance", {})

        # Log performance summary
        if performance:
            logger.info(
                f"Generation performance: {performance.get('throughput_traj_per_sec', 0):.2f} traj/s, "
                + f"mean time: {performance.get('total_time_mean', 0):.3f}s"
            )

        logger.info(f"Trajectories saved to: {output_path}")
        return output_path, performance


class TrajectoryEvaluator:
    """Evaluate generated trajectories against real data with smart caching"""

    def __init__(self, config: PipelineConfig):
        self.config = config
        self.evaluation_cache = {}  # Cache full evaluations by (file, od_source)

    def evaluate_trajectories(
        self,
        generated_file: Path,
        model_type: str,
        od_source: str,
        generation_performance: dict = None,
    ) -> Dict[str, Any]:
        """Evaluate generated trajectories with caching and optional performance metrics

        Args:
            generated_file: Path to generated trajectories
            model_type: Model type identifier
            od_source: OD source (train/test)
            generation_performance: Optional performance metrics from generation
        """
        cache_key = (str(generated_file), od_source)

        # Check if we've already evaluated this exact combination
        if cache_key in self.evaluation_cache:
            logger.info(
                f"Using cached evaluation results for {model_type} ({od_source} OD)"
            )
            cached_results = self.evaluation_cache[cache_key]
            # Update with new performance metrics if provided
            if generation_performance:
                cached_results["generation_performance"] = generation_performance
            return cached_results

        logger.info(f"Evaluating trajectories: {model_type} ({od_source} OD)")

        # Use the programmatic interface with performance metrics
        results = evaluate_trajectories_programmatic(
            generated_file=str(generated_file),
            dataset=self.config.dataset,
            od_source=od_source,  # Pass OD source to load correct real data
            data_dir=getattr(self.config, "data_dir", None),
            grid_size=self.config.grid_size,
            edr_eps=self.config.edr_eps,
            enable_wandb=False,  # We'll handle WandB separately
            wandb_project=None,
            wandb_run_name=None,
            wandb_tags=None,
            generation_performance=generation_performance,  # Pass performance metrics
        )

        # Cache the results
        self.evaluation_cache[cache_key] = results

        # Add our metadata
        results["metadata"]["model_type"] = model_type
        results["metadata"]["od_source"] = od_source
        results["metadata"]["seed"] = self.config.seed

        # Re-save the results file with updated metadata
        # Find the results file path from the eval directory structure
        eval_dirs = sorted(Path("./eval").glob("20*"), key=lambda x: x.stat().st_mtime)
        if eval_dirs:
            latest_results_file = eval_dirs[-1] / "results.json"
            if latest_results_file.exists():
                import json

                with open(latest_results_file, "w") as f:
                    json.dump(results, f, indent=4)
                logger.debug(
                    f"Updated results file with model metadata: {latest_results_file}"
                )

        logger.info(f"Evaluation completed for {model_type} ({od_source} OD)")
        return results


class WandBManager:
    """Manage WandB logging efficiently with background uploads"""

    def __init__(self, config: PipelineConfig):
        self.config = config
        self.runs = {}  # Store run objects
        self.completed_runs = []  # Track completed runs for background sync
        self.sync_thread = None

    def init_run(
        self, run_name: str, tags: List[str], config_dict: Dict[str, Any]
    ) -> str:
        """Initialize a WandB run and return run ID"""
        if not self.config.enable_wandb:
            return "wandb_disabled"

        try:
            # Use offline mode - no network delays
            run = wandb.init(
                project=self.config.wandb_project,
                name=run_name,
                tags=tags,
                config=config_dict,
                reinit=True,
                mode="offline",
            )
            self.runs[run_name] = run
            logger.info(f"WandB run initialized (offline): {run_name}")
            return run.id
        except Exception as e:
            logger.warning(f"Failed to initialize WandB run: {e}")
            return "wandb_failed"

    def log_metrics(self, run_name: str, metrics: Dict[str, Any]):
        """Log metrics to WandB (offline, no upload delay)"""
        if not self.config.enable_wandb or run_name not in self.runs:
            return

        try:
            # Filter out non-scalar metrics
            scalar_metrics = {
                k: v
                for k, v in metrics.items()
                if isinstance(v, (int, float)) and k != "metadata"
            }
            if scalar_metrics:
                self.runs[run_name].log(scalar_metrics)
        except Exception as e:
            logger.warning(f"Failed to log metrics to WandB: {e}")

    def finish_run(self, run_name: str):
        """Finish a WandB run and track for background sync"""
        if not self.config.enable_wandb or run_name not in self.runs:
            return

        try:
            run = self.runs[run_name]
            run_dir = run.dir  # Get the run directory before finishing
            run.finish()
            del self.runs[run_name]

            # Track for background sync
            self.completed_runs.append(run_dir)
            logger.info(f"WandB run completed (offline): {run_name}")
        except Exception as e:
            logger.warning(f"Failed to finish WandB run: {e}")

    def finish_all_runs(self):
        """Finish all remaining WandB runs"""
        for run_name in list(self.runs.keys()):
            self.finish_run(run_name)

    def start_background_sync(self):
        """Start background thread to sync offline runs to WandB"""
        if not self.config.enable_wandb or not self.completed_runs:
            return

        if not self.config.background_sync:
            logger.info(f"📤 {len(self.completed_runs)} WandB runs saved offline")
            logger.info("   To upload: uv run wandb sync wandb/offline-run-*")
            return

        def sync_worker():
            """Background worker to sync runs"""
            logger.info(
                f"📤 Starting background sync of {len(self.completed_runs)} WandB runs..."
            )

            import subprocess

            synced = 0
            failed = 0

            for run_dir in self.completed_runs:
                try:
                    # Use subprocess to run wandb sync via uv
                    result = subprocess.run(
                        ["uv", "run", "wandb", "sync", run_dir],
                        capture_output=True,
                        text=True,
                        timeout=300,  # 5 min timeout per run
                    )
                    if result.returncode == 0:
                        synced += 1
                        logger.info(
                            f"✅ Synced {synced}/{len(self.completed_runs)}: {os.path.basename(run_dir)}"
                        )
                    else:
                        failed += 1
                        logger.warning(f"⚠️  Sync failed: {os.path.basename(run_dir)}")
                except Exception as e:
                    failed += 1
                    logger.warning(f"Error syncing {os.path.basename(run_dir)}: {e}")

            logger.info(
                f"📤 Background sync complete! {synced} synced, {failed} failed"
            )

        # Start background thread (daemon=False so it completes even if main exits)
        self.sync_thread = threading.Thread(target=sync_worker, daemon=False)
        self.sync_thread.start()
        logger.info("📤 Background WandB sync started (non-blocking)")
        logger.info("   Pipeline will exit immediately. Sync continues in background.")


class EvaluationPipeline:
    """Main evaluation pipeline"""

    def __init__(self, config: PipelineConfig, eval_dir: Path):
        self.config = config
        self.eval_dir = eval_dir
        self.models_dir = self.eval_dir / "models"
        self.detector = ModelDetector(self.models_dir)
        self.generator = TrajectoryGenerator(config)
        self.evaluator = TrajectoryEvaluator(config)
        self.wandb_manager = WandBManager(config)
        self.interrupted = False

        # Change to eval directory for all operations
        os.chdir(self.eval_dir)
        logger.info(f"Working directory: {self.eval_dir}")

        # Validate configuration
        self._validate_config()

        # Set random seed
        set_seed(config.seed)

        # Set PyTorch settings
        torch.set_num_threads(torch.get_num_threads())
        torch.backends.cudnn.benchmark = True

    def _validate_config(self):
        """Validate pipeline configuration"""
        logger.info("Validating pipeline configuration...")

        def _raise_permission(path: Path, operation: str, exc: PermissionError):
            raise PermissionError(
                f"Permission denied while trying to {operation}: '{path}'. "
                "This usually means the pipeline is running under a different UID than the files, "
                "or the filesystem applies root-squash / restrictive directory permissions. "
                "Run as the owning user (e.g., your login UID) or relax directory traverse permissions."
            ) from exc

        # Check models directory
        if not self.models_dir.exists():
            raise FileNotFoundError(f"Models directory not found: {self.models_dir}")

        # Check data directory
        # Prefer an explicit config.data_dir, but fall back to common layouts.
        candidate_dirs = []

        if hasattr(self.config, "data_dir") and self.config.data_dir:
            configured = Path(self.config.data_dir)
            if not configured.is_absolute():
                configured = self.eval_dir / configured
            candidate_dirs.append(configured)

        # Common eval layout: <eval-root>/../data/<dataset>
        candidate_dirs.append(self.eval_dir.parent / "data" / self.config.dataset)

        # Project-root layout: <repo>/data/<dataset>
        candidate_dirs.append(PROJECT_ROOT / "data" / self.config.dataset)

        data_dir = next((p for p in candidate_dirs if p.exists()), None)
        if data_dir is None:
            raise FileNotFoundError(
                "Data directory not found. Tried: "
                + ", ".join(str(p) for p in candidate_dirs)
            )

        # Resolve symlink if needed
        try:
            if data_dir.is_symlink():
                data_dir = data_dir.resolve()
        except PermissionError as e:
            _raise_permission(data_dir, "probe dataset symlink", e)

        # Check required data files
        required_files = ["test.csv", "roadmap.geo"]
        for file in required_files:
            file_path = data_dir / file
            if not file_path.exists():
                raise FileNotFoundError(f"Required data file not found: {file_path}")

        # Validate cross-dataset evaluation if configured
        if self.config.cross_dataset_eval:
            cross_data_dir = Path(self.config.cross_dataset_eval)
            if not cross_data_dir.is_absolute():
                # Relative to eval dir
                cross_data_dir = self.eval_dir / cross_data_dir

            if not cross_data_dir.exists():
                raise FileNotFoundError(
                    f"Cross-dataset directory not found: {cross_data_dir}"
                )

            # Resolve symlink if needed
            try:
                if cross_data_dir.is_symlink():
                    cross_data_dir = cross_data_dir.resolve()
            except PermissionError as e:
                _raise_permission(cross_data_dir, "probe cross-dataset symlink", e)

            # Check required cross-dataset files
            for file in required_files:
                file_path = cross_data_dir / file
                if not file_path.exists():
                    raise FileNotFoundError(
                        f"Required cross-dataset file not found: {file_path}"
                    )

            logger.info(
                f"✓ Cross-dataset evaluation enabled: {self.config.cross_dataset_name or cross_data_dir.name}"
            )

        # Check dataset config file (not required, just for validation)
        # Config is in the eval directory itself

        # Validate CUDA device
        if torch.cuda.is_available():
            if self.config.cuda_device >= torch.cuda.device_count():
                raise ValueError(
                    f"CUDA device {self.config.cuda_device} not available. Only {torch.cuda.device_count()} devices found."
                )
        else:
            logger.warning("CUDA not available, using CPU")

        # Validate parameters
        if self.config.num_gene <= 0:
            raise ValueError("num_gene must be positive")

        if self.config.beam_width <= 0:
            raise ValueError("beam_width must be positive")

        if self.config.grid_size <= 0:
            raise ValueError("grid_size must be positive")

        if self.config.edr_eps <= 0:
            raise ValueError("edr_eps must be positive")

        logger.info("Configuration validation passed")

    def _check_existing_results(
        self, model_type: str, od_source: str
    ) -> Optional[Path]:
        """Check if A* generated file exists and return path if found.

        Only returns files that are confirmed to be A* search (beam_search_enabled: false).
        If only beam search files exist, returns None to trigger A* generation.
        """
        if self.config.force:
            return None

        def _csv_has_at_least_n_lines(path: Path, n_lines: int) -> bool:
            try:
                with open(path, "r") as f:
                    for i, _ in enumerate(f, start=1):
                        if i >= n_lines:
                            return True
                return False
            except OSError:
                return False

        # Check for existing generated file
        model_seed = (
            TrajectoryGenerator._infer_seed_from_model_type(model_type)
            or self.config.seed
        )
        gene_dir = Path(f"./gene/{self.config.dataset}/seed{model_seed}")

        # Try multiple patterns to support both Porto and Beijing naming conventions
        patterns = [
            f"*{model_type}*{od_source}.csv",  # Primary: Porto and most Beijing
            f"*{model_type}*{od_source}od*.csv",  # Backward compatibility
            f"{model_type}*{od_source}*.csv",  # Beijing: model_type first
        ]

        generated_files = []
        for pattern in patterns:
            generated_files.extend(gene_dir.glob(pattern))

        if not generated_files:
            return None

        # Check each file to see if it's A* (beam_search_enabled: false)
        # Sort by modification time, newest first
        generated_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)

        import json

        for csv_file in generated_files:
            # Check for corresponding perf.json file
            perf_file = csv_file.parent / f"{csv_file.stem}_perf.json"

            if perf_file.exists():
                try:
                    with open(perf_file) as f:
                        perf_data = json.load(f)

                    # Check if this is an A* file (beam_search_enabled: false)
                    beam_search_enabled = perf_data.get("beam_search_enabled", True)

                    if not beam_search_enabled:
                        expected_traj = int(self.config.num_gene)
                        actual_traj = perf_data.get("num_trajectories")

                        # If perf metadata indicates an incomplete run, ignore it.
                        if isinstance(actual_traj, int) and actual_traj < expected_traj:
                            logger.warning(
                                "Ignoring incomplete generated file %s: %s/%s trajectories",
                                csv_file.name,
                                actual_traj,
                                expected_traj,
                            )
                            continue

                        # Fallback safety if perf is missing/old.
                        if not isinstance(actual_traj, int):
                            # header + expected trajectories
                            if not _csv_has_at_least_n_lines(csv_file, expected_traj + 1):
                                logger.warning(
                                    "Ignoring incomplete generated file %s: fewer than %s lines",
                                    csv_file.name,
                                    expected_traj + 1,
                                )
                                continue

                        logger.info(
                            f"Found existing A* generated file: {csv_file.name}"
                        )
                        return csv_file
                    else:
                        logger.debug(
                            f"Skipping beam search file: {csv_file.name} "
                            f"(looking for A* file)"
                        )
                        continue
                except (json.JSONDecodeError, KeyError) as e:
                    logger.warning(
                        f"Could not parse perf.json for {csv_file.name}: {e}. "
                        f"Assuming beam search, skipping."
                    )
                    continue
            else:
                # No perf.json file - assume it's an old file or beam search
                # Skip it and continue looking
                logger.debug(
                    f"No perf.json found for {csv_file.name}, skipping "
                    f"(looking for A* file)"
                )
                continue

        # No A* files found, only beam search files exist
        logger.info(
            f"No A* generated file found for {model_type} ({od_source} OD). "
            f"Will generate with A* search."
        )
        return None

    def _handle_error(
        self,
        error: Exception,
        context: str,
        model_type: str = None,
        od_source: str = None,
    ):
        """Handle errors with comprehensive logging"""
        error_msg = f"Error in {context}"
        if model_type:
            error_msg += f" for {model_type}"
        if od_source:
            error_msg += f" ({od_source} OD)"
        error_msg += f": {str(error)}"

        logger.error(error_msg)
        logger.error(f"Error type: {type(error).__name__}")

        # Log stack trace for debugging
        import traceback

        logger.error(f"Stack trace:\n{traceback.format_exc()}")

        # Continue execution unless it's a critical error
        if isinstance(error, (FileNotFoundError, ValueError, KeyError)):
            logger.error("Critical error encountered, stopping pipeline")
            raise
        else:
            logger.warning(
                "Non-critical error encountered, continuing with next operation"
            )

    def _build_file_to_model_mapping(self) -> Dict[str, tuple]:
        """Build mapping of generated file -> (model_type, od_source) from eval results"""
        import json

        mapping = {}
        eval_dir = Path("./eval")

        if not eval_dir.exists():
            logger.warning("No eval directory found, cannot build file mapping")
            return mapping

        # Scan all evaluation result directories
        for eval_subdir in eval_dir.iterdir():
            if not eval_subdir.is_dir():
                continue

            results_file = eval_subdir / "results.json"
            if results_file.exists():
                try:
                    with open(results_file) as f:
                        results = json.load(f)

                    metadata = results.get("metadata", {})
                    generated_file = metadata.get("generated_file", "")
                    model_type = metadata.get("model_type", "")
                    od_source = metadata.get("od_source", "")

                    if generated_file and model_type and od_source:
                        # Normalize path to just the filename
                        file_path = Path(generated_file)
                        mapping[file_path.name] = (model_type, od_source)
                        logger.debug(
                            f"Mapped {file_path.name} -> {model_type} ({od_source} OD)"
                        )
                except Exception as e:
                    logger.warning(f"Failed to read {results_file}: {e}")

        logger.info(f"Built mapping for {len(mapping)} generated files")
        return mapping

    def _run_scenario_analysis(self):
        """Run scenario-based analysis on all generated trajectories"""
        from tools.analyze_scenarios import (
            run_scenario_analysis,
            run_cross_model_scenario_analysis,
        )

        # Auto-detect scenarios config if not provided
        if not self.config.scenarios_config:
            # Try config/scenarios_{dataset}.yaml first
            scenarios_config = Path(
                f"./config/scenarios_{self.config.dataset.lower()}.yaml"
            )
            if not scenarios_config.exists():
                # Fallback to config/scenarios.yaml
                scenarios_config = Path("./config/scenarios.yaml")

            if not scenarios_config.exists():
                logger.warning("No scenarios config found, skipping scenario analysis")
                return
        else:
            scenarios_config = Path(self.config.scenarios_config)

        # Copy config to eval directory for reproducibility
        config_copy = Path("./config") / scenarios_config.name
        config_copy.parent.mkdir(exist_ok=True)

        # Only copy if source and destination are different
        import shutil

        if scenarios_config.resolve() != config_copy.resolve():
            shutil.copy(scenarios_config, config_copy)
            logger.info(f"Copied scenarios config to {config_copy}")
        else:
            logger.info(f"Using existing scenarios config: {config_copy}")

        # Build mapping of generated files to models
        file_mapping = self._build_file_to_model_mapping()

        # Output directory for scenarios
        scenarios_output = Path("./scenarios")

        # Step 1: Run individual model analysis for each OD source
        logger.info("\n🎯 Step 1: Analyzing individual models...")
        for od_source in self.config.od_sources:
            logger.info(f"\nRunning scenario analysis for {od_source} OD...")

            try:
                # Find generated files for this OD source
                gene_dir = Path(f"./gene/{self.config.dataset}/seed{self.config.seed}")
                if not gene_dir.exists():
                    logger.warning(f"Gene directory not found: {gene_dir}")
                    continue

                generated_files = list(gene_dir.glob("*.csv"))

                if not generated_files:
                    logger.warning(f"No generated files found in {gene_dir}")
                    continue

                # Filter by OD source using the mapping
                od_files = []
                for gen_file in generated_files:
                    if gen_file.name in file_mapping:
                        _, file_od = file_mapping[gen_file.name]
                        if file_od == od_source:
                            od_files.append(gen_file)

                if not od_files:
                    logger.warning(f"No generated files found for {od_source} OD")
                    continue

                logger.info(f"Found {len(od_files)} files for {od_source} OD")

                # Run analysis on each generated file (one per model)
                for gen_file in od_files:
                    # Get model name from mapping
                    if gen_file.name in file_mapping:
                        model_name, _ = file_mapping[gen_file.name]
                    else:
                        logger.warning(
                            f"No model mapping for {gen_file.name}, skipping"
                        )
                        continue

                    output_dir = scenarios_output / od_source / model_name

                    logger.info(f"  Analyzing {model_name} ({od_source} OD)...")

                    run_scenario_analysis(
                        generated_file=gen_file,
                        dataset=self.config.dataset,
                        od_source=od_source,
                        config_path=config_copy,
                        output_dir=output_dir,
                        model_name=model_name,
                        data_dir=getattr(self.config, "data_dir", None),
                    )

                    logger.info(f"  ✅ Results saved to {output_dir}")

            except Exception as e:
                logger.error(f"Scenario analysis failed for {od_source} OD: {e}")
                import traceback

                traceback.print_exc()
                continue

        # Step 2: Run cross-model aggregation analysis
        logger.info("\n🔄 Step 2: Running cross-model scenario aggregation...")
        try:
            run_cross_model_scenario_analysis(
                eval_dir=self.eval_dir,
                config_path=config_copy,
                od_sources=self.config.od_sources,
            )
            logger.info("✅ Cross-model analysis complete")
        except Exception as e:
            logger.error(f"Cross-model analysis failed: {e}")
            import traceback

            traceback.print_exc()

        logger.info(f"\n✅ Scenario analysis complete! Results in {scenarios_output}/")

    def _run_cross_dataset_evaluation(self):
        """Run cross-dataset evaluation on all trained models"""
        from tools.analyze_scenarios import run_scenario_analysis

        logger.info("\n🌐 Starting cross-dataset evaluation...")
        logger.info(
            f"Evaluating models trained on {self.config.dataset} using {self.config.cross_dataset_name} data"
        )

        # Setup cross-dataset paths
        cross_data_dir = Path(self.config.cross_dataset_eval)
        if not cross_data_dir.is_absolute():
            cross_data_dir = self.eval_dir / cross_data_dir

        # Output directory structure: cross_dataset_eval/{cross_dataset_name}/{od_source}/{model}/
        cross_eval_output = (
            Path("./cross_dataset_eval") / self.config.cross_dataset_name
        )

        # Get scenarios config for cross-dataset
        scenarios_config = None
        if self.config.run_scenarios:
            # Try config/scenarios_{cross_dataset_name}.yaml
            config_path = Path(
                f"./config/scenarios_{self.config.cross_dataset_name.lower()}.yaml"
            )
            if config_path.exists():
                scenarios_config = config_path
                logger.info(f"Using cross-dataset scenarios config: {scenarios_config}")

        # Loop through all models and OD sources
        for od_source in self.config.od_sources:
            logger.info(f"\n📊 Processing {od_source} OD for cross-dataset...")

            for model_file in self.models_dir.iterdir():
                if not model_file.suffix == ".pth":
                    continue

                model_type = self._extract_model_from_filename(model_file.name)
                logger.info(f"  Model: {model_type}")

                try:
                    # Step 1: Generate trajectories on cross-dataset
                    logger.info("  🔄 Generating trajectories on cross-dataset...")
                    generated_file, gen_perf = self.generator.generate_trajectories(
                        model_file, model_type, od_source
                    )

                    # Move generated file to cross-dataset directory
                    cross_gene_dir = cross_eval_output / od_source / model_type / "gene"
                    cross_gene_dir.mkdir(parents=True, exist_ok=True)
                    cross_gen_file = cross_gene_dir / generated_file.name

                    import shutil

                    shutil.copy(generated_file, cross_gen_file)
                    logger.info(f"  💾 Generated: {cross_gen_file}")

                    # Step 2: Evaluate on cross-dataset
                    logger.info("  📈 Evaluating on cross-dataset...")
                    eval_results = evaluate_trajectories_programmatic(
                        generated_file=str(cross_gen_file),
                        dataset=self.config.dataset,
                        od_source=od_source,
                        data_dir=str(cross_data_dir),
                        grid_size=self.config.grid_size,
                        edr_eps=self.config.edr_eps,
                        enable_wandb=self.config.enable_wandb,
                        wandb_project=self.config.wandb_project,
                        wandb_run_name=f"cross-{self.config.cross_dataset_name}-{model_type}-{od_source}",
                        wandb_tags=[
                            "cross-dataset",
                            self.config.cross_dataset_name,
                            model_type,
                            od_source,
                        ],
                        generation_performance=gen_perf,
                        cross_dataset=True,
                        cross_dataset_name=self.config.cross_dataset_name,
                        trained_on_dataset=self.config.dataset,
                    )

                    # Save evaluation results
                    eval_output_dir = (
                        cross_eval_output / od_source / model_type / "eval"
                    )
                    eval_output_dir.mkdir(parents=True, exist_ok=True)

                    import json
                    from datetime import datetime

                    results_file = (
                        eval_output_dir
                        / f"results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
                    )
                    with open(results_file, "w") as f:
                        json.dump(eval_results, f, indent=2)
                    logger.info(f"  💾 Evaluation results: {results_file}")

                    # Step 3: Run scenario analysis if configured
                    if scenarios_config:
                        logger.info("  🎯 Running scenario analysis...")
                        scenario_output = (
                            cross_eval_output / od_source / model_type / "scenarios"
                        )

                        run_scenario_analysis(
                            generated_file=cross_gen_file,
                            dataset=self.config.cross_dataset_name,
                            od_source=od_source,
                            config_path=scenarios_config,
                            output_dir=scenario_output,
                            model_name=model_type,
                            data_dir=cross_data_dir,
                            cross_dataset=True,
                            trained_on_dataset=self.config.dataset,
                        )
                        logger.info(f"  ✅ Scenarios: {scenario_output}")

                except Exception as e:
                    logger.error(
                        f"  ❌ Cross-dataset evaluation failed for {model_type} ({od_source}): {e}"
                    )
                    import traceback

                    traceback.print_exc()
                    continue

        logger.info(
            f"\n✅ Cross-dataset evaluation complete! Results in {cross_eval_output}/"
        )

    def _run_abnormal_detection_analysis(self):
        """Run abnormal trajectory detection and model performance analysis"""

        logger.info("\n🔍 Starting abnormal trajectory detection analysis...")

        # Auto-detect abnormal config if not provided
        if not self.config.abnormal_config:
            abnormal_config = Path("./config/abnormal_detection.yaml")
            if not abnormal_config.exists():
                logger.warning(
                    "No abnormal detection config found, skipping abnormal analysis"
                )
                return
        else:
            abnormal_config = Path(self.config.abnormal_config)

        if not abnormal_config.exists():
            logger.error(f"Abnormal detection config not found: {abnormal_config}")
            return

        # Analyze BOTH datasets: main evaluation dataset AND cross-dataset
        datasets_to_analyze = []

        # Always analyze the main evaluation dataset
        # Prefer an explicit config.data_dir override, but fall back to common layouts.
        candidate_main_dirs = []
        if getattr(self.config, "data_dir", None):
            configured = Path(self.config.data_dir)
            if not configured.is_absolute():
                configured = self.eval_dir / configured
            candidate_main_dirs.append(configured)
        candidate_main_dirs.append(self.eval_dir.parent / "data" / self.config.dataset)
        candidate_main_dirs.append(PROJECT_ROOT / "data" / self.config.dataset)

        main_data_dir = next((p for p in candidate_main_dirs if p.exists()), None)
        if main_data_dir is not None:
            datasets_to_analyze.append(
                {
                    "name": self.config.dataset,
                    "data_dir": main_data_dir,
                    "is_main": True,
                }
            )

        # Also analyze cross-dataset if configured (BJUT_Beijing)
        # NOTE: only include cross-dataset analysis when the 'cross_dataset' phase
        # is enabled. This ensures users can disable cross-dataset evaluation
        # entirely (e.g., via --skip cross_dataset or config) and avoid mixing
        # datasets into abnormal detection workflows.
        if (
            self.config.cross_dataset_eval
            and self.config.cross_dataset_name
            and "cross_dataset" in self.config.phases
        ):
            cross_data_dir = Path(self.config.cross_dataset_eval)
            if not cross_data_dir.is_absolute():
                cross_data_dir = self.eval_dir / cross_data_dir

            if cross_data_dir.exists():
                datasets_to_analyze.append(
                    {
                        "name": self.config.cross_dataset_name,
                        "data_dir": cross_data_dir,
                        "is_main": False,
                    }
                )

        if not datasets_to_analyze:
            logger.error("No datasets found for abnormal analysis")
            return

        logger.info(
            f"Will analyze {len(datasets_to_analyze)} dataset(s): {[d['name'] for d in datasets_to_analyze]}"
        )

        # Analyze each dataset
        for dataset_info in datasets_to_analyze:
            analysis_dataset = dataset_info["name"]
            data_dir = dataset_info["data_dir"]
            is_main = dataset_info["is_main"]

            logger.info(f"\n{'=' * 70}")
            logger.info(
                f"📊 Analyzing dataset: {analysis_dataset} ({'main' if is_main else 'cross-dataset'})"
            )
            logger.info(f"{'=' * 70}")

            # Output directory for abnormal analysis
            abnormal_output = Path("./abnormal") / analysis_dataset

            # Step 1: Run abnormal detection on real data
            logger.info("\n🔍 Step 1: Detecting abnormal trajectories in real data...")
            self._analyze_dataset_abnormalities(
                analysis_dataset, data_dir, abnormal_config, abnormal_output
            )

    def _get_generated_file_for_analysis(
        self, model_type: str, od_source: str, analysis_dataset: str
    ) -> Optional[Path]:
        """Get appropriate generated file (translated if cross-dataset, original otherwise)

        Args:
            model_type: Model type identifier
            od_source: train or test
            analysis_dataset: Dataset being analyzed (Beijing, BJUT_Beijing, Porto, etc.)

        Returns:
            Path to generated file, or None if not found
        """
        # Determine if this is cross-dataset analysis
        is_cross_dataset = analysis_dataset != self.config.dataset

        if is_cross_dataset:
            # Check for translated files
            translated_dir = Path(
                f"./gene_translated/{analysis_dataset}/seed{self.config.seed}"
            )

            if translated_dir.exists():
                translated_files = list(
                    translated_dir.glob(f"*{model_type}*{od_source}*.csv")
                )
                if translated_files:
                    logger.info(
                        f"      ✅ Using translated file for cross-dataset: {translated_files[0].name}"
                    )
                    return translated_files[0]
                else:
                    logger.warning(
                        f"      ⚠️  No translated file for {model_type}/{od_source}. "
                        f"Run 'road_network_translate' phase first!"
                    )
                    return None
            else:
                logger.warning(
                    f"      ⚠️  Translated directory not found: {translated_dir}. "
                    f"Cross-dataset analysis will use original files (may cause ID mismatch!)"
                )
                # Fall through to original files with warning

        # Same-dataset analysis OR fallback: use original generated files
        gene_dir = Path(f"./gene/{self.config.dataset}/seed{self.config.seed}")
        if not gene_dir.exists():
            logger.error(f"      ❌ Gene directory not found: {gene_dir}")
            return None

        generated_files = list(gene_dir.glob(f"*{model_type}*{od_source}*.csv"))
        if not generated_files:
            logger.warning(f"      ⚠️  No generated file for {model_type}/{od_source}")
            return None

        if not is_cross_dataset:
            logger.debug(
                f"      Using original file (same dataset): {generated_files[0].name}"
            )

        return generated_files[0]

    def _analyze_dataset_abnormalities(
        self, dataset_name: str, data_dir: Path, config_path: Path, output_dir: Path
    ):
        """Analyze abnormal trajectories for a single dataset (real + generated)"""
        from tools.analyze_abnormal import run_abnormal_analysis

        import json
        from datetime import datetime

        # Step 1: Detect abnormal trajectories in real data
        for od_source in self.config.od_sources:
            real_file = data_dir / f"{od_source}.csv"
            if not real_file.exists():
                logger.warning(f"Real data file not found: {real_file}, skipping")
                continue

            logger.info(f"\n  📂 Processing {od_source} data...")

            # 1A: Run detection on real data
            detection_output_real = output_dir / od_source / "real_data"
            logger.info("    🔍 Detecting abnormal trajectories in REAL data...")

            try:
                real_results = run_abnormal_analysis(
                    real_file=real_file,
                    dataset=dataset_name,
                    config_path=config_path,
                    output_dir=detection_output_real,
                    data_dir=data_dir,
                )

                # Count UNIQUE abnormal trajectories (not sum of categories to avoid double-counting)
                abnormal_indices = real_results.get("abnormal_indices", {})
                unique_abnormal_ids = set()
                for indices in abnormal_indices.values():
                    unique_abnormal_ids.update(indices)
                real_abnormal_count = len(unique_abnormal_ids)
                real_total = real_results.get("total_trajectories", 0)
                real_rate = (
                    (real_abnormal_count / real_total * 100) if real_total > 0 else 0
                )

                logger.info(
                    f"    ✅ Real data: {real_abnormal_count}/{real_total} ({real_rate:.2f}%) abnormal"
                )

                # Step 2: Detect abnormal trajectories in GENERATED data
                logger.info(
                    "\n    🤖 Detecting abnormal trajectories in GENERATED data..."
                )

                # Store results for comparison
                model_results = {}

                for model_type in self.config.models:
                    # Get appropriate file (translated for cross-dataset, original otherwise)
                    generated_file = self._get_generated_file_for_analysis(
                        model_type=model_type,
                        od_source=od_source,
                        analysis_dataset=dataset_name,
                    )

                    if not generated_file:
                        # Helper already logged appropriate warning
                        continue

                    logger.info(f"      Analyzing {model_type}...")

                    # Run detection on generated trajectories
                    detection_output_gen = (
                        output_dir / od_source / "generated" / model_type
                    )

                    try:
                        gen_results = run_abnormal_analysis(
                            real_file=generated_file,
                            dataset=dataset_name,
                            config_path=config_path,
                            output_dir=detection_output_gen,
                            is_real_data=False,  # Generated data format
                            data_dir=data_dir,
                        )

                        # Count UNIQUE abnormal trajectories (not sum of categories to avoid double-counting)
                        abnormal_indices = gen_results.get("abnormal_indices", {})
                        unique_abnormal_ids = set()
                        for indices in abnormal_indices.values():
                            unique_abnormal_ids.update(indices)
                        gen_abnormal_count = len(unique_abnormal_ids)
                        gen_total = gen_results.get("total_trajectories", 0)
                        gen_rate = (
                            (gen_abnormal_count / gen_total * 100)
                            if gen_total > 0
                            else 0
                        )

                        logger.info(
                            f"        Generated: {gen_abnormal_count}/{gen_total} ({gen_rate:.2f}%) abnormal"
                        )

                        # Store for comparison
                        model_results[model_type] = {
                            "model": model_type,  # Fix: Add model name to results
                            "abnormal_count": gen_abnormal_count,
                            "total_trajectories": gen_total,
                            "abnormal_rate": gen_rate,
                            "abnormal_by_category": {
                                cat: len(indices)
                                for cat, indices in gen_results.get(
                                    "abnormal_indices", {}
                                ).items()
                            },
                        }

                    except Exception as e:
                        logger.error(
                            f"        ❌ Detection failed for {model_type}: {e}"
                        )
                        continue

                # Step 3: Compare and report
                logger.info(f"\n    📊 Comparison Report ({od_source}):")
                logger.info(f"    {'=' * 60}")
                logger.info(
                    f"    Real data:   {real_abnormal_count}/{real_total} ({real_rate:.2f}%)"
                )

                for model_type, results in model_results.items():
                    diff = results["abnormal_rate"] - real_rate
                    symbol = "⚠️ " if diff > 5 else "✅"
                    logger.info(
                        f"    {symbol} {model_type:12s}: {results['abnormal_count']}/{results['total_trajectories']} "
                        f"({results['abnormal_rate']:.2f}%, {diff:+.2f}% vs real)"
                    )

                # Save comparison report
                comparison_output = output_dir / od_source / "comparison_report.json"
                comparison_output.parent.mkdir(parents=True, exist_ok=True)

                comparison_data = {
                    "dataset": dataset_name,
                    "od_source": od_source,
                    "timestamp": datetime.now().isoformat(),
                    "real_data": {
                        "abnormal_count": real_abnormal_count,
                        "total_trajectories": real_total,
                        "abnormal_rate": real_rate,
                        "abnormal_by_category": {
                            cat: len(indices)
                            for cat, indices in real_results.get(
                                "abnormal_indices", {}
                            ).items()
                        },
                    },
                    "generated_data": model_results,
                }

                with open(comparison_output, "w") as f:
                    json.dump(comparison_data, f, indent=2)

                logger.info(f"    💾 Saved comparison report to {comparison_output}")

            except Exception as e:
                logger.error(f"  ❌ Abnormal analysis failed for {od_source}: {e}")
                import traceback

                traceback.print_exc()
                continue

        logger.info(
            f"\n✅ Abnormal analysis complete for {dataset_name}! Results in {output_dir}/"
        )

    def _extract_model_from_filename(self, filename: str) -> str:
        """Extract model type from generated file name"""
        # Example: hoser_vanilla_testod_gene_20241024_123456.csv -> vanilla
        if "vanilla" in filename:
            return "vanilla"
        elif "distilled" in filename:
            return "distilled"
        else:
            return "unknown"

    @phase("generation", critical=True)
    def run_generation(self):
        """Generate trajectories for all models"""
        logger.info("🔄 Generating trajectories...")

        for model_type in self.config.models:
            try:
                model_file = self.detector.find_model_file(model_type)
                if not model_file:
                    error_msg = f"Model file not found for type: {model_type}"
                    logger.error(error_msg)
                    continue

                logger.info(f"Processing model: {model_type} ({model_file.name})")

                for od_source in self.config.od_sources:
                    # Check for interruption
                    if self.interrupted:
                        logger.info("Pipeline interrupted by user")
                        raise KeyboardInterrupt()

                    # Check for existing results
                    existing_file = self._check_existing_results(model_type, od_source)

                    # Generation logic
                    try:
                        if existing_file:
                            generated_file = existing_file
                            logger.info(
                                f"Using existing generated file: {generated_file}"
                            )
                        else:
                            generated_file, generation_performance = (
                                self.generator.generate_trajectories(
                                    model_file, model_type, od_source
                                )
                            )

                            # Log to WandB
                            if self.config.enable_wandb:
                                run_name = f"gene_{model_type}_seed{self.config.seed}_{od_source}od"
                                tags = [
                                    model_type,
                                    f"seed{self.config.seed}",
                                    f"{od_source}_od",
                                    "generation",
                                    "beam4",
                                ]
                                config_dict = {
                                    "dataset": self.config.dataset,
                                    "seed": self.config.seed,
                                    "num_gene": self.config.num_gene,
                                    "model_type": model_type,
                                    "od_source": od_source,
                                    "beam_width": self.config.beam_width,
                                }

                                self.wandb_manager.init_run(run_name, tags, config_dict)

                                # Log generation metrics including performance
                                log_data = {
                                    "num_trajectories_generated": self.config.num_gene,
                                    "output_file": str(generated_file),
                                }
                                if generation_performance:
                                    # Add performance metrics with 'perf/' prefix
                                    perf_log = {
                                        f"perf/{k}": v
                                        for k, v in generation_performance.items()
                                        if isinstance(v, (int, float))
                                    }
                                    log_data.update(perf_log)

                                self.wandb_manager.log_metrics(run_name, log_data)
                                self.wandb_manager.finish_run(run_name)

                    except Exception as e:
                        self._handle_error(e, "generation", model_type, od_source)
                        logger.error(
                            f"Generation failed for {model_type} ({od_source} OD): {str(e)}"
                        )
                        continue

            except Exception as e:
                logger.error(f"Error processing model {model_type}: {str(e)}")
                continue

    @phase("base_eval", critical=True)
    def run_base_eval(self):
        """Evaluate on base dataset (Beijing)"""
        logger.info("📊 Evaluating on base dataset...")

        results_summary = {}
        failed_operations = []

        for model_type in self.config.models:
            for od_source in self.config.od_sources:
                # Check for interruption
                if self.interrupted:
                    logger.info("Pipeline interrupted by user")
                    raise KeyboardInterrupt()

                logger.info(f"Evaluating {model_type} ({od_source} OD)")

                # Find generated file (prefer A* files)
                model_seed = (
                    TrajectoryGenerator._infer_seed_from_model_type(model_type)
                    or self.config.seed
                )
                gene_dir = Path(f"./gene/{self.config.dataset}/seed{model_seed}")

                # Try new naming pattern first: {timestamp}_{model_type}_{od_source}.csv
                pattern = f"*_{model_type}_{od_source}.csv"
                generated_files = list(gene_dir.glob(pattern))

                if not generated_files:
                    # Fallback: try other patterns
                    patterns = [
                        f"*{model_type}*{od_source}.csv",
                        f"*{model_type}*{od_source}od*.csv",
                        f"{model_type}*{od_source}*.csv",
                    ]
                    for p in patterns:
                        generated_files.extend(gene_dir.glob(p))

                    if not generated_files:
                        error_msg = f"No existing generated file found for {model_type} ({od_source} OD)"
                        logger.error(error_msg)
                        failed_operations.append(error_msg)
                        continue

                # Filter for A* files only (check perf.json for beam_search_enabled: false)
                import json

                astar_files = []
                perf_metadata = {}

                for csv_file in generated_files:
                    perf_file = csv_file.parent / f"{csv_file.stem}_perf.json"

                    if not perf_file.exists():
                        continue

                    try:
                        with open(perf_file) as f:
                            perf_data = json.load(f)

                        perf_metadata[csv_file] = perf_data

                        beam_search_enabled = perf_data.get("beam_search_enabled", True)

                        expected_traj = int(self.config.num_gene)
                        actual_traj = perf_data.get("num_trajectories")

                        if not beam_search_enabled and isinstance(actual_traj, int) and actual_traj >= expected_traj:
                            astar_files.append(csv_file)
                        elif not beam_search_enabled and isinstance(actual_traj, int) and actual_traj < expected_traj:
                            logger.warning(
                                "Ignoring incomplete generated file %s: %s/%s trajectories",
                                csv_file.name,
                                actual_traj,
                                expected_traj,
                            )
                    except (json.JSONDecodeError, KeyError):
                        logger.warning(
                            "Invalid perf metadata for %s, skipping A* detection",
                            csv_file.name,
                        )
                        continue

                if astar_files:
                    # Use most recent A* file
                    generated_file = max(astar_files, key=lambda x: x.stat().st_mtime)
                    logger.info(f"Found existing A* file: {generated_file.name}")
                else:
                    error_msg = (
                        f"No complete A* generated file found for {model_type} ({od_source} OD) "
                        f"in {gene_dir} (expected {self.config.num_gene} trajectories)."
                    )
                    logger.error(error_msg)
                    failed_operations.append(error_msg)
                    continue

                generation_performance = perf_metadata.get(generated_file)

                # Evaluation phase
                try:
                    eval_results = self.evaluator.evaluate_trajectories(
                        generated_file,
                        model_type,
                        od_source,
                        generation_performance,
                    )

                    # Log to WandB
                    if self.config.enable_wandb:
                        run_name = (
                            f"eval_{model_type}_seed{self.config.seed}_{od_source}od"
                        )
                        tags = [
                            model_type,
                            f"seed{self.config.seed}",
                            f"{od_source}_od",
                            "evaluation",
                        ]
                        config_dict = {
                            "dataset": self.config.dataset,
                            "seed": self.config.seed,
                            "model_type": model_type,
                            "od_source": od_source,
                            "generated_file": str(generated_file),
                        }

                        self.wandb_manager.init_run(run_name, tags, config_dict)
                        self.wandb_manager.log_metrics(run_name, eval_results)
                        self.wandb_manager.finish_run(run_name)

                    # Store results for summary
                    key = f"{model_type}_{od_source}"
                    results_summary[key] = {
                        "generated_file": str(generated_file),
                        "metrics": eval_results,
                    }

                    logger.info(
                        f"✅ Evaluation complete for {model_type} ({od_source} OD)"
                    )

                except Exception as e:
                    self._handle_error(e, "evaluation", model_type, od_source)
                    failed_operations.append(
                        f"Evaluation failed for {model_type} ({od_source} OD): {str(e)}"
                    )
                    continue

        # Log summary
        if results_summary:
            logger.info("\n=== Base Dataset Evaluation Results ===")
            for key, result in results_summary.items():
                logger.info(f"{key}: {result['metrics']}")

        if failed_operations:
            logger.warning(f"\n⚠️  {len(failed_operations)} operations failed:")
            for op in failed_operations:
                logger.warning(f"  - {op}")

        return results_summary

    @phase("perturbation_correction", critical=False)
    def run_perturbation_correction(self):
        """Evaluate whether predictions correct perturbations (Phase B)."""
        if not getattr(self.config, "perturbation_source_csv", None):
            msg = (
                "Perturbation correction not configured: missing "
                "perturbation_source_csv (config) or --perturbation-source-csv (CLI)"
            )
            # If user is explicitly running only this phase, fail fast.
            if self.config.phases == {"perturbation_correction"}:
                raise ValueError(msg)
            logger.info(msg)
            return

        try:
            from tools.perturbation_correction import (
                PerturbationCorrectionConfig,
                PerturbationTeacherConfig,
                run_perturbation_correction,
            )
        except ModuleNotFoundError:
            # Rarely, environments with a conflicting installed `tools` package
            # or modified sys.path can break the normal import. Fall back to a
            # direct file-based import from this repo.
            import importlib.util

            module_path = (PROJECT_ROOT / "tools" / "perturbation_correction.py").resolve()
            spec = importlib.util.spec_from_file_location(
                "hoser_tools_perturbation_correction", module_path
            )
            if spec is None or spec.loader is None:
                raise
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)

            PerturbationCorrectionConfig = module.PerturbationCorrectionConfig
            PerturbationTeacherConfig = module.PerturbationTeacherConfig
            run_perturbation_correction = module.run_perturbation_correction

        source_csv = Path(str(self.config.perturbation_source_csv))
        if not source_csv.is_absolute():
            source_csv = (self.eval_dir / source_csv).resolve()

        resolved_data_dir = None
        if getattr(self.config, "data_dir", None):
            resolved_data_dir = Path(str(self.config.data_dir))
            if not resolved_data_dir.is_absolute():
                resolved_data_dir = (self.eval_dir / resolved_data_dir).resolve()

        teacher_cfg = None
        if getattr(self.config, "perturbation_lmtad_checkpoint", None):
            lmtad_repo = getattr(self.config, "perturbation_lmtad_repo", None)
            if not lmtad_repo:
                raise ValueError(
                    "perturbation_lmtad_checkpoint set but perturbation_lmtad_repo is missing"
                )

            # `LMTADTeacher` expects `repo_path` to point at the LM-TAD code repo root
            # that contains `code/models/LMTAD.py`.
            lmtad_repo_path = Path(str(lmtad_repo)).resolve()
            expected = lmtad_repo_path / "code" / "models" / "LMTAD.py"
            if not expected.exists():
                raise FileNotFoundError(
                    "LM-TAD repo missing expected file. "
                    f"Expected: {expected} (from perturbation_lmtad_repo={lmtad_repo_path}). "
                    "Set perturbation_lmtad_repo to the LM-TAD repository root (e.g., /home/<user>/LMTAD)."
                )

            device = (
                f"cuda:{self.config.cuda_device}"
                if torch.cuda.is_available()
                else "cpu"
            )
            teacher_cfg = PerturbationTeacherConfig(
                lmtad_repo=lmtad_repo_path,
                lmtad_checkpoint=Path(
                    str(self.config.perturbation_lmtad_checkpoint)
                ).resolve(),
                device=device,
                batch_size=int(self.config.perturbation_lmtad_batch_size),
            )

        cfg = PerturbationCorrectionConfig(
            dataset=self.config.dataset,
            eval_dir=self.eval_dir,
            project_root=PROJECT_ROOT,
            data_dir=resolved_data_dir,
            perturbation_source_csv=source_csv,
            od_source=str(getattr(self.config, "perturbation_od_source", "train")),
            max_entries=getattr(self.config, "perturbation_max_entries", None),
            seed=int(getattr(self.config, "perturbation_seed", 0)),
            use_astar=bool(getattr(self.config, "perturbation_use_astar", False)),
            beam_width=int(self.config.beam_width),
            cuda_device=int(self.config.cuda_device),
            force=bool(self.config.force),
            teacher=teacher_cfg,
        )

        for model_type in self.config.models:
            model_file = self.detector.find_model_file(model_type)
            if not model_file:
                logger.error("Model file not found for type: %s", model_type)
                continue

            run_perturbation_correction(
                cfg=cfg,
                model_type=model_type,
                model_checkpoint=model_file,
            )

    @phase("cross_dataset", critical=False)
    def run_cross_dataset(self):
        """Evaluate on cross-dataset (BJUT)"""
        if not self.config.cross_dataset_eval:
            logger.info("Cross-dataset not configured, skipping")
            return

        logger.info("🌐 Evaluating on cross-dataset...")
        self._run_cross_dataset_evaluation()

    @phase("road_network_translate", critical=False)
    def run_road_network_translate(self):
        """Translate generated trajectories to cross-dataset road network"""
        logger.info("🗺️  Translating road networks for cross-dataset analysis...")

        # Only needed when cross-dataset is configured
        if not self.config.cross_dataset_eval:
            logger.info("No cross-dataset configured, skipping translation")
            return

        # Check if mapping exists, create if not
        mapping_file = Path(
            f"road_mapping_{self.config.dataset.lower()}_to_{self.config.cross_dataset_name.lower().replace(' ', '_')}.json"
        )

        if not mapping_file.exists():
            logger.info(f"  📍 Mapping file not found, creating: {mapping_file}")

            # Import mapping tool
            from tools.map_road_networks import (
                load_road_network_with_coords,
                find_nearest_road_batch,
                save_comprehensive_output,
            )

            # Determine .geo file paths
            source_geo = Path(f"data/{self.config.dataset}/roadmap.geo")
            if not source_geo.exists():
                source_geo = (
                    self.eval_dir.parent / "data" / self.config.dataset / "roadmap.geo"
                )

            target_data_dir = Path(self.config.cross_dataset_eval)
            if not target_data_dir.is_absolute():
                target_data_dir = self.eval_dir / target_data_dir
            target_geo = target_data_dir / "roadmap.geo"

            if not source_geo.exists():
                logger.error(f"❌ Source .geo file not found: {source_geo}")
                return
            if not target_geo.exists():
                logger.error(f"❌ Target .geo file not found: {target_geo}")
                return

            # Create mapping
            source_roads = load_road_network_with_coords(source_geo)
            target_roads = load_road_network_with_coords(target_geo)

            result = find_nearest_road_batch(
                source_roads=source_roads,
                target_roads=target_roads,
                max_distance_m=50.0,
            )

            save_comprehensive_output(
                result=result,
                source_dataset=self.config.dataset,
                target_dataset=self.config.cross_dataset_name,
                source_geo=source_geo,
                target_geo=target_geo,
                max_distance=50.0,
                output_file=mapping_file,
            )

            # Check mapping quality
            mapping_rate = result["statistics"]["mapping_rate_pct"]
            if mapping_rate < 70:
                logger.error(
                    f"❌ Poor mapping quality ({mapping_rate:.1f}%), aborting translation"
                )
                return
            elif mapping_rate < 85:
                logger.warning(f"⚠️  Fair mapping quality ({mapping_rate:.1f}%)")
        else:
            logger.info(f"  ✅ Using existing mapping: {mapping_file}")

        # Load mapping
        import json

        with open(mapping_file, "r") as f:
            mapping = json.load(f)

        # Handle both old format (int->int) and new format (int->dict)
        mapping_int = {}
        for k, v in mapping.items():
            if isinstance(v, dict):
                # New format: {"target_road_id": 123, "distance_m": 5.0}
                mapping_int[int(k)] = int(v["target_road_id"])
            else:
                # Old format: 123
                mapping_int[int(k)] = int(v)

        # Translate all generated files
        from tools.translate_trajectories import translate_trajectory_file

        gene_dir = Path(f"./gene/{self.config.dataset}/seed{self.config.seed}")
        if not gene_dir.exists():
            logger.warning(
                f"Gene directory not found: {gene_dir}, skipping translation"
            )
            return

        gene_files = list(gene_dir.glob("*.csv"))
        if not gene_files:
            logger.warning("No generated files to translate")
            return

        logger.info(f"\n  📦 Found {len(gene_files)} generated files to translate")

        output_dir = Path(
            f"./gene_translated/{self.config.cross_dataset_name}/seed{self.config.seed}"
        )
        output_dir.mkdir(parents=True, exist_ok=True)

        for gene_file in gene_files:
            output_file = output_dir / gene_file.name

            try:
                translate_trajectory_file(
                    input_file=gene_file,
                    mapping=mapping_int,
                    output_file=output_file,
                )
            except Exception as e:
                logger.error(f"  ❌ Translation failed for {gene_file.name}: {e}")
                import traceback

                traceback.print_exc()
                continue

        logger.info(f"\n✅ Translation complete! Files in {output_dir}/")

    @phase("abnormal", critical=False)
    def run_abnormal(self):
        """Detect abnormal trajectories"""
        if not self.config.run_abnormal_detection:
            logger.info("Abnormal detection not configured, skipping")
            return

        logger.info("🔍 Running abnormal detection...")
        self._run_abnormal_detection_analysis()

    @phase("abnormal_od_extract", critical=False)
    def run_abnormal_od_extract(self):
        """Extract abnormal OD pairs from abnormal detection results.

        Primary workflow:
        - If cross-dataset is configured: extract from cross-dataset abnormal detection.
        - Otherwise: extract from the main dataset abnormal detection (e.g., Wang statistical).
        """
        logger.info("📊 Extracting abnormal OD pairs from abnormal detection results...")

        # Determine which dataset's abnormal results to use.
        is_cross_dataset = bool(
            self.config.cross_dataset_name and self.config.cross_dataset_eval
        )

        target_dataset = (
            self.config.cross_dataset_name if is_cross_dataset else self.config.dataset
        )

        # Prefer LM-TAD spatial abnormal OD pairs when available.
        # These are produced by tools/run_lmtad_spatial_pipeline.py and have schema
        # {"od_pairs_by_type": {"route_switch": [...], "detour": [...]}}.
        lmtad_pairs_file = Path(
            f"./abnormal_od_pairs_lmtad_spatial_{target_dataset}.json"
        )

        if lmtad_pairs_file.exists():
            logger.info(
                f"✅ Found LM-TAD spatial OD pairs file, using it: {lmtad_pairs_file}"
            )
            return

        # If the user configured LM-TAD spatial detection (or explicitly provided the
        # LM-TAD source eval directory), we can extract OD pairs directly from the
        # LM-TAD TSV exports without needing the 'abnormal' phase.
        if getattr(self.config, "run_lmtad_spatial_detection", False) or getattr(
            self.config, "lmtad_source_eval_dir", None
        ):
            try:
                from tools.extract_lmtad_spatial_abnormal_od import (
                    extract_spatial_abnormal_od_pairs,
                )

                source_eval_dir = self._detect_lmtad_source_eval_dir()
                if not source_eval_dir:
                    logger.error(
                        "❌ Could not auto-detect LM-TAD source eval directory. "
                        "Set 'lmtad_source_eval_dir' in evaluation.yaml or run the lmtad_spatial_abnormality phase."
                    )
                    return

                # Use a base dataset name for mapping when the eval dataset is a derived variant
                # (e.g., Beijing_per_type_detour). This helps the LM-TAD token→road mapping logic.
                base_dataset = target_dataset
                if isinstance(target_dataset, str) and "_" in target_dataset:
                    candidate = target_dataset.split("_", 1)[0]
                    if (PROJECT_ROOT / "data" / candidate).exists():
                        base_dataset = candidate

                logger.info(
                    f"🔎 Extracting LM-TAD spatial abnormal OD pairs from: {source_eval_dir} (dataset='{base_dataset}')"
                )
                result = extract_spatial_abnormal_od_pairs(
                    tsv_file=source_eval_dir,
                    dataset=base_dataset,
                    source_eval_dir=source_eval_dir,
                )

                import json

                with open(lmtad_pairs_file, "w") as f:
                    json.dump(result, f, indent=2)

                logger.info(
                    f"✅ Saved LM-TAD spatial abnormal OD pairs to {lmtad_pairs_file}"
                )
                return
            except Exception as e:
                logger.error(f"❌ LM-TAD spatial OD extraction failed: {e}")
                import traceback

                traceback.print_exc()
                return

        abnormal_results_dir = Path(f"./abnormal/{target_dataset}")
        if not abnormal_results_dir.exists():
            logger.error(
                f"❌ Dependency not met: Run 'abnormal'/'wang_abnormality' phase first. "
                f"Missing: {abnormal_results_dir}"
            )
            return

        # Find detection result files (try both old and new directory structures)
        detection_files = list(
            abnormal_results_dir.glob("*/real_data/detection_results.json")
        )
        if not detection_files:
            # Try old directory structure
            detection_files = list(
                abnormal_results_dir.glob("*/detection/detection_results.json")
            )

        if not detection_files:
            logger.error(f"❌ No detection results found in {abnormal_results_dir}")
            return

        # If Wang is enabled, ensure we're extracting from Wang statistical results.
        # Wang detection writes `config_file: config/abnormal_detection_statistical.yaml`.
        if getattr(self.config, "run_wang_detection", False):
            import json

            wang_files: list[Path] = []
            for file_path in detection_files:
                try:
                    with open(file_path, "r") as f:
                        payload = json.load(f)
                    config_file = str(payload.get("config_file", ""))
                    if "abnormal_detection_statistical" in config_file:
                        wang_files.append(file_path)
                except Exception:
                    continue

            if not wang_files:
                logger.error(
                    "❌ No Wang statistical detection results found. "
                    "Enable/run the 'wang_abnormality' phase (or run the Wang pipeline) before 'abnormal_od_extract'."
                )
                return

            detection_files = wang_files

        logger.info(f"  Found {len(detection_files)} detection result files")

        # Prepare data file paths
        if is_cross_dataset:
            data_dir = Path(self.config.cross_dataset_eval)
            if not data_dir.is_absolute():
                data_dir = self.eval_dir / data_dir
        else:
            # Prefer explicit eval config data_dir; fall back to common layouts.
            candidate_dirs = []
            if getattr(self.config, "data_dir", None):
                configured = Path(self.config.data_dir)
                if not configured.is_absolute():
                    configured = self.eval_dir / configured
                candidate_dirs.append(configured)
            candidate_dirs.append(self.eval_dir.parent / "data" / self.config.dataset)
            candidate_dirs.append(PROJECT_ROOT / "data" / self.config.dataset)
            data_dir = next((p for p in candidate_dirs if p.exists()), None)
            if data_dir is None:
                logger.error(
                    "❌ Data directory not found for abnormal OD extraction. Tried: "
                    + ", ".join(str(p) for p in candidate_dirs)
                )
                return

        data_files = []
        for det_file in detection_files:
            # Extract split name (train/test) from path
            split = det_file.parent.parent.name
            data_file = data_dir / f"{split}.csv"
            if data_file.exists():
                data_files.append(data_file)

        if len(data_files) != len(detection_files):
            logger.warning(
                "⚠️  Some data files not found, continuing with available files"
            )

        # Import and run extraction
        from tools.extract_abnormal_od_pairs import extract_abnormal_od_pairs

        output_suffix = target_dataset.lower().replace(" ", "_")
        output_file = Path(f"./abnormal_od_pairs_{output_suffix}.json")

        try:
            result = extract_abnormal_od_pairs(
                detection_results_files=detection_files,
                real_data_files=data_files,
                dataset_name=target_dataset,
            )

            # Save result
            import json

            with open(output_file, "w") as f:
                json.dump(result, f, indent=2)

            logger.info(
                f"✅ Extracted {result['total_unique_od_pairs']} abnormal OD pairs"
            )
            logger.info(f"💾 Saved to {output_file}")

        except Exception as e:
            logger.error(f"❌ OD pair extraction failed: {e}")
            import traceback

            traceback.print_exc()

    @phase("abnormal_od_generate", critical=False)
    def run_abnormal_od_generate(self):
        """Generate trajectories for abnormal OD pairs"""
        logger.info("🤖 Generating trajectories for abnormal OD pairs...")

        # Supports both cross-dataset and same-dataset workflows.

        # Check dependency: Need OD pairs file
        od_pairs_files = list(Path(".").glob("abnormal_od_pairs_*.json"))
        if not od_pairs_files:
            logger.error(
                "❌ Dependency not met: Run 'abnormal_od_extract' phase first. "
                "Missing: abnormal_od_pairs_*.json"
            )
            return

        # Prefer dataset-specific OD pairs file if available.
        # If LM-TAD spatial OD pairs exist for the main dataset, prefer those.
        main_dataset_pattern = self.config.dataset.lower().replace(" ", "_")
        lmtad_matching = [
            f
            for f in od_pairs_files
            if "lmtad_spatial" in f.name.lower()
            and main_dataset_pattern in f.name.lower()
        ]
        if lmtad_matching:
            od_pairs_file = lmtad_matching[0]
        elif self.config.cross_dataset_name:
            cross_dataset_pattern = self.config.cross_dataset_name.lower().replace(
                " ", "_"
            )
            matching_cross = [
                f for f in od_pairs_files if cross_dataset_pattern in f.name.lower()
            ]
            od_pairs_file = matching_cross[0] if matching_cross else od_pairs_files[0]
        else:
            matching_main = [
                f for f in od_pairs_files if main_dataset_pattern in f.name.lower()
            ]
            od_pairs_file = matching_main[0] if matching_main else od_pairs_files[0]

        logger.info(f"  Using OD pairs: {od_pairs_file}")

        # Load OD pairs
        import json

        with open(od_pairs_file, "r") as f:
            od_pairs_data = json.load(f)

        # Create flat list of OD pairs (support multiple schemas)
        all_pairs = []
        if "od_pairs_by_category" in od_pairs_data:
            pairs_by_bucket = od_pairs_data.get("od_pairs_by_category", {})
        elif "od_pairs_by_type" in od_pairs_data:
            # LM-TAD spatial workflow
            pairs_by_bucket = od_pairs_data.get("od_pairs_by_type", {})
        else:
            pairs_by_bucket = {}

        for category, pairs in pairs_by_bucket.items():
            # Limit to 20 pairs per bucket for reasonable runtime
            pairs_limited = pairs[:20]
            all_pairs.extend(pairs_limited)
            logger.info(f"  {category}: {len(pairs_limited)} OD pairs")

        # Deduplicate
        unique_pairs = list(set(tuple(p) for p in all_pairs))
        logger.info(f"  Total unique OD pairs: {len(unique_pairs)}")

        if not unique_pairs:
            logger.warning("No OD pairs to generate for")
            return

        # Output directory
        output_dir = Path(
            f"./gene_abnormal/{self.config.dataset}/seed{self.config.seed}"
        )
        output_dir.mkdir(parents=True, exist_ok=True)

        # Generate for each model
        from gene import generate_trajectories_programmatic

        # Use config value if set, otherwise default to 10 for faster testing
        num_traj_per_od = getattr(self.config, "abnormal_num_traj_per_od", 10)

        for model_type in self.config.models:
            try:
                model_file = self.detector.find_model_file(model_type)
                if not model_file:
                    logger.warning(f"  ⚠️  Model file not found for {model_type}")
                    continue

                logger.info(f"\n  {'=' * 60}")
                logger.info(f"  🚀 Generating with {model_type}")
                logger.info(f"  {'=' * 60}")

                # Expand OD pairs (repeat each pair num_traj_per_od times)
                od_list_expanded = []
                for origin, dest in unique_pairs:
                    for _ in range(num_traj_per_od):
                        od_list_expanded.append((origin, dest))

                logger.info(f"    Generating {len(od_list_expanded)} trajectories...")

                output_file = output_dir / f"{model_type}_abnormal_od.csv"

                result = generate_trajectories_programmatic(
                    dataset=self.config.dataset,
                    model_path=str(model_file),
                    od_pairs=od_list_expanded,
                    output_file=str(output_file),
                    seed=self.config.seed,
                    data_dir=self.config.data_dir,
                    cuda_device=self.config.cuda_device,
                    beam_search=self.config.beam_search,
                    beam_width=self.config.beam_width,
                )

                if result.get("output_file"):
                    traj_count = result.get("num_generated", 0)
                    logger.info(
                        f"    ✅ Generated {traj_count} trajectories → {result['output_file']}"
                    )
                else:
                    logger.error("    ❌ Generation failed: No output file produced")

            except Exception as e:
                logger.error(f"  ❌ Error generating with {model_type}: {e}")
                import traceback

                traceback.print_exc()
                continue

        logger.info(f"\n✅ Abnormal OD generation complete! Results in {output_dir}/")

    @phase("abnormal_od_evaluate", critical=False)
    def run_abnormal_od_evaluate(self):
        """Evaluate model performance on abnormal OD pairs"""
        logger.info("📊 Evaluating performance on abnormal OD pairs...")

        # Supports both cross-dataset and same-dataset workflows.
        is_cross_dataset = bool(
            self.config.cross_dataset_name and self.config.cross_dataset_eval
        )

        # Check dependency: Need generated files
        gen_dir = Path(f"./gene_abnormal/{self.config.dataset}/seed{self.config.seed}")
        if not gen_dir.exists():
            logger.error(
                "❌ Dependency not met: Run 'abnormal_od_generate' phase first. "
                f"Missing: {gen_dir}"
            )
            return

        gen_files = list(gen_dir.glob("*_abnormal_od.csv"))
        if not gen_files:
            logger.error(f"❌ No generated files found in {gen_dir}")
            return

        logger.info(f"  Found {len(gen_files)} generated files")

        # Check dependency: Need OD pairs file
        od_pairs_files = list(Path(".").glob("abnormal_od_pairs_*.json"))
        if not od_pairs_files:
            logger.error("❌ Missing abnormal OD pairs file")
            return

        od_pairs_file = od_pairs_files[0]

        # Resolve dataset directory for evaluation
        if is_cross_dataset:
            eval_dataset = self.config.cross_dataset_name
            data_dir = Path(self.config.cross_dataset_eval)
            if not data_dir.is_absolute():
                data_dir = self.eval_dir / data_dir
        else:
            eval_dataset = self.config.dataset
            candidate_dirs = []
            if getattr(self.config, "data_dir", None):
                configured = Path(self.config.data_dir)
                if not configured.is_absolute():
                    configured = self.eval_dir / configured
                candidate_dirs.append(configured)
            candidate_dirs.append(self.eval_dir.parent / "data" / self.config.dataset)
            candidate_dirs.append(PROJECT_ROOT / "data" / self.config.dataset)
            data_dir = next((p for p in candidate_dirs if p.exists()), None)
            if data_dir is None:
                logger.error(
                    "❌ Data directory not found for abnormal OD evaluation. Tried: "
                    + ", ".join(str(p) for p in candidate_dirs)
                )
                return

        # Output directory
        output_dir = Path(f"./eval_abnormal/{self.config.dataset}")
        output_dir.mkdir(parents=True, exist_ok=True)

        # Import evaluation functions
        import json
        from tools.analyze_abnormal import run_abnormal_analysis
        from evaluation import evaluate_trajectories_programmatic

        # Evaluate each model
        all_results = {}

        for gen_file in gen_files:
            model_name = gen_file.stem.replace("_abnormal_od", "")
            logger.info(f"\n  {'=' * 60}")
            logger.info(f"  📊 Evaluating {model_name}")
            logger.info(f"  {'=' * 60}")

            try:
                # Step 1: Abnormal detection
                logger.info("    🔍 Running abnormal detection...")
                detection_output = output_dir / model_name / "detection"
                detection_output.mkdir(parents=True, exist_ok=True)

                # Resolve config path
                abnormal_config_path = Path("config/abnormal_detection.yaml")
                "perturbation_correction",
                if not abnormal_config_path.exists():
                    # Fallback to project root config
                    project_config = PROJECT_ROOT / "config" / "abnormal_detection.yaml"
                    if project_config.exists():
                        logger.info(
                            f"    Using config from project root: {project_config}"
                        )
                        abnormal_config_path = project_config
                    else:
                        logger.warning(
                            f"    ⚠️ Config not found: {abnormal_config_path} or {project_config}"
                        )

                detection_results = run_abnormal_analysis(
                    real_file=gen_file,
                    dataset=self.config.dataset,
                    config_path=abnormal_config_path,
                    output_dir=detection_output,
                    is_real_data=False,  # Analyzing generated abnormal OD data
                    data_dir=(Path(self.config.data_dir) if getattr(self.config, "data_dir", None) else None),
                )

                # Calculate rates
                total_traj = detection_results.get("total_trajectories", 0)
                abnormal_by_category = {}

                for category, indices in detection_results.get(
                    "abnormal_indices", {}
                ).items():
                    count = len(indices)
                    rate = (count / total_traj * 100) if total_traj > 0 else 0
                    abnormal_by_category[category] = {"count": count, "rate": rate}
                    logger.info(f"      {category}: {count}/{total_traj} ({rate:.2f}%)")

                # Step 2: Similarity metrics
                logger.info("    📏 Computing similarity metrics...")
                eval_output = output_dir / model_name / "metrics"
                eval_output.mkdir(parents=True, exist_ok=True)

                # Compare generated trajectories against the corresponding real dataset.
                eval_results = evaluate_trajectories_programmatic(
                    generated_file=str(gen_file),
                    dataset=eval_dataset,
                    od_source="train",  # Abnormal OD pairs come from train set
                    data_dir=str(data_dir),
                )

                metrics = {}
                if eval_results.get("status") == "success":
                    eval_data = eval_results.get("results", {})
                    metrics = {
                        "edr": eval_data.get("edr", 0.0),
                        "dtw": eval_data.get("dtw", 0.0),
                        "hausdorff": eval_data.get("hausdorff", 0.0),
                    }
                    logger.info(f"      EDR: {metrics['edr']:.4f}")

                # Save results
                model_results = {
                    "model": model_name,
                    "total_trajectories": total_traj,
                    "abnormality_detection": abnormal_by_category,
                    "similarity_metrics": metrics,
                }

                all_results[model_name] = model_results

                result_file = output_dir / model_name / "abnormal_od_evaluation.json"
                with open(result_file, "w") as f:
                    json.dump(model_results, f, indent=2)

            except Exception as e:
                logger.error(f"    ❌ Evaluation failed: {e}")
                import traceback

                traceback.print_exc()
                continue

        # Comparison report
        logger.info(f"\n  {'=' * 60}")
        logger.info("  📊 Comparison Report")
        logger.info(f"  {'=' * 60}")

        for model_name, results in all_results.items():
            total_abnormal = sum(
                cat["count"] for cat in results["abnormality_detection"].values()
            )
            total_traj = results["total_trajectories"]
            rate = (total_abnormal / total_traj * 100) if total_traj > 0 else 0
            logger.info(
                f"    {model_name:15s}: {total_abnormal:4d}/{total_traj:4d} ({rate:5.2f}%)"
            )

        # Save comparison
        comparison_report = {
            "dataset": self.config.dataset,
            "od_pairs_file": str(od_pairs_file),
            "model_results": all_results,
        }

        comparison_file = output_dir / "comparison_report.json"
        with open(comparison_file, "w") as f:
            json.dump(comparison_report, f, indent=2)

        logger.info(f"\n✅ Evaluation complete! Results in {output_dir}/")

    @phase("wang_abnormality", critical=False)
    def run_wang_abnormality(self):
        """Run Wang statistical abnormality detection pipeline"""
        if not self.config.run_wang_detection:
            logger.info("Wang statistical detection not configured, skipping")
            return

        logger.info("📊 Running Wang statistical abnormality detection pipeline...")

        # Auto-detect Wang config if not provided
        if not self.config.wang_config:
            wang_config = Path("./config/abnormal_detection_statistical.yaml")
            if not wang_config.exists():
                logger.warning(
                    "No Wang statistical detection config found, skipping Wang analysis"
                )
                return
        else:
            wang_config = Path(self.config.wang_config)

        if not wang_config.exists():
            logger.error(f"Wang detection config not found: {wang_config}")
            return

        # Import the pipeline runner
        from tools.run_wang_detection_pipeline import run_wang_detection_pipeline

        # Resolve data directory (supports eval workspaces that keep datasets outside ../data/{dataset}).
        resolved_data_dir = None
        if getattr(self.config, "data_dir", None):
            resolved_data_dir = Path(self.config.data_dir)
            if not resolved_data_dir.is_absolute():
                resolved_data_dir = (self.eval_dir / resolved_data_dir).resolve()

        # Run the Wang detection pipeline
        try:
            success = run_wang_detection_pipeline(
                eval_dir=self.eval_dir,
                dataset=self.config.dataset,
                data_dir=resolved_data_dir,
                skip_real=False,
                skip_generated=False,
                skip_aggregation=False,
                skip_visualization=False,
            )
            if success:
                logger.info(
                    "✅ Wang statistical detection pipeline completed successfully"
                )
            else:
                logger.error("❌ Wang detection pipeline completed with failures")
                raise RuntimeError("Wang detection pipeline failed")

        except Exception as e:
            logger.error(f"❌ Wang detection pipeline failed: {e}")
            raise

    @phase("lmtad_spatial_abnormality", critical=False)
    def run_lmtad_spatial_abnormality(self):
        """Run LM-TAD spatial abnormality detection pipeline"""
        if not self.config.run_lmtad_spatial_detection:
            logger.info("LM-TAD spatial detection not configured, skipping")
            return

        logger.info("📊 Running LM-TAD spatial abnormality detection pipeline...")

        # Auto-detect LM-TAD source eval directory if not provided
        lmtad_source_eval_dir = self._detect_lmtad_source_eval_dir()

        if not lmtad_source_eval_dir:
            logger.error("Could not find LM-TAD source evaluation directory")
            return

        # Find LM-TAD checkpoint
        lmtad_checkpoint = self._find_lmtad_checkpoint(lmtad_source_eval_dir)
        if not lmtad_checkpoint:
            logger.error("Could not find LM-TAD checkpoint file")
            return

        # Import the pipeline runner
        from tools.run_lmtad_spatial_pipeline import run_lmtad_spatial_pipeline

        resolved_data_dir = None
        if getattr(self.config, "data_dir", None):
            resolved_data_dir = Path(str(self.config.data_dir))
            if not resolved_data_dir.is_absolute():
                resolved_data_dir = (self.eval_dir / resolved_data_dir).resolve()

        # Run the LM-TAD spatial detection pipeline
        try:
            # Load evaluation config for Porto grid configuration
            eval_config = None
            config_path = self.eval_dir / "config" / "evaluation.yaml"
            if config_path.exists():
                import yaml

                with open(config_path, "r") as f:
                    eval_config = yaml.safe_load(f)

            # Prefer an explicitly configured LM-TAD repo path. For backwards
            # compatibility with existing eval configs, fall back to
            # `perturbation_lmtad_repo` if `lmtad_repo` isn't provided.
            resolved_lmtad_repo = getattr(self.config, "lmtad_repo", None)
            if resolved_lmtad_repo is None:
                resolved_lmtad_repo = getattr(self.config, "lmtad_spatial_repo", None)
            if resolved_lmtad_repo is None:
                resolved_lmtad_repo = getattr(self.config, "perturbation_lmtad_repo", None)

            if resolved_lmtad_repo is not None:
                resolved_lmtad_repo = Path(str(resolved_lmtad_repo))
                if not resolved_lmtad_repo.is_absolute():
                    resolved_lmtad_repo = (self.eval_dir / resolved_lmtad_repo).resolve()

            success = run_lmtad_spatial_pipeline(
                eval_dir=self.eval_dir,
                dataset=self.config.dataset,
                lmtad_source_eval_dir=lmtad_source_eval_dir,
                lmtad_checkpoint=lmtad_checkpoint,
                data_dir=resolved_data_dir,
                skip_extraction=False,
                skip_generation=False,
                skip_evaluation=False,
                skip_aggregation=False,
                skip_visualization=False,
                seed=self.config.seed,
                num_traj_per_od=(
                    int(self.config.lmtad_num_trajectories_per_od)
                    if self.config.lmtad_num_trajectories_per_od is not None
                    else 20
                ),
                max_od_pairs=(
                    int(self.config.lmtad_max_od_pairs)
                    if self.config.lmtad_max_od_pairs is not None
                    else 250
                ),
                lmtad_repo=resolved_lmtad_repo,  # Avoid fragile checkpoint-path auto-detection
                force=self.config.force,
                eval_config=eval_config,
                max_duplicate_ratio=(
                    getattr(self.config, "lmtad_max_duplicate_ratio", 0.1)
                ),
            )
            if success:
                logger.info(
                    "✅ LM-TAD spatial detection pipeline completed successfully"
                )
            else:
                logger.error("❌ LM-TAD spatial pipeline completed with failures")
                raise RuntimeError("LM-TAD spatial pipeline failed")
        except Exception as e:
            logger.error(f"❌ LM-TAD spatial pipeline failed: {e}")
            raise

    def _detect_lmtad_source_eval_dir(self) -> Optional[Path]:
        """Auto-detect LM-TAD source evaluation directory"""
        if self.config.lmtad_source_eval_dir:
            return Path(self.config.lmtad_source_eval_dir)

        # Auto-detect: /home/mka299/LMTAD/code/results/LMTAD/{dataset}/run_*/.../eval/
        lmtad_repo = Path("/home/mka299/LMTAD")
        dataset_candidates = [self.config.dataset]
        if isinstance(self.config.dataset, str) and "_" in self.config.dataset:
            dataset_candidates.append(self.config.dataset.split("_", 1)[0])

        # LM-TAD uses a different canonical dataset directory name for Beijing.
        # Support derived dataset variants (e.g., Beijing_per_type_detour) by
        # falling back to the Beijing LM-TAD reference name.
        base_lower = str(dataset_candidates[-1]).lower() if dataset_candidates else ""
        if base_lower == "beijing" and "beijing_hoser_reference" not in dataset_candidates:
            dataset_candidates.append("beijing_hoser_reference")

        dataset_eval_base = None
        for cand in dataset_candidates:
            candidate_base = lmtad_repo / "code" / "results" / "LMTAD" / cand
            if candidate_base.exists():
                dataset_eval_base = candidate_base
                break

        if dataset_eval_base is None:
            return None

        # Find most recent run directory
        run_dirs = sorted(
            dataset_eval_base.glob("run_*"),
            key=lambda x: x.stat().st_mtime,
            reverse=True,
        )

        for run_dir in run_dirs:
            # Look for eval directory with TSV files
            eval_dir = (
                run_dir
                / "outlier_False"
                / "n_layer_8_n_head_12_n_embd_768_lr_0.0003_integer_poe_False"
                / "eval"
            )
            if eval_dir.exists():
                tsv_files = sorted(eval_dir.glob("ckpt_best_outliers_*.tsv"))
                if not tsv_files:
                    tsv_files = sorted(eval_dir.glob("*outliers*.tsv"))
                if not tsv_files:
                    tsv_files = sorted(eval_dir.glob("*.tsv"))
                if tsv_files:
                    return eval_dir

        return None

    def _find_lmtad_checkpoint(self, eval_dir: Path) -> Optional[Path]:
        """Find LM-TAD checkpoint file from evaluation directory"""
        # Checkpoint is typically in parent directory
        checkpoint_dir = eval_dir.parent / "checkpoints"
        if checkpoint_dir.exists():
            # Try common checkpoint names
            for name in ["ckpt_best.pt", "best_model.pt", "checkpoint.pt"]:
                checkpoint = checkpoint_dir / name
                if checkpoint.exists():
                    return checkpoint

        # Also check parent directory directly (common location)
        for name in ["ckpt_best.pt", "best_model.pt"]:
            checkpoint = eval_dir.parent / name
            if checkpoint.exists():
                return checkpoint

        # Also check eval_dir itself
        for name in ["ckpt_best.pt", "best_model.pt"]:
            checkpoint = eval_dir / name
            if checkpoint.exists():
                return checkpoint

        return None

    @phase("scenarios", critical=False)
    def run_scenarios(self):
        """Run scenario analysis"""
        if not self.config.run_scenarios:
            logger.info("Scenarios not configured, skipping")
            return

        logger.info("🎯 Running scenario analysis...")
        self._run_scenario_analysis()

    @phase("paired_analysis", critical=False)
    def run_paired_analysis(self):
        """Run paired statistical tests comparing models on same OD pairs"""
        logger.info("📊 Running paired statistical analysis...")

        # Check dependency: need base_eval results with trajectory metrics
        eval_dir = Path("./eval")
        if not eval_dir.exists():
            logger.warning("No eval directory found, skipping paired analysis")
            return

        # Find all evaluation results with trajectory metrics
        eval_subdirs = [d for d in eval_dir.iterdir() if d.is_dir()]
        if not eval_subdirs:
            logger.warning(
                "No evaluation subdirectories found, skipping paired analysis"
            )
            return

        # Check for trajectory_metrics.json in at least one eval dir
        has_trajectory_metrics = any(
            (subdir / "trajectory_metrics.json").exists() for subdir in eval_subdirs
        )
        if not has_trajectory_metrics:
            logger.warning(
                "No trajectory_metrics.json files found. "
                "Run base_eval phase first to generate trajectory-level metrics."
            )
            return

        # Auto-detect model pairs to compare
        # Group evaluations by (model_type, od_source)
        from collections import defaultdict
        import json

        evals_by_od = defaultdict(lambda: defaultdict(list))

        for subdir in eval_subdirs:
            results_file = subdir / "results.json"
            traj_metrics_file = subdir / "trajectory_metrics.json"

            if not results_file.exists() or not traj_metrics_file.exists():
                continue

            try:
                with open(results_file) as f:
                    results = json.load(f)

                metadata = results.get("metadata", {})
                model_type = metadata.get("model_type", "unknown")
                od_source = metadata.get("od_source", "unknown")

                if model_type != "unknown" and od_source != "unknown":
                    evals_by_od[od_source][model_type].append(subdir)

            except Exception as e:
                logger.warning(f"Failed to read {results_file}: {e}")
                continue

        if not evals_by_od:
            logger.warning("No valid evaluation results found for paired analysis")
            return

        # Import the paired comparison tool
        from tools.compare_models_paired_analysis import (
            load_trajectory_metrics,
            match_trajectory_pairs,
            perform_paired_comparison,
            generate_markdown_summary,
        )

        # Output directory for paired analysis results
        paired_output_dir = Path("./paired_analysis")
        paired_output_dir.mkdir(exist_ok=True)

        # For each OD source, compare models pairwise
        comparison_count = 0
        for od_source, models_dict in evals_by_od.items():
            logger.info(f"\n📂 Processing {od_source} OD")

            model_types = list(models_dict.keys())
            if len(model_types) < 2:
                logger.info(
                    f"  Only 1 model found for {od_source} OD, skipping comparisons"
                )
                continue

            # Compare each pair of models
            for i in range(len(model_types)):
                for j in range(i + 1, len(model_types)):
                    model1_type = model_types[i]
                    model2_type = model_types[j]

                    # Get evaluation directories (use most recent if multiple)
                    eval1_dir = sorted(
                        models_dict[model1_type], key=lambda x: x.stat().st_mtime
                    )[-1]
                    eval2_dir = sorted(
                        models_dict[model2_type], key=lambda x: x.stat().st_mtime
                    )[-1]

                    logger.info(f"  Comparing {model1_type} vs {model2_type}")

                    try:
                        # Load trajectory metrics
                        data1 = load_trajectory_metrics(eval1_dir)
                        data2 = load_trajectory_metrics(eval2_dir)

                        # Match trajectory pairs
                        matched1, matched2 = match_trajectory_pairs(
                            data1["trajectory_metrics"], data2["trajectory_metrics"]
                        )

                        if len(matched1) == 0:
                            logger.warning(
                                "    No matching trajectory pairs found, skipping"
                            )
                            continue

                        # Perform paired comparison
                        metrics_to_compare = [
                            "hausdorff_km",
                            "hausdorff_norm",
                            "dtw_km",
                            "dtw_norm",
                            "edr",
                        ]
                        comparison_results = perform_paired_comparison(
                            matched_metrics1=matched1,
                            matched_metrics2=matched2,
                            model1_name=model1_type,
                            model2_name=model2_type,
                            metrics_to_compare=metrics_to_compare,
                            alpha=0.05,
                        )

                        # Prepare output
                        output_data = {
                            "model1_name": model1_type,
                            "model2_name": model2_type,
                            "model1_eval_dir": str(eval1_dir),
                            "model2_eval_dir": str(eval2_dir),
                            "n_matched_pairs": len(matched1),
                            "alpha": 0.05,
                            "metrics": comparison_results,
                            "metadata": {
                                "model1_metadata": data1.get("metadata", {}),
                                "model2_metadata": data2.get("metadata", {}),
                            },
                        }

                        # Save results
                        comparison_output_dir = (
                            paired_output_dir
                            / od_source
                            / f"{model1_type}_vs_{model2_type}"
                        )
                        comparison_output_dir.mkdir(parents=True, exist_ok=True)

                        # Convert numpy types to native Python types for JSON serialization
                        def convert_numpy_types(obj):
                            """Recursively convert numpy types to native Python types."""
                            import numpy as np

                            if isinstance(obj, dict):
                                return {
                                    k: convert_numpy_types(v) for k, v in obj.items()
                                }
                            elif isinstance(obj, list):
                                return [convert_numpy_types(item) for item in obj]
                            elif isinstance(obj, (np.integer, np.floating)):
                                return float(obj)
                            elif isinstance(obj, np.bool_):
                                return bool(obj)
                            elif isinstance(obj, np.ndarray):
                                return obj.tolist()
                            return obj

                        output_data_serializable = convert_numpy_types(output_data)

                        output_file = comparison_output_dir / "paired_comparison.json"
                        with open(output_file, "w") as f:
                            json.dump(output_data_serializable, f, indent=2)

                        # Generate markdown summary
                        generate_markdown_summary(output_data, output_file)

                        logger.info(f"    ✅ Results saved to {comparison_output_dir}")
                        comparison_count += 1

                    except Exception as e:
                        logger.error(
                            f"    ❌ Paired analysis failed for {model1_type} vs {model2_type}: {e}"
                        )
                        import traceback

                        traceback.print_exc()
                        continue

        if comparison_count > 0:
            logger.info(
                f"\n✅ Paired analysis complete! {comparison_count} comparisons saved to {paired_output_dir}/"
            )
        else:
            logger.warning("No paired comparisons were completed")

    def run(self):
        """Execute all enabled phases in order"""
        logger.info("Starting HOSER Distillation Evaluation Pipeline")
        logger.info(f"Configuration: {self.config.__dict__}")

        try:
            self._prepare_execution_context()
            failures = self._execute_phases()

            if failures:
                self._log_phase_failures(failures)
                return False

            logger.info("\n✅ Pipeline completed successfully!")
            return True
        except Exception as e:
            logger.error(f"Pipeline failed with critical error: {str(e)}")
            import traceback

            logger.error(f"Stack trace:\n{traceback.format_exc()}")
            raise
        finally:
            self._cleanup_after_run()

    def _prepare_execution_context(self) -> None:
        """Prepare models and phase configuration before execution."""
        self._normalize_phase_configuration()
        self._ensure_models_loaded()
        logger.info(f"Models to process: {self.config.models}")
        logger.info(f"OD sources: {self.config.od_sources}")

    def _normalize_phase_configuration(self) -> None:
        """Ensure phase configuration is consistent."""
        phases = getattr(self.config, "phases", set())
        if phases is None:
            phases = set()

        self.config.phases = set(phases)

        # Keep phase set consistent with feature flags, unless the operator
        # explicitly requested a phase subset via --only.
        if not getattr(self.config, "phases_explicit", False):
            if getattr(self.config, "run_abnormal_detection", False):
                self.config.phases.add("abnormal")
            if getattr(self.config, "run_wang_detection", False):
                self.config.phases.add("wang_abnormality")
            if getattr(self.config, "run_lmtad_spatial_detection", False):
                self.config.phases.add("lmtad_spatial_abnormality")
            if getattr(self.config, "run_scenarios", False):
                self.config.phases.add("scenarios")
        logger.info(f"Enabled phases: {sorted(self.config.phases)}")

    def _ensure_models_loaded(self) -> None:
        """Populate model list if necessary before executing phases."""
        if not self.config.models:
            self.config.models = self.detector.detect_models()

        if not self.config.models:
            raise ValueError("No models configured for evaluation")

    def _execute_phases(self) -> List[str]:
        """Execute phases in defined order and collect non-critical failures."""
        failures: List[str] = []
        for phase_name in self._phase_execution_order():
            if not self._should_run_phase(phase_name):
                logger.info(f"⏭️  Skipping phase: {phase_name}")
                continue

            phase_info = PHASE_REGISTRY.get(phase_name)
            if not phase_info:
                logger.warning(f"⚠️  Phase not registered: {phase_name}")
                continue

            logger.info(f"\n{'=' * 70}")
            logger.info(f"🚀 Running phase: {phase_name}")
            logger.info(f"{'=' * 70}")

            try:
                phase_info["func"](self)
                logger.info(f"✅ Phase {phase_name} completed")
            except Exception as exc:  # noqa: BLE001
                logger.error(f"❌ Phase {phase_name} failed: {exc}")
                if phase_info.get("critical"):
                    logger.error("Critical phase failed, stopping pipeline")
                    raise

                logger.warning("Non-critical phase failed, continuing")
                failures.append(f"{phase_name}: {exc}")

        return failures

    def _phase_execution_order(self) -> List[str]:
        """Return the canonical execution order for phases."""
        default_order = [
            "generation",
            "base_eval",
            "perturbation_correction",
            "paired_analysis",
            "cross_dataset",
            "road_network_translate",
            "abnormal",
            # Abnormality sources should run before extracting abnormal OD pairs.
            # In particular, Wang writes `abnormal/<dataset>/*/real_data/detection_results.json`.
            "wang_abnormality",
            "lmtad_spatial_abnormality",
            "abnormal_od_extract",
            "abnormal_od_generate",
            "abnormal_od_evaluate",
            "scenarios",
        ]
        extras = [phase for phase in self.config.phases if phase not in default_order]
        return default_order + sorted(extras)

    def _should_run_phase(self, phase_name: str) -> bool:
        """Determine whether a phase is enabled for execution."""
        return phase_name in self.config.phases

    def _log_phase_failures(self, failures: List[str]) -> None:
        """Summarize non-critical phase failures for operator visibility."""
        logger.warning(f"\n⚠️  {len(failures)} phase(s) failed:")
        for failure in failures:
            logger.warning(f"  - {failure}")

    def _cleanup_after_run(self) -> None:
        """Ensure WandB resources are finalized regardless of outcome."""
        try:
            self.wandb_manager.finish_all_runs()
            self.wandb_manager.start_background_sync()
        except Exception as exc:  # noqa: BLE001
            logger.warning(f"Error during WandB cleanup: {exc}")


def main():
    """Main entry point"""
    parser = argparse.ArgumentParser(description="HOSER Evaluation Pipeline")
    parser.add_argument(
        "--eval-dir", type=str, help="Evaluation directory (default: current directory)"
    )
    parser.add_argument(
        "--config",
        type=str,
        help="Path to YAML configuration file (default: config/evaluation.yaml in eval dir)",
    )
    parser.add_argument("--seed", type=int, help="Random seed (overrides config)")
    parser.add_argument(
        "--models",
        type=str,
        help="Comma-separated list of models to run (overrides config)",
    )
    parser.add_argument(
        "--od-source", type=str, help="OD sources: train,test (overrides config)"
    )
    parser.add_argument(
        "--skip-gene", action="store_true", help="Skip generation (overrides config)"
    )
    parser.add_argument(
        "--skip-eval", action="store_true", help="Skip evaluation (overrides config)"
    )
    parser.add_argument(
        "--only",
        type=str,
        help=(
            "Run only these phases (comma-separated). Available: "
            "generation,base_eval,paired_analysis,cross_dataset,road_network_translate,abnormal,"
            "abnormal_od_extract,abnormal_od_generate,abnormal_od_evaluate,wang_abnormality,"
            "lmtad_spatial_abnormality,scenarios,perturbation_correction."
        ),
    )
    parser.add_argument(
        "--skip",
        type=str,
        help=(
            "Skip these phases (comma-separated). Example: --skip generation,base_eval"
        ),
    )
    parser.add_argument(
        "--force", action="store_true", help="Force re-run (overrides config)"
    )
    parser.add_argument("--cuda", type=int, help="CUDA device (overrides config)")
    parser.add_argument(
        "--num-gene", type=int, help="Number of trajectories (overrides config)"
    )
    parser.add_argument(
        "--use-astar",
        action="store_true",
        help="Use A* search instead of beam search (original HOSER method)",
    )
    parser.add_argument(
        "--wandb-project", type=str, help="WandB project (overrides config)"
    )
    parser.add_argument(
        "--no-wandb",
        action="store_true",
        help="Disable WandB logging (overrides config)",
    )
    parser.add_argument(
        "--verbose", action="store_true", help="Verbose output (overrides config)"
    )
    parser.add_argument(
        "--run-scenarios",
        action="store_true",
        help="Run scenario analysis after evaluation (optional)",
    )
    parser.add_argument(
        "--scenarios-config",
        type=str,
        help="Path to scenarios config YAML (default: auto-detect from config/)",
    )
    parser.add_argument(
        "--cross-dataset",
        type=str,
        help="Path to cross-dataset for evaluation (e.g., ../data/BJUT_Beijing)",
    )
    parser.add_argument(
        "--cross-dataset-name",
        type=str,
        default=None,
        help="Name of the cross-dataset (only set this when running cross-dataset eval)",
    )
    parser.add_argument(
        "--run-abnormal",
        action="store_true",
        help="Run abnormal trajectory detection analysis",
    )
    parser.add_argument(
        "--abnormal-config",
        type=str,
        help="Path to abnormal detection config YAML (default: config/abnormal_detection.yaml)",
    )
    parser.add_argument(
        "--run-wang",
        action="store_true",
        help="Run Wang statistical abnormality detection and visualization",
    )
    parser.add_argument(
        "--wang-config",
        type=str,
        help="Path to Wang detection config YAML (default: config/abnormal_detection_statistical.yaml)",
    )
    parser.add_argument(
        "--run-lmtad-spatial",
        action="store_true",
        help="Run LM-TAD spatial abnormality detection and visualization",
    )
    parser.add_argument(
        "--lmtad-spatial-config",
        type=str,
        help="Path to LM-TAD spatial config YAML (optional)",
    )
    parser.add_argument(
        "--lmtad-source-eval-dir",
        type=str,
        help="Path to LM-TAD source evaluation directory (auto-detected if not provided)",
    )
    parser.add_argument(
        "--lmtad-num-trajectories-per-od",
        type=int,
        help="Number of trajectories per OD pair for LM-TAD spatial evaluation (default: 20, for testing use smaller values)",
    )
    parser.add_argument(
        "--lmtad-max-od-pairs",
        type=int,
        help="Maximum OD pairs to sample for LM-TAD spatial evaluation (default: 250, for testing use smaller values)",
    )
    parser.add_argument(
        "--lmtad-max-duplicate-ratio",
        type=float,
        help="Maximum duplicate ratio allowed for trajectories during LM-TAD validation (default: 0.1)",
    )
    parser.add_argument(
        "--abnormal-num-traj-per-od",
        type=int,
        default=10,
        help="Number of trajectories per OD pair for abnormal_od_generate phase (default: 10)",
    )

    # Phase B: perturbation correction (CLI overrides eval-dir/config/evaluation.yaml)
    parser.add_argument(
        "--perturbation-source-csv",
        type=str,
        help="Path to perturbed train.csv (overrides config)",
    )
    parser.add_argument(
        "--perturbation-od-source",
        type=str,
        help="OD source for timestamp selection: train|test (overrides config)",
    )
    parser.add_argument(
        "--perturbation-max-entries",
        type=int,
        help="Max abnormal rows to sample for Phase B (overrides config)",
    )
    parser.add_argument(
        "--perturbation-seed",
        type=int,
        help="Sampling seed for Phase B (overrides config)",
    )
    parser.add_argument(
        "--perturbation-use-astar",
        action="store_true",
        help="Use A* search for Phase B generation (overrides config)",
    )
    parser.add_argument(
        "--perturbation-lmtad-checkpoint",
        type=str,
        help="LM-TAD checkpoint to enable teacher perplexity scoring (overrides config)",
    )
    parser.add_argument(
        "--perturbation-lmtad-repo",
        type=str,
        help="Path to LM-TAD repo (required if checkpoint is set) (overrides config)",
    )
    parser.add_argument(
        "--perturbation-lmtad-batch-size",
        type=int,
        help="Batch size for LM-TAD evaluation (overrides config)",
    )

    args = parser.parse_args()

    # Configure logging level
    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    # Determine evaluation directory
    if args.eval_dir:
        eval_dir = Path(args.eval_dir).resolve()
    else:
        # Use current directory if it looks like an eval dir
        current_dir = Path.cwd()
        if (current_dir / "models").exists() and (current_dir / "config").exists():
            eval_dir = current_dir
        else:
            parser.error(
                "No --eval-dir specified and current directory doesn't appear to be an evaluation directory"
            )

    if not eval_dir.exists():
        parser.error(f"Evaluation directory not found: {eval_dir}")

    # Determine config path
    if args.config:
        config_path = args.config
    else:
        config_path = eval_dir / "config" / "evaluation.yaml"
        if not config_path.exists():
            parser.error(f"Config file not found: {config_path}")

    # Create configuration from YAML file
    config = PipelineConfig(str(config_path), eval_dir)

    # Apply phase control (BEFORE other overrides)
    if hasattr(args, "only") and args.only:
        config.phases = {p.strip() for p in args.only.split(",") if p.strip()}
        config.phases_explicit = True
        logger.info(f"Running only phases: {config.phases}")

        # If the user explicitly requested a phase via --only, treat that as an
        # explicit enable signal for the corresponding feature flag.
        if "abnormal" in config.phases:
            config.run_abnormal_detection = True
        if "wang_abnormality" in config.phases:
            config.run_wang_detection = True
        if "lmtad_spatial_abnormality" in config.phases:
            config.run_lmtad_spatial_detection = True
        if "scenarios" in config.phases:
            config.run_scenarios = True

    if hasattr(args, "skip") and args.skip:
        skip_phases = {p.strip() for p in args.skip.split(",") if p.strip()}
        config.phases -= skip_phases
        logger.info(f"Skipping phases: {skip_phases}")

    # Override with command line arguments if provided
    if args.seed is not None:
        config.seed = args.seed
    if args.models is not None:
        config.models = args.models.split(",")
    if args.od_source is not None:
        config.od_sources = args.od_source.split(",")
    if args.force:
        config.force = True
    if args.cuda is not None:
        config.cuda_device = args.cuda
    if args.num_gene is not None:
        config.num_gene = args.num_gene
    if args.use_astar:
        config.beam_search = False
    if args.wandb_project is not None:
        config.wandb_project = args.wandb_project
    if args.no_wandb:
        config.enable_wandb = False
    if args.verbose:
        config.verbose = True
    if args.run_scenarios:
        config.run_scenarios = True
    if args.scenarios_config:
        config.scenarios_config = args.scenarios_config
    if args.cross_dataset:
        config.cross_dataset_eval = args.cross_dataset
    if args.cross_dataset_name:
        config.cross_dataset_name = args.cross_dataset_name
    if args.run_abnormal:
        config.run_abnormal_detection = True
    if args.abnormal_config:
        config.abnormal_config = args.abnormal_config
    if args.run_wang:
        config.run_wang_detection = True
    if args.wang_config:
        config.wang_config = args.wang_config
    if args.run_lmtad_spatial:
        config.run_lmtad_spatial_detection = True
    if args.lmtad_spatial_config:
        config.lmtad_spatial_config = args.lmtad_spatial_config
    if (
        hasattr(args, "lmtad_max_duplicate_ratio")
        and args.lmtad_max_duplicate_ratio is not None
    ):
        config.lmtad_max_duplicate_ratio = args.lmtad_max_duplicate_ratio
    if args.lmtad_source_eval_dir:
        config.lmtad_source_eval_dir = args.lmtad_source_eval_dir
    if args.lmtad_num_trajectories_per_od is not None:
        config.lmtad_num_trajectories_per_od = args.lmtad_num_trajectories_per_od
    if args.lmtad_max_od_pairs is not None:
        config.lmtad_max_od_pairs = args.lmtad_max_od_pairs
    if args.abnormal_num_traj_per_od is not None:
        config.abnormal_num_traj_per_od = args.abnormal_num_traj_per_od

    # Phase B overrides
    if getattr(args, "perturbation_source_csv", None):
        config.perturbation_source_csv = args.perturbation_source_csv
    if getattr(args, "perturbation_od_source", None):
        config.perturbation_od_source = args.perturbation_od_source
    if getattr(args, "perturbation_max_entries", None) is not None:
        config.perturbation_max_entries = args.perturbation_max_entries
    if getattr(args, "perturbation_seed", None) is not None:
        config.perturbation_seed = args.perturbation_seed
    if getattr(args, "perturbation_use_astar", False):
        config.perturbation_use_astar = True
    if getattr(args, "perturbation_lmtad_checkpoint", None):
        config.perturbation_lmtad_checkpoint = args.perturbation_lmtad_checkpoint
    if getattr(args, "perturbation_lmtad_repo", None):
        config.perturbation_lmtad_repo = args.perturbation_lmtad_repo
    if getattr(args, "perturbation_lmtad_batch_size", None) is not None:
        config.perturbation_lmtad_batch_size = args.perturbation_lmtad_batch_size

    # Run pipeline
    try:
        pipeline = EvaluationPipeline(config, eval_dir)
        success = pipeline.run()

        if success:
            logger.info("✅ Pipeline completed successfully!")
            sys.exit(0)
        else:
            logger.warning("⚠️  Pipeline completed with some failures")
            sys.exit(1)
    except Exception as e:
        logger.error(f"Pipeline failed: {str(e)}")
        sys.exit(1)


if __name__ == "__main__":
    main()