evaluate_models.py

"""
Evaluate and Export Models

This script evaluates the trained models and exports the best ones for deployment.
"""

import sys
import os
import json
import joblib
import logging
import pandas as pd
import numpy as np
from pathlib import Path
from datetime import datetime
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.StreamHandler(),
        logging.FileHandler('model_evaluation.log')
    ]
)
logger = logging.getLogger(__name__)

class ModelEvaluator:
    """Class for evaluating and exporting trained models."""
    
    def __init__(self, model_dir='models', output_dir='models/best'):
        """Initialize the model evaluator."""
        self.model_dir = Path(model_dir)
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)
    
    def find_model_dirs(self):
        """Find all model directories."""
        return [d for d in self.model_dir.glob('*') if d.is_dir()]
    
    def load_model_results(self, model_dir):
        """Load model results from a directory."""
        results_file = model_dir / 'results.json'
        if not results_file.exists():
            return None
            
        with open(results_file, 'r') as f:
            return json.load(f)
    
    def evaluate_models(self):
        """Evaluate all models and select the best ones."""
        logger.info("Starting model evaluation...")
        
        best_land_model = None
        best_yield_model = None
        best_land_score = -np.inf
        best_yield_score = -np.inf
        
        # Find all model directories
        model_dirs = self.find_model_dirs()
        if not model_dirs:
            logger.warning("No model directories found.")
            return None, None
        
        # Evaluate each model directory
        for model_dir in model_dirs:
            logger.info(f"\nEvaluating models in: {model_dir.name}")
            
            # Skip if not a valid model directory
            results = self.load_model_results(model_dir)
            if not results:
                logger.warning(f"No results found in {model_dir.name}")
                continue
            
            # Check if this is a land value or yield model
            is_yield_model = 'yield' in model_dir.name.lower()
            
            # Get the best model info
            best_model_name = results.get('best_model')
            best_score = results.get('best_score', -np.inf)
            
            if not best_model_name or best_score < 0:
                logger.warning(f"No valid best model found in {model_dir.name}")
                continue
            
            # Update the best model if this one is better
            if is_yield_model and best_score > best_yield_score:
                best_yield_score = best_score
                best_yield_model = {
                    'dir': model_dir,
                    'name': best_model_name,
                    'score': best_score,
                    'results': results
                }
                logger.info(f"New best yield model: {best_model_name} (R²: {best_score:.4f})")
            elif not is_yield_model and best_score > best_land_score:
                best_land_score = best_score
                best_land_model = {
                    'dir': model_dir,
                    'name': best_model_name,
                    'score': best_score,
                    'results': results
                }
                logger.info(f"New best land value model: {best_model_name} (R²: {best_score:.4f})")
        
        return best_land_model, best_yield_model
    
    def export_best_models(self, land_model, yield_model):
        """Export the best models for deployment."""
        logger.info("\nExporting best models...")
        
        exported_models = {}
        
        # Export land value model if available
        if land_model:
            model_path = land_model['dir'] / f"{land_model['name']}.joblib"
            if model_path.exists():
                output_path = self.output_dir / 'land_value_model.pkl'
                self._export_model(model_path, output_path)
                exported_models['land_value'] = {
                    'path': str(output_path),
                    'model': land_model['name'],
                    'r2_score': land_model['score']
                }
        
        # Export yield model if available
        if yield_model:
            model_path = yield_model['dir'] / f"{yield_model['name']}.joblib"
            if model_path.exists():
                output_path = self.output_dir / 'yield_model.pkl'
                self._export_model(model_path, output_path)
                exported_models['yield'] = {
                    'path': str(output_path),
                    'model': yield_model['name'],
                    'r2_score': yield_model['score']
                }
        
        # Save export summary
        if exported_models:
            summary_path = self.output_dir / 'model_summary.json'
            with open(summary_path, 'w') as f:
                json.dump(exported_models, f, indent=2)
            logger.info(f"Saved model summary to: {summary_path}")
        
        return exported_models
    
    def _export_model(self, src_path, dst_path):
        """Export a model file with error handling."""
        try:
            # Load and immediately save to ensure compatibility
            model = joblib.load(src_path)
            joblib.dump(model, dst_path)
            logger.info(f"Exported model: {src_path.name} -> {dst_path}")
            return True
        except Exception as e:
            logger.error(f"Error exporting {src_path}: {e}")
            return False

def main():
    """Run the model evaluation and export pipeline."""
    try:
        logger.info("=== Model Evaluation and Export ===\n")
        
        # Initialize evaluator
        evaluator = ModelEvaluator()
        
        # Find and evaluate models
        land_model, yield_model = evaluator.evaluate_models()
        
        if not land_model and not yield_model:
            logger.error("No valid models found for export.")
            return 1
        
        # Export the best models
        exported_models = evaluator.export_best_models(land_model, yield_model)
        
        # Print summary
        logger.info("\n=== Model Export Summary ===")
        for model_type, info in exported_models.items():
            logger.info(f"{model_type.upper()} - Model: {info['model']}, R²: {info['r2_score']:.4f}")
        
        logger.info("\n=== Model evaluation and export complete ===")
        return 0
        
    except Exception as e:
        logger.error(f"Error in model evaluation: {e}", exc_info=True)
        return 1

if __name__ == "__main__":
    sys.exit(main())