diff --git a/README.md b/README.md index ead6659..a160cc9 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ A professional, scalable machine learning library with modular architecture for - **Confusion matrices** and classification reports - **ROC AUC** and other advanced metrics - **Performance tracking** and comparison +- **Visualizations**: Automatic generation of Confusion Matrix, ROC Curve, and Residuals plots ### 🔍 Exploration - **Statistical summaries** and data profiling @@ -78,10 +79,11 @@ classifier.fit(X_train, y_train, tune_hyperparameters=True) # Make predictions y_pred = classifier.predict(X_test) -# Evaluate +# Evaluate with visualization evaluator = ModelEvaluator() -metrics = evaluator.evaluate_classification(y_test, y_pred) +metrics = evaluator.evaluate_classification(y_test, y_pred, y_pred_proba=classifier.predict_proba(X_test), plot=True) evaluator.print_report() +# Plots are saved in the 'plots/' directory ``` ### Clustering Example diff --git a/mltools/evaluation/evaluator.py b/mltools/evaluation/evaluator.py index 76b55a7..ffe0016 100644 --- a/mltools/evaluation/evaluator.py +++ b/mltools/evaluation/evaluator.py @@ -2,13 +2,17 @@ import numpy as np import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns from typing import Dict, Any, Optional import warnings +import os from sklearn.metrics import ( accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix, classification_report, - mean_squared_error, mean_absolute_error, r2_score + mean_squared_error, mean_absolute_error, r2_score, + roc_curve, precision_recall_curve ) from mltools.utils import Config, get_logger @@ -25,6 +29,7 @@ class ModelEvaluator: - Classification and regression support - Confusion matrix analysis - Performance reports + - Visualizations (Confusion Matrix, ROC Curve, Residuals) """ def __init__(self, config: Optional[Config] = None): @@ -37,12 +42,18 @@ def __init__(self, config: Optional[Config] = None): self.config = config or Config() self.logger = get_logger('ModelEvaluator') self.results = {} + + # Create plots directory if it doesn't exist + self.plots_dir = 'plots' + if not os.path.exists(self.plots_dir): + os.makedirs(self.plots_dir) def evaluate_classification( self, y_true: np.ndarray, y_pred: np.ndarray, - y_pred_proba: Optional[np.ndarray] = None + y_pred_proba: Optional[np.ndarray] = None, + plot: bool = False ) -> Dict[str, Any]: """ Evaluate classification model @@ -51,6 +62,7 @@ def evaluate_classification( y_true: True labels y_pred: Predicted labels y_pred_proba: Predicted probabilities (optional) + plot: Whether to generate and save plots Returns: Dictionary of evaluation metrics @@ -78,7 +90,8 @@ def evaluate_classification( self.logger.warning(f"Could not compute ROC AUC: {str(e)}") metrics['roc_auc'] = None - metrics['confusion_matrix'] = confusion_matrix(y_true, y_pred).tolist() + cm = confusion_matrix(y_true, y_pred) + metrics['confusion_matrix'] = cm.tolist() try: report = classification_report(y_true, y_pred, output_dict=True, zero_division=0) @@ -89,12 +102,18 @@ def evaluate_classification( self.results = metrics self._log_results(metrics) + if plot: + self.plot_confusion_matrix(cm) + if y_pred_proba is not None and len(np.unique(y_true)) == 2: + self.plot_roc_curve(y_true, y_pred_proba[:, 1]) + return metrics def evaluate_regression( self, y_true: np.ndarray, - y_pred: np.ndarray + y_pred: np.ndarray, + plot: bool = False ) -> Dict[str, Any]: """ Evaluate regression model @@ -102,6 +121,7 @@ def evaluate_regression( Parameters: y_true: True values y_pred: Predicted values + plot: Whether to generate and save plots Returns: Dictionary of evaluation metrics @@ -122,8 +142,56 @@ def evaluate_regression( self.results = metrics self._log_results(metrics) + if plot: + self.plot_residuals(y_true, y_pred) + return metrics + def plot_confusion_matrix(self, cm: np.ndarray, filename: str = 'confusion_matrix.png'): + """Plot and save confusion matrix""" + plt.figure(figsize=(10, 7)) + sns.heatmap(cm, annot=True, fmt='d', cmap='Blues') + plt.title('Confusion Matrix') + plt.ylabel('Actual') + plt.xlabel('Predicted') + path = os.path.join(self.plots_dir, filename) + plt.savefig(path) + plt.close() + self.logger.info(f"Confusion matrix plot saved to {path}") + + def plot_roc_curve(self, y_true: np.ndarray, y_score: np.ndarray, filename: str = 'roc_curve.png'): + """Plot and save ROC curve""" + fpr, tpr, _ = roc_curve(y_true, y_score) + roc_auc = roc_auc_score(y_true, y_score) + + plt.figure(figsize=(10, 7)) + plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (area = {roc_auc:.2f})') + plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--') + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title('Receiver Operating Characteristic (ROC)') + plt.legend(loc="lower right") + path = os.path.join(self.plots_dir, filename) + plt.savefig(path) + plt.close() + self.logger.info(f"ROC curve plot saved to {path}") + + def plot_residuals(self, y_true: np.ndarray, y_pred: np.ndarray, filename: str = 'residuals.png'): + """Plot and save residuals plot""" + residuals = y_true - y_pred + plt.figure(figsize=(10, 7)) + plt.scatter(y_pred, residuals, alpha=0.5) + plt.axhline(y=0, color='r', linestyle='--') + plt.xlabel('Predicted Values') + plt.ylabel('Residuals') + plt.title('Residuals Plot') + path = os.path.join(self.plots_dir, filename) + plt.savefig(path) + plt.close() + self.logger.info(f"Residuals plot saved to {path}") + def _log_results(self, metrics: Dict[str, Any]): """Log evaluation results""" self.logger.info("Evaluation Results:")