Skip to content

feat(evaluation): add automatic visualization for model evaluation #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
raofal-msodeh wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(evaluation): add automatic visualization for model evaluation #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ A professional, scalable machine learning library with modular architecture for
- **Confusion matrices** and classification reports
- **ROC AUC** and other advanced metrics
- **Performance tracking** and comparison
- **Visualizations**: Automatic generation of Confusion Matrix, ROC Curve, and Residuals plots

### 🔍 Exploration
- **Statistical summaries** and data profiling
Expand Down Expand Up @@ -78,10 +79,11 @@ classifier.fit(X_train, y_train, tune_hyperparameters=True)
# Make predictions
y_pred = classifier.predict(X_test)

# Evaluate
# Evaluate with visualization
evaluator = ModelEvaluator()
metrics = evaluator.evaluate_classification(y_test, y_pred)
metrics = evaluator.evaluate_classification(y_test, y_pred, y_pred_proba=classifier.predict_proba(X_test), plot=True)
evaluator.print_report()
# Plots are saved in the 'plots/' directory
```

### Clustering Example
Expand Down
76 changes: 72 additions & 4 deletions mltools/evaluation/evaluator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from typing import Dict, Any, Optional
import warnings
import os

from sklearn.metrics import (
accuracy_score, precision_score, recall_score, f1_score,
roc_auc_score, confusion_matrix, classification_report,
mean_squared_error, mean_absolute_error, r2_score
mean_squared_error, mean_absolute_error, r2_score,
roc_curve, precision_recall_curve
)

from mltools.utils import Config, get_logger
Expand All @@ -25,6 +29,7 @@ class ModelEvaluator:
- Classification and regression support
- Confusion matrix analysis
- Performance reports
- Visualizations (Confusion Matrix, ROC Curve, Residuals)
"""

def __init__(self, config: Optional[Config] = None):
Expand All @@ -37,12 +42,18 @@ def __init__(self, config: Optional[Config] = None):
self.config = config or Config()
self.logger = get_logger('ModelEvaluator')
self.results = {}

# Create plots directory if it doesn't exist
self.plots_dir = 'plots'
if not os.path.exists(self.plots_dir):
os.makedirs(self.plots_dir)
Comment on lines +45 to +49

@coderabbitai coderabbitai Bot Apr 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Avoid creating plots/ as an unconditional side effect of __init__.

The directory is created on every ModelEvaluator() instantiation, even when the caller never sets plot=True. This pollutes the current working directory (which is whatever the user happens to be running from) and surprises consumers who only want metrics. Defer creation to the point where a plot is actually being saved, and use exist_ok=True to avoid the TOCTOU between os.path.exists and os.makedirs.

🛠️ Proposed fix 
     def __init__(self, config: Optional[Config] = None):
         ...
         self.config = config or Config()
         self.logger = get_logger('ModelEvaluator')
         self.results = {}
-        
-        # Create plots directory if it doesn't exist
-        self.plots_dir = 'plots'
-        if not os.path.exists(self.plots_dir):
-            os.makedirs(self.plots_dir)
+        self.plots_dir = 'plots'
+
+    def _ensure_plots_dir(self) -> None:
+        os.makedirs(self.plots_dir, exist_ok=True)

Then call self._ensure_plots_dir() at the start of each plot_* helper before computing the path.

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@mltools/evaluation/evaluator.py` around lines 45 - 49, The constructor
ModelEvaluator.__init__ currently creates the 'plots' directory unconditionally
via self.plots_dir — defer this side effect: remove the
os.path.exists/os.makedirs calls from __init__, keep self.plots_dir = 'plots',
add a helper method _ensure_plots_dir that calls os.makedirs(self.plots_dir,
exist_ok=True), and call self._ensure_plots_dir() at the start of each plotting
helper (any methods named plot_*). This ensures the directory is created only
when a plot is actually saved and avoids the TOCTOU race by using exist_ok=True.


def evaluate_classification(
self,
y_true: np.ndarray,
y_pred: np.ndarray,
y_pred_proba: Optional[np.ndarray] = None
y_pred_proba: Optional[np.ndarray] = None,
plot: bool = False
) -> Dict[str, Any]:
"""
Evaluate classification model
Expand All @@ -51,6 +62,7 @@ def evaluate_classification(
y_true: True labels
y_pred: Predicted labels
y_pred_proba: Predicted probabilities (optional)
plot: Whether to generate and save plots

Returns:
Dictionary of evaluation metrics
Expand Down Expand Up @@ -78,7 +90,8 @@ def evaluate_classification(
self.logger.warning(f"Could not compute ROC AUC: {str(e)}")
metrics['roc_auc'] = None

metrics['confusion_matrix'] = confusion_matrix(y_true, y_pred).tolist()
cm = confusion_matrix(y_true, y_pred)
metrics['confusion_matrix'] = cm.tolist()

try:
report = classification_report(y_true, y_pred, output_dict=True, zero_division=0)
Expand All @@ -89,19 +102,26 @@ def evaluate_classification(
self.results = metrics
self._log_results(metrics)

if plot:
self.plot_confusion_matrix(cm)
if y_pred_proba is not None and len(np.unique(y_true)) == 2:
self.plot_roc_curve(y_true, y_pred_proba[:, 1])
Comment on lines +105 to +108

@coderabbitai coderabbitai Bot Apr 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

y_pred_proba[:, 1] assumes a 2-D probability array.

If a caller passes a 1-D array of positive-class probabilities for a binary problem (a common shape returned by some estimators / wrappers), line 108 will raise IndexError: too many indices for array. The ROC AUC computation on line 82 has the same issue, but here the failure happens after evaluate_classification has already logged metrics, which is more confusing. Consider normalizing the shape once and reusing it.

🛠️ Proposed fix 
-        if plot:
-            self.plot_confusion_matrix(cm)
-            if y_pred_proba is not None and len(np.unique(y_true)) == 2:
-                self.plot_roc_curve(y_true, y_pred_proba[:, 1])
+        if plot:
+            self.plot_confusion_matrix(cm)
+            if y_pred_proba is not None and len(np.unique(y_true)) == 2:
+                proba_pos = y_pred_proba[:, 1] if y_pred_proba.ndim == 2 else y_pred_proba
+                self.plot_roc_curve(y_true, proba_pos)
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@mltools/evaluation/evaluator.py` around lines 105 - 108, The code assumes
y_pred_proba is 2-D (using y_pred_proba[:, 1]) which breaks if callers pass a
1-D array of positive-class probabilities; update evaluate_classification (and
any ROC/AUC sites) to normalize y_pred_proba once: if y_pred_proba is 1-D treat
it as the positive-class probabilities, otherwise extract column 1 from a 2-D
array, store that normalized 1-D array (e.g., pos_probs) and use pos_probs for
ROC/AUC calls and for plot_roc_curve to avoid IndexError and duplicate logic;
reference symbols: evaluate_classification, y_pred_proba, plot_roc_curve, and
any ROC/AUC computation lines.


return metrics

def evaluate_regression(
self,
y_true: np.ndarray,
y_pred: np.ndarray
y_pred: np.ndarray,
plot: bool = False
) -> Dict[str, Any]:
"""
Evaluate regression model

Parameters:
y_true: True values
y_pred: Predicted values
plot: Whether to generate and save plots

Returns:
Dictionary of evaluation metrics
Expand All @@ -122,8 +142,56 @@ def evaluate_regression(
self.results = metrics
self._log_results(metrics)

if plot:
self.plot_residuals(y_true, y_pred)

return metrics

def plot_confusion_matrix(self, cm: np.ndarray, filename: str = 'confusion_matrix.png'):
"""Plot and save confusion matrix"""
plt.figure(figsize=(10, 7))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix')
plt.ylabel('Actual')
plt.xlabel('Predicted')
path = os.path.join(self.plots_dir, filename)
plt.savefig(path)
plt.close()
self.logger.info(f"Confusion matrix plot saved to {path}")

def plot_roc_curve(self, y_true: np.ndarray, y_score: np.ndarray, filename: str = 'roc_curve.png'):
"""Plot and save ROC curve"""
fpr, tpr, _ = roc_curve(y_true, y_score)
roc_auc = roc_auc_score(y_true, y_score)

plt.figure(figsize=(10, 7))
plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver Operating Characteristic (ROC)')
plt.legend(loc="lower right")
path = os.path.join(self.plots_dir, filename)
plt.savefig(path)
plt.close()
self.logger.info(f"ROC curve plot saved to {path}")

def plot_residuals(self, y_true: np.ndarray, y_pred: np.ndarray, filename: str = 'residuals.png'):
"""Plot and save residuals plot"""
residuals = y_true - y_pred
plt.figure(figsize=(10, 7))
plt.scatter(y_pred, residuals, alpha=0.5)
plt.axhline(y=0, color='r', linestyle='--')
plt.xlabel('Predicted Values')
plt.ylabel('Residuals')
plt.title('Residuals Plot')
path = os.path.join(self.plots_dir, filename)
plt.savefig(path)
plt.close()
self.logger.info(f"Residuals plot saved to {path}")

def _log_results(self, metrics: Dict[str, Any]):
"""Log evaluation results"""
self.logger.info("Evaluation Results:")
Expand Down