diff --git a/accuracies.txt b/accuracies.txt deleted file mode 100644 index b43b5ff..0000000 --- a/accuracies.txt +++ /dev/null @@ -1,110 +0,0 @@ -Abbott, Andrew: 0.848 accuracy (554 pitches) -Alcantara, Sandy: 0.898 accuracy (578 pitches) -Allen, Logan: 0.939 accuracy (527 pitches) -Anderson, Tyler: 0.932 accuracy (468 pitches) -Bassitt, Chris: 0.792 accuracy (600 pitches) -Baz, Shane: 0.754 accuracy (540 pitches) -Bello, Brayan: 0.869 accuracy (528 pitches) -Berríos, José: 0.765 accuracy (548 pitches) -Bibee, Tanner: 0.798 accuracy (623 pitches) -Boyd, Matthew: 0.909 accuracy (561 pitches) -Bradley, Taj: 0.722 accuracy (475 pitches) -Brown, Hunter: 0.755 accuracy (592 pitches) -Bubic, Kris: 0.844 accuracy (404 pitches) -Buehler, Walker: 0.780 accuracy (451 pitches) -Burke, Sean: 0.820 accuracy (506 pitches) -Cabrera, Edward: 0.762 accuracy (450 pitches) -Cameron, Noah: 0.861 accuracy (433 pitches) -Castillo, Luis: 0.837 accuracy (614 pitches) -Cease, Dylan: 0.721 accuracy (623 pitches) -Cecconi, Slade: 0.569 accuracy (413 pitches) -Corbin, Patrick: 0.894 accuracy (518 pitches) -Crochet, Garrett: 0.910 accuracy (664 pitches) -Elder, Bryce: 0.912 accuracy (511 pitches) -Falter, Bailey: 0.964 accuracy (415 pitches) -Fedde, Erick: 0.822 accuracy (489 pitches) -Flaherty, Jack: 0.952 accuracy (579 pitches) -Freeland, Kyle: 0.817 accuracy (524 pitches) -Fried, Max: 0.866 accuracy (671 pitches) -Gallen, Zac: 0.960 accuracy (642 pitches) -Gausman, Kevin: 0.913 accuracy (619 pitches) -Gilbert, Logan: 0.835 accuracy (436 pitches) -Giolito, Lucas: 0.665 accuracy (474 pitches) -Gore, MacKenzie: 0.946 accuracy (578 pitches) -Gray, Sonny: 0.948 accuracy (576 pitches) -Heaney, Andrew: 0.959 accuracy (416 pitches) -Hendricks, Kyle: 0.986 accuracy (515 pitches) -Holmes, Clay: 0.979 accuracy (569 pitches) -Holmes, Grant: 0.783 accuracy (411 pitches) -Imanaga, Shota: 0.938 accuracy (435 pitches) -Irvin, Jake: 0.680 accuracy (607 pitches) -Keller, Mitch: 0.685 accuracy (574 pitches) -Kelly, Merrill: 0.876 accuracy (580 pitches) -Kikuchi, Yusei: 0.899 accuracy (636 pitches) -Kirby, George: 0.791 accuracy (417 pitches) -Kochanowicz, Jack: 0.649 accuracy (387 pitches) -Kremer, Dean: 0.714 accuracy (539 pitches) -Leiter, Jack: 0.646 accuracy (543 pitches) -Liberatore, Matthew: 0.980 accuracy (497 pitches) -Littell, Zack: 0.967 accuracy (547 pitches) -Lodolo, Nick: 1.000 accuracy (510 pitches) -Lord, Brad: 0.871 accuracy (426 pitches) -Lorenzen, Michael: 0.444 accuracy (491 pitches) -Lugo, Seth: 0.652 accuracy (506 pitches) -Luzardo, Jesús: 0.924 accuracy (618 pitches) -Martin, Davis: 0.731 accuracy (472 pitches) -Martinez, Nick: 0.836 accuracy (531 pitches) -May, Dustin: 0.716 accuracy (447 pitches) -Mikolas, Miles: 0.852 accuracy (514 pitches) -Mize, Casey: 0.679 accuracy (480 pitches) -Morton, Charlie: 0.808 accuracy (510 pitches) -Márquez, Germán: 0.766 accuracy (441 pitches) -Nelson, Ryne: 0.738 accuracy (515 pitches) -Ober, Bailey: 0.925 accuracy (494 pitches) -Paddack, Chris: 0.830 accuracy (542 pitches) -Pallante, Andre: 0.889 accuracy (558 pitches) -Parker, Mitchell: 0.819 accuracy (541 pitches) -Patrick, Chad: 0.744 accuracy (402 pitches) -Pepiot, Ryan: 0.869 accuracy (528 pitches) -Peralta, Freddy: 0.997 accuracy (617 pitches) -Peterson, David: 0.822 accuracy (540 pitches) -Pfaadt, Brandon: 0.473 accuracy (611 pitches) -Pivetta, Nick: 0.892 accuracy (602 pitches) -Povich, Cade: 0.810 accuracy (394 pitches) -Priester, Quinn: 0.725 accuracy (494 pitches) -Quantrill, Cal: 0.814 accuracy (409 pitches) -Quintana, Jose: 0.922 accuracy (435 pitches) -Rasmussen, Drew: 0.827 accuracy (463 pitches) -Ray, Robbie: 0.943 accuracy (628 pitches) -Rea, Colin: 0.803 accuracy (507 pitches) -Rodriguez, Eduardo: 0.979 accuracy (533 pitches) -Rodón, Carlos: 0.950 accuracy (656 pitches) -Ryan, Joe: 0.967 accuracy (570 pitches) -Sale, Chris: 0.978 accuracy (406 pitches) -Sears, JP: 1.000 accuracy (500 pitches) -Senzatela, Antonio: 0.576 accuracy (491 pitches) -Severino, Luis: 0.540 accuracy (546 pitches) -Singer, Brady: 0.932 accuracy (604 pitches) -Skenes, Paul: 0.935 accuracy (627 pitches) -Skubal, Tarik: 0.937 accuracy (605 pitches) -Smith, Shane: 0.337 accuracy (475 pitches) -Soriano, José: 0.927 accuracy (587 pitches) -Springs, Jeffrey: 0.933 accuracy (569 pitches) -Strider, Spencer: 0.744 accuracy (422 pitches) -Sugano, Tomoyuki: 0.739 accuracy (513 pitches) -Suárez, Ranger: 0.914 accuracy (486 pitches) -Sánchez, Cristopher: 0.959 accuracy (591 pitches) -Taillon, Jameson: 0.854 accuracy (396 pitches) -Valdez, Framber: 0.983 accuracy (596 pitches) -Verlander, Justin: 0.993 accuracy (537 pitches) -Vásquez, Randy: 0.430 accuracy (430 pitches) -Wacha, Michael: 0.779 accuracy (566 pitches) -Walker, Taijuan: 0.915 accuracy (446 pitches) -Warren, Will: 0.780 accuracy (576 pitches) -Webb, Logan: 0.869 accuracy (656 pitches) -Wheeler, Zack: 0.895 accuracy (507 pitches) -Williams, Gavin: 0.949 accuracy (631 pitches) -Woo, Bryan: 0.935 accuracy (566 pitches) -Woods Richardson, Simeon: 0.752 accuracy (391 pitches) -Yamamoto, Yoshinobu: 0.864 accuracy (558 pitches) -deGrom, Jacob: 0.925 accuracy (548 pitches) diff --git a/shap_baseball_toolkit.py b/shap_baseball_toolkit.py deleted file mode 100644 index 46d0638..0000000 --- a/shap_baseball_toolkit.py +++ /dev/null @@ -1,124 +0,0 @@ -import numpy as np -import pandas as pd -import shap -import matplotlib.pyplot as plt -import torch -from typing import Dict, List, Any - - -def run_shap_analysis(model, X_train, X_test, y_test, feature_names, player_ids): - """ - Run SHAP analysis on a trained model. - - Args: - model: The trained model to analyze - X_train: Training data used to fit the model - X_test: Test data to analyze - y_test: True labels for test data - feature_names: List of feature names - player_ids: List/array of player IDs aligned with X_test - - Returns: - dict: Dictionary containing SHAP values and related data - """ - # Initialize SHAP explainer - if hasattr(model, 'predict_proba'): - explainer = shap.KernelExplainer( - model.predict_proba, shap.sample(X_train, 100)) - else: - # For PyTorch models - def f(x): - with torch.no_grad(): - return model(torch.tensor(x, dtype=torch.float32)).numpy() - explainer = shap.KernelExplainer(f, shap.sample(X_train.values, 100)) - - # Calculate SHAP values for each class - shap_values = explainer.shap_values(X_test) - - # Get predictions - if hasattr(model, 'predict_proba'): - proba = model.predict_proba(X_test) - else: - with torch.no_grad(): - logits = model(torch.tensor(X_test.values, dtype=torch.float32)) - proba = torch.softmax(logits, dim=1).numpy() - - return { - "shap_values_list": shap_values, - "proba": proba, - "feature_names": feature_names - } - - -def inspect_player(shap_values_list, X_test, feature_names, proba, y_test, player_ids, target_player=None): - """ - Analyze SHAP values for a specific player. - """ - # 1) pick the row index (target player or worst prediction by loss) - if target_player is not None: - idxs = np.where(np.asarray(player_ids) == target_player)[0] - if len(idxs) == 0: - raise ValueError(f"Player {target_player} not found") - idx = int(idxs[0]) - else: - losses = -np.log(proba[np.arange(len(y_test)), y_test.astype(int)]) - idx = int(np.argmax(losses)) - - # 2) choose class to explain: predicted class - pred_class = int(np.argmax(proba[idx])) - - # 3) robustly get the SHAP vector for this row - sv = shap_values_list - if isinstance(sv, list): - # classic multiclass: list length C, each (N, F) - player_shap = sv[pred_class][idx] - elif isinstance(sv, np.ndarray): - if sv.ndim == 2 and sv.shape[0] == len(X_test): - # (N, F): per-row attributions (e.g., ranked_outputs=1) - player_shap = sv[idx] - elif sv.ndim == 3: - # Try (C, N, F) - if sv.shape[0] <= 10 and sv.shape[1] == len(X_test): - player_shap = sv[pred_class, idx, :] - # Try (N, C, F) <-- your case: (6, 51, 110) - elif sv.shape[0] == len(X_test) and sv.shape[2] == len(feature_names): - player_shap = sv[idx, pred_class, :] - # Try (N, F, C) - elif sv.shape[0] == len(X_test) and sv.shape[1] == len(feature_names): - player_shap = sv[idx, :, pred_class] - else: - raise ValueError(f"Unexpected 3D SHAP shape {sv.shape}") - else: - raise ValueError(f"Unexpected SHAP shape {sv.shape}") - else: - raise TypeError(f"Unsupported SHAP container type: {type(sv)}") - assert player_shap.shape[0] == len(feature_names), "SHAP vector length mismatch" - - # 4) plot a compact bar chart of top contributions (instead of summary_plot on a single row) - contrib = pd.DataFrame({ - "feature": feature_names, - "shap_value": player_shap, - "abs_val": np.abs(player_shap) - }).sort_values("abs_val", ascending=False).head(20) - - plt.figure(figsize=(10, 6)) - plt.barh(contrib["feature"][::-1], contrib["shap_value"][::-1]) - plt.xlabel("SHAP contribution (predicted class)") - plt.title(f"{player_ids[idx]} — row {idx} — pred class {pred_class}") - plt.tight_layout() - - import os - os.makedirs('outputs', exist_ok=True) - outpath = f'outputs/example_player_shap_{str(player_ids[idx]).replace(" ", "_")}.png' - plt.savefig(outpath, dpi=200) - plt.close() - - return { - 'player_id': player_ids[idx], - 'row_index': idx, - 'predicted_class': pred_class, - 'true_label': int(y_test[idx]), - 'predicted_proba': proba[idx].tolist(), - 'top_features': contrib[["feature", "shap_value"]].to_dict(orient="records"), - 'plot_path': outpath - }