Outliers implementation

scripts/__init__.py

@@ -1 +1,3 @@
-from .Data_filter_Jess import * 
+from .preprocessing import search_variable_name
+from .Data_filter_Jess import * 
+

scripts/preprocessing.py

@@ -0,0 +1,73 @@
+import numpy as np
+import pandas
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.ensemble import IsolationForest
+
+def search_variable_name(df, name):
+    "Use this function to search for column names based on a key world"
+
+    df.columns = df.columns.str.lower()
+    interesting_cols = [col for col in list(df.columns) if name in col]
+    good_interesting_cols = []
+    for interesting_col in interesting_cols:
+        if df[interesting_col].isna().sum()/len(df[interesting_col]) < 0.99:
+            good_interesting_cols.append(interesting_col)
+    return good_interesting_cols
+
+def plots_for_outliers(df, selected_cols, save):
+    """Plots seaborns pairplot and boxplot for selected variables
+
+    Inputs: dataframe, selected columns, binary value if either save or not the images
+
+    Returns: nothing, either shows or saves the figures
+    """
+
+    if len(selected_cols) > 8: 
+        print("Too many features selected. Please choose 8 or fewer.")
+        return
+
+    # --- Pairplot ---
+    pairplot = sns.pairplot(df[selected_cols])
+    if save: 
+        pairplot.savefig(f'../data/PAIR_{",".join(selected_cols)}.png')
+        plt.close(pairplot.fig)
+    else:
+        plt.show()
+
+    # --- Boxplot ---
+    plt.figure(figsize=(8, 6))
+    sns.boxplot(data=df[selected_cols])
+    plt.xticks(rotation=90)
+    if save: 
+        plt.savefig(f'../data/BOX_{",".join(selected_cols)}.png')
+        plt.close()
+    else:
+        plt.show()
+
+def isoforest_for_outliers(df, selected_cols):
+    """Perform Isolation Forest algo by scikit-learn to detect outliers"
+
+    Inputs: dataframe and selected column
+
+    Returns: indices of the dataframe there were identified as outliers
+    """
+
+    n_estimators = 100  # Number of trees
+    contamination = 0.05 # Expected proportion of anomalies
+    sample_size = df.shape[0]
+
+    iso_forest = IsolationForest(n_estimators=n_estimators,
+                                contamination=contamination,
+                                max_samples=sample_size,
+                                random_state=42)
+    features = df[selected_cols]
+    iso_forest.fit(features)
+    iso_forest_labels = iso_forest.predict(features)
+
+    if np.any(iso_forest_labels<0):
+        print('Anomalies detected')
+        np.where(iso_forest_labels<0)
+
+    else:
+        print('No anomalies detected')