diff --git a/scripts/Imputing.py b/scripts/Imputing.py index e69de29..4b6bbda 100644 --- a/scripts/Imputing.py +++ b/scripts/Imputing.py @@ -0,0 +1,29 @@ + +import pandas as pd +import numpy as np +from sklearn.impute import KNNImputer +#this function imputes missing values from any df with KNNImputer +def knn_impute_numeric(df, n_neighbors=5): + + #select only the numerical variables to do the imputer + numeric_cols = df.select_dtypes(include=['float','int']) + print(f": shape of df with only numeric features={numeric_cols.shape}") + + #Impute the numerical variables + imputer = KNNImputer(missing_values=np.nan, n_neighbors=n_neighbors, keep_empty_features=True) + imputed_numeric_data = imputer.fit_transform(numeric_cols) + + # Create a new DataFrame with the imputed numeric data + imputed_numeric_df = pd.DataFrame(imputed_numeric_data, columns=numeric_cols.columns, index=df.index) + + # Combine the imputed numeric columns with the non-numeric columns to ensure that we keep them + non_numeric_cols = df.select_dtypes(exclude=['float','int']) + imputed_df = pd.concat([imputed_numeric_df, non_numeric_cols], axis=1) + + # Ensure the column order is the same as the original dataframe + imputed_df = imputed_df[df.columns] + + return imputed_df + +#imputed_df = knn_impute_numeric(filtered_df, n_neighbors=5) +#imputed_df.head() diff --git a/scripts/Scaling.py b/scripts/Scaling.py index e69de29..6e15393 100644 --- a/scripts/Scaling.py +++ b/scripts/Scaling.py @@ -0,0 +1,31 @@ + +#This function scales numerical values +from sklearn.preprocessing import RobustScaler +import pandas as pd +def scaler_numeric(df, target_col=''): + + #separate the nutriscore and the rest of the values to do the scaling + X = df.drop([target_col], axis = 1) + y = df[target_col] + + #select only the numerical variables to do the scaling + X_numeric = X.select_dtypes(include=['float','int']) + print(f": shape of df with only numeric features={X_numeric.shape}") + + #scale the numerical values and put the scaled numerical data into a dataframe + scaler = RobustScaler() + X_scaled = scaler.fit_transform(X_numeric) + X_scaled_df = pd.DataFrame(X_scaled, columns=X_numeric.columns, index=X_numeric.index) + + #combine the scaled df with the nutriscore + X_non_numeric = X.select_dtypes(exclude=['float','int']) + X_processed = pd.concat([X_scaled_df, X_non_numeric], axis=1) + scaled_df = pd.concat([X_processed, y], axis=1) + + # Ensure the column order is the same as the original dataframe + scaled_df = scaled_df[df.columns] + + return scaled_df + +#scaled_df = scaler_numeric(imputed_df, target_col='nutriscore_score') +#scaled_df.head()