diff --git a/aidrin/__init__.py b/aidrin/__init__.py index df2b0625..5ee9f287 100644 --- a/aidrin/__init__.py +++ b/aidrin/__init__.py @@ -13,7 +13,7 @@ def create_app(): @app.context_processor def inject_version(): return dict(app_version=__version__) # global variable to access version in templates - app.secret_key = "aidrin" + app.secret_key = os.environ.get("AIDRIN_SECRET_KEY", "change-me-in-production") #Updated this env variable from static hardcoded to dynamic # Celery Config app.config["CELERY"] = { "broker_url": "redis://localhost:6379/0", @@ -51,6 +51,7 @@ def inject_version(): max_age_seconds = 3600 # 1 hour files_removed = 0 + for filename in os.listdir(UPLOAD_FOLDER): file_path = os.path.join(UPLOAD_FOLDER, filename) try: diff --git a/aidrin/structured_data_metrics/correlation_score.py b/aidrin/structured_data_metrics/correlation_score.py index 7e0afca6..a9c69095 100644 --- a/aidrin/structured_data_metrics/correlation_score.py +++ b/aidrin/structured_data_metrics/correlation_score.py @@ -8,7 +8,7 @@ from celery import Task, shared_task from celery.exceptions import SoftTimeLimitExceeded from dython.nominal import associations - +import logging from aidrin.file_handling.file_parser import read_file matplotlib.use("Agg") @@ -36,7 +36,9 @@ def calc_correlations(self: Task, columns: List[str], file_info): categorical_correlation = associations( df[categorical_columns], nom_nom_assoc=NOMINAL_NOMINAL_ASSOC, plot=False ) - print(categorical_correlation["corr"]) + #print(categorical_correlation["corr"]) + logger=logging.getLogger(__name__) + logger.info(f"Categorical correlation matrix: {categorical_correlation['corr']}")# added logging for categorical correlation matrix # Create a subplot with 1 row and 1 column _, axes = plt.subplots(1, 1, figsize=(8, 8)) diff --git a/aidrin/structured_data_metrics/feature_relevance.py b/aidrin/structured_data_metrics/feature_relevance.py index e708caa2..aa49a737 100644 --- a/aidrin/structured_data_metrics/feature_relevance.py +++ b/aidrin/structured_data_metrics/feature_relevance.py @@ -1,11 +1,13 @@ import base64 import io - +import logging +from venv import logger import matplotlib.pyplot as plt import numpy as np import pandas as pd from celery import Task, shared_task from celery.exceptions import SoftTimeLimitExceeded + from sklearn.preprocessing import LabelEncoder from aidrin.file_handling.file_parser import read_file @@ -151,22 +153,27 @@ @shared_task(bind=True, ignore_result=False) def data_cleaning(self: Task, cat_cols, num_cols, target_col, file_info): try: - print(f"Starting data_cleaning with cat_cols: {cat_cols}, num_cols: {num_cols}, target_col: {target_col}") + #print(f"Starting data_cleaning with cat_cols: {cat_cols}, num_cols: {num_cols}, target_col: {target_col}") + + logger = logging.getLogger(__name__) + logger.info(f"Starting data_cleaning with cat_cols: {cat_cols}, num_cols: {num_cols}, target_col: {target_col}") try: df = read_file(file_info) - print(f"File read successfully. DataFrame shape: {df.shape}") - print(f"DataFrame columns: {list(df.columns)}") - print(f"DataFrame dtypes: {df.dtypes.to_dict()}") + #changed the logging type from print to logging method as a standard approach protocol + logger.info(f"File read successfully. DataFrame shape: {df.shape}") + logger.info(f"DataFrame columns: {list(df.columns)}") + logger.info(f"DataFrame dtypes: {df.dtypes.to_dict()}") + except Exception as e: - print(f"Error reading file: {e}") + logger.error(f"Error reading file: {e}") return { "Error": "Failed to read the file. Please check the file path and type." } # Filter DataFrame to include only the specified columns selected_columns = [target_col] + cat_cols + num_cols - print(f"Selected columns: {selected_columns}") + logger.info(f"Selected columns: {selected_columns}") # Check if all columns exist missing_columns = [col for col in selected_columns if col not in df.columns] @@ -180,133 +187,133 @@ def data_cleaning(self: Task, cat_cols, num_cols, target_col, file_info): } df_filtered = df[selected_columns].copy() - print(f"Filtered DataFrame shape: {df_filtered.shape}") - + logger.info(f"Filtered DataFrame shape: {df_filtered.shape}") + # Fill missing values more robustly if cat_cols: # Only process if there are categorical columns for col in cat_cols: try: - print(f"Processing categorical column: {col}") - print(f"Column {col} unique values before fillna: {df_filtered[col].nunique()}") + logger.info(f"Processing categorical column: {col}") + logger.info(f"Column {col} unique values before fillna: {df_filtered[col].nunique()}") df_filtered[col] = df_filtered[col].fillna("Missing") - print(f"Column {col} unique values after fillna: {df_filtered[col].nunique()}") + logger.info(f"Column {col} unique values after fillna: {df_filtered[col].nunique()}") except Exception as e: - print(f"Warning: Error filling missing values in categorical column {col}: {e}") + logger.warning(f"Warning: Error filling missing values in categorical column {col}: {e}") # Fallback: replace NaN with a default value df_filtered[col] = df_filtered[col].astype(str).replace('nan', 'Missing') else: - print("No categorical columns to process") + logger.info("No categorical columns to process") if num_cols: # Only process if there are numerical columns for col in num_cols: try: - print(f"Processing numerical column: {col}") - print(f"Column {col} data type: {df_filtered[col].dtype}") + logger.info(f"Processing numerical column: {col}") + logger.info(f"Column {col} data type: {df_filtered[col].dtype}") # Calculate mean safely col_mean = df_filtered[col].mean() if pd.isna(col_mean): col_mean = 0.0 - print(f"Column {col} mean: {col_mean}") + logger.info(f"Column {col} mean: {col_mean}") df_filtered[col] = df_filtered[col].fillna(col_mean) except Exception as e: - print(f"Warning: Error filling missing values in numerical column {col}: {e}") + logger.warning(f"Warning: Error filling missing values in numerical column {col}: {e}") # Fallback: replace NaN with 0 df_filtered[col] = df_filtered[col].fillna(0.0) else: - print("No numerical columns to process") + logger.info("No numerical columns to process") # One-hot encode categorical columns only if they exist if cat_cols: try: - print(f"Starting one-hot encoding for {len(cat_cols)} categorical columns...") + logger.info(f"Starting one-hot encoding for {len(cat_cols)} categorical columns...") df_filtered = pd.get_dummies(df_filtered, columns=cat_cols) - print(f"One-hot encoding completed. DataFrame now has {df_filtered.shape[1]} columns.") + logger.info(f"One-hot encoding completed. DataFrame now has {df_filtered.shape[1]} columns.") except Exception as e: - print(f"Error during one-hot encoding: {e}") + logger.error(f"Error during one-hot encoding: {e}") return {"Error": f"One-hot encoding failed: {str(e)}"} else: - print("No categorical columns to encode") + logger.info("No categorical columns to encode") # Encode target variable if categorical if pd.api.types.is_object_dtype(df_filtered[target_col]) or isinstance(df_filtered[target_col].dtype, pd.StringDtype): try: - print(f"Encoding target column {target_col}...") + logger.info(f"Encoding target column {target_col}...") le_target = LabelEncoder() df_filtered[target_col] = le_target.fit_transform(df_filtered[target_col]) - print(f"Target column {target_col} encoded successfully.") + logger.info(f"Target column {target_col} encoded successfully.") except Exception as e: - print(f"Error encoding target column: {e}") + logger.error(f"Error encoding target column: {e}") return {"Error": f"Target column encoding failed: {str(e)}"} # Convert to JSON more safely try: - print("Converting DataFrame to JSON format...") + logger.info("Converting DataFrame to JSON format...") result = df_filtered.to_dict(orient="list") - print(f"Data cleaning completed successfully. Final data shape: {df_filtered.shape}") + logger.info(f"Data cleaning completed successfully. Final data shape: {df_filtered.shape}") return result except Exception as e: - print(f"Error converting to JSON: {e}") + logger.error(f"Error converting to JSON: {e}") return {"Error": f"JSON conversion failed: {str(e)}"} except SoftTimeLimitExceeded: - print("Data Cleaning task timed out.") + logger.info("Data Cleaning task timed out.") raise Exception("Data Cleaning task timed out.") except Exception as e: - print(f"Error occurred during data cleaning: {e}") + logger.error(f"Error occurred during data cleaning: {e}") return {"Error": f"Data cleaning failed: {str(e)}"} @shared_task(bind=True, ignore_result=False) def pearson_correlation(self: Task, df_json, target_col) -> dict: try: - print(f"Starting pearson_correlation with target_col: {target_col}") - print(f"Input df_json type: {type(df_json)}") + logger.info(f"Starting pearson_correlation with target_col: {target_col}") + logger.info(f"Input df_json type: {type(df_json)}") if isinstance(df_json, dict): - print(f"Input df_json keys: {list(df_json.keys())}") + logger.info(f"Input df_json keys: {list(df_json.keys())}") # Convert JSON back to DataFrame with proper error handling try: df = pd.DataFrame.from_dict(df_json) - print(f"DataFrame created successfully. Shape: {df.shape}") - print(f"DataFrame columns: {list(df.columns)}") - print(f"DataFrame dtypes: {df.dtypes.to_dict()}") + logger.info(f"DataFrame created successfully. Shape: {df.shape}") + logger.info(f"DataFrame columns: {list(df.columns)}") + logger.info(f"DataFrame dtypes: {df.dtypes.to_dict()}") except Exception as e: - print(f"Error converting JSON to DataFrame: {e}") + logger.error(f"Error converting JSON to DataFrame: {e}") return {"Error": f"Failed to convert data: {str(e)}"} # Ensure target column exists if target_col not in df.columns: - print(f"Target column '{target_col}' not found. Available columns: {list(df.columns)}") + logger.error(f"Target column '{target_col}' not found. Available columns: {list(df.columns)}") return {"Error": f"Target column '{target_col}' not found in the data"} # Get columns excluding target column cols = df.columns.difference([target_col]) if len(cols) == 0: - print("No feature columns found for correlation analysis") + logger.info("No feature columns found for correlation analysis") return {"Error": "No feature columns found for correlation analysis"} - print(f"Processing {len(cols)} feature columns: {list(cols)}") + logger.info(f"Processing {len(cols)} feature columns: {list(cols)}") correlations = {} - for col in cols: + for col in cols: if col != target_col: try: - print(f"Processing column: {col}") - print(f"Column {col} dtype: {df[col].dtype}") - print(f"Target column {target_col} dtype: {df[target_col].dtype}") + logger.info(f"Processing column: {col}") + logger.info(f"Column {col} dtype: {df[col].dtype}") + logger.info(f"Target column {target_col} dtype: {df[target_col].dtype}") # Ensure both columns are numeric if not pd.api.types.is_numeric_dtype(df[col]) or not pd.api.types.is_numeric_dtype(df[target_col]): - print(f"Warning: Skipping column '{col}' - non-numeric data types") + logger.warning(f"Skipping column '{col}' - non-numeric data types") continue # Remove any NaN values for this specific column pair valid_data = df[[col, target_col]].dropna() if len(valid_data) < 2: - print(f"Warning: Skipping column '{col}' - insufficient valid data after removing NaN values") + logger.warning(f"Skipping column '{col}' - insufficient valid data after removing NaN values") continue - print(f"Column {col} valid data points: {len(valid_data)}") + logger.info(f"Column {col} valid data points: {len(valid_data)}") # Calculate covariance cov = np.cov(valid_data[col], valid_data[target_col], ddof=0)[0, 1] @@ -316,7 +323,7 @@ def pearson_correlation(self: Task, df_json, target_col) -> dict: # Check for division by zero if std_dev_col == 0 or std_dev_target == 0: - print(f"Warning: Skipping column '{col}' - zero standard deviation") + logger.warning(f"Warning: Skipping column '{col}' - zero standard deviation") continue # Calculate Pearson correlation coefficient @@ -325,25 +332,25 @@ def pearson_correlation(self: Task, df_json, target_col) -> dict: # Ensure correlation is a valid number if np.isfinite(corr): correlations[col] = float(corr) # Convert to Python float for JSON serialization - print(f"Column {col} correlation: {corr}") + logger.info(f"Column {col} correlation: {corr}") else: - print(f"Warning: Skipping column '{col}' - invalid correlation value: {corr}") + logger.warning(f"Warning: Skipping column '{col}' - invalid correlation value: {corr}") except Exception as e: - print(f"Warning: Error calculating correlation for column '{col}': {e}") + logger.warning(f"Warning: Error calculating correlation for column '{col}': {e}") continue if not correlations: - print("No valid correlations could be calculated") + logger.error("No valid correlations could be calculated") return {"Error": "No valid correlations could be calculated"} - print(f"Successfully calculated correlations for {len(correlations)} features") + logger.info(f"Successfully calculated correlations for {len(correlations)} features") return correlations except SoftTimeLimitExceeded: raise Exception("Pearson Correlation task timed out.") except Exception as e: - print(f"Unexpected error in pearson_correlation: {e}") + logger.error(f"Unexpected error in pearson_correlation: {e}") return {"Error": f"Correlation calculation failed: {str(e)}"} diff --git a/aidrin/structured_data_metrics/privacy_measure.py b/aidrin/structured_data_metrics/privacy_measure.py index 4ac5b826..b5beabbc 100644 --- a/aidrin/structured_data_metrics/privacy_measure.py +++ b/aidrin/structured_data_metrics/privacy_measure.py @@ -74,7 +74,8 @@ def generate_single_attribute_MM_risk_scores(df, id_col, eval_cols, task=None): # Drop rows with missing values selected_df = selected_df.dropna() rows_after_dropna = len(selected_df) - print(rows_after_dropna) + logger = logging.getLogger(__name__) + logger.info(f"rows left after droping NaN values {rows_after_dropna}")# changed the logging type from print to logging method as a standard approach protocol if rows_after_dropna == 0: raise ValueError("After removing missing values, no data remains. Please check your data quality or select different columns.") @@ -250,7 +251,9 @@ def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None) rows_after_dropna = len(selected_df) if rows_after_dropna == 0: - print("DEBUG: About to raise ValueError - no data remains after dropna") + + logger = logging.getLogger(__name__) + logger.info("About to raise ValueError - no data remains after dropna")# changed the logging type from print to logging method as a standard approach protocol raise ValueError("After removing missing values, no data remains. Please check your data quality or select different columns.") # Check data quality for quasi-identifiers