Skip to content

Fix logging refactor #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Tushar-sketch-bit wants to merge 2 commits into from
Closed

Fix logging refactor #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
80e232b
Loaded the secret key from an environment variable instead of hardcod…
Tushar-sketch-bit Mar 16, 2026
45f91b9
Replaced debug print() statements with logging in metric modules to e…
Tushar-sketch-bit Mar 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion aidrin/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def create_app():
@app.context_processor
def inject_version():
return dict(app_version=__version__) # global variable to access version in templates
app.secret_key = "aidrin"
app.secret_key = os.environ.get("AIDRIN_SECRET_KEY", "change-me-in-production") #Updated this env variable from static hardcoded to dynamic
# Celery Config
app.config["CELERY"] = {
"broker_url": "redis://localhost:6379/0",
Expand Down Expand Up @@ -51,6 +51,7 @@ def inject_version():
max_age_seconds = 3600 # 1 hour
files_removed = 0


for filename in os.listdir(UPLOAD_FOLDER):
file_path = os.path.join(UPLOAD_FOLDER, filename)
try:
Expand Down
6 changes: 4 additions & 2 deletions aidrin/structured_data_metrics/correlation_score.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from celery import Task, shared_task
from celery.exceptions import SoftTimeLimitExceeded
from dython.nominal import associations

import logging
from aidrin.file_handling.file_parser import read_file

matplotlib.use("Agg")
Expand Down Expand Up @@ -36,7 +36,9 @@ def calc_correlations(self: Task, columns: List[str], file_info):
categorical_correlation = associations(
df[categorical_columns], nom_nom_assoc=NOMINAL_NOMINAL_ASSOC, plot=False
)
print(categorical_correlation["corr"])
#print(categorical_correlation["corr"])
logger=logging.getLogger(__name__)
logger.info(f"Categorical correlation matrix: {categorical_correlation['corr']}")# added logging for categorical correlation matrix

# Create a subplot with 1 row and 1 column
_, axes = plt.subplots(1, 1, figsize=(8, 8))
Expand Down
117 changes: 62 additions & 55 deletions aidrin/structured_data_metrics/feature_relevance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import base64
import io

import logging
from venv import logger
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from celery import Task, shared_task
from celery.exceptions import SoftTimeLimitExceeded

from sklearn.preprocessing import LabelEncoder

from aidrin.file_handling.file_parser import read_file
Expand Down Expand Up @@ -151,22 +153,27 @@
@shared_task(bind=True, ignore_result=False)
def data_cleaning(self: Task, cat_cols, num_cols, target_col, file_info):
try:
print(f"Starting data_cleaning with cat_cols: {cat_cols}, num_cols: {num_cols}, target_col: {target_col}")
#print(f"Starting data_cleaning with cat_cols: {cat_cols}, num_cols: {num_cols}, target_col: {target_col}")

logger = logging.getLogger(__name__)
logger.info(f"Starting data_cleaning with cat_cols: {cat_cols}, num_cols: {num_cols}, target_col: {target_col}")

try:
df = read_file(file_info)
print(f"File read successfully. DataFrame shape: {df.shape}")
print(f"DataFrame columns: {list(df.columns)}")
print(f"DataFrame dtypes: {df.dtypes.to_dict()}")
#changed the logging type from print to logging method as a standard approach protocol
logger.info(f"File read successfully. DataFrame shape: {df.shape}")
logger.info(f"DataFrame columns: {list(df.columns)}")
logger.info(f"DataFrame dtypes: {df.dtypes.to_dict()}")

except Exception as e:
print(f"Error reading file: {e}")
logger.error(f"Error reading file: {e}")
return {
"Error": "Failed to read the file. Please check the file path and type."
}

# Filter DataFrame to include only the specified columns
selected_columns = [target_col] + cat_cols + num_cols
print(f"Selected columns: {selected_columns}")
logger.info(f"Selected columns: {selected_columns}")

# Check if all columns exist
missing_columns = [col for col in selected_columns if col not in df.columns]
Expand All @@ -180,133 +187,133 @@ def data_cleaning(self: Task, cat_cols, num_cols, target_col, file_info):
}

df_filtered = df[selected_columns].copy()
print(f"Filtered DataFrame shape: {df_filtered.shape}")

logger.info(f"Filtered DataFrame shape: {df_filtered.shape}")
# Fill missing values more robustly
if cat_cols: # Only process if there are categorical columns
for col in cat_cols:
try:
print(f"Processing categorical column: {col}")
print(f"Column {col} unique values before fillna: {df_filtered[col].nunique()}")
logger.info(f"Processing categorical column: {col}")
logger.info(f"Column {col} unique values before fillna: {df_filtered[col].nunique()}")
df_filtered[col] = df_filtered[col].fillna("Missing")
print(f"Column {col} unique values after fillna: {df_filtered[col].nunique()}")
logger.info(f"Column {col} unique values after fillna: {df_filtered[col].nunique()}")
except Exception as e:
print(f"Warning: Error filling missing values in categorical column {col}: {e}")
logger.warning(f"Warning: Error filling missing values in categorical column {col}: {e}")
# Fallback: replace NaN with a default value
df_filtered[col] = df_filtered[col].astype(str).replace('nan', 'Missing')
else:
print("No categorical columns to process")
logger.info("No categorical columns to process")

if num_cols: # Only process if there are numerical columns
for col in num_cols:
try:
print(f"Processing numerical column: {col}")
print(f"Column {col} data type: {df_filtered[col].dtype}")
logger.info(f"Processing numerical column: {col}")
logger.info(f"Column {col} data type: {df_filtered[col].dtype}")
# Calculate mean safely
col_mean = df_filtered[col].mean()
if pd.isna(col_mean):
col_mean = 0.0
print(f"Column {col} mean: {col_mean}")
logger.info(f"Column {col} mean: {col_mean}")
df_filtered[col] = df_filtered[col].fillna(col_mean)
except Exception as e:
print(f"Warning: Error filling missing values in numerical column {col}: {e}")
logger.warning(f"Warning: Error filling missing values in numerical column {col}: {e}")
# Fallback: replace NaN with 0
df_filtered[col] = df_filtered[col].fillna(0.0)
else:
print("No numerical columns to process")
logger.info("No numerical columns to process")

# One-hot encode categorical columns only if they exist
if cat_cols:
try:
print(f"Starting one-hot encoding for {len(cat_cols)} categorical columns...")
logger.info(f"Starting one-hot encoding for {len(cat_cols)} categorical columns...")
df_filtered = pd.get_dummies(df_filtered, columns=cat_cols)
print(f"One-hot encoding completed. DataFrame now has {df_filtered.shape[1]} columns.")
logger.info(f"One-hot encoding completed. DataFrame now has {df_filtered.shape[1]} columns.")
except Exception as e:
print(f"Error during one-hot encoding: {e}")
logger.error(f"Error during one-hot encoding: {e}")
return {"Error": f"One-hot encoding failed: {str(e)}"}
else:
print("No categorical columns to encode")
logger.info("No categorical columns to encode")

# Encode target variable if categorical
if pd.api.types.is_object_dtype(df_filtered[target_col]) or isinstance(df_filtered[target_col].dtype, pd.StringDtype):
try:
print(f"Encoding target column {target_col}...")
logger.info(f"Encoding target column {target_col}...")
le_target = LabelEncoder()
df_filtered[target_col] = le_target.fit_transform(df_filtered[target_col])
print(f"Target column {target_col} encoded successfully.")
logger.info(f"Target column {target_col} encoded successfully.")
except Exception as e:
print(f"Error encoding target column: {e}")
logger.error(f"Error encoding target column: {e}")
return {"Error": f"Target column encoding failed: {str(e)}"}

# Convert to JSON more safely
try:
print("Converting DataFrame to JSON format...")
logger.info("Converting DataFrame to JSON format...")
result = df_filtered.to_dict(orient="list")
print(f"Data cleaning completed successfully. Final data shape: {df_filtered.shape}")
logger.info(f"Data cleaning completed successfully. Final data shape: {df_filtered.shape}")
return result
except Exception as e:
print(f"Error converting to JSON: {e}")
logger.error(f"Error converting to JSON: {e}")
return {"Error": f"JSON conversion failed: {str(e)}"}

except SoftTimeLimitExceeded:
print("Data Cleaning task timed out.")
logger.info("Data Cleaning task timed out.")
raise Exception("Data Cleaning task timed out.")
except Exception as e:
print(f"Error occurred during data cleaning: {e}")
logger.error(f"Error occurred during data cleaning: {e}")
return {"Error": f"Data cleaning failed: {str(e)}"}


@shared_task(bind=True, ignore_result=False)
def pearson_correlation(self: Task, df_json, target_col) -> dict:
try:
print(f"Starting pearson_correlation with target_col: {target_col}")
print(f"Input df_json type: {type(df_json)}")
logger.info(f"Starting pearson_correlation with target_col: {target_col}")
logger.info(f"Input df_json type: {type(df_json)}")
if isinstance(df_json, dict):
print(f"Input df_json keys: {list(df_json.keys())}")
logger.info(f"Input df_json keys: {list(df_json.keys())}")

# Convert JSON back to DataFrame with proper error handling
try:
df = pd.DataFrame.from_dict(df_json)
print(f"DataFrame created successfully. Shape: {df.shape}")
print(f"DataFrame columns: {list(df.columns)}")
print(f"DataFrame dtypes: {df.dtypes.to_dict()}")
logger.info(f"DataFrame created successfully. Shape: {df.shape}")
logger.info(f"DataFrame columns: {list(df.columns)}")
logger.info(f"DataFrame dtypes: {df.dtypes.to_dict()}")
except Exception as e:
print(f"Error converting JSON to DataFrame: {e}")
logger.error(f"Error converting JSON to DataFrame: {e}")
return {"Error": f"Failed to convert data: {str(e)}"}

# Ensure target column exists
if target_col not in df.columns:
print(f"Target column '{target_col}' not found. Available columns: {list(df.columns)}")
logger.error(f"Target column '{target_col}' not found. Available columns: {list(df.columns)}")
return {"Error": f"Target column '{target_col}' not found in the data"}

# Get columns excluding target column
cols = df.columns.difference([target_col])
if len(cols) == 0:
print("No feature columns found for correlation analysis")
logger.info("No feature columns found for correlation analysis")
return {"Error": "No feature columns found for correlation analysis"}

print(f"Processing {len(cols)} feature columns: {list(cols)}")
logger.info(f"Processing {len(cols)} feature columns: {list(cols)}")

correlations = {}
for col in cols:
for col in cols:
if col != target_col:
try:
print(f"Processing column: {col}")
print(f"Column {col} dtype: {df[col].dtype}")
print(f"Target column {target_col} dtype: {df[target_col].dtype}")
logger.info(f"Processing column: {col}")
logger.info(f"Column {col} dtype: {df[col].dtype}")
logger.info(f"Target column {target_col} dtype: {df[target_col].dtype}")

# Ensure both columns are numeric
if not pd.api.types.is_numeric_dtype(df[col]) or not pd.api.types.is_numeric_dtype(df[target_col]):
print(f"Warning: Skipping column '{col}' - non-numeric data types")
logger.warning(f"Skipping column '{col}' - non-numeric data types")
continue

# Remove any NaN values for this specific column pair
valid_data = df[[col, target_col]].dropna()
if len(valid_data) < 2:
print(f"Warning: Skipping column '{col}' - insufficient valid data after removing NaN values")
logger.warning(f"Skipping column '{col}' - insufficient valid data after removing NaN values")
continue

print(f"Column {col} valid data points: {len(valid_data)}")
logger.info(f"Column {col} valid data points: {len(valid_data)}")

# Calculate covariance
cov = np.cov(valid_data[col], valid_data[target_col], ddof=0)[0, 1]
Expand All @@ -316,7 +323,7 @@ def pearson_correlation(self: Task, df_json, target_col) -> dict:

# Check for division by zero
if std_dev_col == 0 or std_dev_target == 0:
print(f"Warning: Skipping column '{col}' - zero standard deviation")
logger.warning(f"Warning: Skipping column '{col}' - zero standard deviation")
continue

# Calculate Pearson correlation coefficient
Expand All @@ -325,25 +332,25 @@ def pearson_correlation(self: Task, df_json, target_col) -> dict:
# Ensure correlation is a valid number
if np.isfinite(corr):
correlations[col] = float(corr) # Convert to Python float for JSON serialization
print(f"Column {col} correlation: {corr}")
logger.info(f"Column {col} correlation: {corr}")
else:
print(f"Warning: Skipping column '{col}' - invalid correlation value: {corr}")
logger.warning(f"Warning: Skipping column '{col}' - invalid correlation value: {corr}")

except Exception as e:
print(f"Warning: Error calculating correlation for column '{col}': {e}")
logger.warning(f"Warning: Error calculating correlation for column '{col}': {e}")
continue

if not correlations:
print("No valid correlations could be calculated")
logger.error("No valid correlations could be calculated")
return {"Error": "No valid correlations could be calculated"}

print(f"Successfully calculated correlations for {len(correlations)} features")
logger.info(f"Successfully calculated correlations for {len(correlations)} features")
return correlations

except SoftTimeLimitExceeded:
raise Exception("Pearson Correlation task timed out.")
except Exception as e:
print(f"Unexpected error in pearson_correlation: {e}")
logger.error(f"Unexpected error in pearson_correlation: {e}")
return {"Error": f"Correlation calculation failed: {str(e)}"}


Expand Down
7 changes: 5 additions & 2 deletions aidrin/structured_data_metrics/privacy_measure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ def generate_single_attribute_MM_risk_scores(df, id_col, eval_cols, task=None):
# Drop rows with missing values
selected_df = selected_df.dropna()
rows_after_dropna = len(selected_df)
print(rows_after_dropna)
logger = logging.getLogger(__name__)
logger.info(f"rows left after droping NaN values {rows_after_dropna}")# changed the logging type from print to logging method as a standard approach protocol
if rows_after_dropna == 0:
raise ValueError("After removing missing values, no data remains. Please check your data quality or select different columns.")

Expand Down Expand Up @@ -250,7 +251,9 @@ def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None)
rows_after_dropna = len(selected_df)

if rows_after_dropna == 0:
print("DEBUG: About to raise ValueError - no data remains after dropna")

logger = logging.getLogger(__name__)
logger.info("About to raise ValueError - no data remains after dropna")# changed the logging type from print to logging method as a standard approach protocol
raise ValueError("After removing missing values, no data remains. Please check your data quality or select different columns.")

# Check data quality for quasi-identifiers
Expand Down
Loading