From cfcb2dd2f8c9fe4cd24dd21cd803e30b8a682134 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 16 Jun 2026 18:42:50 -0400 Subject: [PATCH 01/23] Add automated Readiness Report tab with aggregated readiness metrics Introduce a non-interactive Readiness Report in the inspector that runs without user feature or target selection. Add GET /readiness-report to aggregate dataset overview (per-feature profiles, histograms, categorical charts), data quality scorecards, impact-on-AI signals from vectorized all-pairs correlation (leakage, redundancy, isolated features), and automated fairness & bias checks with auto-selected columns. Wire the new sidebar panel and lazy-loaded UI renderers, add chart helpers for histograms and categorical distributions, and include info-tooltips that explain each readiness metric. --- web/routes/metrics.py | 920 +++++++++++++++++++++++ web/routes/utils.py | 78 +- web/static/css/theme.css | 21 +- web/static/js/inspector.js | 975 ++++++++++++++++++++++++- web/templates/_components/sidebar.html | 8 + web/templates/inspector.html | 1 + 6 files changed, 1977 insertions(+), 26 deletions(-) diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 6e3a8def..fb8b8caa 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -1,7 +1,9 @@ import json import logging +import os import time +import pandas as pd from celery.result import AsyncResult from web.telemetry import get_tracer, trace_metric from flask import ( @@ -57,6 +59,8 @@ get_result_or_default, is_metric_cache_valid, store_result, + summary_histograms, + categorical_distribution_charts, ) metrics_bp = Blueprint("metrics", __name__) @@ -137,6 +141,922 @@ def data_quality(): return get_result_or_default("metrics.data_quality", file_path, file_name) +# --------------------------------------------------------------------------- +# Readiness Report (aggregated, non-interactive) +# --------------------------------------------------------------------------- + +def _grade_label(score): + """Map a 0–1 readiness score to a coarse status label.""" + if score is None: + return "unknown" + if score >= 0.9: + return "good" + if score >= 0.7: + return "warning" + return "poor" + + +# Dataset-overview readiness thresholds (per-feature profile) +_OVERVIEW_MISSING_WARNING = 0.05 +_OVERVIEW_MISSING_POOR = 0.20 +_OVERVIEW_DOMINANT_WARNING = 0.95 +_OVERVIEW_HIGH_CARDINALITY = 50 +_OVERVIEW_ID_UNIQUE_RATIO = 0.9 +_OVERVIEW_CAT_TOP_N = 5 + + +def _classify_feature_type(series): + """Map a pandas Series to a coarse feature type label.""" + if pd.api.types.is_bool_dtype(series): + return "boolean" + if pd.api.types.is_datetime64_any_dtype(series): + return "datetime" + if pd.api.types.is_numeric_dtype(series): + return "numerical" + return "categorical" + + +def _feature_readiness_status(pct_missing, n_unique, n_rows, feat_type, pct_dominant): + """Derive a per-feature readiness status from profile statistics.""" + if n_unique <= 1: + return "poor" + if pct_missing is not None and pct_missing > _OVERVIEW_MISSING_POOR: + return "poor" + if n_rows > 0 and n_unique / n_rows >= _OVERVIEW_ID_UNIQUE_RATIO: + return "poor" + if pct_missing is not None and pct_missing > _OVERVIEW_MISSING_WARNING: + return "warning" + if feat_type == "categorical" and n_unique > _OVERVIEW_HIGH_CARDINALITY: + return "warning" + if pct_dominant is not None and pct_dominant > _OVERVIEW_DOMINANT_WARNING: + return "warning" + return "good" + + +def _feature_summary(series, feat_type, n_unique): + """Build a compact, type-specific summary string for one feature.""" + non_null = series.dropna() + if len(non_null) == 0: + return "all missing" + + if feat_type == "numerical": + mean = non_null.mean() + std = non_null.std() + if pd.notna(std) and std > 0: + return f"mean {mean:.2g}, std {std:.2g}" + return f"min {non_null.min():.2g} – max {non_null.max():.2g}" + + if feat_type == "categorical": + vc = non_null.value_counts(normalize=True) + top_val = vc.index[0] + top_pct = vc.iloc[0] * 100 + return f"{top_val} ({top_pct:.0f}%), {n_unique} categories" + + if feat_type == "datetime": + return f"{non_null.min()} – {non_null.max()}" + + if feat_type == "boolean": + true_pct = (non_null.astype(bool)).mean() * 100 + return f"True {true_pct:.0f}%, False {100 - true_pct:.0f}%" + + return f"{n_unique} unique values" + + +def _build_feature_profiles(df): + """Compute a per-column readiness profile for every feature in *df*.""" + n_rows = len(df) + profiles = [] + type_counts = {"numerical": 0, "categorical": 0, "datetime": 0, "boolean": 0} + + for col in df.columns: + series = df[col] + feat_type = _classify_feature_type(series) + type_counts[feat_type] = type_counts.get(feat_type, 0) + 1 + + n_missing = int(series.isnull().sum()) + pct_missing = round(n_missing / n_rows, 4) if n_rows else 0.0 + n_unique = int(series.nunique(dropna=True)) + + pct_dominant = None + non_null = series.dropna() + if len(non_null) > 0: + pct_dominant = round( + float(non_null.value_counts(normalize=True).iloc[0]), 4 + ) + + profiles.append({ + "feature": str(col), + "type": feat_type, + "dtype": str(series.dtype), + "pct_missing": pct_missing, + "n_unique": n_unique, + "pct_dominant": pct_dominant, + "status": _feature_readiness_status( + pct_missing, n_unique, n_rows, feat_type, pct_dominant + ), + "summary": _feature_summary(series, feat_type, n_unique), + }) + + return profiles, type_counts + + +def _build_categorical_distributions(df, top_n=_OVERVIEW_CAT_TOP_N): + """Top-*n* value counts (with percentages) for each categorical column.""" + distributions = {} + for col in df.columns: + if _classify_feature_type(df[col]) != "categorical": + continue + vc = df[col].value_counts(dropna=True) + total = len(df[col].dropna()) + if total == 0: + distributions[str(col)] = [] + continue + entries = [] + for val, count in vc.head(top_n).items(): + entries.append({ + "value": str(val), + "count": int(count), + "pct": round(float(count) / total, 4), + }) + distributions[str(col)] = entries + return distributions + + +def _build_dataset_overview_section(file_info): + """Build the dataset-overview portion of the readiness report. + + Returns file metadata, per-feature readiness profiles, numerical describe() + summary, categorical top-value distributions, and numerical histograms. + """ + file_path, file_name, file_type = file_info + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + n_rows = len(df) + profiles, type_counts = _build_feature_profiles(df) + + file_size_bytes = None + if file_path and os.path.exists(file_path): + try: + file_size_bytes = os.path.getsize(file_path) + except OSError: + pass + + memory_bytes = int(df.memory_usage(deep=True).sum()) + + numerical_summary = {} + num_df = df.select_dtypes(include="number") + if not num_df.empty: + numerical_summary = num_df.describe().map( + lambda x: round(x, 2) if x == 0 or abs(x) >= 0.001 else f"{x:.2e}" + ).to_dict() + for v in numerical_summary.values(): + for old_key in list(v.keys()): + if old_key in ["25%", "50%", "75%"]: + new_key = old_key.replace("%", "th percentile") + v[new_key] = v.pop(old_key) + + return { + "file_metadata": { + "file_name": file_name, + "file_type": file_type, + "file_size_bytes": file_size_bytes, + "memory_bytes": memory_bytes, + "rows": n_rows, + "columns": len(df.columns), + "numerical_count": type_counts.get("numerical", 0), + "categorical_count": type_counts.get("categorical", 0), + "datetime_count": type_counts.get("datetime", 0), + "boolean_count": type_counts.get("boolean", 0), + }, + "feature_profiles": profiles, + "numerical_summary": numerical_summary, + "categorical_distributions": _build_categorical_distributions(df), + "categorical_charts": categorical_distribution_charts(df), + "histograms": summary_histograms(df, figsize=(7, 4.5)), + "profile_thresholds": { + "missing_warning": _OVERVIEW_MISSING_WARNING, + "missing_poor": _OVERVIEW_MISSING_POOR, + "dominant_warning": _OVERVIEW_DOMINANT_WARNING, + "high_cardinality": _OVERVIEW_HIGH_CARDINALITY, + "id_unique_ratio": _OVERVIEW_ID_UNIQUE_RATIO, + }, + } + + +def _build_data_quality_section(file_info): + """Compute the data-quality portion of the readiness report. + + Runs completeness, outliers, and duplicity (the same functions backing the + Data Quality tab), then derives readiness-oriented KPIs (normalized so that + higher is always better), an overall grade, and a "needs attention" list. + + Returns a JSON-serializable dict, or ``{"error": str}`` on failure. + """ + section = {} + + # --- Completeness ----------------------------------------------------- + compl = completeness(file_info) + compl_scores = compl.get("Completeness scores", {}) or {} + overall_completeness = compl.get("Overall Completeness") + + # --- Outliers --------------------------------------------------------- + out = outliers(file_info) + out_scores_raw = out.get("Outlier scores", {}) if isinstance(out, dict) else {} + overall_outlier = None + out_scores = {} + if isinstance(out_scores_raw, dict): + overall_outlier = out_scores_raw.get("Overall outlier score") + out_scores = { + k: v for k, v in out_scores_raw.items() if k != "Overall outlier score" + } + outliers_error = out.get("Error") if isinstance(out, dict) else None + + # --- Duplicity -------------------------------------------------------- + dup = duplicity(file_info) + overall_duplicity = ( + dup.get("Duplicity scores", {}).get("Overall duplicity of the dataset") + if isinstance(dup, dict) + else None + ) + + # --- Normalized KPIs (higher = better) -------------------------------- + completeness_kpi = overall_completeness + uniqueness_kpi = (1 - overall_duplicity) if overall_duplicity is not None else None + outlier_clean_kpi = (1 - overall_outlier) if overall_outlier is not None else None + + kpis = [ + { + "id": "completeness", + "label": "Completeness", + "value": completeness_kpi, + "status": _grade_label(completeness_kpi), + "hint": "Share of non-missing values across the dataset.", + }, + { + "id": "uniqueness", + "label": "Uniqueness", + "value": uniqueness_kpi, + "status": _grade_label(uniqueness_kpi), + "hint": "1 − proportion of duplicate rows.", + }, + { + "id": "outlier_cleanliness", + "label": "Outlier-cleanliness", + "value": outlier_clean_kpi, + "status": _grade_label(outlier_clean_kpi), + "hint": "1 − mean outlier proportion (IQR method) across numerical features.", + }, + ] + + present = [k["value"] for k in kpis if k["value"] is not None] + grade = sum(present) / len(present) if present else None + + # --- Needs attention -------------------------------------------------- + incomplete = sorted( + ( + {"feature": col, "completeness": score} + for col, score in compl_scores.items() + if isinstance(score, (int, float)) and score < 1.0 + ), + key=lambda x: x["completeness"], + ) + high_outliers = sorted( + ( + {"feature": col, "outlier_proportion": score} + for col, score in out_scores.items() + if isinstance(score, (int, float)) and score > 0 + ), + key=lambda x: x["outlier_proportion"], + reverse=True, + ) + + section = { + "grade": grade, + "grade_status": _grade_label(grade), + "kpis": kpis, + "needs_attention": { + "incomplete_features": incomplete, + "outlier_features": high_outliers, + "duplicate_rows": ( + overall_duplicity if overall_duplicity not in (None, 0) else 0 + ), + }, + "details": { + "completeness": { + "overall": overall_completeness, + "scores": compl_scores, + "visualization": compl.get("Completeness Visualization"), + }, + "outliers": { + "overall": overall_outlier, + "scores": out_scores, + "visualization": out.get("Outliers Visualization") if isinstance(out, dict) else None, + "error": outliers_error, + }, + "duplicity": {"overall": overall_duplicity}, + }, + } + return section + + +# Impact-on-AI tuning constants +_CORR_MAX_COLUMNS = 25 # cap analysed columns to keep heatmaps readable +_CORR_HIGH_CARD_MAX = 50 # drop categorical columns with more unique values +_CORR_ID_UNIQUE_RATIO = 0.9 # drop categorical columns that look like IDs +_CORR_REDUNDANT_THRESHOLD = 0.8 # |score| at/above which a pair is "redundant" +_CORR_LEAKAGE_THRESHOLD = 0.95 # |score| at/above which a pair is "leakage risk" +_CORR_ISOLATED_THRESHOLD = 0.1 # max |score| below which a feature is "isolated" + + +def _prune_columns_for_corr(df): + """Select columns worth feeding into the all-pairs correlation analysis. + + Drops columns that are useless or pathological for correlation: + constants, ID-like / high-cardinality categoricals. Numerical columns are + always kept (they are cheap to correlate). The result is capped at + ``_CORR_MAX_COLUMNS`` (numerical prioritized) to keep the computation and + heatmaps tractable. + + Returns ``(kept_columns, dropped)`` where *dropped* is a list of + ``{"feature": str, "reason": str}``. + """ + n_rows = max(len(df), 1) + numeric_cols = list(df.select_dtypes(exclude=["object", "string", "category"]).columns) + categorical_cols = list(df.select_dtypes(include=["object", "string", "category"]).columns) + + kept_numeric = [] + kept_categorical = [] + dropped = [] + + for col in numeric_cols: + if df[col].nunique(dropna=True) <= 1: + dropped.append({"feature": col, "reason": "constant column"}) + else: + kept_numeric.append(col) + + for col in categorical_cols: + nunique = df[col].nunique(dropna=True) + if nunique <= 1: + dropped.append({"feature": col, "reason": "constant column"}) + elif nunique / n_rows >= _CORR_ID_UNIQUE_RATIO: + dropped.append({"feature": col, "reason": "ID-like (near-unique values)"}) + elif nunique > _CORR_HIGH_CARD_MAX: + dropped.append({"feature": col, "reason": f"high cardinality ({nunique} categories)"}) + else: + kept_categorical.append(col) + + # Cap total columns, prioritizing numerical features + kept = kept_numeric + kept_categorical + if len(kept) > _CORR_MAX_COLUMNS: + for col in kept[_CORR_MAX_COLUMNS:]: + dropped.append({"feature": col, "reason": "exceeded column cap"}) + kept = kept[:_CORR_MAX_COLUMNS] + + return kept, dropped + + +def _pairwise_signals(scores): + """Derive readiness signals from a flat ``{"a vs b": score}`` mapping. + + Collapses the symmetric/asymmetric directional entries into one record per + unordered pair (keeping the largest-magnitude score), then classifies pairs + as redundant or leakage-risk and flags features that are not meaningfully + related to anything else ("isolated"). + """ + pair_max = {} + for key, val in scores.items(): + if " vs " not in key or not isinstance(val, (int, float)): + continue + a, b = key.split(" vs ", 1) + if a == b: + continue + ukey = tuple(sorted([a, b])) + abs_score = abs(val) + if ukey not in pair_max or abs_score > pair_max[ukey]["abs_score"]: + pair_max[ukey] = { + "a": ukey[0], + "b": ukey[1], + "score": round(float(val), 3), + "abs_score": abs_score, + } + + pairs = list(pair_max.values()) + pairs.sort(key=lambda p: p["abs_score"], reverse=True) + + leakage = [ + {"a": p["a"], "b": p["b"], "score": p["score"]} + for p in pairs + if p["abs_score"] >= _CORR_LEAKAGE_THRESHOLD + ] + redundant = [ + {"a": p["a"], "b": p["b"], "score": p["score"]} + for p in pairs + if _CORR_REDUNDANT_THRESHOLD <= p["abs_score"] < _CORR_LEAKAGE_THRESHOLD + ] + top = [{"a": p["a"], "b": p["b"], "score": p["score"]} for p in pairs[:8]] + + # Per-feature connectivity: the strongest relationship each feature has + connectivity = {} + for p in pairs: + connectivity[p["a"]] = max(connectivity.get(p["a"], 0.0), p["abs_score"]) + connectivity[p["b"]] = max(connectivity.get(p["b"], 0.0), p["abs_score"]) + isolated = sorted( + f for f, c in connectivity.items() if c < _CORR_ISOLATED_THRESHOLD + ) + + return { + "redundant": redundant, + "leakage": leakage, + "top": top, + "isolated": isolated, + } + + +def _build_impact_on_ai_section(file_info): + """Compute the Impact-on-AI portion of the readiness report. + + Runs an automated, non-interactive all-pairs correlation analysis (numerical + via vectorized pandas correlation, categorical via Theil's U) over a pruned, + capped set of columns, then derives redundancy / leakage / isolation signals. + + Returns a JSON-serializable dict, or ``{"error": str}`` on failure. + """ + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + kept, dropped = _prune_columns_for_corr(df) + if len(kept) < 2: + return { + "error": "Not enough usable columns for correlation analysis after pruning.", + "columns_analyzed": len(kept), + "columns_dropped": dropped, + } + + corr = calc_correlations(kept, file_info) + if isinstance(corr, dict) and "Message" in corr: + return {"error": corr["Message"], "columns_dropped": dropped} + + scores = corr.get("Correlation Scores", {}) if isinstance(corr, dict) else {} + signals = _pairwise_signals(scores) + cat = corr.get("Correlations Analysis Categorical", {}) or {} + num = corr.get("Correlations Analysis Numerical", {}) or {} + + return { + "columns_analyzed": len(kept), + "columns_dropped": dropped, + "redundant_pairs": signals["redundant"], + "leakage_pairs": signals["leakage"], + "isolated_features": signals["isolated"], + "top_pairs": signals["top"], + "details": { + "categorical_visualization": cat.get( + "Correlations Analysis Categorical Visualization" + ), + "numerical_visualization": num.get( + "Correlations Analysis Numerical Visualization" + ), + "numerical_method": num.get("Method"), + }, + } + + +# Fairness & Bias tuning constants +_FAIRNESS_SENSITIVE_MIN_UNIQUE = 2 +_FAIRNESS_SENSITIVE_MAX_UNIQUE = 30 +_FAIRNESS_MAX_SENSITIVE_COLS = 5 +_FAIRNESS_ID_UNIQUE_RATIO = 0.9 +_FAIRNESS_TARGET_MAX_UNIQUE = 30 +_FAIRNESS_REP_RATIO_FLAG = 3.0 # probability ratio at/above which representation is flagged +_FAIRNESS_MINORITY_SHARE = 0.05 # class share below which a label is "minority" +_FAIRNESS_IMBALANCE_GOOD = 0.5 # imbalance degree below this → good +_FAIRNESS_IMBALANCE_WARNING = 1.0 # below this → warning, else poor +_FAIRNESS_TSD_FLAG = 0.15 # TSD above this → outcome-rate disparity flagged +_FAIRNESS_SENSITIVE_NAME_HINTS = ( + "gender", "sex", "race", "ethnic", "age", "religion", "marital", + "disability", "national", "origin", "minority", +) +_FAIRNESS_TARGET_NAME_HINTS = ( + "label", "target", "class", "outcome", "decision", "y", "income", +) + + +def _fairness_name_hint_score(col_name, hints): + """Return a higher score when *col_name* matches an earlier hint substring.""" + lower = col_name.lower() + best = 0 + for i, hint in enumerate(hints): + if hint in lower: + best = max(best, len(hints) - i) + return best + + +def _auto_select_fairness_columns(df): + """Pick sensitive attributes and a target column for automated fairness checks. + + Returns a dict with selected columns, exclusion reasons, and the + auto-selected positive class (mode of the target) for CDD. + """ + n_rows = max(len(df), 1) + cat_cols = list(df.select_dtypes(include=["object", "string", "category"]).columns) + + sensitive_candidates = [] + excluded_sensitive = [] + + for col in cat_cols: + nunique = df[col].nunique(dropna=True) + if nunique < _FAIRNESS_SENSITIVE_MIN_UNIQUE: + excluded_sensitive.append({"feature": col, "reason": "constant column"}) + elif nunique > _FAIRNESS_SENSITIVE_MAX_UNIQUE: + excluded_sensitive.append({ + "feature": col, + "reason": f"high cardinality ({nunique} categories, max {_FAIRNESS_SENSITIVE_MAX_UNIQUE})", + }) + elif nunique / n_rows >= _FAIRNESS_ID_UNIQUE_RATIO: + excluded_sensitive.append({"feature": col, "reason": "ID-like (near-unique values)"}) + else: + sensitive_candidates.append({ + "feature": col, + "nunique": int(nunique), + "name_hint_score": _fairness_name_hint_score(col, _FAIRNESS_SENSITIVE_NAME_HINTS), + }) + + sensitive_candidates.sort( + key=lambda c: (-c["name_hint_score"], c["nunique"], c["feature"]) + ) + selected_sensitive = [c["feature"] for c in sensitive_candidates[:_FAIRNESS_MAX_SENSITIVE_COLS]] + for c in sensitive_candidates[_FAIRNESS_MAX_SENSITIVE_COLS:]: + excluded_sensitive.append({"feature": c["feature"], "reason": "exceeded sensitive-column cap"}) + + # Target: low-cardinality columns (2–30 unique), prefer name hints; exclude sensitives + target_candidates = [] + sensitive_set = set(selected_sensitive) + for col in df.columns: + if col in sensitive_set: + continue + nunique = df[col].nunique(dropna=True) + if _FAIRNESS_SENSITIVE_MIN_UNIQUE <= nunique <= _FAIRNESS_TARGET_MAX_UNIQUE: + target_candidates.append({ + "feature": col, + "nunique": int(nunique), + "name_hint_score": _fairness_name_hint_score(col, _FAIRNESS_TARGET_NAME_HINTS), + }) + + target_candidates.sort( + key=lambda c: (-c["name_hint_score"], c["nunique"], c["feature"]) + ) + target_col = target_candidates[0]["feature"] if target_candidates else None + target_reason = None + if target_col: + tc = target_candidates[0] + if tc["name_hint_score"] > 0: + target_reason = "matched target name hint" + elif tc == target_candidates[-1] and len(target_candidates) == 1: + target_reason = "only eligible low-cardinality column" + else: + target_reason = "lowest-cardinality eligible column (after name-hint priority)" + + positive_class = None + positive_reason = None + if target_col is not None: + target_series = df[target_col].dropna() + if len(target_series) > 0: + positive_class = target_series.mode().iloc[0] + positive_reason = "most frequent class in auto-selected target" + + primary_sensitive = selected_sensitive[0] if selected_sensitive else None + + return { + "sensitive_columns": selected_sensitive, + "primary_sensitive": primary_sensitive, + "target_column": target_col, + "positive_class": positive_class, + "selection_criteria": { + "sensitive_attributes": { + "rule": ( + f"Categorical columns with {_FAIRNESS_SENSITIVE_MIN_UNIQUE}–" + f"{_FAIRNESS_SENSITIVE_MAX_UNIQUE} unique values, excluding " + "ID-like columns; prefer name hints " + f"{list(_FAIRNESS_SENSITIVE_NAME_HINTS)}; " + f"capped at {_FAIRNESS_MAX_SENSITIVE_COLS}." + ), + "name_hints": list(_FAIRNESS_SENSITIVE_NAME_HINTS), + "selected": selected_sensitive, + "excluded": excluded_sensitive, + }, + "target_column": { + "rule": ( + f"Columns with {_FAIRNESS_SENSITIVE_MIN_UNIQUE}–{_FAIRNESS_TARGET_MAX_UNIQUE} " + "unique values, excluding auto-selected sensitive columns; " + "prefer name hints " + f"{list(_FAIRNESS_TARGET_NAME_HINTS)}." + ), + "name_hints": list(_FAIRNESS_TARGET_NAME_HINTS), + "selected": target_col, + "reason": target_reason, + }, + "positive_class": { + "rule": "Most frequent value in the auto-selected target (used for CDD only).", + "selected": str(positive_class) if positive_class is not None else None, + "reason": positive_reason, + }, + "thresholds": { + "representation_ratio_flag": _FAIRNESS_REP_RATIO_FLAG, + "minority_class_share": _FAIRNESS_MINORITY_SHARE, + "imbalance_degree_good": _FAIRNESS_IMBALANCE_GOOD, + "imbalance_degree_warning": _FAIRNESS_IMBALANCE_WARNING, + "tsd_disparity_flag": _FAIRNESS_TSD_FLAG, + }, + }, + } + + +def _parse_representation_flags(ratios_dict): + """Extract per-column max probability ratio and flag extreme imbalance.""" + by_column = {} + if not isinstance(ratios_dict, dict) or "Error" in ratios_dict: + return [], None, ratios_dict.get("Error") if isinstance(ratios_dict, dict) else None + + for key, ratio in ratios_dict.items(): + if not isinstance(ratio, (int, float)) or "Column: '" not in key: + continue + # Key format: Column: 'col', Probability ratio for 'A' to 'B' + try: + col_part = key.split("Column: '", 1)[1] + col = col_part.split("',", 1)[0] + except IndexError: + continue + entry = by_column.setdefault(col, {"column": col, "max_ratio": 0.0, "flagged_pairs": []}) + r = float(ratio) + if r > entry["max_ratio"]: + entry["max_ratio"] = round(r, 3) + if r >= _FAIRNESS_REP_RATIO_FLAG: + pair_part = key.split("Probability ratio for ", 1)[-1] + entry["flagged_pairs"].append({"pair": pair_part, "ratio": round(r, 3)}) + + summaries = sorted(by_column.values(), key=lambda x: x["max_ratio"], reverse=True) + worst = summaries[0]["max_ratio"] if summaries else None + rep_balance_kpi = min(1.0, 1.0 / worst) if worst and worst > 0 else None + return summaries, rep_balance_kpi, None + + +def _imbalance_status(id_score): + if id_score is None: + return "unknown" + if id_score < _FAIRNESS_IMBALANCE_GOOD: + return "good" + if id_score < _FAIRNESS_IMBALANCE_WARNING: + return "warning" + return "poor" + + +def _build_fairness_bias_section(file_info): + """Compute the Fairness & Bias portion of the readiness report. + + Auto-selects sensitive attributes and a target, then runs representation + rate, class imbalance, statistical rate, and conditional demographic + disparity using the same functions as the Fairness & Bias tab. + """ + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + selection = _auto_select_fairness_columns(df) + sensitive_cols = selection["sensitive_columns"] + target_col = selection["target_column"] + primary_sensitive = selection["primary_sensitive"] + positive_class = selection["positive_class"] + + kpis = [] + needs_attention = { + "representation_imbalance": [], + "minority_classes": [], + "outcome_disparities": [], + "cdd_disparities": [], + } + details = {} + + # --- Representation Rate (all selected sensitive columns) --------------- + rep_error = None + rep_balance_kpi = None + if sensitive_cols: + ratios = calculate_representation_rate(sensitive_cols, file_info) + rep_summaries, rep_balance_kpi, rep_error = _parse_representation_flags(ratios) + needs_attention["representation_imbalance"] = [ + s for s in rep_summaries if s["max_ratio"] >= _FAIRNESS_REP_RATIO_FLAG + ] + rep_visualizations = {} + for col in sensitive_cols: + try: + vis = create_representation_rate_vis([col], file_info) + if isinstance(vis, str): + rep_visualizations[col] = vis + except Exception: + pass + details["representation_rate"] = { + "ratios": ratios if not rep_error else None, + "summaries": rep_summaries, + "visualizations": rep_visualizations, + "error": rep_error, + } + else: + details["representation_rate"] = { + "error": "No eligible sensitive-attribute columns found.", + } + + kpis.append({ + "id": "representation_balance", + "label": "Representation balance", + "value": rep_balance_kpi, + "status": _grade_label(rep_balance_kpi), + "hint": f"1 / worst group probability ratio (flagged when ratio ≥ {_FAIRNESS_REP_RATIO_FLAG}).", + }) + + # --- Class Imbalance (auto target) -------------------------------------- + imbalance_degree = None + ci_error = None + if target_col: + ci_dict = _compute_class_imbalance(df, target_col, "EU") + if "Error" in ci_dict: + ci_error = ci_dict["Error"] + else: + imb = ci_dict.get("Imbalance degree") or {} + imbalance_degree = imb.get("Imbalance Degree score") + details["class_imbalance"] = { + "visualization": ci_dict.get("Class Imbalance Visualization"), + "imbalance_degree": imbalance_degree, + } + # Minority classes + vc = df[target_col].value_counts(normalize=True, dropna=True) + for cls, share in vc.items(): + if share < _FAIRNESS_MINORITY_SHARE: + needs_attention["minority_classes"].append({ + "class": str(cls), + "share": round(float(share), 4), + }) + else: + details["class_imbalance"] = {"error": "No eligible target column found."} + + label_balance_kpi = ( + max(0.0, 1.0 - min(float(imbalance_degree) / 2.0, 1.0)) + if imbalance_degree is not None + else None + ) + kpis.append({ + "id": "label_balance", + "label": "Label balance", + "value": label_balance_kpi, + "status": _imbalance_status(imbalance_degree), + "hint": ( + f"Derived from Imbalance Degree (EU); 0 = balanced. " + f"Good < {_FAIRNESS_IMBALANCE_GOOD}, warning < {_FAIRNESS_IMBALANCE_WARNING}." + ), + "raw_imbalance_degree": imbalance_degree, + }) + + # --- Statistical Rate (primary sensitive + target) ---------------------- + disparity_kpi = None + if primary_sensitive and target_col: + sr = calculate_statistical_rates(target_col, primary_sensitive, file_info) + if isinstance(sr, dict) and "Error" in sr: + details["statistical_rate"] = {"error": sr["Error"]} + else: + tsd_scores = sr.get("TSD scores") or {} + flagged = [ + {"class": str(cls), "tsd": round(float(score), 4)} + for cls, score in tsd_scores.items() + if isinstance(score, (int, float)) and float(score) >= _FAIRNESS_TSD_FLAG + ] + flagged.sort(key=lambda x: x["tsd"], reverse=True) + needs_attention["outcome_disparities"] = flagged + max_tsd = max( + (float(v) for v in tsd_scores.values() if isinstance(v, (int, float))), + default=None, + ) + disparity_kpi = max(0.0, 1.0 - min(max_tsd, 1.0)) if max_tsd is not None else None + details["statistical_rate"] = { + "sensitive": primary_sensitive, + "target": target_col, + "tsd_scores": tsd_scores, + "visualization": sr.get("Statistical Rate Visualization"), + } + else: + details["statistical_rate"] = { + "error": "Requires both a sensitive attribute and a target column.", + } + + kpis.append({ + "id": "outcome_parity", + "label": "Outcome parity", + "value": disparity_kpi, + "status": _grade_label(disparity_kpi), + "hint": ( + f"1 − max TSD across classes (flagged when TSD ≥ {_FAIRNESS_TSD_FLAG}). " + "Uses primary sensitive attribute." + ), + }) + + # --- Conditional Demographic Disparity ---------------------------------- + if primary_sensitive and target_col and positive_class is not None: + try: + cdd = conditional_demographic_disparity( + df[target_col].tolist(), + df[primary_sensitive].tolist(), + positive_class, + ) + if isinstance(cdd, dict) and "Error" in cdd: + details["cdd"] = {"error": cdd["Error"]} + else: + disparities = (cdd or {}).get("Disparities") or {} + cdd_flagged = [ + {"group": str(grp), "disparity": info.get("disparity")} + for grp, info in disparities.items() + if str(info.get("disparity", "")).lower() == "true" + ] + needs_attention["cdd_disparities"] = cdd_flagged + details["cdd"] = { + "sensitive": primary_sensitive, + "target": target_col, + "positive_class": str(positive_class), + "disparities": disparities, + } + except Exception as e: + details["cdd"] = {"error": str(e)} + else: + details["cdd"] = { + "error": "Requires sensitive attribute, target, and auto-positive class.", + } + + present = [k["value"] for k in kpis if k["value"] is not None] + grade = sum(present) / len(present) if present else None + + return { + "grade": grade, + "grade_status": _grade_label(grade), + "auto_selection": selection, + "kpis": kpis, + "needs_attention": needs_attention, + "details": details, + } + + +@metrics_bp.route("/readiness-report", methods=["GET"]) +def readiness_report(): + """Return an aggregated, non-interactive data-readiness report as JSON. + + Covers dataset overview, Data Quality, Impact-on-AI, and Fairness & Bias. + Designed to be extended with more pillars over time. + """ + file_path = session.get("uploaded_file_path") + file_name = session.get("uploaded_file_name") + file_type = session.get("uploaded_file_type") + + if not file_path: + return jsonify({"success": False, "message": "No file uploaded"}), 200 + + file_info = (file_path, file_name, file_type) + start_time = time.time() + try: + try: + dataset_overview_section = _build_dataset_overview_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — dataset overview error: %s", e, exc_info=True) + dataset_overview_section = {"error": f"{type(e).__name__}: {e}"} + + try: + data_quality_section = _build_data_quality_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — data quality error: %s", e, exc_info=True) + data_quality_section = {"error": f"{type(e).__name__}: {e}"} + + try: + impact_section = _build_impact_on_ai_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — impact on AI error: %s", e, exc_info=True) + impact_section = {"error": f"{type(e).__name__}: {e}"} + + try: + fairness_section = _build_fairness_bias_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — fairness & bias error: %s", e, exc_info=True) + fairness_section = {"error": f"{type(e).__name__}: {e}"} + + response = ensure_json_serializable({ + "success": True, + "dataset_overview": dataset_overview_section, + "data_quality": data_quality_section, + "impact_on_ai": impact_section, + "fairness_bias": fairness_section, + }) + metric_time_log.info("Readiness report built in %.2f seconds", time.time() - start_time) + return jsonify(response) + except Exception as e: + metric_time_log.error("Readiness report error: %s", e, exc_info=True) + return jsonify({"success": False, "message": f"{type(e).__name__}: {e}"}), 200 + + # --------------------------------------------------------------------------- # Fairness # --------------------------------------------------------------------------- diff --git a/web/routes/utils.py b/web/routes/utils.py index d355f401..6e192435 100644 --- a/web/routes/utils.py +++ b/web/routes/utils.py @@ -193,14 +193,14 @@ def ensure_json_serializable(obj): return obj -def summary_histograms(df): +def summary_histograms(df, figsize=(4, 3), dpi=150): """Generate base64-encoded KDE distribution plots for all numeric columns.""" text_color = "#6b7280" curve_color = "#4485F4" line_graphs = {} for column in df.select_dtypes(include="number").columns: - fig, ax = plt.subplots(figsize=(4, 3)) + fig, ax = plt.subplots(figsize=figsize) fig.patch.set_alpha(0) ax.set_facecolor("none") @@ -214,7 +214,7 @@ def summary_histograms(df): fig.tight_layout(pad=0.5) img_buffer = io.BytesIO() - fig.savefig(img_buffer, format="png", dpi=150, transparent=True) + fig.savefig(img_buffer, format="png", dpi=dpi, transparent=True) img_buffer.seek(0) encoded_img = base64.b64encode(img_buffer.read()).decode("utf-8") @@ -224,3 +224,75 @@ def summary_histograms(df): img_buffer.close() return line_graphs + + +def categorical_distribution_charts(df, top_n=10): + """Generate base64-encoded pie charts for categorical columns. + + Shows the top *top_n* values per column; remaining values are grouped + into an "Other" slice when applicable. + """ + text_color = "#6b7280" + palette = [ + "#4485F4", "#34A853", "#FBBC05", "#EA4335", "#9C27B0", + "#00ACC1", "#FF7043", "#8D6E63", "#78909C", "#AB47BC", + ] + charts = {} + + cat_cols = df.select_dtypes(include=["object", "string", "category"]).columns + for column in cat_cols: + series = df[column].dropna() + if len(series) == 0: + continue + + vc = series.value_counts() + if len(vc) > top_n: + top = vc.head(top_n) + other_count = int(vc.iloc[top_n:].sum()) + labels = [str(v) for v in top.index] + sizes = [int(v) for v in top.values] + if other_count > 0: + labels.append("Other") + sizes.append(other_count) + else: + labels = [str(v) for v in vc.index] + sizes = [int(v) for v in vc.values] + + n_slices = len(labels) + colors = [palette[i % len(palette)] for i in range(n_slices)] + + fig, ax = plt.subplots(figsize=(5, 5)) + fig.patch.set_alpha(0) + + wedges, _, autotexts = ax.pie( + sizes, + labels=None, + autopct=lambda pct: f"{pct:.1f}%" if pct >= 4 else "", + colors=colors, + startangle=90, + pctdistance=0.75, + textprops={"color": text_color, "fontsize": 8}, + ) + for t in autotexts: + t.set_fontsize(7) + + ax.legend( + wedges, + labels, + loc="center left", + bbox_to_anchor=(1, 0.5), + fontsize=8, + frameon=False, + labelcolor=text_color, + ) + ax.set_title(str(column), fontsize=10, color=text_color, pad=8) + fig.tight_layout() + + img_buffer = io.BytesIO() + fig.savefig(img_buffer, format="png", dpi=150, transparent=True, bbox_inches="tight") + img_buffer.seek(0) + charts[str(column)] = base64.b64encode(img_buffer.read()).decode("utf-8") + plt.close(fig) + img_buffer.close() + + return charts diff --git a/web/static/css/theme.css b/web/static/css/theme.css index 44af6f02..b45e9e96 100644 --- a/web/static/css/theme.css +++ b/web/static/css/theme.css @@ -313,10 +313,29 @@ html.dark .info-text { } .info-icon:hover .info-text, -.info-text:hover { +.info-text:hover, +.info-icon:focus-within .info-text { display: block !important; } +/* Readiness report: show tooltip below icon (avoids clipping in tables) */ +#panel-readiness-report .info-icon--below .info-text { + left: auto; + right: 0; + top: calc(100% + 6px); + margin-left: 0; +} + +#panel-readiness-report th .info-icon, +#panel-readiness-report .inline-flex .info-icon { + flex-shrink: 0; +} + +/* Let readiness tooltips extend outside scrollable table wrappers when possible */ +#panel-readiness-report table { + overflow: visible; +} + .info-text a { color: var(--brandBlue) !important; text-decoration: underline !important; diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 8dd1d788..998d777a 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -10,6 +10,7 @@ let activePanel = "data-overview"; let codeMirrorEditor = null; let lastMetricResult = null; // Store last result for JSON download +let _readinessReportLoaded = false; // Lazy-load guard for the Readiness Report panel /** * Show a metric panel by ID, hiding all others. @@ -62,6 +63,13 @@ function showPanel(panelId, pushHistory) { initCodeMirror(); } + // Lazy load the data overview + data quality into the Readiness Report + // panel on first open + if (panelId === "readiness-report" && !_readinessReportLoaded) { + _readinessReportLoaded = true; + loadReadinessReport(); + } + // Close mobile sidebar after selection const sidebar = document.getElementById("sidebar"); if (sidebar && window.innerWidth < 640) { @@ -1907,11 +1915,24 @@ function submitCustomMetric() { /** * Render histogram images in the data overview panel. * @param {Object} histograms - Dict of {column_theme: base64_img} from /summary-statistics + * @param {string} [containerId] + * @param {boolean} [skipHeading] + * @param {string} [layout] - "compact" (default) or "large" for readiness report */ -function renderWorkspaceHistograms(histograms) { - const container = document.getElementById("workspace-histograms"); +function renderWorkspaceHistograms(histograms, containerId, skipHeading, layout) { + const container = document.getElementById( + containerId || "workspace-histograms", + ); if (!container) return; + const isLarge = layout === "large"; + const gridClass = isLarge + ? "grid grid-cols-1 sm:grid-cols-2 gap-6" + : "grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4"; + const imgClass = isLarge + ? "w-full h-auto object-contain" + : "w-full"; + // Always use the light variant — CSS filter handles dark mode const columns = {}; for (const [key, base64] of Object.entries(histograms)) { @@ -1926,14 +1947,17 @@ function renderWorkspaceHistograms(histograms) { return; } - let html = - '

Feature Distributions

'; - html += '
'; + let html = ""; + if (!skipHeading) { + html += + '

Feature Distributions

'; + } + html += `
`; for (const [colName, base64] of Object.entries(columns)) { html += `
- Distribution of ${colName} + Distribution of ${colName}
${colName}
`; @@ -1943,26 +1967,47 @@ function renderWorkspaceHistograms(histograms) { container.innerHTML = html; } -// ==================== Workspace Init ==================== +/** + * Render categorical distribution pie charts (base64 PNG per column). + */ +function renderCategoricalPieCharts(charts, containerId) { + const container = document.getElementById(containerId); + if (!container || !charts) return; + + const entries = Object.entries(charts); + if (entries.length === 0) { + container.innerHTML = ""; + return; + } + + let html = '
'; + for (const [colName, base64] of entries) { + html += ` +
+ Distribution of ${colName} +
`; + } + html += "
"; + container.innerHTML = html; +} /** - * Initialize the workspace after file upload. - * Fetches summary statistics and populates feature dropdowns. + * Fetch summary statistics from the backend and render the stat cards, + * summary table, and feature-distribution histograms into the given + * containers. Reused by both the Data Overview panel and the + * Readiness Report panel. + * + * @param {string} summaryContainerId - element ID for the stats/table. + * @param {string} histogramsContainerId - element ID for the histograms. */ -function initWorkspace() { - // Restore panel from URL hash, or default to data-overview - const hash = location.hash.replace("#", ""); - const initialPanel = - hash && document.getElementById("panel-" + hash) ? hash : "data-overview"; - showPanel(initialPanel, false); // false = don't push to history on init - // Replace current history entry so back button works from the first panel - history.replaceState({ panel: initialPanel }, "", "#" + initialPanel); +function loadDataOverview(summaryContainerId, histogramsContainerId) { + const summaryId = summaryContainerId || "workspace-summary"; + const histogramsId = histogramsContainerId || "workspace-histograms"; - // Fetch summary statistics fetch("/summary-statistics") .then((r) => r.json()) .then((data) => { - const container = document.getElementById("workspace-summary"); + const container = document.getElementById(summaryId); if (!container) return; if (data.success) { @@ -2038,19 +2083,905 @@ function initWorkspace() { container.innerHTML = html; - // Render histograms in the data overview panel + // Render histograms below the summary table if (data.histograms) { - renderWorkspaceHistograms(data.histograms); + renderWorkspaceHistograms(data.histograms, histogramsId); } } else { container.innerHTML = `

Could not load summary: ${data.message}

`; } }) .catch((err) => { - const container = document.getElementById("workspace-summary"); + const container = document.getElementById(summaryId); if (container) container.innerHTML = `

Error loading summary: ${err.message}

`; }); +} + +/** + * Map a readiness status string to Tailwind color classes. + */ +function _dqStatusClasses(status) { + switch (status) { + case "good": + return { + text: "text-green-700 dark:text-green-400", + bar: "bg-green-500", + badge: + "bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400", + }; + case "warning": + return { + text: "text-amber-700 dark:text-amber-400", + bar: "bg-amber-500", + badge: + "bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-400", + }; + case "poor": + return { + text: "text-red-700 dark:text-red-400", + bar: "bg-red-500", + badge: "bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400", + }; + default: + return { + text: "text-gray-500 dark:text-gray-400", + bar: "bg-gray-400", + badge: + "bg-gray-100 text-gray-700 dark:bg-gray-700 dark:text-gray-300", + }; + } +} + +/** Format a 0–1 score as a whole percentage, or "N/A" if missing. */ +function _pct(value) { + if (value === null || value === undefined || isNaN(value)) return "N/A"; + return `${Math.round(value * 100)}%`; +} + +/** + * Fetch the aggregated readiness report and render the Data Quality + * scorecard (KPI tiles + overall grade + "needs attention" lists + + * a collapsible details view with the original charts). + */ +function loadReadinessReport() { + const overviewContainer = document.getElementById("readiness-summary"); + const dqContainer = document.getElementById("readiness-data-quality"); + const impactContainer = document.getElementById("readiness-impact"); + const fairnessContainer = document.getElementById("readiness-fairness"); + + fetch("/readiness-report") + .then((r) => r.json()) + .then((resp) => { + if (!resp.success) { + const msg = `

Could not load readiness report: ${resp.message || "unknown error"}

`; + if (overviewContainer) overviewContainer.innerHTML = msg; + if (dqContainer) dqContainer.innerHTML = msg; + if (impactContainer) impactContainer.innerHTML = msg; + if (fairnessContainer) fairnessContainer.innerHTML = msg; + return; + } + if (overviewContainer) + renderReadinessDatasetOverview(overviewContainer, resp.dataset_overview || {}); + if (dqContainer) + renderReadinessDataQuality(dqContainer, resp.data_quality || {}); + if (impactContainer) + renderReadinessImpact(impactContainer, resp.impact_on_ai || {}); + if (fairnessContainer) + renderReadinessFairness(fairnessContainer, resp.fairness_bias || {}); + }) + .catch((err) => { + const msg = `

Error loading readiness report: ${err.message}

`; + if (overviewContainer) overviewContainer.innerHTML = msg; + if (dqContainer) dqContainer.innerHTML = msg; + if (impactContainer) impactContainer.innerHTML = msg; + if (fairnessContainer) fairnessContainer.innerHTML = msg; + }); +} + +/** Escape text for safe inclusion in readiness info tooltips. */ +function _escapeHtml(s) { + return String(s) + .replace(/&/g, "&") + .replace(/i${_escapeHtml(text)}`; +} + +/** Table header cell with label + info tooltip. */ +function _readinessTh(label, infoKey, alignRight) { + const align = alignRight ? " text-right" : ""; + return `${label}${_readinessInfoIcon(infoKey)}`; +} + +/** Format byte count as a human-readable size string. */ +function _formatBytes(bytes) { + if (bytes == null || isNaN(bytes)) return "—"; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +/** Badge HTML for per-feature readiness status. */ +function _profileStatusBadge(status) { + const cls = _dqStatusClasses(status); + const label = status === "good" ? "Good" : status === "warning" ? "Warning" : status === "poor" ? "Poor" : "—"; + return `${label}`; +} + +/** + * Render the dataset overview: file metadata, KPI tiles, per-feature readiness + * profile, and collapsible detailed statistics / distributions / histograms. + */ +function renderReadinessDatasetOverview(container, overview) { + if (overview.error) { + container.innerHTML = `

Dataset overview unavailable: ${overview.error}

`; + return; + } + + const meta = overview.file_metadata || {}; + const profiles = overview.feature_profiles || []; + + // --- File metadata --- + let html = ` +
+

${meta.file_name || "Dataset"}

+
+
Type: ${meta.file_type || "—"}
+
Size: ${_formatBytes(meta.file_size_bytes)}
+
Memory: ${_formatBytes(meta.memory_bytes)}
+
Rows: ${(meta.rows || 0).toLocaleString()}
+
Columns: ${meta.columns || 0}
+
Numerical: ${meta.numerical_count || 0}
+
Categorical: ${meta.categorical_count || 0}
+
Other: ${(meta.datetime_count || 0) + (meta.boolean_count || 0)}
+
+
`; + + // --- KPI tiles --- + html += ` +
+
+
${(meta.rows || 0).toLocaleString()}
+
Records
+
+
+
${meta.columns || 0}
+
Features
+
+
+
${meta.numerical_count || 0}
+
Numerical
+
+
+
${meta.categorical_count || 0}
+
Categorical
+
+
`; + + // --- Per-feature readiness profile --- + const poorCount = profiles.filter((p) => p.status === "poor").length; + const warnCount = profiles.filter((p) => p.status === "warning").length; + + html += ` +
+

Per-feature readiness profile${_readinessInfoIcon("feature_profile")}

+ + ${poorCount ? `${poorCount} poor` : ""} + ${warnCount ? `${poorCount ? " · " : ""}${warnCount} warning` : ""} + ${!poorCount && !warnCount ? "all good" : ""} + +
`; + + html += `
`; + html += ``; + html += ``; + html += ``; + html += _readinessTh("% missing", "pct_missing", true); + html += _readinessTh("# unique", "n_unique", true); + html += _readinessTh("% dominant", "pct_dominant", true); + html += _readinessTh("Status", "profile_status", false); + html += ``; + html += ``; + + profiles.forEach((p, i) => { + const stripe = i % 2 === 0 ? "bg-white dark:bg-gray-800" : "bg-gray-50 dark:bg-gray-700/50"; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + }); + html += `
FeatureTypeDtypeSummary
${p.feature}${p.type}${p.dtype}${_pct(p.pct_missing)}${p.n_unique}${p.pct_dominant != null ? _pct(p.pct_dominant) : "—"}${_profileStatusBadge(p.status)}${p.summary || "—"}
`; + + // --- Collapsible detailed statistics --- + let detailsInner = ""; + + const numSummary = overview.numerical_summary || {}; + const numFeatures = Object.keys(numSummary); + if (numFeatures.length > 0) { + const allStats = Object.keys(numSummary[numFeatures[0]] || {}); + const preferredOrder = [ + "count", "min", "25th percentile", "50th percentile", "mean", + "75th percentile", "max", "std", + ]; + const statKeys = preferredOrder + .filter((s) => allStats.includes(s)) + .concat(allStats.filter((s) => !preferredOrder.includes(s))); + + detailsInner += `

Numerical summary statistics

`; + detailsInner += `
`; + detailsInner += ``; + statKeys.forEach((s) => { + detailsInner += ``; + }); + detailsInner += ``; + numFeatures.forEach((feat, i) => { + const stripe = i % 2 === 0 ? "bg-white dark:bg-gray-800" : "bg-gray-50 dark:bg-gray-700/50"; + detailsInner += ``; + statKeys.forEach((s) => { + detailsInner += ``; + }); + detailsInner += ``; + }); + detailsInner += `
Feature${s}
${feat}${numSummary[feat][s] ?? "—"}
`; + } + + const catCharts = overview.categorical_charts || {}; + const catChartCols = Object.keys(catCharts); + if (catChartCols.length > 0) { + detailsInner += `

Categorical value distributions

`; + detailsInner += `
`; + } + + if (overview.histograms && Object.keys(overview.histograms).length > 0) { + detailsInner += `

Feature distributions (numerical)

`; + detailsInner += `
`; + } + + if (detailsInner) { + html += ` +
+ + Show detailed statistics & distributions + +
${detailsInner}
+
`; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; + + if (catChartCols.length > 0) { + renderCategoricalPieCharts(overview.categorical_charts, "readiness-categorical-charts"); + } + if (overview.histograms) { + renderWorkspaceHistograms( + overview.histograms, + "readiness-histograms-inner", + true, + "large", + ); + } +} + +/** + * Render the Data Quality scorecard into the given container. + */ +function renderReadinessDataQuality(container, dq) { + if (dq.error) { + container.innerHTML = `

Data quality unavailable: ${dq.error}

`; + return; + } + const kpis = dq.kpis || []; + const gradeCls = _dqStatusClasses(dq.grade_status); + + // --- Overall grade + KPI tiles --- + let html = ` +
+ Overall data quality grade${_readinessInfoIcon("overall_dq_grade")} + ${_pct(dq.grade)} +
+
`; + + kpis.forEach((k) => { + const cls = _dqStatusClasses(k.status); + const widthPct = + k.value === null || k.value === undefined + ? 0 + : Math.max(0, Math.min(100, Math.round(k.value * 100))); + html += ` +
+
+ ${k.label}${_readinessInfoIcon(k.id)} + ${_pct(k.value)} +
+
+
+
+

${k.hint || ""}

+
`; + }); + html += "
"; + + // --- Needs attention --- + const na = dq.needs_attention || {}; + const incomplete = na.incomplete_features || []; + const outlierFeats = na.outlier_features || []; + const dupRows = na.duplicate_rows || 0; + + const naItems = []; + if (incomplete.length) { + const top = incomplete + .slice(0, 6) + .map( + (f) => + `
  • ${f.feature}${_pct(f.completeness)} complete
    • `, + ) + .join(""); + const more = + incomplete.length > 6 + ? `
  • +${incomplete.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Incomplete features (${incomplete.length})${_readinessInfoIcon("completeness")}

    +
      ${top}${more}
    +
    `); + } + if (outlierFeats.length) { + const top = outlierFeats + .slice(0, 6) + .map( + (f) => + `
  • ${f.feature}${_pct(f.outlier_proportion)} outliers
    • `, + ) + .join(""); + const more = + outlierFeats.length > 6 + ? `
  • +${outlierFeats.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Features with outliers (${outlierFeats.length})${_readinessInfoIcon("outlier_cleanliness")}

    +
      ${top}${more}
    +
    `); + } + if (dupRows && dupRows > 0) { + naItems.push(` +
    +

    Duplicate rows${_readinessInfoIcon("uniqueness")}

    +

    ${_pct(dupRows)} of rows are exact duplicates.

    +
    `); + } + + if (naItems.length) { + html += ` +
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } else { + html += ` +
    +

    No data quality issues detected — all features complete, no duplicates, no outliers.

    +
    `; + } + + // --- Collapsible details (original charts) --- + const det = dq.details || {}; + let detailsInner = ""; + if (det.completeness && det.completeness.visualization) { + detailsInner += ` +
    +

    Completeness by feature

    + Completeness chart +
    `; + } + if (det.outliers && det.outliers.visualization) { + detailsInner += ` +
    +

    Outliers by feature

    + Outliers chart +
    `; + } else if (det.outliers && det.outliers.error) { + detailsInner += `

    Outliers: ${det.outliers.error}

    `; + } + + if (detailsInner) { + html += ` +
    + + Show detailed charts + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; +} + +/** + * Render the Impact on AI scorecard (automated all-pairs correlation signals: + * redundancy, leakage risk, isolated features) into the given container. + */ +function renderReadinessImpact(container, impact) { + if (impact.error) { + container.innerHTML = `

    Impact on AI unavailable: ${impact.error}

    `; + return; + } + + const redundant = impact.redundant_pairs || []; + const leakage = impact.leakage_pairs || []; + const isolated = impact.isolated_features || []; + const topPairs = impact.top_pairs || []; + const dropped = impact.columns_dropped || []; + const analyzed = impact.columns_analyzed || 0; + + const fmtScore = (s) => (typeof s === "number" ? s.toFixed(2) : s); + const fmtPair = (p) => + `
  • ${p.a} \u2194 ${p.b}${fmtScore(p.score)}
    • `; + + // --- Stat tiles --- + const tiles = [ + { label: "Features analyzed", value: analyzed, status: "neutral", infoKey: "features_analyzed" }, + { + label: "Leakage-risk pairs", + value: leakage.length, + status: leakage.length ? "poor" : "good", + infoKey: "leakage_risk_pairs", + }, + { + label: "Redundant pairs", + value: redundant.length, + status: redundant.length ? "warning" : "good", + infoKey: "redundant_pairs", + }, + { + label: "Isolated features", + value: isolated.length, + status: isolated.length ? "warning" : "good", + infoKey: "isolated_features", + }, + ]; + + let html = '
    '; + tiles.forEach((t) => { + const cls = _dqStatusClasses(t.status); + const valColor = + t.status === "neutral" ? "text-gray-900 dark:text-white" : cls.text; + html += ` +
    +
    ${t.value}
    +
    ${t.label}${_readinessInfoIcon(t.infoKey)}
    +
    `; + }); + html += "
    "; + + // --- Needs attention --- + const naItems = []; + if (leakage.length) { + const items = leakage.slice(0, 6).map(fmtPair).join(""); + const more = + leakage.length > 6 + ? `
  • +${leakage.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Leakage risk (|score| ≥ 0.95)${_readinessInfoIcon("leakage_risk_pairs")}

    +
      ${items}${more}
    +
    `); + } + if (redundant.length) { + const items = redundant.slice(0, 6).map(fmtPair).join(""); + const more = + redundant.length > 6 + ? `
  • +${redundant.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Redundant pairs (|score| ≥ 0.8)${_readinessInfoIcon("redundant_pairs")}

    +
      ${items}${more}
    +
    `); + } + if (isolated.length) { + const items = isolated + .slice(0, 10) + .map((f) => `
  • ${f}
    • `) + .join(""); + const more = + isolated.length > 10 + ? `
  • +${isolated.length - 10} more
    • ` + : ""; + naItems.push(` +
    +

    Isolated features (no strong relationships)${_readinessInfoIcon("isolated_features")}

    +
      ${items}${more}
    +
    `); + } + + if (naItems.length) { + html += ` +
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } else { + html += ` +
    +

    No redundancy, leakage risk, or isolated features detected.

    +
    `; + } + + // --- Most-related pairs table --- + if (topPairs.length) { + html += + '

    Most-related feature pairs' + + _readinessInfoIcon("most_related_pairs") + + "

    "; + html += + '
    '; + html += + ''; + topPairs.forEach((p, i) => { + const stripe = + i % 2 === 0 + ? "bg-white dark:bg-gray-800" + : "bg-gray-50 dark:bg-gray-700/50"; + html += ``; + }); + html += "
    Feature AFeature BScore
    ${p.a}${p.b}${fmtScore(p.score)}
    "; + } + + // --- Collapsible details (heatmaps + excluded columns) --- + const det = impact.details || {}; + let detailsInner = ""; + if (det.numerical_visualization) { + const method = det.numerical_method ? ` (${det.numerical_method})` : ""; + detailsInner += ` +
    +

    Numerical correlation${method}

    + Numerical correlation heatmap +
    `; + } + if (det.categorical_visualization) { + detailsInner += ` +
    +

    Categorical correlation (Theil's U)

    + Categorical correlation heatmap +
    `; + } + if (dropped.length) { + const items = dropped + .map( + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ) + .join(""); + detailsInner += ` +
    +

    Excluded columns (${dropped.length})

    +
      ${items}
    +
    `; + } + if (detailsInner) { + html += ` +
    + + Show correlation heatmaps & excluded columns + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; +} + +/** + * Render the Fairness & Bias scorecard (auto-selected columns, four metrics, + * selection criteria, needs-attention lists, collapsible charts). + */ +function renderReadinessFairness(container, fb) { + if (fb.error) { + container.innerHTML = `

    Fairness & Bias unavailable: ${fb.error}

    `; + return; + } + + const sel = fb.auto_selection || {}; + const criteria = sel.selection_criteria || {}; + const sensCrit = criteria.sensitive_attributes || {}; + const targetCrit = criteria.target_column || {}; + const posCrit = criteria.positive_class || {}; + const thresholds = criteria.thresholds || {}; + const kpis = fb.kpis || []; + const na = fb.needs_attention || {}; + const gradeCls = _dqStatusClasses(fb.grade_status); + + // --- Auto-selection criteria (transparent) --- + let html = ` +
    +

    Auto-selection criteria

    +
      +
    • Sensitive attributes: ${sensCrit.selected?.length ? sensCrit.selected.join(", ") : "none"}
      + ${sensCrit.rule || ""}
      • +
    • Target column: ${targetCrit.selected || "none"}${targetCrit.reason ? ` (${targetCrit.reason})` : ""}
      + ${targetCrit.rule || ""}
      • +
    • CDD positive class: ${posCrit.selected ?? "none"}${posCrit.reason ? ` (${posCrit.reason})` : ""}
      + ${posCrit.rule || ""}
      • +
    • Primary sensitive (statistical rate & CDD): ${sel.primary_sensitive || "none"}
      • +
    • Flags: + representation ratio ≥ ${thresholds.representation_ratio_flag ?? "—"}, + minority class < ${_pct(thresholds.minority_class_share)}, + TSD ≥ ${thresholds.tsd_disparity_flag ?? "—"}, + imbalance degree good/warning < ${thresholds.imbalance_degree_good ?? "—"} / ${thresholds.imbalance_degree_warning ?? "—"} +
      • +
    +
    `; + + // --- Overall grade + KPI tiles --- + html += ` +
    + Overall fairness grade${_readinessInfoIcon("overall_fairness_grade")} + ${_pct(fb.grade)} +
    +
    `; + + kpis.forEach((k) => { + const cls = _dqStatusClasses(k.status); + const displayVal = + k.id === "label_balance" && k.raw_imbalance_degree != null + ? `ID ${Number(k.raw_imbalance_degree).toFixed(2)}` + : _pct(k.value); + const widthPct = + k.value === null || k.value === undefined + ? 0 + : Math.max(0, Math.min(100, Math.round(k.value * 100))); + html += ` +
    +
    + ${k.label}${_readinessInfoIcon(k.id)} + ${displayVal} +
    +
    +
    +
    +

    ${k.hint || ""}

    +
    `; + }); + html += "
    "; + + // --- Needs attention --- + const naItems = []; + const repImbalance = na.representation_imbalance || []; + if (repImbalance.length) { + const items = repImbalance + .slice(0, 5) + .map( + (s) => + `
  • ${s.column}max ratio ${s.max_ratio}
    • `, + ) + .join(""); + naItems.push(` +
    +

    Representation imbalance (${repImbalance.length})${_readinessInfoIcon("representation_imbalance")}

    +
      ${items}
    +
    `); + } + const minorities = na.minority_classes || []; + if (minorities.length) { + const items = minorities + .map( + (m) => + `
  • ${m.class}${_pct(m.share)} share
    • `, + ) + .join(""); + naItems.push(` +
    +

    Minority classes (${minorities.length})${_readinessInfoIcon("minority_classes")}

    +
      ${items}
    +
    `); + } + const outcomeDisp = na.outcome_disparities || []; + if (outcomeDisp.length) { + const items = outcomeDisp + .map( + (d) => + `
  • ${d.class}TSD ${d.tsd}
    • `, + ) + .join(""); + naItems.push(` +
    +

    Outcome-rate disparities (${outcomeDisp.length})${_readinessInfoIcon("outcome_disparities")}

    +
      ${items}
    +
    `); + } + const cddDisp = na.cdd_disparities || []; + if (cddDisp.length) { + const items = cddDisp + .map((d) => `
  • ${d.group}
    • `) + .join(""); + naItems.push(` +
    +

    CDD flagged groups (${cddDisp.length})${_readinessInfoIcon("cdd_disparities")}

    +
      ${items}
    +
    `); + } + + if (naItems.length) { + html += ` +
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } else { + html += ` +
    +

    No fairness issues detected under the automated thresholds.

    +
    `; + } + + // --- Collapsible details (charts) --- + const det = fb.details || {}; + let detailsInner = ""; + + const repVis = det.representation_rate?.visualizations || {}; + for (const [col, b64] of Object.entries(repVis)) { + detailsInner += ` +
    +

    Representation rate — ${col}

    + Representation ${col} +
    `; + } + if (det.representation_rate?.error && !Object.keys(repVis).length) { + detailsInner += `

    Representation rate: ${det.representation_rate.error}

    `; + } + + if (det.class_imbalance?.visualization) { + detailsInner += ` +
    +

    Class imbalance — ${targetCrit.selected || "target"}

    + Class imbalance +
    `; + } else if (det.class_imbalance?.error) { + detailsInner += `

    Class imbalance: ${det.class_imbalance.error}

    `; + } + + if (det.statistical_rate?.visualization) { + detailsInner += ` +
    +

    Statistical rate — ${det.statistical_rate.sensitive} × ${det.statistical_rate.target}

    + Statistical rate +
    `; + } else if (det.statistical_rate?.error) { + detailsInner += `

    Statistical rate: ${det.statistical_rate.error}

    `; + } + + if (det.cdd?.disparities && !det.cdd.error) { + const rows = Object.entries(det.cdd.disparities) + .map( + ([grp, info]) => + `${grp}${info.disparity}`, + ) + .join(""); + detailsInner += ` +
    +

    Conditional demographic disparity (positive: ${det.cdd.positive_class})

    + ${rows}
    GroupDisparity
    +
    `; + } else if (det.cdd?.error) { + detailsInner += `

    CDD: ${det.cdd.error}

    `; + } + + const excluded = sensCrit.excluded || []; + if (excluded.length) { + const items = excluded + .map( + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ) + .join(""); + detailsInner += ` +
    +

    Excluded sensitive candidates (${excluded.length})

    +
      ${items}
    +
    `; + } + + if (detailsInner) { + html += ` +
    + + Show detailed charts & CDD table + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; +} + +// ==================== Workspace Init ==================== + +/** + * Initialize the workspace after file upload. + * Fetches summary statistics and populates feature dropdowns. + */ +function initWorkspace() { + // Restore panel from URL hash, or default to data-overview + const hash = location.hash.replace("#", ""); + const initialPanel = + hash && document.getElementById("panel-" + hash) ? hash : "data-overview"; + showPanel(initialPanel, false); // false = don't push to history on init + // Replace current history entry so back button works from the first panel + history.replaceState({ panel: initialPanel }, "", "#" + initialPanel); + + // Fetch + render summary statistics into the Data Overview panel + loadDataOverview(); // Populate feature dropdowns via /feature-set (same as metric.js does) fetch("/feature-set", { method: "POST" }) diff --git a/web/templates/_components/sidebar.html b/web/templates/_components/sidebar.html index f7c6dd3f..35ab5af2 100644 --- a/web/templates/_components/sidebar.html +++ b/web/templates/_components/sidebar.html @@ -18,6 +18,14 @@ Data Overview + + +
    diff --git a/web/templates/inspector.html b/web/templates/inspector.html index 792ba5fb..498ac5f7 100644 --- a/web/templates/inspector.html +++ b/web/templates/inspector.html @@ -56,6 +56,7 @@
    {% include '_panels/_data_overview.html' %} + {% include '_panels/_readiness_report.html' %} {% include '_panels/_data_quality.html' %} {% include '_panels/_feature_relevance.html' %} {% include '_panels/_correlation_analysis.html' %} From 91daa888002ad81fa6ef69bb31bc6db99ed5cc31 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 16 Jun 2026 18:42:50 -0400 Subject: [PATCH 02/23] Add --- web/routes/metrics.py | 920 +++++++++++++++++ web/routes/utils.py | 78 +- web/static/css/theme.css | 21 +- web/static/js/inspector.js | 975 ++++++++++++++++++- web/templates/_components/sidebar.html | 8 + web/templates/_panels/_readiness_report.html | 55 ++ web/templates/inspector.html | 1 + 7 files changed, 2032 insertions(+), 26 deletions(-) create mode 100644 web/templates/_panels/_readiness_report.html diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 6e3a8def..fb8b8caa 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -1,7 +1,9 @@ import json import logging +import os import time +import pandas as pd from celery.result import AsyncResult from web.telemetry import get_tracer, trace_metric from flask import ( @@ -57,6 +59,8 @@ get_result_or_default, is_metric_cache_valid, store_result, + summary_histograms, + categorical_distribution_charts, ) metrics_bp = Blueprint("metrics", __name__) @@ -137,6 +141,922 @@ def data_quality(): return get_result_or_default("metrics.data_quality", file_path, file_name) +# --------------------------------------------------------------------------- +# Readiness Report (aggregated, non-interactive) +# --------------------------------------------------------------------------- + +def _grade_label(score): + """Map a 0–1 readiness score to a coarse status label.""" + if score is None: + return "unknown" + if score >= 0.9: + return "good" + if score >= 0.7: + return "warning" + return "poor" + + +# Dataset-overview readiness thresholds (per-feature profile) +_OVERVIEW_MISSING_WARNING = 0.05 +_OVERVIEW_MISSING_POOR = 0.20 +_OVERVIEW_DOMINANT_WARNING = 0.95 +_OVERVIEW_HIGH_CARDINALITY = 50 +_OVERVIEW_ID_UNIQUE_RATIO = 0.9 +_OVERVIEW_CAT_TOP_N = 5 + + +def _classify_feature_type(series): + """Map a pandas Series to a coarse feature type label.""" + if pd.api.types.is_bool_dtype(series): + return "boolean" + if pd.api.types.is_datetime64_any_dtype(series): + return "datetime" + if pd.api.types.is_numeric_dtype(series): + return "numerical" + return "categorical" + + +def _feature_readiness_status(pct_missing, n_unique, n_rows, feat_type, pct_dominant): + """Derive a per-feature readiness status from profile statistics.""" + if n_unique <= 1: + return "poor" + if pct_missing is not None and pct_missing > _OVERVIEW_MISSING_POOR: + return "poor" + if n_rows > 0 and n_unique / n_rows >= _OVERVIEW_ID_UNIQUE_RATIO: + return "poor" + if pct_missing is not None and pct_missing > _OVERVIEW_MISSING_WARNING: + return "warning" + if feat_type == "categorical" and n_unique > _OVERVIEW_HIGH_CARDINALITY: + return "warning" + if pct_dominant is not None and pct_dominant > _OVERVIEW_DOMINANT_WARNING: + return "warning" + return "good" + + +def _feature_summary(series, feat_type, n_unique): + """Build a compact, type-specific summary string for one feature.""" + non_null = series.dropna() + if len(non_null) == 0: + return "all missing" + + if feat_type == "numerical": + mean = non_null.mean() + std = non_null.std() + if pd.notna(std) and std > 0: + return f"mean {mean:.2g}, std {std:.2g}" + return f"min {non_null.min():.2g} – max {non_null.max():.2g}" + + if feat_type == "categorical": + vc = non_null.value_counts(normalize=True) + top_val = vc.index[0] + top_pct = vc.iloc[0] * 100 + return f"{top_val} ({top_pct:.0f}%), {n_unique} categories" + + if feat_type == "datetime": + return f"{non_null.min()} – {non_null.max()}" + + if feat_type == "boolean": + true_pct = (non_null.astype(bool)).mean() * 100 + return f"True {true_pct:.0f}%, False {100 - true_pct:.0f}%" + + return f"{n_unique} unique values" + + +def _build_feature_profiles(df): + """Compute a per-column readiness profile for every feature in *df*.""" + n_rows = len(df) + profiles = [] + type_counts = {"numerical": 0, "categorical": 0, "datetime": 0, "boolean": 0} + + for col in df.columns: + series = df[col] + feat_type = _classify_feature_type(series) + type_counts[feat_type] = type_counts.get(feat_type, 0) + 1 + + n_missing = int(series.isnull().sum()) + pct_missing = round(n_missing / n_rows, 4) if n_rows else 0.0 + n_unique = int(series.nunique(dropna=True)) + + pct_dominant = None + non_null = series.dropna() + if len(non_null) > 0: + pct_dominant = round( + float(non_null.value_counts(normalize=True).iloc[0]), 4 + ) + + profiles.append({ + "feature": str(col), + "type": feat_type, + "dtype": str(series.dtype), + "pct_missing": pct_missing, + "n_unique": n_unique, + "pct_dominant": pct_dominant, + "status": _feature_readiness_status( + pct_missing, n_unique, n_rows, feat_type, pct_dominant + ), + "summary": _feature_summary(series, feat_type, n_unique), + }) + + return profiles, type_counts + + +def _build_categorical_distributions(df, top_n=_OVERVIEW_CAT_TOP_N): + """Top-*n* value counts (with percentages) for each categorical column.""" + distributions = {} + for col in df.columns: + if _classify_feature_type(df[col]) != "categorical": + continue + vc = df[col].value_counts(dropna=True) + total = len(df[col].dropna()) + if total == 0: + distributions[str(col)] = [] + continue + entries = [] + for val, count in vc.head(top_n).items(): + entries.append({ + "value": str(val), + "count": int(count), + "pct": round(float(count) / total, 4), + }) + distributions[str(col)] = entries + return distributions + + +def _build_dataset_overview_section(file_info): + """Build the dataset-overview portion of the readiness report. + + Returns file metadata, per-feature readiness profiles, numerical describe() + summary, categorical top-value distributions, and numerical histograms. + """ + file_path, file_name, file_type = file_info + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + n_rows = len(df) + profiles, type_counts = _build_feature_profiles(df) + + file_size_bytes = None + if file_path and os.path.exists(file_path): + try: + file_size_bytes = os.path.getsize(file_path) + except OSError: + pass + + memory_bytes = int(df.memory_usage(deep=True).sum()) + + numerical_summary = {} + num_df = df.select_dtypes(include="number") + if not num_df.empty: + numerical_summary = num_df.describe().map( + lambda x: round(x, 2) if x == 0 or abs(x) >= 0.001 else f"{x:.2e}" + ).to_dict() + for v in numerical_summary.values(): + for old_key in list(v.keys()): + if old_key in ["25%", "50%", "75%"]: + new_key = old_key.replace("%", "th percentile") + v[new_key] = v.pop(old_key) + + return { + "file_metadata": { + "file_name": file_name, + "file_type": file_type, + "file_size_bytes": file_size_bytes, + "memory_bytes": memory_bytes, + "rows": n_rows, + "columns": len(df.columns), + "numerical_count": type_counts.get("numerical", 0), + "categorical_count": type_counts.get("categorical", 0), + "datetime_count": type_counts.get("datetime", 0), + "boolean_count": type_counts.get("boolean", 0), + }, + "feature_profiles": profiles, + "numerical_summary": numerical_summary, + "categorical_distributions": _build_categorical_distributions(df), + "categorical_charts": categorical_distribution_charts(df), + "histograms": summary_histograms(df, figsize=(7, 4.5)), + "profile_thresholds": { + "missing_warning": _OVERVIEW_MISSING_WARNING, + "missing_poor": _OVERVIEW_MISSING_POOR, + "dominant_warning": _OVERVIEW_DOMINANT_WARNING, + "high_cardinality": _OVERVIEW_HIGH_CARDINALITY, + "id_unique_ratio": _OVERVIEW_ID_UNIQUE_RATIO, + }, + } + + +def _build_data_quality_section(file_info): + """Compute the data-quality portion of the readiness report. + + Runs completeness, outliers, and duplicity (the same functions backing the + Data Quality tab), then derives readiness-oriented KPIs (normalized so that + higher is always better), an overall grade, and a "needs attention" list. + + Returns a JSON-serializable dict, or ``{"error": str}`` on failure. + """ + section = {} + + # --- Completeness ----------------------------------------------------- + compl = completeness(file_info) + compl_scores = compl.get("Completeness scores", {}) or {} + overall_completeness = compl.get("Overall Completeness") + + # --- Outliers --------------------------------------------------------- + out = outliers(file_info) + out_scores_raw = out.get("Outlier scores", {}) if isinstance(out, dict) else {} + overall_outlier = None + out_scores = {} + if isinstance(out_scores_raw, dict): + overall_outlier = out_scores_raw.get("Overall outlier score") + out_scores = { + k: v for k, v in out_scores_raw.items() if k != "Overall outlier score" + } + outliers_error = out.get("Error") if isinstance(out, dict) else None + + # --- Duplicity -------------------------------------------------------- + dup = duplicity(file_info) + overall_duplicity = ( + dup.get("Duplicity scores", {}).get("Overall duplicity of the dataset") + if isinstance(dup, dict) + else None + ) + + # --- Normalized KPIs (higher = better) -------------------------------- + completeness_kpi = overall_completeness + uniqueness_kpi = (1 - overall_duplicity) if overall_duplicity is not None else None + outlier_clean_kpi = (1 - overall_outlier) if overall_outlier is not None else None + + kpis = [ + { + "id": "completeness", + "label": "Completeness", + "value": completeness_kpi, + "status": _grade_label(completeness_kpi), + "hint": "Share of non-missing values across the dataset.", + }, + { + "id": "uniqueness", + "label": "Uniqueness", + "value": uniqueness_kpi, + "status": _grade_label(uniqueness_kpi), + "hint": "1 − proportion of duplicate rows.", + }, + { + "id": "outlier_cleanliness", + "label": "Outlier-cleanliness", + "value": outlier_clean_kpi, + "status": _grade_label(outlier_clean_kpi), + "hint": "1 − mean outlier proportion (IQR method) across numerical features.", + }, + ] + + present = [k["value"] for k in kpis if k["value"] is not None] + grade = sum(present) / len(present) if present else None + + # --- Needs attention -------------------------------------------------- + incomplete = sorted( + ( + {"feature": col, "completeness": score} + for col, score in compl_scores.items() + if isinstance(score, (int, float)) and score < 1.0 + ), + key=lambda x: x["completeness"], + ) + high_outliers = sorted( + ( + {"feature": col, "outlier_proportion": score} + for col, score in out_scores.items() + if isinstance(score, (int, float)) and score > 0 + ), + key=lambda x: x["outlier_proportion"], + reverse=True, + ) + + section = { + "grade": grade, + "grade_status": _grade_label(grade), + "kpis": kpis, + "needs_attention": { + "incomplete_features": incomplete, + "outlier_features": high_outliers, + "duplicate_rows": ( + overall_duplicity if overall_duplicity not in (None, 0) else 0 + ), + }, + "details": { + "completeness": { + "overall": overall_completeness, + "scores": compl_scores, + "visualization": compl.get("Completeness Visualization"), + }, + "outliers": { + "overall": overall_outlier, + "scores": out_scores, + "visualization": out.get("Outliers Visualization") if isinstance(out, dict) else None, + "error": outliers_error, + }, + "duplicity": {"overall": overall_duplicity}, + }, + } + return section + + +# Impact-on-AI tuning constants +_CORR_MAX_COLUMNS = 25 # cap analysed columns to keep heatmaps readable +_CORR_HIGH_CARD_MAX = 50 # drop categorical columns with more unique values +_CORR_ID_UNIQUE_RATIO = 0.9 # drop categorical columns that look like IDs +_CORR_REDUNDANT_THRESHOLD = 0.8 # |score| at/above which a pair is "redundant" +_CORR_LEAKAGE_THRESHOLD = 0.95 # |score| at/above which a pair is "leakage risk" +_CORR_ISOLATED_THRESHOLD = 0.1 # max |score| below which a feature is "isolated" + + +def _prune_columns_for_corr(df): + """Select columns worth feeding into the all-pairs correlation analysis. + + Drops columns that are useless or pathological for correlation: + constants, ID-like / high-cardinality categoricals. Numerical columns are + always kept (they are cheap to correlate). The result is capped at + ``_CORR_MAX_COLUMNS`` (numerical prioritized) to keep the computation and + heatmaps tractable. + + Returns ``(kept_columns, dropped)`` where *dropped* is a list of + ``{"feature": str, "reason": str}``. + """ + n_rows = max(len(df), 1) + numeric_cols = list(df.select_dtypes(exclude=["object", "string", "category"]).columns) + categorical_cols = list(df.select_dtypes(include=["object", "string", "category"]).columns) + + kept_numeric = [] + kept_categorical = [] + dropped = [] + + for col in numeric_cols: + if df[col].nunique(dropna=True) <= 1: + dropped.append({"feature": col, "reason": "constant column"}) + else: + kept_numeric.append(col) + + for col in categorical_cols: + nunique = df[col].nunique(dropna=True) + if nunique <= 1: + dropped.append({"feature": col, "reason": "constant column"}) + elif nunique / n_rows >= _CORR_ID_UNIQUE_RATIO: + dropped.append({"feature": col, "reason": "ID-like (near-unique values)"}) + elif nunique > _CORR_HIGH_CARD_MAX: + dropped.append({"feature": col, "reason": f"high cardinality ({nunique} categories)"}) + else: + kept_categorical.append(col) + + # Cap total columns, prioritizing numerical features + kept = kept_numeric + kept_categorical + if len(kept) > _CORR_MAX_COLUMNS: + for col in kept[_CORR_MAX_COLUMNS:]: + dropped.append({"feature": col, "reason": "exceeded column cap"}) + kept = kept[:_CORR_MAX_COLUMNS] + + return kept, dropped + + +def _pairwise_signals(scores): + """Derive readiness signals from a flat ``{"a vs b": score}`` mapping. + + Collapses the symmetric/asymmetric directional entries into one record per + unordered pair (keeping the largest-magnitude score), then classifies pairs + as redundant or leakage-risk and flags features that are not meaningfully + related to anything else ("isolated"). + """ + pair_max = {} + for key, val in scores.items(): + if " vs " not in key or not isinstance(val, (int, float)): + continue + a, b = key.split(" vs ", 1) + if a == b: + continue + ukey = tuple(sorted([a, b])) + abs_score = abs(val) + if ukey not in pair_max or abs_score > pair_max[ukey]["abs_score"]: + pair_max[ukey] = { + "a": ukey[0], + "b": ukey[1], + "score": round(float(val), 3), + "abs_score": abs_score, + } + + pairs = list(pair_max.values()) + pairs.sort(key=lambda p: p["abs_score"], reverse=True) + + leakage = [ + {"a": p["a"], "b": p["b"], "score": p["score"]} + for p in pairs + if p["abs_score"] >= _CORR_LEAKAGE_THRESHOLD + ] + redundant = [ + {"a": p["a"], "b": p["b"], "score": p["score"]} + for p in pairs + if _CORR_REDUNDANT_THRESHOLD <= p["abs_score"] < _CORR_LEAKAGE_THRESHOLD + ] + top = [{"a": p["a"], "b": p["b"], "score": p["score"]} for p in pairs[:8]] + + # Per-feature connectivity: the strongest relationship each feature has + connectivity = {} + for p in pairs: + connectivity[p["a"]] = max(connectivity.get(p["a"], 0.0), p["abs_score"]) + connectivity[p["b"]] = max(connectivity.get(p["b"], 0.0), p["abs_score"]) + isolated = sorted( + f for f, c in connectivity.items() if c < _CORR_ISOLATED_THRESHOLD + ) + + return { + "redundant": redundant, + "leakage": leakage, + "top": top, + "isolated": isolated, + } + + +def _build_impact_on_ai_section(file_info): + """Compute the Impact-on-AI portion of the readiness report. + + Runs an automated, non-interactive all-pairs correlation analysis (numerical + via vectorized pandas correlation, categorical via Theil's U) over a pruned, + capped set of columns, then derives redundancy / leakage / isolation signals. + + Returns a JSON-serializable dict, or ``{"error": str}`` on failure. + """ + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + kept, dropped = _prune_columns_for_corr(df) + if len(kept) < 2: + return { + "error": "Not enough usable columns for correlation analysis after pruning.", + "columns_analyzed": len(kept), + "columns_dropped": dropped, + } + + corr = calc_correlations(kept, file_info) + if isinstance(corr, dict) and "Message" in corr: + return {"error": corr["Message"], "columns_dropped": dropped} + + scores = corr.get("Correlation Scores", {}) if isinstance(corr, dict) else {} + signals = _pairwise_signals(scores) + cat = corr.get("Correlations Analysis Categorical", {}) or {} + num = corr.get("Correlations Analysis Numerical", {}) or {} + + return { + "columns_analyzed": len(kept), + "columns_dropped": dropped, + "redundant_pairs": signals["redundant"], + "leakage_pairs": signals["leakage"], + "isolated_features": signals["isolated"], + "top_pairs": signals["top"], + "details": { + "categorical_visualization": cat.get( + "Correlations Analysis Categorical Visualization" + ), + "numerical_visualization": num.get( + "Correlations Analysis Numerical Visualization" + ), + "numerical_method": num.get("Method"), + }, + } + + +# Fairness & Bias tuning constants +_FAIRNESS_SENSITIVE_MIN_UNIQUE = 2 +_FAIRNESS_SENSITIVE_MAX_UNIQUE = 30 +_FAIRNESS_MAX_SENSITIVE_COLS = 5 +_FAIRNESS_ID_UNIQUE_RATIO = 0.9 +_FAIRNESS_TARGET_MAX_UNIQUE = 30 +_FAIRNESS_REP_RATIO_FLAG = 3.0 # probability ratio at/above which representation is flagged +_FAIRNESS_MINORITY_SHARE = 0.05 # class share below which a label is "minority" +_FAIRNESS_IMBALANCE_GOOD = 0.5 # imbalance degree below this → good +_FAIRNESS_IMBALANCE_WARNING = 1.0 # below this → warning, else poor +_FAIRNESS_TSD_FLAG = 0.15 # TSD above this → outcome-rate disparity flagged +_FAIRNESS_SENSITIVE_NAME_HINTS = ( + "gender", "sex", "race", "ethnic", "age", "religion", "marital", + "disability", "national", "origin", "minority", +) +_FAIRNESS_TARGET_NAME_HINTS = ( + "label", "target", "class", "outcome", "decision", "y", "income", +) + + +def _fairness_name_hint_score(col_name, hints): + """Return a higher score when *col_name* matches an earlier hint substring.""" + lower = col_name.lower() + best = 0 + for i, hint in enumerate(hints): + if hint in lower: + best = max(best, len(hints) - i) + return best + + +def _auto_select_fairness_columns(df): + """Pick sensitive attributes and a target column for automated fairness checks. + + Returns a dict with selected columns, exclusion reasons, and the + auto-selected positive class (mode of the target) for CDD. + """ + n_rows = max(len(df), 1) + cat_cols = list(df.select_dtypes(include=["object", "string", "category"]).columns) + + sensitive_candidates = [] + excluded_sensitive = [] + + for col in cat_cols: + nunique = df[col].nunique(dropna=True) + if nunique < _FAIRNESS_SENSITIVE_MIN_UNIQUE: + excluded_sensitive.append({"feature": col, "reason": "constant column"}) + elif nunique > _FAIRNESS_SENSITIVE_MAX_UNIQUE: + excluded_sensitive.append({ + "feature": col, + "reason": f"high cardinality ({nunique} categories, max {_FAIRNESS_SENSITIVE_MAX_UNIQUE})", + }) + elif nunique / n_rows >= _FAIRNESS_ID_UNIQUE_RATIO: + excluded_sensitive.append({"feature": col, "reason": "ID-like (near-unique values)"}) + else: + sensitive_candidates.append({ + "feature": col, + "nunique": int(nunique), + "name_hint_score": _fairness_name_hint_score(col, _FAIRNESS_SENSITIVE_NAME_HINTS), + }) + + sensitive_candidates.sort( + key=lambda c: (-c["name_hint_score"], c["nunique"], c["feature"]) + ) + selected_sensitive = [c["feature"] for c in sensitive_candidates[:_FAIRNESS_MAX_SENSITIVE_COLS]] + for c in sensitive_candidates[_FAIRNESS_MAX_SENSITIVE_COLS:]: + excluded_sensitive.append({"feature": c["feature"], "reason": "exceeded sensitive-column cap"}) + + # Target: low-cardinality columns (2–30 unique), prefer name hints; exclude sensitives + target_candidates = [] + sensitive_set = set(selected_sensitive) + for col in df.columns: + if col in sensitive_set: + continue + nunique = df[col].nunique(dropna=True) + if _FAIRNESS_SENSITIVE_MIN_UNIQUE <= nunique <= _FAIRNESS_TARGET_MAX_UNIQUE: + target_candidates.append({ + "feature": col, + "nunique": int(nunique), + "name_hint_score": _fairness_name_hint_score(col, _FAIRNESS_TARGET_NAME_HINTS), + }) + + target_candidates.sort( + key=lambda c: (-c["name_hint_score"], c["nunique"], c["feature"]) + ) + target_col = target_candidates[0]["feature"] if target_candidates else None + target_reason = None + if target_col: + tc = target_candidates[0] + if tc["name_hint_score"] > 0: + target_reason = "matched target name hint" + elif tc == target_candidates[-1] and len(target_candidates) == 1: + target_reason = "only eligible low-cardinality column" + else: + target_reason = "lowest-cardinality eligible column (after name-hint priority)" + + positive_class = None + positive_reason = None + if target_col is not None: + target_series = df[target_col].dropna() + if len(target_series) > 0: + positive_class = target_series.mode().iloc[0] + positive_reason = "most frequent class in auto-selected target" + + primary_sensitive = selected_sensitive[0] if selected_sensitive else None + + return { + "sensitive_columns": selected_sensitive, + "primary_sensitive": primary_sensitive, + "target_column": target_col, + "positive_class": positive_class, + "selection_criteria": { + "sensitive_attributes": { + "rule": ( + f"Categorical columns with {_FAIRNESS_SENSITIVE_MIN_UNIQUE}–" + f"{_FAIRNESS_SENSITIVE_MAX_UNIQUE} unique values, excluding " + "ID-like columns; prefer name hints " + f"{list(_FAIRNESS_SENSITIVE_NAME_HINTS)}; " + f"capped at {_FAIRNESS_MAX_SENSITIVE_COLS}." + ), + "name_hints": list(_FAIRNESS_SENSITIVE_NAME_HINTS), + "selected": selected_sensitive, + "excluded": excluded_sensitive, + }, + "target_column": { + "rule": ( + f"Columns with {_FAIRNESS_SENSITIVE_MIN_UNIQUE}–{_FAIRNESS_TARGET_MAX_UNIQUE} " + "unique values, excluding auto-selected sensitive columns; " + "prefer name hints " + f"{list(_FAIRNESS_TARGET_NAME_HINTS)}." + ), + "name_hints": list(_FAIRNESS_TARGET_NAME_HINTS), + "selected": target_col, + "reason": target_reason, + }, + "positive_class": { + "rule": "Most frequent value in the auto-selected target (used for CDD only).", + "selected": str(positive_class) if positive_class is not None else None, + "reason": positive_reason, + }, + "thresholds": { + "representation_ratio_flag": _FAIRNESS_REP_RATIO_FLAG, + "minority_class_share": _FAIRNESS_MINORITY_SHARE, + "imbalance_degree_good": _FAIRNESS_IMBALANCE_GOOD, + "imbalance_degree_warning": _FAIRNESS_IMBALANCE_WARNING, + "tsd_disparity_flag": _FAIRNESS_TSD_FLAG, + }, + }, + } + + +def _parse_representation_flags(ratios_dict): + """Extract per-column max probability ratio and flag extreme imbalance.""" + by_column = {} + if not isinstance(ratios_dict, dict) or "Error" in ratios_dict: + return [], None, ratios_dict.get("Error") if isinstance(ratios_dict, dict) else None + + for key, ratio in ratios_dict.items(): + if not isinstance(ratio, (int, float)) or "Column: '" not in key: + continue + # Key format: Column: 'col', Probability ratio for 'A' to 'B' + try: + col_part = key.split("Column: '", 1)[1] + col = col_part.split("',", 1)[0] + except IndexError: + continue + entry = by_column.setdefault(col, {"column": col, "max_ratio": 0.0, "flagged_pairs": []}) + r = float(ratio) + if r > entry["max_ratio"]: + entry["max_ratio"] = round(r, 3) + if r >= _FAIRNESS_REP_RATIO_FLAG: + pair_part = key.split("Probability ratio for ", 1)[-1] + entry["flagged_pairs"].append({"pair": pair_part, "ratio": round(r, 3)}) + + summaries = sorted(by_column.values(), key=lambda x: x["max_ratio"], reverse=True) + worst = summaries[0]["max_ratio"] if summaries else None + rep_balance_kpi = min(1.0, 1.0 / worst) if worst and worst > 0 else None + return summaries, rep_balance_kpi, None + + +def _imbalance_status(id_score): + if id_score is None: + return "unknown" + if id_score < _FAIRNESS_IMBALANCE_GOOD: + return "good" + if id_score < _FAIRNESS_IMBALANCE_WARNING: + return "warning" + return "poor" + + +def _build_fairness_bias_section(file_info): + """Compute the Fairness & Bias portion of the readiness report. + + Auto-selects sensitive attributes and a target, then runs representation + rate, class imbalance, statistical rate, and conditional demographic + disparity using the same functions as the Fairness & Bias tab. + """ + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + selection = _auto_select_fairness_columns(df) + sensitive_cols = selection["sensitive_columns"] + target_col = selection["target_column"] + primary_sensitive = selection["primary_sensitive"] + positive_class = selection["positive_class"] + + kpis = [] + needs_attention = { + "representation_imbalance": [], + "minority_classes": [], + "outcome_disparities": [], + "cdd_disparities": [], + } + details = {} + + # --- Representation Rate (all selected sensitive columns) --------------- + rep_error = None + rep_balance_kpi = None + if sensitive_cols: + ratios = calculate_representation_rate(sensitive_cols, file_info) + rep_summaries, rep_balance_kpi, rep_error = _parse_representation_flags(ratios) + needs_attention["representation_imbalance"] = [ + s for s in rep_summaries if s["max_ratio"] >= _FAIRNESS_REP_RATIO_FLAG + ] + rep_visualizations = {} + for col in sensitive_cols: + try: + vis = create_representation_rate_vis([col], file_info) + if isinstance(vis, str): + rep_visualizations[col] = vis + except Exception: + pass + details["representation_rate"] = { + "ratios": ratios if not rep_error else None, + "summaries": rep_summaries, + "visualizations": rep_visualizations, + "error": rep_error, + } + else: + details["representation_rate"] = { + "error": "No eligible sensitive-attribute columns found.", + } + + kpis.append({ + "id": "representation_balance", + "label": "Representation balance", + "value": rep_balance_kpi, + "status": _grade_label(rep_balance_kpi), + "hint": f"1 / worst group probability ratio (flagged when ratio ≥ {_FAIRNESS_REP_RATIO_FLAG}).", + }) + + # --- Class Imbalance (auto target) -------------------------------------- + imbalance_degree = None + ci_error = None + if target_col: + ci_dict = _compute_class_imbalance(df, target_col, "EU") + if "Error" in ci_dict: + ci_error = ci_dict["Error"] + else: + imb = ci_dict.get("Imbalance degree") or {} + imbalance_degree = imb.get("Imbalance Degree score") + details["class_imbalance"] = { + "visualization": ci_dict.get("Class Imbalance Visualization"), + "imbalance_degree": imbalance_degree, + } + # Minority classes + vc = df[target_col].value_counts(normalize=True, dropna=True) + for cls, share in vc.items(): + if share < _FAIRNESS_MINORITY_SHARE: + needs_attention["minority_classes"].append({ + "class": str(cls), + "share": round(float(share), 4), + }) + else: + details["class_imbalance"] = {"error": "No eligible target column found."} + + label_balance_kpi = ( + max(0.0, 1.0 - min(float(imbalance_degree) / 2.0, 1.0)) + if imbalance_degree is not None + else None + ) + kpis.append({ + "id": "label_balance", + "label": "Label balance", + "value": label_balance_kpi, + "status": _imbalance_status(imbalance_degree), + "hint": ( + f"Derived from Imbalance Degree (EU); 0 = balanced. " + f"Good < {_FAIRNESS_IMBALANCE_GOOD}, warning < {_FAIRNESS_IMBALANCE_WARNING}." + ), + "raw_imbalance_degree": imbalance_degree, + }) + + # --- Statistical Rate (primary sensitive + target) ---------------------- + disparity_kpi = None + if primary_sensitive and target_col: + sr = calculate_statistical_rates(target_col, primary_sensitive, file_info) + if isinstance(sr, dict) and "Error" in sr: + details["statistical_rate"] = {"error": sr["Error"]} + else: + tsd_scores = sr.get("TSD scores") or {} + flagged = [ + {"class": str(cls), "tsd": round(float(score), 4)} + for cls, score in tsd_scores.items() + if isinstance(score, (int, float)) and float(score) >= _FAIRNESS_TSD_FLAG + ] + flagged.sort(key=lambda x: x["tsd"], reverse=True) + needs_attention["outcome_disparities"] = flagged + max_tsd = max( + (float(v) for v in tsd_scores.values() if isinstance(v, (int, float))), + default=None, + ) + disparity_kpi = max(0.0, 1.0 - min(max_tsd, 1.0)) if max_tsd is not None else None + details["statistical_rate"] = { + "sensitive": primary_sensitive, + "target": target_col, + "tsd_scores": tsd_scores, + "visualization": sr.get("Statistical Rate Visualization"), + } + else: + details["statistical_rate"] = { + "error": "Requires both a sensitive attribute and a target column.", + } + + kpis.append({ + "id": "outcome_parity", + "label": "Outcome parity", + "value": disparity_kpi, + "status": _grade_label(disparity_kpi), + "hint": ( + f"1 − max TSD across classes (flagged when TSD ≥ {_FAIRNESS_TSD_FLAG}). " + "Uses primary sensitive attribute." + ), + }) + + # --- Conditional Demographic Disparity ---------------------------------- + if primary_sensitive and target_col and positive_class is not None: + try: + cdd = conditional_demographic_disparity( + df[target_col].tolist(), + df[primary_sensitive].tolist(), + positive_class, + ) + if isinstance(cdd, dict) and "Error" in cdd: + details["cdd"] = {"error": cdd["Error"]} + else: + disparities = (cdd or {}).get("Disparities") or {} + cdd_flagged = [ + {"group": str(grp), "disparity": info.get("disparity")} + for grp, info in disparities.items() + if str(info.get("disparity", "")).lower() == "true" + ] + needs_attention["cdd_disparities"] = cdd_flagged + details["cdd"] = { + "sensitive": primary_sensitive, + "target": target_col, + "positive_class": str(positive_class), + "disparities": disparities, + } + except Exception as e: + details["cdd"] = {"error": str(e)} + else: + details["cdd"] = { + "error": "Requires sensitive attribute, target, and auto-positive class.", + } + + present = [k["value"] for k in kpis if k["value"] is not None] + grade = sum(present) / len(present) if present else None + + return { + "grade": grade, + "grade_status": _grade_label(grade), + "auto_selection": selection, + "kpis": kpis, + "needs_attention": needs_attention, + "details": details, + } + + +@metrics_bp.route("/readiness-report", methods=["GET"]) +def readiness_report(): + """Return an aggregated, non-interactive data-readiness report as JSON. + + Covers dataset overview, Data Quality, Impact-on-AI, and Fairness & Bias. + Designed to be extended with more pillars over time. + """ + file_path = session.get("uploaded_file_path") + file_name = session.get("uploaded_file_name") + file_type = session.get("uploaded_file_type") + + if not file_path: + return jsonify({"success": False, "message": "No file uploaded"}), 200 + + file_info = (file_path, file_name, file_type) + start_time = time.time() + try: + try: + dataset_overview_section = _build_dataset_overview_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — dataset overview error: %s", e, exc_info=True) + dataset_overview_section = {"error": f"{type(e).__name__}: {e}"} + + try: + data_quality_section = _build_data_quality_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — data quality error: %s", e, exc_info=True) + data_quality_section = {"error": f"{type(e).__name__}: {e}"} + + try: + impact_section = _build_impact_on_ai_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — impact on AI error: %s", e, exc_info=True) + impact_section = {"error": f"{type(e).__name__}: {e}"} + + try: + fairness_section = _build_fairness_bias_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — fairness & bias error: %s", e, exc_info=True) + fairness_section = {"error": f"{type(e).__name__}: {e}"} + + response = ensure_json_serializable({ + "success": True, + "dataset_overview": dataset_overview_section, + "data_quality": data_quality_section, + "impact_on_ai": impact_section, + "fairness_bias": fairness_section, + }) + metric_time_log.info("Readiness report built in %.2f seconds", time.time() - start_time) + return jsonify(response) + except Exception as e: + metric_time_log.error("Readiness report error: %s", e, exc_info=True) + return jsonify({"success": False, "message": f"{type(e).__name__}: {e}"}), 200 + + # --------------------------------------------------------------------------- # Fairness # --------------------------------------------------------------------------- diff --git a/web/routes/utils.py b/web/routes/utils.py index d355f401..6e192435 100644 --- a/web/routes/utils.py +++ b/web/routes/utils.py @@ -193,14 +193,14 @@ def ensure_json_serializable(obj): return obj -def summary_histograms(df): +def summary_histograms(df, figsize=(4, 3), dpi=150): """Generate base64-encoded KDE distribution plots for all numeric columns.""" text_color = "#6b7280" curve_color = "#4485F4" line_graphs = {} for column in df.select_dtypes(include="number").columns: - fig, ax = plt.subplots(figsize=(4, 3)) + fig, ax = plt.subplots(figsize=figsize) fig.patch.set_alpha(0) ax.set_facecolor("none") @@ -214,7 +214,7 @@ def summary_histograms(df): fig.tight_layout(pad=0.5) img_buffer = io.BytesIO() - fig.savefig(img_buffer, format="png", dpi=150, transparent=True) + fig.savefig(img_buffer, format="png", dpi=dpi, transparent=True) img_buffer.seek(0) encoded_img = base64.b64encode(img_buffer.read()).decode("utf-8") @@ -224,3 +224,75 @@ def summary_histograms(df): img_buffer.close() return line_graphs + + +def categorical_distribution_charts(df, top_n=10): + """Generate base64-encoded pie charts for categorical columns. + + Shows the top *top_n* values per column; remaining values are grouped + into an "Other" slice when applicable. + """ + text_color = "#6b7280" + palette = [ + "#4485F4", "#34A853", "#FBBC05", "#EA4335", "#9C27B0", + "#00ACC1", "#FF7043", "#8D6E63", "#78909C", "#AB47BC", + ] + charts = {} + + cat_cols = df.select_dtypes(include=["object", "string", "category"]).columns + for column in cat_cols: + series = df[column].dropna() + if len(series) == 0: + continue + + vc = series.value_counts() + if len(vc) > top_n: + top = vc.head(top_n) + other_count = int(vc.iloc[top_n:].sum()) + labels = [str(v) for v in top.index] + sizes = [int(v) for v in top.values] + if other_count > 0: + labels.append("Other") + sizes.append(other_count) + else: + labels = [str(v) for v in vc.index] + sizes = [int(v) for v in vc.values] + + n_slices = len(labels) + colors = [palette[i % len(palette)] for i in range(n_slices)] + + fig, ax = plt.subplots(figsize=(5, 5)) + fig.patch.set_alpha(0) + + wedges, _, autotexts = ax.pie( + sizes, + labels=None, + autopct=lambda pct: f"{pct:.1f}%" if pct >= 4 else "", + colors=colors, + startangle=90, + pctdistance=0.75, + textprops={"color": text_color, "fontsize": 8}, + ) + for t in autotexts: + t.set_fontsize(7) + + ax.legend( + wedges, + labels, + loc="center left", + bbox_to_anchor=(1, 0.5), + fontsize=8, + frameon=False, + labelcolor=text_color, + ) + ax.set_title(str(column), fontsize=10, color=text_color, pad=8) + fig.tight_layout() + + img_buffer = io.BytesIO() + fig.savefig(img_buffer, format="png", dpi=150, transparent=True, bbox_inches="tight") + img_buffer.seek(0) + charts[str(column)] = base64.b64encode(img_buffer.read()).decode("utf-8") + plt.close(fig) + img_buffer.close() + + return charts diff --git a/web/static/css/theme.css b/web/static/css/theme.css index 44af6f02..b45e9e96 100644 --- a/web/static/css/theme.css +++ b/web/static/css/theme.css @@ -313,10 +313,29 @@ html.dark .info-text { } .info-icon:hover .info-text, -.info-text:hover { +.info-text:hover, +.info-icon:focus-within .info-text { display: block !important; } +/* Readiness report: show tooltip below icon (avoids clipping in tables) */ +#panel-readiness-report .info-icon--below .info-text { + left: auto; + right: 0; + top: calc(100% + 6px); + margin-left: 0; +} + +#panel-readiness-report th .info-icon, +#panel-readiness-report .inline-flex .info-icon { + flex-shrink: 0; +} + +/* Let readiness tooltips extend outside scrollable table wrappers when possible */ +#panel-readiness-report table { + overflow: visible; +} + .info-text a { color: var(--brandBlue) !important; text-decoration: underline !important; diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 8dd1d788..998d777a 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -10,6 +10,7 @@ let activePanel = "data-overview"; let codeMirrorEditor = null; let lastMetricResult = null; // Store last result for JSON download +let _readinessReportLoaded = false; // Lazy-load guard for the Readiness Report panel /** * Show a metric panel by ID, hiding all others. @@ -62,6 +63,13 @@ function showPanel(panelId, pushHistory) { initCodeMirror(); } + // Lazy load the data overview + data quality into the Readiness Report + // panel on first open + if (panelId === "readiness-report" && !_readinessReportLoaded) { + _readinessReportLoaded = true; + loadReadinessReport(); + } + // Close mobile sidebar after selection const sidebar = document.getElementById("sidebar"); if (sidebar && window.innerWidth < 640) { @@ -1907,11 +1915,24 @@ function submitCustomMetric() { /** * Render histogram images in the data overview panel. * @param {Object} histograms - Dict of {column_theme: base64_img} from /summary-statistics + * @param {string} [containerId] + * @param {boolean} [skipHeading] + * @param {string} [layout] - "compact" (default) or "large" for readiness report */ -function renderWorkspaceHistograms(histograms) { - const container = document.getElementById("workspace-histograms"); +function renderWorkspaceHistograms(histograms, containerId, skipHeading, layout) { + const container = document.getElementById( + containerId || "workspace-histograms", + ); if (!container) return; + const isLarge = layout === "large"; + const gridClass = isLarge + ? "grid grid-cols-1 sm:grid-cols-2 gap-6" + : "grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4"; + const imgClass = isLarge + ? "w-full h-auto object-contain" + : "w-full"; + // Always use the light variant — CSS filter handles dark mode const columns = {}; for (const [key, base64] of Object.entries(histograms)) { @@ -1926,14 +1947,17 @@ function renderWorkspaceHistograms(histograms) { return; } - let html = - '

    Feature Distributions

    '; - html += '
    '; + let html = ""; + if (!skipHeading) { + html += + '

    Feature Distributions

    '; + } + html += `
    `; for (const [colName, base64] of Object.entries(columns)) { html += `
    - Distribution of ${colName} + Distribution of ${colName}
    ${colName}
    `; @@ -1943,26 +1967,47 @@ function renderWorkspaceHistograms(histograms) { container.innerHTML = html; } -// ==================== Workspace Init ==================== +/** + * Render categorical distribution pie charts (base64 PNG per column). + */ +function renderCategoricalPieCharts(charts, containerId) { + const container = document.getElementById(containerId); + if (!container || !charts) return; + + const entries = Object.entries(charts); + if (entries.length === 0) { + container.innerHTML = ""; + return; + } + + let html = '
    '; + for (const [colName, base64] of entries) { + html += ` +
    + Distribution of ${colName} +
    `; + } + html += "
    "; + container.innerHTML = html; +} /** - * Initialize the workspace after file upload. - * Fetches summary statistics and populates feature dropdowns. + * Fetch summary statistics from the backend and render the stat cards, + * summary table, and feature-distribution histograms into the given + * containers. Reused by both the Data Overview panel and the + * Readiness Report panel. + * + * @param {string} summaryContainerId - element ID for the stats/table. + * @param {string} histogramsContainerId - element ID for the histograms. */ -function initWorkspace() { - // Restore panel from URL hash, or default to data-overview - const hash = location.hash.replace("#", ""); - const initialPanel = - hash && document.getElementById("panel-" + hash) ? hash : "data-overview"; - showPanel(initialPanel, false); // false = don't push to history on init - // Replace current history entry so back button works from the first panel - history.replaceState({ panel: initialPanel }, "", "#" + initialPanel); +function loadDataOverview(summaryContainerId, histogramsContainerId) { + const summaryId = summaryContainerId || "workspace-summary"; + const histogramsId = histogramsContainerId || "workspace-histograms"; - // Fetch summary statistics fetch("/summary-statistics") .then((r) => r.json()) .then((data) => { - const container = document.getElementById("workspace-summary"); + const container = document.getElementById(summaryId); if (!container) return; if (data.success) { @@ -2038,19 +2083,905 @@ function initWorkspace() { container.innerHTML = html; - // Render histograms in the data overview panel + // Render histograms below the summary table if (data.histograms) { - renderWorkspaceHistograms(data.histograms); + renderWorkspaceHistograms(data.histograms, histogramsId); } } else { container.innerHTML = `

    Could not load summary: ${data.message}

    `; } }) .catch((err) => { - const container = document.getElementById("workspace-summary"); + const container = document.getElementById(summaryId); if (container) container.innerHTML = `

    Error loading summary: ${err.message}

    `; }); +} + +/** + * Map a readiness status string to Tailwind color classes. + */ +function _dqStatusClasses(status) { + switch (status) { + case "good": + return { + text: "text-green-700 dark:text-green-400", + bar: "bg-green-500", + badge: + "bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400", + }; + case "warning": + return { + text: "text-amber-700 dark:text-amber-400", + bar: "bg-amber-500", + badge: + "bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-400", + }; + case "poor": + return { + text: "text-red-700 dark:text-red-400", + bar: "bg-red-500", + badge: "bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400", + }; + default: + return { + text: "text-gray-500 dark:text-gray-400", + bar: "bg-gray-400", + badge: + "bg-gray-100 text-gray-700 dark:bg-gray-700 dark:text-gray-300", + }; + } +} + +/** Format a 0–1 score as a whole percentage, or "N/A" if missing. */ +function _pct(value) { + if (value === null || value === undefined || isNaN(value)) return "N/A"; + return `${Math.round(value * 100)}%`; +} + +/** + * Fetch the aggregated readiness report and render the Data Quality + * scorecard (KPI tiles + overall grade + "needs attention" lists + + * a collapsible details view with the original charts). + */ +function loadReadinessReport() { + const overviewContainer = document.getElementById("readiness-summary"); + const dqContainer = document.getElementById("readiness-data-quality"); + const impactContainer = document.getElementById("readiness-impact"); + const fairnessContainer = document.getElementById("readiness-fairness"); + + fetch("/readiness-report") + .then((r) => r.json()) + .then((resp) => { + if (!resp.success) { + const msg = `

    Could not load readiness report: ${resp.message || "unknown error"}

    `; + if (overviewContainer) overviewContainer.innerHTML = msg; + if (dqContainer) dqContainer.innerHTML = msg; + if (impactContainer) impactContainer.innerHTML = msg; + if (fairnessContainer) fairnessContainer.innerHTML = msg; + return; + } + if (overviewContainer) + renderReadinessDatasetOverview(overviewContainer, resp.dataset_overview || {}); + if (dqContainer) + renderReadinessDataQuality(dqContainer, resp.data_quality || {}); + if (impactContainer) + renderReadinessImpact(impactContainer, resp.impact_on_ai || {}); + if (fairnessContainer) + renderReadinessFairness(fairnessContainer, resp.fairness_bias || {}); + }) + .catch((err) => { + const msg = `

    Error loading readiness report: ${err.message}

    `; + if (overviewContainer) overviewContainer.innerHTML = msg; + if (dqContainer) dqContainer.innerHTML = msg; + if (impactContainer) impactContainer.innerHTML = msg; + if (fairnessContainer) fairnessContainer.innerHTML = msg; + }); +} + +/** Escape text for safe inclusion in readiness info tooltips. */ +function _escapeHtml(s) { + return String(s) + .replace(/&/g, "&") + .replace(/i${_escapeHtml(text)}`; +} + +/** Table header cell with label + info tooltip. */ +function _readinessTh(label, infoKey, alignRight) { + const align = alignRight ? " text-right" : ""; + return `${label}${_readinessInfoIcon(infoKey)}`; +} + +/** Format byte count as a human-readable size string. */ +function _formatBytes(bytes) { + if (bytes == null || isNaN(bytes)) return "—"; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +/** Badge HTML for per-feature readiness status. */ +function _profileStatusBadge(status) { + const cls = _dqStatusClasses(status); + const label = status === "good" ? "Good" : status === "warning" ? "Warning" : status === "poor" ? "Poor" : "—"; + return `${label}`; +} + +/** + * Render the dataset overview: file metadata, KPI tiles, per-feature readiness + * profile, and collapsible detailed statistics / distributions / histograms. + */ +function renderReadinessDatasetOverview(container, overview) { + if (overview.error) { + container.innerHTML = `

    Dataset overview unavailable: ${overview.error}

    `; + return; + } + + const meta = overview.file_metadata || {}; + const profiles = overview.feature_profiles || []; + + // --- File metadata --- + let html = ` +
    +

    ${meta.file_name || "Dataset"}

    +
    +
    Type: ${meta.file_type || "—"}
    +
    Size: ${_formatBytes(meta.file_size_bytes)}
    +
    Memory: ${_formatBytes(meta.memory_bytes)}
    +
    Rows: ${(meta.rows || 0).toLocaleString()}
    +
    Columns: ${meta.columns || 0}
    +
    Numerical: ${meta.numerical_count || 0}
    +
    Categorical: ${meta.categorical_count || 0}
    +
    Other: ${(meta.datetime_count || 0) + (meta.boolean_count || 0)}
    +
    +
    `; + + // --- KPI tiles --- + html += ` +
    +
    +
    ${(meta.rows || 0).toLocaleString()}
    +
    Records
    +
    +
    +
    ${meta.columns || 0}
    +
    Features
    +
    +
    +
    ${meta.numerical_count || 0}
    +
    Numerical
    +
    +
    +
    ${meta.categorical_count || 0}
    +
    Categorical
    +
    +
    `; + + // --- Per-feature readiness profile --- + const poorCount = profiles.filter((p) => p.status === "poor").length; + const warnCount = profiles.filter((p) => p.status === "warning").length; + + html += ` +
    +

    Per-feature readiness profile${_readinessInfoIcon("feature_profile")}

    + + ${poorCount ? `${poorCount} poor` : ""} + ${warnCount ? `${poorCount ? " · " : ""}${warnCount} warning` : ""} + ${!poorCount && !warnCount ? "all good" : ""} + +
    `; + + html += `
    `; + html += ``; + html += ``; + html += ``; + html += _readinessTh("% missing", "pct_missing", true); + html += _readinessTh("# unique", "n_unique", true); + html += _readinessTh("% dominant", "pct_dominant", true); + html += _readinessTh("Status", "profile_status", false); + html += ``; + html += ``; + + profiles.forEach((p, i) => { + const stripe = i % 2 === 0 ? "bg-white dark:bg-gray-800" : "bg-gray-50 dark:bg-gray-700/50"; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + }); + html += `
    FeatureTypeDtypeSummary
    ${p.feature}${p.type}${p.dtype}${_pct(p.pct_missing)}${p.n_unique}${p.pct_dominant != null ? _pct(p.pct_dominant) : "—"}${_profileStatusBadge(p.status)}${p.summary || "—"}
    `; + + // --- Collapsible detailed statistics --- + let detailsInner = ""; + + const numSummary = overview.numerical_summary || {}; + const numFeatures = Object.keys(numSummary); + if (numFeatures.length > 0) { + const allStats = Object.keys(numSummary[numFeatures[0]] || {}); + const preferredOrder = [ + "count", "min", "25th percentile", "50th percentile", "mean", + "75th percentile", "max", "std", + ]; + const statKeys = preferredOrder + .filter((s) => allStats.includes(s)) + .concat(allStats.filter((s) => !preferredOrder.includes(s))); + + detailsInner += `

    Numerical summary statistics

    `; + detailsInner += `
    `; + detailsInner += ``; + statKeys.forEach((s) => { + detailsInner += ``; + }); + detailsInner += ``; + numFeatures.forEach((feat, i) => { + const stripe = i % 2 === 0 ? "bg-white dark:bg-gray-800" : "bg-gray-50 dark:bg-gray-700/50"; + detailsInner += ``; + statKeys.forEach((s) => { + detailsInner += ``; + }); + detailsInner += ``; + }); + detailsInner += `
    Feature${s}
    ${feat}${numSummary[feat][s] ?? "—"}
    `; + } + + const catCharts = overview.categorical_charts || {}; + const catChartCols = Object.keys(catCharts); + if (catChartCols.length > 0) { + detailsInner += `

    Categorical value distributions

    `; + detailsInner += `
    `; + } + + if (overview.histograms && Object.keys(overview.histograms).length > 0) { + detailsInner += `

    Feature distributions (numerical)

    `; + detailsInner += `
    `; + } + + if (detailsInner) { + html += ` +
    + + Show detailed statistics & distributions + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; + + if (catChartCols.length > 0) { + renderCategoricalPieCharts(overview.categorical_charts, "readiness-categorical-charts"); + } + if (overview.histograms) { + renderWorkspaceHistograms( + overview.histograms, + "readiness-histograms-inner", + true, + "large", + ); + } +} + +/** + * Render the Data Quality scorecard into the given container. + */ +function renderReadinessDataQuality(container, dq) { + if (dq.error) { + container.innerHTML = `

    Data quality unavailable: ${dq.error}

    `; + return; + } + const kpis = dq.kpis || []; + const gradeCls = _dqStatusClasses(dq.grade_status); + + // --- Overall grade + KPI tiles --- + let html = ` +
    + Overall data quality grade${_readinessInfoIcon("overall_dq_grade")} + ${_pct(dq.grade)} +
    +
    `; + + kpis.forEach((k) => { + const cls = _dqStatusClasses(k.status); + const widthPct = + k.value === null || k.value === undefined + ? 0 + : Math.max(0, Math.min(100, Math.round(k.value * 100))); + html += ` +
    +
    + ${k.label}${_readinessInfoIcon(k.id)} + ${_pct(k.value)} +
    +
    +
    +
    +

    ${k.hint || ""}

    +
    `; + }); + html += "
    "; + + // --- Needs attention --- + const na = dq.needs_attention || {}; + const incomplete = na.incomplete_features || []; + const outlierFeats = na.outlier_features || []; + const dupRows = na.duplicate_rows || 0; + + const naItems = []; + if (incomplete.length) { + const top = incomplete + .slice(0, 6) + .map( + (f) => + `
  • ${f.feature}${_pct(f.completeness)} complete
    • `, + ) + .join(""); + const more = + incomplete.length > 6 + ? `
  • +${incomplete.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Incomplete features (${incomplete.length})${_readinessInfoIcon("completeness")}

    +
      ${top}${more}
    +
    `); + } + if (outlierFeats.length) { + const top = outlierFeats + .slice(0, 6) + .map( + (f) => + `
  • ${f.feature}${_pct(f.outlier_proportion)} outliers
    • `, + ) + .join(""); + const more = + outlierFeats.length > 6 + ? `
  • +${outlierFeats.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Features with outliers (${outlierFeats.length})${_readinessInfoIcon("outlier_cleanliness")}

    +
      ${top}${more}
    +
    `); + } + if (dupRows && dupRows > 0) { + naItems.push(` +
    +

    Duplicate rows${_readinessInfoIcon("uniqueness")}

    +

    ${_pct(dupRows)} of rows are exact duplicates.

    +
    `); + } + + if (naItems.length) { + html += ` +
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } else { + html += ` +
    +

    No data quality issues detected — all features complete, no duplicates, no outliers.

    +
    `; + } + + // --- Collapsible details (original charts) --- + const det = dq.details || {}; + let detailsInner = ""; + if (det.completeness && det.completeness.visualization) { + detailsInner += ` +
    +

    Completeness by feature

    + Completeness chart +
    `; + } + if (det.outliers && det.outliers.visualization) { + detailsInner += ` +
    +

    Outliers by feature

    + Outliers chart +
    `; + } else if (det.outliers && det.outliers.error) { + detailsInner += `

    Outliers: ${det.outliers.error}

    `; + } + + if (detailsInner) { + html += ` +
    + + Show detailed charts + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; +} + +/** + * Render the Impact on AI scorecard (automated all-pairs correlation signals: + * redundancy, leakage risk, isolated features) into the given container. + */ +function renderReadinessImpact(container, impact) { + if (impact.error) { + container.innerHTML = `

    Impact on AI unavailable: ${impact.error}

    `; + return; + } + + const redundant = impact.redundant_pairs || []; + const leakage = impact.leakage_pairs || []; + const isolated = impact.isolated_features || []; + const topPairs = impact.top_pairs || []; + const dropped = impact.columns_dropped || []; + const analyzed = impact.columns_analyzed || 0; + + const fmtScore = (s) => (typeof s === "number" ? s.toFixed(2) : s); + const fmtPair = (p) => + `
  • ${p.a} \u2194 ${p.b}${fmtScore(p.score)}
    • `; + + // --- Stat tiles --- + const tiles = [ + { label: "Features analyzed", value: analyzed, status: "neutral", infoKey: "features_analyzed" }, + { + label: "Leakage-risk pairs", + value: leakage.length, + status: leakage.length ? "poor" : "good", + infoKey: "leakage_risk_pairs", + }, + { + label: "Redundant pairs", + value: redundant.length, + status: redundant.length ? "warning" : "good", + infoKey: "redundant_pairs", + }, + { + label: "Isolated features", + value: isolated.length, + status: isolated.length ? "warning" : "good", + infoKey: "isolated_features", + }, + ]; + + let html = '
    '; + tiles.forEach((t) => { + const cls = _dqStatusClasses(t.status); + const valColor = + t.status === "neutral" ? "text-gray-900 dark:text-white" : cls.text; + html += ` +
    +
    ${t.value}
    +
    ${t.label}${_readinessInfoIcon(t.infoKey)}
    +
    `; + }); + html += "
    "; + + // --- Needs attention --- + const naItems = []; + if (leakage.length) { + const items = leakage.slice(0, 6).map(fmtPair).join(""); + const more = + leakage.length > 6 + ? `
  • +${leakage.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Leakage risk (|score| ≥ 0.95)${_readinessInfoIcon("leakage_risk_pairs")}

    +
      ${items}${more}
    +
    `); + } + if (redundant.length) { + const items = redundant.slice(0, 6).map(fmtPair).join(""); + const more = + redundant.length > 6 + ? `
  • +${redundant.length - 6} more
    • ` + : ""; + naItems.push(` +
    +

    Redundant pairs (|score| ≥ 0.8)${_readinessInfoIcon("redundant_pairs")}

    +
      ${items}${more}
    +
    `); + } + if (isolated.length) { + const items = isolated + .slice(0, 10) + .map((f) => `
  • ${f}
    • `) + .join(""); + const more = + isolated.length > 10 + ? `
  • +${isolated.length - 10} more
    • ` + : ""; + naItems.push(` +
    +

    Isolated features (no strong relationships)${_readinessInfoIcon("isolated_features")}

    +
      ${items}${more}
    +
    `); + } + + if (naItems.length) { + html += ` +
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } else { + html += ` +
    +

    No redundancy, leakage risk, or isolated features detected.

    +
    `; + } + + // --- Most-related pairs table --- + if (topPairs.length) { + html += + '

    Most-related feature pairs' + + _readinessInfoIcon("most_related_pairs") + + "

    "; + html += + '
    '; + html += + ''; + topPairs.forEach((p, i) => { + const stripe = + i % 2 === 0 + ? "bg-white dark:bg-gray-800" + : "bg-gray-50 dark:bg-gray-700/50"; + html += ``; + }); + html += "
    Feature AFeature BScore
    ${p.a}${p.b}${fmtScore(p.score)}
    "; + } + + // --- Collapsible details (heatmaps + excluded columns) --- + const det = impact.details || {}; + let detailsInner = ""; + if (det.numerical_visualization) { + const method = det.numerical_method ? ` (${det.numerical_method})` : ""; + detailsInner += ` +
    +

    Numerical correlation${method}

    + Numerical correlation heatmap +
    `; + } + if (det.categorical_visualization) { + detailsInner += ` +
    +

    Categorical correlation (Theil's U)

    + Categorical correlation heatmap +
    `; + } + if (dropped.length) { + const items = dropped + .map( + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ) + .join(""); + detailsInner += ` +
    +

    Excluded columns (${dropped.length})

    +
      ${items}
    +
    `; + } + if (detailsInner) { + html += ` +
    + + Show correlation heatmaps & excluded columns + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; +} + +/** + * Render the Fairness & Bias scorecard (auto-selected columns, four metrics, + * selection criteria, needs-attention lists, collapsible charts). + */ +function renderReadinessFairness(container, fb) { + if (fb.error) { + container.innerHTML = `

    Fairness & Bias unavailable: ${fb.error}

    `; + return; + } + + const sel = fb.auto_selection || {}; + const criteria = sel.selection_criteria || {}; + const sensCrit = criteria.sensitive_attributes || {}; + const targetCrit = criteria.target_column || {}; + const posCrit = criteria.positive_class || {}; + const thresholds = criteria.thresholds || {}; + const kpis = fb.kpis || []; + const na = fb.needs_attention || {}; + const gradeCls = _dqStatusClasses(fb.grade_status); + + // --- Auto-selection criteria (transparent) --- + let html = ` +
    +

    Auto-selection criteria

    +
      +
    • Sensitive attributes: ${sensCrit.selected?.length ? sensCrit.selected.join(", ") : "none"}
      + ${sensCrit.rule || ""}
      • +
    • Target column: ${targetCrit.selected || "none"}${targetCrit.reason ? ` (${targetCrit.reason})` : ""}
      + ${targetCrit.rule || ""}
      • +
    • CDD positive class: ${posCrit.selected ?? "none"}${posCrit.reason ? ` (${posCrit.reason})` : ""}
      + ${posCrit.rule || ""}
      • +
    • Primary sensitive (statistical rate & CDD): ${sel.primary_sensitive || "none"}
      • +
    • Flags: + representation ratio ≥ ${thresholds.representation_ratio_flag ?? "—"}, + minority class < ${_pct(thresholds.minority_class_share)}, + TSD ≥ ${thresholds.tsd_disparity_flag ?? "—"}, + imbalance degree good/warning < ${thresholds.imbalance_degree_good ?? "—"} / ${thresholds.imbalance_degree_warning ?? "—"} +
      • +
    +
    `; + + // --- Overall grade + KPI tiles --- + html += ` +
    + Overall fairness grade${_readinessInfoIcon("overall_fairness_grade")} + ${_pct(fb.grade)} +
    +
    `; + + kpis.forEach((k) => { + const cls = _dqStatusClasses(k.status); + const displayVal = + k.id === "label_balance" && k.raw_imbalance_degree != null + ? `ID ${Number(k.raw_imbalance_degree).toFixed(2)}` + : _pct(k.value); + const widthPct = + k.value === null || k.value === undefined + ? 0 + : Math.max(0, Math.min(100, Math.round(k.value * 100))); + html += ` +
    +
    + ${k.label}${_readinessInfoIcon(k.id)} + ${displayVal} +
    +
    +
    +
    +

    ${k.hint || ""}

    +
    `; + }); + html += "
    "; + + // --- Needs attention --- + const naItems = []; + const repImbalance = na.representation_imbalance || []; + if (repImbalance.length) { + const items = repImbalance + .slice(0, 5) + .map( + (s) => + `
  • ${s.column}max ratio ${s.max_ratio}
    • `, + ) + .join(""); + naItems.push(` +
    +

    Representation imbalance (${repImbalance.length})${_readinessInfoIcon("representation_imbalance")}

    +
      ${items}
    +
    `); + } + const minorities = na.minority_classes || []; + if (minorities.length) { + const items = minorities + .map( + (m) => + `
  • ${m.class}${_pct(m.share)} share
    • `, + ) + .join(""); + naItems.push(` +
    +

    Minority classes (${minorities.length})${_readinessInfoIcon("minority_classes")}

    +
      ${items}
    +
    `); + } + const outcomeDisp = na.outcome_disparities || []; + if (outcomeDisp.length) { + const items = outcomeDisp + .map( + (d) => + `
  • ${d.class}TSD ${d.tsd}
    • `, + ) + .join(""); + naItems.push(` +
    +

    Outcome-rate disparities (${outcomeDisp.length})${_readinessInfoIcon("outcome_disparities")}

    +
      ${items}
    +
    `); + } + const cddDisp = na.cdd_disparities || []; + if (cddDisp.length) { + const items = cddDisp + .map((d) => `
  • ${d.group}
    • `) + .join(""); + naItems.push(` +
    +

    CDD flagged groups (${cddDisp.length})${_readinessInfoIcon("cdd_disparities")}

    +
      ${items}
    +
    `); + } + + if (naItems.length) { + html += ` +
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } else { + html += ` +
    +

    No fairness issues detected under the automated thresholds.

    +
    `; + } + + // --- Collapsible details (charts) --- + const det = fb.details || {}; + let detailsInner = ""; + + const repVis = det.representation_rate?.visualizations || {}; + for (const [col, b64] of Object.entries(repVis)) { + detailsInner += ` +
    +

    Representation rate — ${col}

    + Representation ${col} +
    `; + } + if (det.representation_rate?.error && !Object.keys(repVis).length) { + detailsInner += `

    Representation rate: ${det.representation_rate.error}

    `; + } + + if (det.class_imbalance?.visualization) { + detailsInner += ` +
    +

    Class imbalance — ${targetCrit.selected || "target"}

    + Class imbalance +
    `; + } else if (det.class_imbalance?.error) { + detailsInner += `

    Class imbalance: ${det.class_imbalance.error}

    `; + } + + if (det.statistical_rate?.visualization) { + detailsInner += ` +
    +

    Statistical rate — ${det.statistical_rate.sensitive} × ${det.statistical_rate.target}

    + Statistical rate +
    `; + } else if (det.statistical_rate?.error) { + detailsInner += `

    Statistical rate: ${det.statistical_rate.error}

    `; + } + + if (det.cdd?.disparities && !det.cdd.error) { + const rows = Object.entries(det.cdd.disparities) + .map( + ([grp, info]) => + `${grp}${info.disparity}`, + ) + .join(""); + detailsInner += ` +
    +

    Conditional demographic disparity (positive: ${det.cdd.positive_class})

    + ${rows}
    GroupDisparity
    +
    `; + } else if (det.cdd?.error) { + detailsInner += `

    CDD: ${det.cdd.error}

    `; + } + + const excluded = sensCrit.excluded || []; + if (excluded.length) { + const items = excluded + .map( + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ) + .join(""); + detailsInner += ` +
    +

    Excluded sensitive candidates (${excluded.length})

    +
      ${items}
    +
    `; + } + + if (detailsInner) { + html += ` +
    + + Show detailed charts & CDD table + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; +} + +// ==================== Workspace Init ==================== + +/** + * Initialize the workspace after file upload. + * Fetches summary statistics and populates feature dropdowns. + */ +function initWorkspace() { + // Restore panel from URL hash, or default to data-overview + const hash = location.hash.replace("#", ""); + const initialPanel = + hash && document.getElementById("panel-" + hash) ? hash : "data-overview"; + showPanel(initialPanel, false); // false = don't push to history on init + // Replace current history entry so back button works from the first panel + history.replaceState({ panel: initialPanel }, "", "#" + initialPanel); + + // Fetch + render summary statistics into the Data Overview panel + loadDataOverview(); // Populate feature dropdowns via /feature-set (same as metric.js does) fetch("/feature-set", { method: "POST" }) diff --git a/web/templates/_components/sidebar.html b/web/templates/_components/sidebar.html index f7c6dd3f..35ab5af2 100644 --- a/web/templates/_components/sidebar.html +++ b/web/templates/_components/sidebar.html @@ -18,6 +18,14 @@ Data Overview + + +
    diff --git a/web/templates/_panels/_readiness_report.html b/web/templates/_panels/_readiness_report.html new file mode 100644 index 00000000..1bca345e --- /dev/null +++ b/web/templates/_panels/_readiness_report.html @@ -0,0 +1,55 @@ + + diff --git a/web/templates/inspector.html b/web/templates/inspector.html index 792ba5fb..498ac5f7 100644 --- a/web/templates/inspector.html +++ b/web/templates/inspector.html @@ -56,6 +56,7 @@
    {% include '_panels/_data_overview.html' %} + {% include '_panels/_readiness_report.html' %} {% include '_panels/_data_quality.html' %} {% include '_panels/_feature_relevance.html' %} {% include '_panels/_correlation_analysis.html' %} From 2f45cbc4a3e58cfb261820c9a32d765656981e99 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 16 Jun 2026 18:45:53 -0400 Subject: [PATCH 03/23] Add .idea in the gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d5fb9145..9d07e71f 100644 --- a/.gitignore +++ b/.gitignore @@ -160,7 +160,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # datasets metadata/ From ea7ad80b5f893d0a2db526e13e1dc396022ba453 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 24 Jun 2026 10:42:20 -0400 Subject: [PATCH 04/23] Add data governance in the readiness report --- aidrin/structured_data_metrics/add_noise.py | 18 +- web/routes/metrics.py | 705 ++++++++++++++++++- web/static/js/inspector.js | 277 ++++++++ web/templates/_panels/_readiness_report.html | 14 + 4 files changed, 1004 insertions(+), 10 deletions(-) diff --git a/aidrin/structured_data_metrics/add_noise.py b/aidrin/structured_data_metrics/add_noise.py index a1686b8d..90b88bf6 100644 --- a/aidrin/structured_data_metrics/add_noise.py +++ b/aidrin/structured_data_metrics/add_noise.py @@ -17,7 +17,7 @@ def add_laplace_noise(data, epsilon): raise Exception("Epsilon cannot be 0") -def return_noisy_stats(add_noise_columns, epsilon, file_info): +def return_noisy_stats(add_noise_columns, epsilon, file_info, save_output=True): # Convert JSON back to DataFrame if needed, otherwise use DataFrame directly import pandas as pd @@ -107,13 +107,15 @@ def return_noisy_stats(add_noise_columns, epsilon, file_info): # Encode the combined image as base64 combined_image_base64 = base64.b64encode(img_buf.getvalue()).decode("utf-8") img_buf.close() - try: - # Create the new directory - os.makedirs("noisy", exist_ok=True) - df_drop_na.to_csv("noisy/noisy_data.csv", index=False) - stat_dict["Noisy file saved"] = "Successful" - except Exception: - stat_dict["Noisy file saved"] = "Error" + if save_output: + try: + os.makedirs("noisy", exist_ok=True) + df_drop_na.to_csv("noisy/noisy_data.csv", index=False) + stat_dict["Noisy file saved"] = "Successful" + except Exception: + stat_dict["Noisy file saved"] = "Error" + else: + stat_dict["Noisy file saved"] = "Skipped (readiness report preview only)" stat_dict["DP Statistics Visualization"] = combined_image_base64 diff --git a/web/routes/metrics.py b/web/routes/metrics.py index fb8b8caa..99bee0e2 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -1,5 +1,6 @@ import json import logging +import math import os import time @@ -46,6 +47,8 @@ compute_k_anonymity, compute_l_diversity, compute_t_closeness, + generate_multiple_attribute_MM_risk_scores, + generate_single_attribute_MM_risk_scores, ) from aidrin.structured_data_metrics.representation_rate import ( calculate_representation_rate, @@ -1002,12 +1005,703 @@ def _build_fairness_bias_section(file_info): } +# --------------------------------------------------------------------------- +# Data Governance (readiness report) +# --------------------------------------------------------------------------- + +_GOV_QI_MIN_UNIQUE = 2 +_GOV_QI_MAX_UNIQUE = 50 +_GOV_QI_MAX_COUNT = 4 +_GOV_ID_UNIQUE_RATIO = 0.9 +_GOV_NUMERIC_QI_MAX_UNIQUE = 20 +_GOV_MM_QI_MAX_UNIQUE = 100 +_GOV_MISSING_MAX = 0.20 +_GOV_SENSITIVE_MIN_UNIQUE = 2 +_GOV_SENSITIVE_MAX_UNIQUE = 30 +_GOV_HIPAA_MAX_COLUMNS = 40 +_GOV_HIPAA_MISSING_MAX = 0.50 +_GOV_DP_MAX_FEATURES = 2 +_GOV_DP_EPSILON = 0.1 +_GOV_SMALL_SAMPLE = 30 + +_GOV_K_GOOD = 5 +_GOV_K_WARNING = 2 +_GOV_L_GOOD = 3 +_GOV_L_WARNING = 2 +_GOV_T_GOOD = 0.10 +_GOV_T_WARNING = 0.20 +_GOV_MM_SINGLE_GOOD = 0.3 +_GOV_MM_SINGLE_WARNING = 0.6 +_GOV_MM_MULTI_GOOD = 0.5 +_GOV_MM_MULTI_WARNING = 0.8 + +_GOV_QI_NAME_HINTS = ( + "zip", "postal", "gender", "sex", "age", "race", "ethnic", "city", + "state", "country", "birth", "dob", "county", "region", "marital", +) +_GOV_SENSITIVE_NAME_HINTS = ( + "diagnosis", "disease", "condition", "salary", "income", "religion", + "health", "treatment", "medication", "outcome", "disability", +) +_GOV_ID_NAME_HINTS = ( + "id", "uuid", "guid", "index", "record", "patient", "user", "member", + "row", "case", +) +_GOV_HIPAA_NAME_HINTS = ( + "name", "address", "email", "phone", "ssn", "medical", "account", + "notes", "text", "comment", +) +_GOV_DP_NAME_HINTS = ( + "age", "income", "salary", "score", "amount", "value", "rate", "count", + "weight", "height", +) +_GOV_HIPAA_SERIOUS_TYPES = frozenset({ + "US_SSN", "MEDICAL_IDS", "VALID_POSTAL_CODE", +}) +_SYNTHETIC_ID_COL = "__aidrin_row_index__" + + +def _gov_name_hint_score(col_name, hints): + return _fairness_name_hint_score(col_name, hints) + + +def _column_pct_missing(series, n_rows): + if n_rows <= 0: + return 1.0 + return float(series.isna().sum()) / n_rows + + +def _column_entropy_norm(series): + """Normalized Shannon entropy of value counts in [0, 1].""" + vc = series.dropna().value_counts(normalize=True) + if len(vc) <= 1: + return 0.0 + ent = -sum(p * math.log2(p) for p in vc if p > 0) + max_ent = math.log2(len(vc)) + return ent / max_ent if max_ent > 0 else 0.0 + + +def _k_anonymity_status(k_val): + if k_val is None: + return "unknown" + if k_val >= _GOV_K_GOOD: + return "good" + if k_val >= _GOV_K_WARNING: + return "warning" + return "poor" + + +def _l_diversity_status(l_val): + if l_val is None: + return "unknown" + if l_val >= _GOV_L_GOOD: + return "good" + if l_val >= _GOV_L_WARNING: + return "warning" + return "poor" + + +def _t_closeness_status(t_val): + if t_val is None: + return "unknown" + if t_val <= _GOV_T_GOOD: + return "good" + if t_val <= _GOV_T_WARNING: + return "warning" + return "poor" + + +def _mm_risk_status(mean_risk, good=_GOV_MM_SINGLE_GOOD, warning=_GOV_MM_SINGLE_WARNING): + if mean_risk is None: + return "unknown" + if mean_risk < good: + return "good" + if mean_risk < warning: + return "warning" + return "poor" + + +def _hipaa_status(detected_phi): + if not detected_phi: + return "good", 1.0 + serious = any( + t in _GOV_HIPAA_SERIOUS_TYPES + for info in detected_phi.values() + for t in (info.get("potential_types_detected") or []) + ) + if serious: + return "poor", 0.0 + return "warning", 0.5 + + +def _auto_select_governance_columns(df, fairness_target=None): + """Pick columns for automated privacy and HIPAA checks in the readiness report.""" + n_rows = max(len(df), 1) + excluded = [] + + # --- Sensitive attribute (before QIs so it can be excluded) ------------ + sensitive_candidates = [] + for col in df.columns: + feat_type = _classify_feature_type(df[col]) + if feat_type not in ("categorical", "boolean"): + continue + nunique = df[col].nunique(dropna=True) + pct_miss = _column_pct_missing(df[col], n_rows) + if nunique < _GOV_SENSITIVE_MIN_UNIQUE: + continue + if nunique > _GOV_SENSITIVE_MAX_UNIQUE: + excluded.append({ + "feature": col, "role": "sensitive", + "reason": f"high cardinality ({nunique} categories)", + }) + continue + if pct_miss > _GOV_MISSING_MAX: + excluded.append({ + "feature": col, "role": "sensitive", + "reason": f"high missingness ({pct_miss:.0%})", + }) + continue + hint = _gov_name_hint_score(col, _GOV_SENSITIVE_NAME_HINTS) + fairness_boost = 1 if fairness_target and col == fairness_target else 0 + sensitive_candidates.append({ + "feature": col, + "nunique": int(nunique), + "name_hint_score": hint + fairness_boost, + "entropy": _column_entropy_norm(df[col]), + }) + + sensitive_candidates.sort( + key=lambda c: (-c["name_hint_score"], -c["entropy"], c["feature"]) + ) + sensitive_col = ( + sensitive_candidates[0]["feature"] if sensitive_candidates else None + ) + + # --- ID column ----------------------------------------------------------- + id_candidates = [] + for col in df.columns: + if df[col].nunique(dropna=True) == n_rows and n_rows > 0: + id_candidates.append({ + "feature": col, + "name_hint_score": _gov_name_hint_score(col, _GOV_ID_NAME_HINTS), + }) + id_candidates.sort(key=lambda c: (-c["name_hint_score"], c["feature"])) + id_synthetic = not id_candidates + id_col = id_candidates[0]["feature"] if id_candidates else _SYNTHETIC_ID_COL + + # --- Quasi-identifiers --------------------------------------------------- + qi_candidates = [] + for col in df.columns: + if col == sensitive_col: + continue + if not id_synthetic and col == id_col: + continue + feat_type = _classify_feature_type(df[col]) + if feat_type == "datetime": + excluded.append({"feature": col, "role": "quasi-identifier", "reason": "datetime column"}) + continue + nunique = df[col].nunique(dropna=True) + pct_miss = _column_pct_missing(df[col], n_rows) + if nunique < _GOV_QI_MIN_UNIQUE: + excluded.append({"feature": col, "role": "quasi-identifier", "reason": "constant column"}) + continue + if pct_miss > _GOV_MISSING_MAX: + excluded.append({ + "feature": col, "role": "quasi-identifier", + "reason": f"high missingness ({pct_miss:.0%})", + }) + continue + if nunique / n_rows >= _GOV_ID_UNIQUE_RATIO: + excluded.append({"feature": col, "role": "quasi-identifier", "reason": "ID-like (near-unique values)"}) + continue + if feat_type in ("categorical", "boolean"): + if nunique > _GOV_QI_MAX_UNIQUE: + excluded.append({ + "feature": col, "role": "quasi-identifier", + "reason": f"high cardinality ({nunique} categories)", + }) + continue + elif feat_type == "numerical": + if nunique > _GOV_NUMERIC_QI_MAX_UNIQUE: + excluded.append({ + "feature": col, "role": "quasi-identifier", + "reason": f"continuous numeric ({nunique} unique values)", + }) + continue + else: + excluded.append({"feature": col, "role": "quasi-identifier", "reason": f"unsupported type ({feat_type})"}) + continue + + qi_candidates.append({ + "feature": col, + "nunique": int(nunique), + "feat_type": feat_type, + "name_hint_score": _gov_name_hint_score(col, _GOV_QI_NAME_HINTS), + }) + + qi_candidates.sort( + key=lambda c: (-c["name_hint_score"], c["nunique"], c["feature"]) + ) + quasi_identifiers = [c["feature"] for c in qi_candidates[:_GOV_QI_MAX_COUNT]] + for c in qi_candidates[_GOV_QI_MAX_COUNT:]: + excluded.append({"feature": c["feature"], "role": "quasi-identifier", "reason": "exceeded QI cap"}) + + mm_quasi_identifiers = [ + c["feature"] for c in qi_candidates[:_GOV_QI_MAX_COUNT] + if c["feat_type"] in ("categorical", "boolean") + and c["nunique"] <= _GOV_MM_QI_MAX_UNIQUE + ] + + # --- HIPAA scan columns -------------------------------------------------- + hipaa_candidates = [] + for col in df.columns: + nunique = df[col].nunique(dropna=True) + if nunique <= 1: + continue + pct_miss = _column_pct_missing(df[col], n_rows) + feat_type = _classify_feature_type(df[col]) + is_text = feat_type in ("categorical",) or str(df[col].dtype) in ("object", "string") + if is_text and pct_miss <= _GOV_HIPAA_MISSING_MAX: + avg_len = df[col].dropna().astype(str).str.len().mean() + hipaa_candidates.append({ + "feature": col, + "name_hint_score": _gov_name_hint_score(col, _GOV_HIPAA_NAME_HINTS), + "avg_len": avg_len if pd.notna(avg_len) else 0, + }) + + hipaa_candidates.sort( + key=lambda c: (-c["name_hint_score"], -c["avg_len"], c["feature"]) + ) + hipaa_scan_columns = [c["feature"] for c in hipaa_candidates[:_GOV_HIPAA_MAX_COLUMNS]] + if not hipaa_scan_columns: + hipaa_scan_columns = [ + col for col in df.columns if df[col].nunique(dropna=True) > 1 + ][: _GOV_HIPAA_MAX_COLUMNS] + + # --- DP numerical features ----------------------------------------------- + dp_candidates = [] + for col in df.columns: + if not pd.api.types.is_numeric_dtype(df[col]): + continue + nunique = df[col].nunique(dropna=True) + if nunique <= 1: + continue + if nunique / n_rows >= _GOV_ID_UNIQUE_RATIO: + continue + if _column_pct_missing(df[col], n_rows) > _GOV_MISSING_MAX: + continue + dp_candidates.append({ + "feature": col, + "name_hint_score": _gov_name_hint_score(col, _GOV_DP_NAME_HINTS), + "pct_missing": _column_pct_missing(df[col], n_rows), + "nunique": int(nunique), + }) + dp_candidates.sort( + key=lambda c: (-c["name_hint_score"], c["pct_missing"], c["nunique"], c["feature"]) + ) + dp_features = [c["feature"] for c in dp_candidates[:_GOV_DP_MAX_FEATURES]] + + return { + "quasi_identifiers": quasi_identifiers, + "mm_quasi_identifiers": mm_quasi_identifiers, + "sensitive_attribute": sensitive_col, + "id_column": id_col, + "id_synthetic": id_synthetic, + "hipaa_scan_columns": hipaa_scan_columns, + "dp_features": dp_features, + "dp_epsilon": _GOV_DP_EPSILON, + "selection_criteria": { + "quasi_identifiers": { + "rule": ( + f"Categorical/boolean with {_GOV_QI_MIN_UNIQUE}–{_GOV_QI_MAX_UNIQUE} " + f"unique values; discrete numeric with {_GOV_QI_MIN_UNIQUE}–" + f"{_GOV_NUMERIC_QI_MAX_UNIQUE}; exclude ID-like, datetime, " + f"and >{_GOV_MISSING_MAX:.0%} missing; prefer name hints; " + f"capped at {_GOV_QI_MAX_COUNT}." + ), + "name_hints": list(_GOV_QI_NAME_HINTS), + "selected": quasi_identifiers, + "excluded": [e for e in excluded if e["role"] == "quasi-identifier"], + }, + "sensitive_attribute": { + "rule": ( + f"Categorical/boolean with {_GOV_SENSITIVE_MIN_UNIQUE}–" + f"{_GOV_SENSITIVE_MAX_UNIQUE} unique values, excluding QIs; " + "prefer sensitive name hints and fairness target when eligible." + ), + "name_hints": list(_GOV_SENSITIVE_NAME_HINTS), + "selected": sensitive_col, + "excluded": [e for e in excluded if e["role"] == "sensitive"], + }, + "id_column": { + "rule": ( + "Exact-unique column preferred (name hints: id, uuid, patient, …); " + "otherwise synthetic row index for linkage-risk scoring only." + ), + "selected": id_col, + "synthetic": id_synthetic, + }, + "hipaa_scan_columns": { + "rule": ( + f"Text-like columns up to {_GOV_HIPAA_MAX_COLUMNS}, prefer HIPAA " + "name hints; fallback to all non-constant columns." + ), + "name_hints": list(_GOV_HIPAA_NAME_HINTS), + "selected": hipaa_scan_columns, + }, + "dp_features": { + "rule": ( + f"Up to {_GOV_DP_MAX_FEATURES} numerical, non-ID-like columns " + f"(illustrative only, ε={_GOV_DP_EPSILON})." + ), + "selected": dp_features, + "epsilon": _GOV_DP_EPSILON, + }, + "thresholds": { + "k_good": _GOV_K_GOOD, + "k_warning": _GOV_K_WARNING, + "l_good": _GOV_L_GOOD, + "l_warning": _GOV_L_WARNING, + "t_good": _GOV_T_GOOD, + "t_warning": _GOV_T_WARNING, + "mm_single_good": _GOV_MM_SINGLE_GOOD, + "mm_single_warning": _GOV_MM_SINGLE_WARNING, + "mm_multi_good": _GOV_MM_MULTI_GOOD, + "mm_multi_warning": _GOV_MM_MULTI_WARNING, + "small_sample_rows": _GOV_SMALL_SAMPLE, + }, + }, + } + + +def _build_data_governance_section(file_info): + """Compute the Data Governance portion of the readiness report.""" + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + n_rows = len(df) + fairness_sel = _auto_select_fairness_columns(df) + selection = _auto_select_governance_columns( + df, fairness_target=fairness_sel.get("target_column") + ) + qi = selection["quasi_identifiers"] + mm_qis = selection["mm_quasi_identifiers"] + sensitive = selection["sensitive_attribute"] + id_col = selection["id_column"] + id_synthetic = selection["id_synthetic"] + + work_df = df.copy() + if id_synthetic: + work_df[_SYNTHETIC_ID_COL] = range(len(work_df)) + id_col = _SYNTHETIC_ID_COL + + kpis = [] + needs_attention = { + "low_anonymity": [], + "hipaa_phi": [], + "high_linkage_risk": [], + "attribute_disclosure": [], + } + details = {} + small_sample = n_rows < _GOV_SMALL_SAMPLE + + # --- k-Anonymity --------------------------------------------------------- + k_val = None + if qi: + k_res = compute_k_anonymity(qi, work_df) + if "Error" not in k_res: + k_val = k_res.get("k-Value") + if k_val is not None and k_val < _GOV_K_WARNING: + needs_attention["low_anonymity"].append({ + "metric": "k-Anonymity", + "value": k_val, + "detail": f"Minimum equivalence class size on QIs: {', '.join(qi)}", + }) + details["k_anonymity"] = { + "quasi_identifiers": qi, + "k_value": k_val, + "descriptive_statistics": k_res.get("descriptive_statistics"), + "visualization": k_res.get("k-Anonymity Visualization"), + } + else: + details["k_anonymity"] = {"error": k_res.get("Error")} + else: + details["k_anonymity"] = {"error": "No eligible quasi-identifier columns found."} + + k_kpi = min(float(k_val) / _GOV_K_GOOD, 1.0) if k_val is not None else None + kpis.append({ + "id": "anonymity_k", + "label": "Anonymity (k)", + "value": k_kpi, + "status": _k_anonymity_status(k_val), + "hint": f"k-Value ≥ {_GOV_K_GOOD} good, ≥ {_GOV_K_WARNING} warning (min group size on auto QIs).", + "raw_k": k_val, + }) + + # --- l-Diversity --------------------------------------------------------- + l_val = None + if qi and sensitive: + l_res = compute_l_diversity(qi, sensitive, work_df) + if "Error" not in l_res: + l_val = l_res.get("l-Value") + if l_val is not None and l_val < _GOV_L_WARNING: + needs_attention["attribute_disclosure"].append({ + "metric": "l-Diversity", + "value": l_val, + "detail": f"Sensitive attribute '{sensitive}' lacks diversity in some QI groups", + }) + details["l_diversity"] = { + "quasi_identifiers": qi, + "sensitive_attribute": sensitive, + "l_value": l_val, + "descriptive_statistics": l_res.get("descriptive_statistics"), + "visualization": l_res.get("l-Diversity Visualization"), + } + else: + details["l_diversity"] = {"error": l_res.get("Error")} + else: + details["l_diversity"] = { + "error": "Requires quasi-identifiers and a sensitive attribute.", + } + + l_kpi = min(float(l_val) / _GOV_L_GOOD, 1.0) if l_val is not None else None + kpis.append({ + "id": "diversity_l", + "label": "Attribute diversity (l)", + "value": l_kpi, + "status": _l_diversity_status(l_val), + "hint": f"l-Value ≥ {_GOV_L_GOOD} good, ≥ {_GOV_L_WARNING} warning (min distinct sensitive values per QI group).", + "raw_l": l_val, + }) + + # --- t-Closeness --------------------------------------------------------- + t_val = None + if qi and sensitive: + t_res = compute_t_closeness(qi, sensitive, work_df) + if "Error" not in t_res: + t_val = t_res.get("t-Value") + if t_val is not None and t_val > _GOV_T_WARNING: + needs_attention["attribute_disclosure"].append({ + "metric": "t-Closeness", + "value": t_val, + "detail": f"Sensitive distribution diverges from global on '{sensitive}'", + }) + details["t_closeness"] = { + "quasi_identifiers": qi, + "sensitive_attribute": sensitive, + "t_value": t_val, + "descriptive_statistics": t_res.get("descriptive_statistics"), + "visualization": t_res.get("t-Closeness Visualization"), + } + else: + details["t_closeness"] = {"error": t_res.get("Error")} + else: + details["t_closeness"] = { + "error": "Requires quasi-identifiers and a sensitive attribute.", + } + + t_kpi = max(0.0, 1.0 - float(t_val) / _GOV_T_WARNING) if t_val is not None else None + kpis.append({ + "id": "distribution_t", + "label": "Distribution leakage (t)", + "value": t_kpi, + "status": _t_closeness_status(t_val), + "hint": f"Max TVD ≤ {_GOV_T_GOOD} good, ≤ {_GOV_T_WARNING} warning (lower is better).", + "raw_t": t_val, + }) + + # --- Entropy risk -------------------------------------------------------- + if qi: + e_res = compute_entropy_risk(qi, work_df) + if "Error" not in e_res: + details["entropy_risk"] = { + "quasi_identifiers": qi, + "entropy_value": e_res.get("Entropy-Value"), + "descriptive_statistics": e_res.get("descriptive_statistics"), + "visualization": e_res.get("Entropy Risk Visualization"), + } + else: + details["entropy_risk"] = {"error": e_res.get("Error")} + else: + details["entropy_risk"] = {"error": "No eligible quasi-identifier columns found."} + + # --- Single-attribute MM risk ---------------------------------------------- + worst_single_mean = None + single_by_qi = {} + if mm_qis: + for q in mm_qis: + try: + s_res = generate_single_attribute_MM_risk_scores(work_df, id_col, [q]) + if "Error" in s_res: + single_by_qi[q] = {"error": s_res["Error"]} + continue + stats = (s_res.get("Descriptive statistics of the risk scores") or {}).get(q, {}) + mean_risk = stats.get("mean") + if mean_risk is not None: + mean_risk = float(mean_risk) + single_by_qi[q] = {"mean_risk": round(mean_risk, 4), "stats": stats} + if worst_single_mean is None or mean_risk > worst_single_mean: + worst_single_mean = mean_risk + if mean_risk >= _GOV_MM_SINGLE_WARNING: + needs_attention["high_linkage_risk"].append({ + "metric": "Single-attribute risk", + "feature": q, + "mean_risk": round(mean_risk, 4), + }) + except Exception as exc: + single_by_qi[q] = {"error": str(exc)} + details["single_attribute_risk"] = { + "id_column": id_col, + "by_quasi_identifier": single_by_qi, + } + else: + details["single_attribute_risk"] = { + "error": "No categorical quasi-identifiers eligible for MM risk scoring.", + } + + single_kpi = ( + max(0.0, 1.0 - worst_single_mean) if worst_single_mean is not None else None + ) + kpis.append({ + "id": "single_linkage_risk", + "label": "Worst single-field risk", + "value": single_kpi, + "status": _mm_risk_status(worst_single_mean), + "hint": ( + f"1 − worst mean MM risk across QIs; good < {_GOV_MM_SINGLE_GOOD}, " + f"warning < {_GOV_MM_SINGLE_WARNING}." + ), + "raw_worst_mean": round(worst_single_mean, 4) if worst_single_mean is not None else None, + }) + + # --- Multiple-attribute MM risk ------------------------------------------ + multi_mean = None + if mm_qis: + try: + m_res = generate_multiple_attribute_MM_risk_scores(work_df, id_col, mm_qis) + if "Error" not in m_res: + m_stats = m_res.get("Descriptive statistics of the risk scores") or {} + multi_mean = m_stats.get("mean") + if multi_mean is not None: + multi_mean = float(multi_mean) + if multi_mean >= _GOV_MM_MULTI_WARNING: + needs_attention["high_linkage_risk"].append({ + "metric": "Multiple-attribute risk", + "features": mm_qis, + "mean_risk": round(multi_mean, 4), + }) + details["multiple_attribute_risk"] = { + "id_column": id_col, + "quasi_identifiers": mm_qis, + "mean_risk": round(multi_mean, 4) if multi_mean is not None else None, + "dataset_risk_score": m_res.get("Dataset Risk Score"), + "stats": m_stats, + "visualization": m_res.get("Multiple attribute risk scoring Visualization"), + } + else: + details["multiple_attribute_risk"] = {"error": m_res.get("Error")} + except Exception as exc: + details["multiple_attribute_risk"] = {"error": str(exc)} + else: + details["multiple_attribute_risk"] = { + "error": "No categorical quasi-identifiers eligible for combined MM risk.", + } + + multi_kpi = max(0.0, 1.0 - multi_mean) if multi_mean is not None else None + kpis.append({ + "id": "linkage_risk", + "label": "Combined linkage risk", + "value": multi_kpi, + "status": _mm_risk_status( + multi_mean, good=_GOV_MM_MULTI_GOOD, warning=_GOV_MM_MULTI_WARNING + ), + "hint": ( + f"1 − mean MM risk on combined QIs; good < {_GOV_MM_MULTI_GOOD}, " + f"warning < {_GOV_MM_MULTI_WARNING}." + ), + "raw_mean": round(multi_mean, 4) if multi_mean is not None else None, + }) + + # --- HIPAA --------------------------------------------------------------- + hipaa_cols = selection["hipaa_scan_columns"] + detected_phi = {} + if hipaa_cols: + detected_phi = detect_hipaa_identifiers(work_df, hipaa_cols) + for col, info in detected_phi.items(): + types = info.get("potential_types_detected") or [] + serious = [t for t in types if t in _GOV_HIPAA_SERIOUS_TYPES] + needs_attention["hipaa_phi"].append({ + "column": col, + "total_flags": info.get("total_flags", 0), + "types": types, + "serious": bool(serious), + "examples": info.get("examples") or [], + }) + details["hipaa"] = { + "columns_scanned": hipaa_cols, + "detected": detected_phi, + } + else: + details["hipaa"] = {"error": "No columns available to scan."} + + hipaa_status, hipaa_kpi = _hipaa_status(detected_phi) + kpis.append({ + "id": "phi_exposure", + "label": "PHI exposure", + "value": hipaa_kpi, + "status": hipaa_status, + "hint": "Pattern scan for HIPAA-like identifiers (SSN, medical IDs, postal codes, etc.). Not full Safe Harbor certification.", + "columns_flagged": len(detected_phi), + }) + + # --- Differential privacy (illustrative) ----------------------------------- + dp_features = selection["dp_features"] + if dp_features: + try: + dp_res = return_noisy_stats( + dp_features, _GOV_DP_EPSILON, work_df, save_output=False + ) + if "Error" not in dp_res: + details["differential_privacy"] = { + "features": dp_features, + "epsilon": _GOV_DP_EPSILON, + "illustrative": True, + "visualization": dp_res.get("DP Statistics Visualization"), + "summary": { + k: v for k, v in dp_res.items() + if k.endswith("(before noise)") or k.endswith("(after noise)") + }, + } + else: + details["differential_privacy"] = {"error": dp_res.get("Error")} + except Exception as exc: + details["differential_privacy"] = {"error": str(exc)} + else: + details["differential_privacy"] = { + "error": "No eligible numerical columns for illustrative DP demo.", + } + + grade_kpis = [k for k in kpis if k["value"] is not None] + grade = sum(k["value"] for k in grade_kpis) / len(grade_kpis) if grade_kpis else None + + return { + "grade": grade, + "grade_status": _grade_label(grade), + "small_sample_warning": small_sample, + "auto_selection": selection, + "kpis": kpis, + "needs_attention": needs_attention, + "details": details, + } + + @metrics_bp.route("/readiness-report", methods=["GET"]) def readiness_report(): """Return an aggregated, non-interactive data-readiness report as JSON. - Covers dataset overview, Data Quality, Impact-on-AI, and Fairness & Bias. - Designed to be extended with more pillars over time. + Covers dataset overview, Data Quality, Impact-on-AI, Fairness & Bias, + and Data Governance. Designed to be extended with more pillars over time. """ file_path = session.get("uploaded_file_path") file_name = session.get("uploaded_file_name") @@ -1043,12 +1737,19 @@ def readiness_report(): metric_time_log.error("Readiness report — fairness & bias error: %s", e, exc_info=True) fairness_section = {"error": f"{type(e).__name__}: {e}"} + try: + governance_section = _build_data_governance_section(file_info) + except Exception as e: + metric_time_log.error("Readiness report — data governance error: %s", e, exc_info=True) + governance_section = {"error": f"{type(e).__name__}: {e}"} + response = ensure_json_serializable({ "success": True, "dataset_overview": dataset_overview_section, "data_quality": data_quality_section, "impact_on_ai": impact_section, "fairness_bias": fairness_section, + "data_governance": governance_section, }) metric_time_log.info("Readiness report built in %.2f seconds", time.time() - start_time) return jsonify(response) diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 998d777a..534c820e 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2149,6 +2149,7 @@ function loadReadinessReport() { const dqContainer = document.getElementById("readiness-data-quality"); const impactContainer = document.getElementById("readiness-impact"); const fairnessContainer = document.getElementById("readiness-fairness"); + const governanceContainer = document.getElementById("readiness-governance"); fetch("/readiness-report") .then((r) => r.json()) @@ -2159,6 +2160,7 @@ function loadReadinessReport() { if (dqContainer) dqContainer.innerHTML = msg; if (impactContainer) impactContainer.innerHTML = msg; if (fairnessContainer) fairnessContainer.innerHTML = msg; + if (governanceContainer) governanceContainer.innerHTML = msg; return; } if (overviewContainer) @@ -2169,6 +2171,8 @@ function loadReadinessReport() { renderReadinessImpact(impactContainer, resp.impact_on_ai || {}); if (fairnessContainer) renderReadinessFairness(fairnessContainer, resp.fairness_bias || {}); + if (governanceContainer) + renderReadinessGovernance(governanceContainer, resp.data_governance || {}); }) .catch((err) => { const msg = `

    Error loading readiness report: ${err.message}

    `; @@ -2176,6 +2180,7 @@ function loadReadinessReport() { if (dqContainer) dqContainer.innerHTML = msg; if (impactContainer) impactContainer.innerHTML = msg; if (fairnessContainer) fairnessContainer.innerHTML = msg; + if (governanceContainer) governanceContainer.innerHTML = msg; }); } @@ -2233,6 +2238,28 @@ const _READINESS_METRIC_INFO = { "Target classes whose outcome rates vary most across sensitive groups (high TSD). Suggests uneven outcomes by group.", cdd_disparities: "Sensitive groups flagged by Conditional Demographic Disparity — rejected outcomes outweigh accepted ones disproportionately (positive class is auto-selected as the most frequent target value).", + overall_governance_grade: + "Average of governance KPIs (anonymity, diversity, distribution leakage, linkage risk, PHI exposure). Higher suggests lower privacy and compliance risk under automated checks.", + anonymity_k: + "Minimum equivalence-class size (k) on auto-selected quasi-identifiers. Higher k means each QI combination appears in at least k rows — harder to re-identify individuals.", + diversity_l: + "Minimum l-diversity on the auto-selected sensitive attribute within QI groups. Higher l means more distinct sensitive values per group — harder to infer a specific sensitive value.", + distribution_t: + "Maximum t-closeness (TVD) between group and global sensitive-attribute distributions. Lower t means groups do not reveal unusually skewed sensitive information.", + single_linkage_risk: + "Worst mean Marketer/Prosecutor re-identification risk across single quasi-identifiers. Higher risk means one field alone can identify many individuals.", + linkage_risk: + "Mean MM re-identification risk when all auto-selected quasi-identifiers are combined — the realistic linkage-attack scenario.", + phi_exposure: + "HIPAA-style pattern scan on auto-selected text columns. Flags potential SSNs, medical IDs, postal codes, emails, etc. Not a full regulatory certification.", + low_anonymity: + "Privacy metrics (e.g. k-Anonymity) below warning thresholds — small equivalence classes increase re-identification risk.", + hipaa_phi: + "Columns where HIPAA-like identifier patterns were detected during the automated scan.", + high_linkage_risk: + "Quasi-identifiers or QI combinations with high Marketer/Prosecutor re-identification risk scores.", + attribute_disclosure: + "l-Diversity or t-Closeness signals suggesting sensitive-attribute values may be inferable within QI groups.", }; /** @@ -2965,6 +2992,256 @@ function renderReadinessFairness(container, fb) { container.innerHTML = html; } +/** + * Render the Data Governance scorecard (auto-selected columns, privacy KPIs, + * HIPAA flags, needs-attention lists, collapsible charts). + */ +function renderReadinessGovernance(container, gov) { + if (gov.error) { + container.innerHTML = `

    Data Governance unavailable: ${gov.error}

    `; + return; + } + + const sel = gov.auto_selection || {}; + const criteria = sel.selection_criteria || {}; + const qiCrit = criteria.quasi_identifiers || {}; + const sensCrit = criteria.sensitive_attribute || {}; + const idCrit = criteria.id_column || {}; + const hipaaCrit = criteria.hipaa_scan_columns || {}; + const dpCrit = criteria.dp_features || {}; + const thresholds = criteria.thresholds || {}; + const kpis = gov.kpis || []; + const na = gov.needs_attention || {}; + const gradeCls = _dqStatusClasses(gov.grade_status); + + let html = ""; + + if (gov.small_sample_warning) { + html += ` +
    + Small dataset (< ${thresholds.small_sample_rows ?? 30} rows) — privacy metrics may be unstable. +
    `; + } + + html += ` +
    +

    Auto-selection criteria

    +
      +
    • Quasi-identifiers: ${qiCrit.selected?.length ? qiCrit.selected.join(", ") : "none"}
      + ${qiCrit.rule || ""}
      • +
    • Sensitive attribute: ${sensCrit.selected || "none"}
      + ${sensCrit.rule || ""}
      • +
    • ID column: ${idCrit.selected || "none"}${idCrit.synthetic ? " (synthetic row index)" : ""}
      + ${idCrit.rule || ""}
      • +
    • HIPAA scan columns: ${(hipaaCrit.selected || []).length} column(s)${hipaaCrit.selected?.length ? ` — ${hipaaCrit.selected.slice(0, 5).join(", ")}${hipaaCrit.selected.length > 5 ? "…" : ""}` : ""}
      • +
    • Thresholds: + k ≥ ${thresholds.k_good ?? "—"}/${thresholds.k_warning ?? "—"}, + l ≥ ${thresholds.l_good ?? "—"}/${thresholds.l_warning ?? "—"}, + t ≤ ${thresholds.t_good ?? "—"}/${thresholds.t_warning ?? "—"}, + MM single < ${thresholds.mm_single_good ?? "—"}/${thresholds.mm_single_warning ?? "—"}, + MM combined < ${thresholds.mm_multi_good ?? "—"}/${thresholds.mm_multi_warning ?? "—"} +
      • +
    +
    `; + + html += ` +
    + Overall governance grade${_readinessInfoIcon("overall_governance_grade")} + ${_pct(gov.grade)} +
    +
    `; + + kpis.forEach((k) => { + const cls = _dqStatusClasses(k.status); + let displayVal = _pct(k.value); + if (k.id === "anonymity_k" && k.raw_k != null) displayVal = `k=${k.raw_k}`; + else if (k.id === "diversity_l" && k.raw_l != null) displayVal = `l=${k.raw_l}`; + else if (k.id === "distribution_t" && k.raw_t != null) displayVal = `t=${Number(k.raw_t).toFixed(3)}`; + else if (k.id === "single_linkage_risk" && k.raw_worst_mean != null) + displayVal = `${Number(k.raw_worst_mean).toFixed(2)} risk`; + else if (k.id === "linkage_risk" && k.raw_mean != null) + displayVal = `${Number(k.raw_mean).toFixed(2)} risk`; + else if (k.id === "phi_exposure" && k.columns_flagged != null) + displayVal = k.columns_flagged === 0 ? "None" : `${k.columns_flagged} col(s)`; + + const widthPct = + k.value === null || k.value === undefined + ? 0 + : Math.max(0, Math.min(100, Math.round(k.value * 100))); + html += ` +
    +
    + ${k.label}${_readinessInfoIcon(k.id)} + ${displayVal} +
    +
    +
    +
    +

    ${k.hint || ""}

    +
    `; + }); + html += "
    "; + + const naItems = []; + const lowAnon = na.low_anonymity || []; + if (lowAnon.length) { + const items = lowAnon + .map( + (x) => + `
  • ${x.metric}${x.value}
    • `, + ) + .join(""); + naItems.push(` +
    +

    Low anonymity (${lowAnon.length})${_readinessInfoIcon("low_anonymity")}

    +
      ${items}
    +
    `); + } + const hipaaPhi = na.hipaa_phi || []; + if (hipaaPhi.length) { + const items = hipaaPhi + .slice(0, 6) + .map( + (x) => + `
  • ${x.column}${(x.types || []).join(", ") || x.total_flags + " flags"}
    • `, + ) + .join(""); + naItems.push(` +
    +

    HIPAA pattern matches (${hipaaPhi.length})${_readinessInfoIcon("hipaa_phi")}

    +
      ${items}
    +
    `); + } + const linkage = na.high_linkage_risk || []; + if (linkage.length) { + const items = linkage + .map((x) => { + const label = x.feature || (x.features || []).join(", "); + return `
  • ${x.metric}: ${label}${x.mean_risk}
    • `; + }) + .join(""); + naItems.push(` +
    +

    High linkage risk (${linkage.length})${_readinessInfoIcon("high_linkage_risk")}

    +
      ${items}
    +
    `); + } + const attrDisc = na.attribute_disclosure || []; + if (attrDisc.length) { + const items = attrDisc + .map( + (x) => + `
  • ${x.metric}${x.value}
    • `, + ) + .join(""); + naItems.push(` +
    +

    Attribute disclosure risk (${attrDisc.length})${_readinessInfoIcon("attribute_disclosure")}

    +
      ${items}
    +
    `); + } + + if (naItems.length) { + html += ` +
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } else { + html += ` +
    +

    No governance issues detected under the automated thresholds.

    +
    `; + } + + const det = gov.details || {}; + let detailsInner = ""; + + const chartMetrics = [ + ["k_anonymity", "k-Anonymity", "visualization"], + ["l_diversity", "l-Diversity", "visualization"], + ["t_closeness", "t-Closeness", "visualization"], + ["entropy_risk", "Entropy risk", "visualization"], + ["multiple_attribute_risk", "Multiple-attribute linkage risk", "visualization"], + ["differential_privacy", "Differential privacy (illustrative)", "visualization"], + ]; + chartMetrics.forEach(([key, title, visKey]) => { + const block = det[key]; + if (block?.[visKey]) { + detailsInner += ` +
    +

    ${title}

    + ${title} +
    `; + } else if (block?.error) { + detailsInner += `

    ${title}: ${block.error}

    `; + } + }); + + const singleRisk = det.single_attribute_risk?.by_quasi_identifier || {}; + const singleRows = Object.entries(singleRisk) + .filter(([, v]) => v.mean_risk != null) + .map( + ([q, v]) => + `${q}${v.mean_risk}`, + ) + .join(""); + if (singleRows) { + detailsInner += ` +
    +

    Single-attribute MM risk by quasi-identifier

    + ${singleRows}
    Quasi-identifierMean risk
    +
    `; + } + + const hipaaDet = det.hipaa?.detected || {}; + const hipaaRows = Object.entries(hipaaDet) + .map( + ([col, info]) => + `${col}${(info.potential_types_detected || []).join(", ")}${info.total_flags}`, + ) + .join(""); + if (hipaaRows) { + detailsInner += ` +
    +

    HIPAA scan results

    + ${hipaaRows}
    ColumnTypesFlags
    +
    `; + } + + const qiExcluded = qiCrit.excluded || []; + if (qiExcluded.length) { + const items = qiExcluded + .map( + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ) + .join(""); + detailsInner += ` +
    +

    Excluded quasi-identifier candidates (${qiExcluded.length})

    +
      ${items}
    +
    `; + } + + if (dpCrit.selected?.length) { + detailsInner += `

    DP demo features: ${dpCrit.selected.join(", ")} (ε=${dpCrit.epsilon ?? "—"}, illustrative only).

    `; + } + + if (detailsInner) { + html += ` +
    + + Show detailed charts & tables + +
    ${detailsInner}
    +
    `; + } + + container.classList.remove("text-center", "py-8"); + container.innerHTML = html; +} + // ==================== Workspace Init ==================== /** diff --git a/web/templates/_panels/_readiness_report.html b/web/templates/_panels/_readiness_report.html index 1bca345e..f3ddc474 100644 --- a/web/templates/_panels/_readiness_report.html +++ b/web/templates/_panels/_readiness_report.html @@ -52,4 +52,18 @@

    Fairness

    Computing fairness metrics...

    + + +
    +

    Data Governance

    +

    + Automated privacy and compliance checks with auto-selected quasi-identifiers, sensitive attributes, and HIPAA scan columns (no user input). +

    +
    +
    + +
    +

    Computing governance metrics...

    +
    +
    From c149091ab620abd3d067e5a313769329f897d177 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 24 Jun 2026 12:47:30 -0400 Subject: [PATCH 05/23] Load readiness report sections progressively with hybrid fetching Add per-section GET /readiness-report/
    endpoints and refactor the full-report route to reuse shared section builders. Update the readiness UI to render dataset overview first, then fetch and display Data Quality, Impact on AI, Fairness, and Governance in parallel while other sections keep their loading spinners. --- web/routes/metrics.py | 121 +++++++++++++++++++++++-------------- web/static/js/inspector.js | 105 ++++++++++++++++++++++---------- 2 files changed, 146 insertions(+), 80 deletions(-) diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 99bee0e2..286d98cb 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -1696,63 +1696,90 @@ def _build_data_governance_section(file_info): } -@metrics_bp.route("/readiness-report", methods=["GET"]) -def readiness_report(): - """Return an aggregated, non-interactive data-readiness report as JSON. - - Covers dataset overview, Data Quality, Impact-on-AI, Fairness & Bias, - and Data Governance. Designed to be extended with more pillars over time. - """ +_READINESS_SECTION_BUILDERS = { + "dataset-overview": _build_dataset_overview_section, + "data-quality": _build_data_quality_section, + "impact-on-ai": _build_impact_on_ai_section, + "fairness-bias": _build_fairness_bias_section, + "data-governance": _build_data_governance_section, +} + +_READINESS_SECTION_RESPONSE_KEYS = { + "dataset-overview": "dataset_overview", + "data-quality": "data_quality", + "impact-on-ai": "impact_on_ai", + "fairness-bias": "fairness_bias", + "data-governance": "data_governance", +} + + +def _readiness_file_info(): + """Return ``(file_path, file_name, file_type)`` from session, or *None*.""" file_path = session.get("uploaded_file_path") - file_name = session.get("uploaded_file_name") - file_type = session.get("uploaded_file_type") - if not file_path: + return None + return ( + file_path, + session.get("uploaded_file_name"), + session.get("uploaded_file_type"), + ) + + +def _build_readiness_section(section, file_info): + """Build one readiness-report section; return error dict on failure.""" + builder = _READINESS_SECTION_BUILDERS.get(section) + if builder is None: + return None + try: + return builder(file_info) + except Exception as e: + metric_time_log.error( + "Readiness report — %s error: %s", section, e, exc_info=True + ) + return {"error": f"{type(e).__name__}: {e}"} + + +@metrics_bp.route("/readiness-report/
    ", methods=["GET"]) +def readiness_report_section(section): + """Return a single readiness-report section as JSON (for progressive UI loading).""" + if section not in _READINESS_SECTION_BUILDERS: + return jsonify({"success": False, "message": f"Unknown section: {section}"}), 404 + + file_info = _readiness_file_info() + if file_info is None: return jsonify({"success": False, "message": "No file uploaded"}), 200 - file_info = (file_path, file_name, file_type) start_time = time.time() - try: - try: - dataset_overview_section = _build_dataset_overview_section(file_info) - except Exception as e: - metric_time_log.error("Readiness report — dataset overview error: %s", e, exc_info=True) - dataset_overview_section = {"error": f"{type(e).__name__}: {e}"} + data = _build_readiness_section(section, file_info) + metric_time_log.info( + "Readiness report section %s built in %.2f seconds", + section, + time.time() - start_time, + ) + return jsonify(ensure_json_serializable({ + "success": True, + "section": section, + "data": data, + })) - try: - data_quality_section = _build_data_quality_section(file_info) - except Exception as e: - metric_time_log.error("Readiness report — data quality error: %s", e, exc_info=True) - data_quality_section = {"error": f"{type(e).__name__}: {e}"} - try: - impact_section = _build_impact_on_ai_section(file_info) - except Exception as e: - metric_time_log.error("Readiness report — impact on AI error: %s", e, exc_info=True) - impact_section = {"error": f"{type(e).__name__}: {e}"} +@metrics_bp.route("/readiness-report", methods=["GET"]) +def readiness_report(): + """Return the full readiness report as JSON (all sections in one response).""" + file_info = _readiness_file_info() + if file_info is None: + return jsonify({"success": False, "message": "No file uploaded"}), 200 - try: - fairness_section = _build_fairness_bias_section(file_info) - except Exception as e: - metric_time_log.error("Readiness report — fairness & bias error: %s", e, exc_info=True) - fairness_section = {"error": f"{type(e).__name__}: {e}"} + start_time = time.time() + try: + response = {"success": True} + for slug in _READINESS_SECTION_BUILDERS: + response[_READINESS_SECTION_RESPONSE_KEYS[slug]] = _build_readiness_section( + slug, file_info + ) - try: - governance_section = _build_data_governance_section(file_info) - except Exception as e: - metric_time_log.error("Readiness report — data governance error: %s", e, exc_info=True) - governance_section = {"error": f"{type(e).__name__}: {e}"} - - response = ensure_json_serializable({ - "success": True, - "dataset_overview": dataset_overview_section, - "data_quality": data_quality_section, - "impact_on_ai": impact_section, - "fairness_bias": fairness_section, - "data_governance": governance_section, - }) metric_time_log.info("Readiness report built in %.2f seconds", time.time() - start_time) - return jsonify(response) + return jsonify(ensure_json_serializable(response)) except Exception as e: metric_time_log.error("Readiness report error: %s", e, exc_info=True) return jsonify({"success": False, "message": f"{type(e).__name__}: {e}"}), 200 diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index cf5e2c6d..50e642ad 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2189,51 +2189,90 @@ function _pct(value) { return `${Math.round(value * 100)}%`; } +/** Show an error message inside one readiness section container. */ +function _readinessSectionError(container, message) { + if (!container) return; + container.classList.add("text-center", "py-8"); + container.innerHTML = `

    ${message}

    `; +} + /** - * Fetch the aggregated readiness report and render the Data Quality - * scorecard (KPI tiles + overall grade + "needs attention" lists + - * a collapsible details view with the original charts). + * Fetch one readiness-report section and render it when ready. + * @param {string} section - URL slug (e.g. "data-quality") + * @param {HTMLElement} container + * @param {Function} renderFn - (container, data) => void + * @returns {Promise} */ -function loadReadinessReport() { - const overviewContainer = document.getElementById("readiness-summary"); - const dqContainer = document.getElementById("readiness-data-quality"); - const impactContainer = document.getElementById("readiness-impact"); - const fairnessContainer = document.getElementById("readiness-fairness"); - const governanceContainer = document.getElementById("readiness-governance"); - - fetch("/readiness-report") +function _fetchReadinessSection(section, container, renderFn) { + if (!container) return Promise.resolve(); + return fetch(`/readiness-report/${section}`) .then((r) => r.json()) .then((resp) => { if (!resp.success) { - const msg = `

    Could not load readiness report: ${resp.message || "unknown error"}

    `; - if (overviewContainer) overviewContainer.innerHTML = msg; - if (dqContainer) dqContainer.innerHTML = msg; - if (impactContainer) impactContainer.innerHTML = msg; - if (fairnessContainer) fairnessContainer.innerHTML = msg; - if (governanceContainer) governanceContainer.innerHTML = msg; + _readinessSectionError( + container, + `Could not load ${section.replace(/-/g, " ")}: ${resp.message || "unknown error"}`, + ); return; } - if (overviewContainer) - renderReadinessDatasetOverview(overviewContainer, resp.dataset_overview || {}); - if (dqContainer) - renderReadinessDataQuality(dqContainer, resp.data_quality || {}); - if (impactContainer) - renderReadinessImpact(impactContainer, resp.impact_on_ai || {}); - if (fairnessContainer) - renderReadinessFairness(fairnessContainer, resp.fairness_bias || {}); - if (governanceContainer) - renderReadinessGovernance(governanceContainer, resp.data_governance || {}); + renderFn(container, resp.data || {}); }) .catch((err) => { - const msg = `

    Error loading readiness report: ${err.message}

    `; - if (overviewContainer) overviewContainer.innerHTML = msg; - if (dqContainer) dqContainer.innerHTML = msg; - if (impactContainer) impactContainer.innerHTML = msg; - if (fairnessContainer) fairnessContainer.innerHTML = msg; - if (governanceContainer) governanceContainer.innerHTML = msg; + _readinessSectionError(container, `Error loading section: ${err.message}`); }); } +/** + * Load the readiness report with hybrid progressive rendering: + * dataset overview first, then remaining sections in parallel. + */ +function loadReadinessReport() { + const overviewContainer = document.getElementById("readiness-summary"); + + const parallelSections = [ + { + section: "data-quality", + container: document.getElementById("readiness-data-quality"), + render: renderReadinessDataQuality, + }, + { + section: "impact-on-ai", + container: document.getElementById("readiness-impact"), + render: renderReadinessImpact, + }, + { + section: "fairness-bias", + container: document.getElementById("readiness-fairness"), + render: renderReadinessFairness, + }, + { + section: "data-governance", + container: document.getElementById("readiness-governance"), + render: renderReadinessGovernance, + }, + ]; + + const loadParallelSections = () => { + Promise.all( + parallelSections.map(({ section, container, render }) => + _fetchReadinessSection(section, container, render), + ), + ); + }; + + if (!overviewContainer) { + loadParallelSections(); + return; + } + + // Hybrid: overview first, then parallel for the rest (spinners stay until each resolves). + _fetchReadinessSection( + "dataset-overview", + overviewContainer, + renderReadinessDatasetOverview, + ).finally(loadParallelSections); +} + /** Escape text for safe inclusion in readiness info tooltips. */ function _escapeHtml(s) { return String(s) From 3238d367bd35644985bfab4d7ebd3ba1ab133dda Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 24 Jun 2026 14:00:32 -0400 Subject: [PATCH 06/23] Standardize readiness report section layout across scorecards Align Data Quality, Impact on AI, Fairness, and Governance sections on a consistent structure: auto-selection criteria (or analysis scope), overall grade with KPI tiles, then needs attention. Add Impact section grading and column-pruning metadata on the backend, and move supplementary charts and tables into collapsible details. --- web/routes/metrics.py | 100 +++++++++++++++++++-- web/static/js/inspector.js | 177 +++++++++++++++++++++++-------------- 2 files changed, 205 insertions(+), 72 deletions(-) diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 286d98cb..734dae6c 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -438,6 +438,17 @@ def _build_data_quality_section(file_info): section = { "grade": grade, "grade_status": _grade_label(grade), + "auto_selection": { + "selection_criteria": { + "analysis_scope": { + "rule": ( + "Completeness, outliers, and duplicity run on the full dataset " + "automatically — no column selection required." + ), + "selected": "all columns", + }, + }, + }, "kpis": kpis, "needs_attention": { "incomplete_features": incomplete, @@ -607,13 +618,92 @@ def _build_impact_on_ai_section(file_info): cat = corr.get("Correlations Analysis Categorical", {}) or {} num = corr.get("Correlations Analysis Numerical", {}) or {} + leakage_pairs = signals["leakage"] + redundant_pairs = signals["redundant"] + isolated_features = signals["isolated"] + top_pairs = signals["top"] + n_kept = len(kept) + + leakage_kpi = 1.0 if not leakage_pairs else max(0.0, 1.0 - len(leakage_pairs) / 5.0) + redundancy_kpi = max(0.0, 1.0 - min(len(redundant_pairs) / 10.0, 1.0)) + informativeness_kpi = ( + max(0.0, 1.0 - len(isolated_features) / n_kept) if n_kept else None + ) + + kpis = [ + { + "id": "leakage_safety", + "label": "Leakage safety", + "value": leakage_kpi, + "status": "good" if not leakage_pairs else ("warning" if len(leakage_pairs) < 3 else "poor"), + "hint": ( + f"No pairs with |score| ≥ {_CORR_LEAKAGE_THRESHOLD}; " + f"{len(leakage_pairs)} leakage-risk pair(s) found." + ), + "raw_count": len(leakage_pairs), + }, + { + "id": "redundancy", + "label": "Redundancy", + "value": redundancy_kpi, + "status": _grade_label(redundancy_kpi), + "hint": ( + f"1 − min(redundant pairs / 10, 1); " + f"{len(redundant_pairs)} pair(s) with |score| ≥ {_CORR_REDUNDANT_THRESHOLD}." + ), + "raw_count": len(redundant_pairs), + }, + { + "id": "informativeness", + "label": "Informativeness", + "value": informativeness_kpi, + "status": _grade_label(informativeness_kpi), + "hint": ( + f"Share of analyzed features with a strong relationship " + f"(max |score| ≥ {_CORR_ISOLATED_THRESHOLD})." + ), + "raw_count": len(isolated_features), + }, + ] + + present = [k["value"] for k in kpis if k["value"] is not None] + grade = sum(present) / len(present) if present else None + return { - "columns_analyzed": len(kept), + "grade": grade, + "grade_status": _grade_label(grade), + "columns_analyzed": n_kept, + "auto_selection": { + "columns_selected": kept, + "selection_criteria": { + "columns_analyzed": { + "rule": ( + f"Prune constants, ID-like categoricals (unique ratio ≥ " + f"{_CORR_ID_UNIQUE_RATIO}), and high-cardinality categoricals " + f"(>{_CORR_HIGH_CARD_MAX} categories); cap at {_CORR_MAX_COLUMNS} " + "columns (numerical prioritized)." + ), + "selected": kept, + "excluded": dropped, + }, + "thresholds": { + "redundant_threshold": _CORR_REDUNDANT_THRESHOLD, + "leakage_threshold": _CORR_LEAKAGE_THRESHOLD, + "isolated_threshold": _CORR_ISOLATED_THRESHOLD, + }, + }, + }, + "kpis": kpis, + "needs_attention": { + "leakage_pairs": leakage_pairs, + "redundant_pairs": redundant_pairs, + "isolated_features": isolated_features, + }, + "top_pairs": top_pairs, "columns_dropped": dropped, - "redundant_pairs": signals["redundant"], - "leakage_pairs": signals["leakage"], - "isolated_features": signals["isolated"], - "top_pairs": signals["top"], + "redundant_pairs": redundant_pairs, + "leakage_pairs": leakage_pairs, + "isolated_features": isolated_features, "details": { "categorical_visualization": cat.get( "Correlations Analysis Categorical Visualization" diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 50e642ad..0a014869 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2295,6 +2295,8 @@ const _READINESS_METRIC_INFO = { "Readiness verdict for this feature. Poor: high missingness, constant, or ID-like. Warning: moderate issues. Good: no major issues detected.", overall_dq_grade: "Average of the data-quality KPIs (completeness, uniqueness, outlier-cleanliness). Higher is better — indicates how clean the dataset is overall.", + analysis_scope: + "This section evaluates every column automatically; no features or targets are chosen by the user.", completeness: "Overall share of non-missing values across all features. Same measure as the Completeness metric on the Data Quality tab.", uniqueness: @@ -2311,6 +2313,14 @@ const _READINESS_METRIC_INFO = { "Features whose strongest correlation to any other feature is below 0.1. May be uninformative noise, identifiers, or weakly related fields worth reviewing.", most_related_pairs: "The feature pairs with the highest absolute correlation scores from the automated scan — quick view of the strongest relationships in the data.", + overall_impact_grade: + "Average of impact KPIs (leakage safety, redundancy, informativeness). Higher suggests healthier feature structure for modeling.", + leakage_safety: + "Whether any feature pairs exceed the leakage-risk correlation threshold (|score| ≥ 0.95). Fewer or no pairs is better.", + redundancy: + "Derived from the count of highly correlated redundant pairs (|score| ≥ 0.8). Lower redundancy is generally preferable.", + informativeness: + "Share of analyzed features that have at least one meaningful correlation to another feature — flags isolated or uninformative columns.", overall_fairness_grade: "Average of fairness KPIs (representation balance, label balance, outcome parity). Higher suggests more balanced representation and outcomes under the automated checks.", representation_balance: @@ -2552,17 +2562,29 @@ function renderReadinessDataQuality(container, dq) { return; } const kpis = dq.kpis || []; - const gradeCls = _dqStatusClasses(dq.grade_status); + const gradeCls = _dqStatusClasses(dq.grade_status); + const scopeCrit = + (dq.auto_selection || {}).selection_criteria?.analysis_scope || {}; - // --- Overall grade + KPI tiles --- - let html = ` + // --- Analysis scope (no column auto-selection required) --- + let html = ` +
    +

    Auto-selection criteria

    +
      +
    • Analysis scope: ${scopeCrit.selected || "all columns"}${_readinessInfoIcon("analysis_scope")}
      + ${scopeCrit.rule || "All columns are evaluated automatically."}
      • +
    +
    `; + + // --- Overall grade + KPI tiles --- + html += `
    Overall data quality grade${_readinessInfoIcon("overall_dq_grade")} ${_pct(dq.grade)}
    `; - kpis.forEach((k) => { + kpis.forEach((k) => { const cls = _dqStatusClasses(k.status); const widthPct = k.value === null || k.value === undefined @@ -2580,10 +2602,10 @@ function renderReadinessDataQuality(container, dq) {

    ${k.hint || ""}

    `; }); - html += "
    "; + html += "
    "; - // --- Needs attention --- - const na = dq.needs_attention || {}; + // --- Needs attention --- + const na = dq.needs_attention || {}; const incomplete = na.incomplete_features || []; const outlierFeats = na.outlier_features || []; const dupRows = na.duplicate_rows || 0; @@ -2646,8 +2668,8 @@ function renderReadinessDataQuality(container, dq) {
    `; } - // --- Collapsible details (original charts) --- - const det = dq.details || {}; + // --- Collapsible details (original charts) --- + const det = dq.details || {}; let detailsInner = ""; if (det.completeness && det.completeness.visualization) { detailsInner += ` @@ -2690,49 +2712,73 @@ function renderReadinessImpact(container, impact) { return; } - const redundant = impact.redundant_pairs || []; - const leakage = impact.leakage_pairs || []; - const isolated = impact.isolated_features || []; + const autoSel = impact.auto_selection || {}; + const crit = autoSel.selection_criteria || {}; + const colCrit = crit.columns_analyzed || {}; + const thresholds = crit.thresholds || {}; + const kpis = impact.kpis || []; + const na = impact.needs_attention || {}; + const gradeCls = _dqStatusClasses(impact.grade_status); + + const leakage = na.leakage_pairs || impact.leakage_pairs || []; + const redundant = na.redundant_pairs || impact.redundant_pairs || []; + const isolated = na.isolated_features || impact.isolated_features || []; const topPairs = impact.top_pairs || []; - const dropped = impact.columns_dropped || []; - const analyzed = impact.columns_analyzed || 0; + const dropped = colCrit.excluded || impact.columns_dropped || []; + const analyzed = impact.columns_analyzed || (colCrit.selected || []).length; const fmtScore = (s) => (typeof s === "number" ? s.toFixed(2) : s); const fmtPair = (p) => `
  • ${p.a} \u2194 ${p.b}${fmtScore(p.score)}
    • `; - // --- Stat tiles --- - const tiles = [ - { label: "Features analyzed", value: analyzed, status: "neutral", infoKey: "features_analyzed" }, - { - label: "Leakage-risk pairs", - value: leakage.length, - status: leakage.length ? "poor" : "good", - infoKey: "leakage_risk_pairs", - }, - { - label: "Redundant pairs", - value: redundant.length, - status: redundant.length ? "warning" : "good", - infoKey: "redundant_pairs", - }, - { - label: "Isolated features", - value: isolated.length, - status: isolated.length ? "warning" : "good", - infoKey: "isolated_features", - }, - ]; + const selectedCols = colCrit.selected || []; + const selectedPreview = + selectedCols.length > 0 + ? `${selectedCols.slice(0, 8).join(", ")}${selectedCols.length > 8 ? "…" : ""}` + : "none"; + + // --- Auto-selection criteria --- + let html = ` +
    +

    Auto-selection criteria

    + +
    `; + + // --- Overall grade + KPI tiles --- + html += ` +
    + Overall impact grade${_readinessInfoIcon("overall_impact_grade")} + ${_pct(impact.grade)} +
    +
    `; - let html = '
    '; - tiles.forEach((t) => { - const cls = _dqStatusClasses(t.status); - const valColor = - t.status === "neutral" ? "text-gray-900 dark:text-white" : cls.text; + kpis.forEach((k) => { + const cls = _dqStatusClasses(k.status); + const displayVal = + k.raw_count != null ? `${k.raw_count} flagged` : _pct(k.value); + const widthPct = + k.value === null || k.value === undefined + ? 0 + : Math.max(0, Math.min(100, Math.round(k.value * 100))); html += ` -
    -
    ${t.value}
    -
    ${t.label}${_readinessInfoIcon(t.infoKey)}
    +
    +
    + ${k.label}${_readinessInfoIcon(k.id)} + ${displayVal} +
    +
    +
    +
    +

    ${k.hint || ""}

    `; }); html += "
    "; @@ -2766,7 +2812,7 @@ function renderReadinessImpact(container, impact) { if (isolated.length) { const items = isolated .slice(0, 10) - .map((f) => `
  • ${f}
    • `) + .map((f) => `
  • ${typeof f === "string" ? f : f.feature || f}
    • `) .join(""); const more = isolated.length > 10 @@ -2792,29 +2838,29 @@ function renderReadinessImpact(container, impact) {
    `; } - // --- Most-related pairs table --- + // --- Collapsible details --- + const det = impact.details || {}; + let detailsInner = ""; + if (topPairs.length) { - html += - '

    Most-related feature pairs' + + detailsInner += + '

    Most-related feature pairs' + _readinessInfoIcon("most_related_pairs") + "

    "; - html += - '
    '; - html += + detailsInner += + '
    '; + detailsInner += ''; topPairs.forEach((p, i) => { const stripe = i % 2 === 0 ? "bg-white dark:bg-gray-800" : "bg-gray-50 dark:bg-gray-700/50"; - html += ``; + detailsInner += ``; }); - html += "
    Feature AFeature BScore
    ${p.a}${p.b}${fmtScore(p.score)}
    ${p.a}${p.b}${fmtScore(p.score)}
    "; + detailsInner += "
    "; } - // --- Collapsible details (heatmaps + excluded columns) --- - const det = impact.details || {}; - let detailsInner = ""; if (det.numerical_visualization) { const method = det.numerical_method ? ` (${det.numerical_method})` : ""; detailsInner += ` @@ -2843,11 +2889,12 @@ function renderReadinessImpact(container, impact) { `; } + if (detailsInner) { html += `
    - Show correlation heatmaps & excluded columns + Show detailed charts & tables
    ${detailsInner}
    `; @@ -3103,16 +3150,7 @@ function renderReadinessGovernance(container, gov) { const na = gov.needs_attention || {}; const gradeCls = _dqStatusClasses(gov.grade_status); - let html = ""; - - if (gov.small_sample_warning) { - html += ` -
    - Small dataset (< ${thresholds.small_sample_rows ?? 30} rows) — privacy metrics may be unstable. -
    `; - } - - html += ` + let html = `

    Auto-selection criteria

    `; From e0afed03d2c4be7c64cb930b2e68f0b52bf32994 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 24 Jun 2026 15:24:01 -0400 Subject: [PATCH 07/23] Contextualize readiness report needs-attention flags with column context Enrich backend needs_attention payloads with target, sensitive, and quasi-identifier metadata (including worst k-anonymity groups) so each flag names the columns and groups that triggered it. Refactor Fairness, Governance, and Impact renderers to shared NA helpers for consistent, context-rich display. --- web/routes/metrics.py | 90 ++++++++- web/static/js/inspector.js | 396 ++++++++++++++++++++++++++----------- 2 files changed, 361 insertions(+), 125 deletions(-) diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 734dae6c..bedc3ec5 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -986,6 +986,7 @@ def _build_fairness_bias_section(file_info): for cls, share in vc.items(): if share < _FAIRNESS_MINORITY_SHARE: needs_attention["minority_classes"].append({ + "target_column": target_col, "class": str(cls), "share": round(float(share), 4), }) @@ -1018,7 +1019,12 @@ def _build_fairness_bias_section(file_info): else: tsd_scores = sr.get("TSD scores") or {} flagged = [ - {"class": str(cls), "tsd": round(float(score), 4)} + { + "target_column": target_col, + "sensitive_column": primary_sensitive, + "class": str(cls), + "tsd": round(float(score), 4), + } for cls, score in tsd_scores.items() if isinstance(score, (int, float)) and float(score) >= _FAIRNESS_TSD_FLAG ] @@ -1064,7 +1070,13 @@ def _build_fairness_bias_section(file_info): else: disparities = (cdd or {}).get("Disparities") or {} cdd_flagged = [ - {"group": str(grp), "disparity": info.get("disparity")} + { + "sensitive_column": primary_sensitive, + "target_column": target_col, + "positive_class": str(positive_class), + "group": str(grp), + "disparity": info.get("disparity"), + } for grp, info in disparities.items() if str(info.get("disparity", "")).lower() == "true" ] @@ -1113,6 +1125,7 @@ def _build_fairness_bias_section(file_info): _GOV_DP_MAX_FEATURES = 2 _GOV_DP_EPSILON = 0.1 _GOV_SMALL_SAMPLE = 30 +_GOV_WORST_GROUPS_MAX = 5 _GOV_K_GOOD = 5 _GOV_K_WARNING = 2 @@ -1224,6 +1237,36 @@ def _hipaa_status(detected_phi): return "warning", 0.5 +def _worst_equivalence_classes(df, quasi_identifiers, max_groups=_GOV_WORST_GROUPS_MAX): + """Return the smallest equivalence classes on *quasi_identifiers*.""" + if not quasi_identifiers: + return [], 0 + + data = df.replace("?", pd.NA) + clean = data.dropna(subset=quasi_identifiers) + if clean.empty: + return [], 0 + + counts = clean.groupby(quasi_identifiers, dropna=False).size() + if counts.empty: + return [], 0 + + singleton_count = int((counts == 1).sum()) + worst_groups = [] + for keys, size in counts.nsmallest(max_groups).items(): + if len(quasi_identifiers) == 1: + key_tuple = (keys,) if not isinstance(keys, tuple) else keys + else: + key_tuple = keys if isinstance(keys, tuple) else (keys,) + qi_values = { + qi: str(val) if pd.notna(val) else "?" + for qi, val in zip(quasi_identifiers, key_tuple) + } + worst_groups.append({"size": int(size), "qi_values": qi_values}) + + return worst_groups, singleton_count + + def _auto_select_governance_columns(df, fairness_target=None): """Pick columns for automated privacy and HIPAA checks in the readiness report.""" n_rows = max(len(df), 1) @@ -1503,10 +1546,17 @@ def _build_data_governance_section(file_info): if "Error" not in k_res: k_val = k_res.get("k-Value") if k_val is not None and k_val < _GOV_K_WARNING: + worst_groups, singleton_count = _worst_equivalence_classes(work_df, qi) needs_attention["low_anonymity"].append({ "metric": "k-Anonymity", "value": k_val, - "detail": f"Minimum equivalence class size on QIs: {', '.join(qi)}", + "quasi_identifiers": list(qi), + "singleton_count": singleton_count, + "worst_groups": worst_groups, + "detail": ( + f"Minimum group size {k_val} on quasi-identifiers: " + f"{', '.join(qi)}" + ), }) details["k_anonymity"] = { "quasi_identifiers": qi, @@ -1539,7 +1589,12 @@ def _build_data_governance_section(file_info): needs_attention["attribute_disclosure"].append({ "metric": "l-Diversity", "value": l_val, - "detail": f"Sensitive attribute '{sensitive}' lacks diversity in some QI groups", + "sensitive_attribute": sensitive, + "quasi_identifiers": list(qi), + "detail": ( + f"Sensitive '{sensitive}' has only {l_val} distinct value(s) " + f"in some groups defined by ({', '.join(qi)})" + ), }) details["l_diversity"] = { "quasi_identifiers": qi, @@ -1575,7 +1630,12 @@ def _build_data_governance_section(file_info): needs_attention["attribute_disclosure"].append({ "metric": "t-Closeness", "value": t_val, - "detail": f"Sensitive distribution diverges from global on '{sensitive}'", + "sensitive_attribute": sensitive, + "quasi_identifiers": list(qi), + "detail": ( + f"Distribution of '{sensitive}' diverges from global " + f"(max TVD {t_val}) within groups of ({', '.join(qi)})" + ), }) details["t_closeness"] = { "quasi_identifiers": qi, @@ -1637,7 +1697,9 @@ def _build_data_governance_section(file_info): needs_attention["high_linkage_risk"].append({ "metric": "Single-attribute risk", "feature": q, + "quasi_identifiers": [q], "mean_risk": round(mean_risk, 4), + "detail": f"Quasi-identifier '{q}' — mean MM risk {mean_risk:.2f}", }) except Exception as exc: single_by_qi[q] = {"error": str(exc)} @@ -1645,6 +1707,17 @@ def _build_data_governance_section(file_info): "id_column": id_col, "by_quasi_identifier": single_by_qi, } + if needs_attention["low_anonymity"] and single_by_qi: + worst_qi = None + for q, info in single_by_qi.items(): + mr = info.get("mean_risk") + if mr is not None and (worst_qi is None or mr > worst_qi[1]): + worst_qi = (q, mr) + if worst_qi: + needs_attention["low_anonymity"][0]["worst_single_qi"] = { + "feature": worst_qi[0], + "mean_risk": worst_qi[1], + } else: details["single_attribute_risk"] = { "error": "No categorical quasi-identifiers eligible for MM risk scoring.", @@ -1678,8 +1751,13 @@ def _build_data_governance_section(file_info): if multi_mean >= _GOV_MM_MULTI_WARNING: needs_attention["high_linkage_risk"].append({ "metric": "Multiple-attribute risk", - "features": mm_qis, + "features": list(mm_qis), + "quasi_identifiers": list(mm_qis), "mean_risk": round(multi_mean, 4), + "detail": ( + f"Combined QIs ({', '.join(mm_qis)}) — " + f"mean MM risk {multi_mean:.2f}" + ), }) details["multiple_attribute_risk"] = { "id_column": id_col, diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 0a014869..74c62bfb 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2281,6 +2281,64 @@ function _escapeHtml(s) { .replace(/"/g, """); } +/** Format quasi-identifier key/value pairs for display. */ +function _formatQiValues(qiValues) { + if (!qiValues || typeof qiValues !== "object") return ""; + return Object.entries(qiValues) + .map(([k, v]) => `${k}=${v}`) + .join(", "); +} + +/** + * One needs-attention list row with optional secondary line (context/detail). + */ +function _readinessNaRow(primary, secondary, value) { + const valHtml = value + ? `${value}` + : ""; + const sub = + secondary + ? `

    ${_escapeHtml(secondary)}

    ` + : ""; + return `
  • +
    + ${primary} + ${valHtml} +
    ${sub} +
    • `; +} + +/** Needs-attention subsection with title and list body. */ +function _readinessNaBlock(title, infoKey, contextHtml, listHtml, tone) { + const titleCls = + tone === "red" + ? "text-red-700 dark:text-red-400" + : tone === "amber" + ? "text-amber-700 dark:text-amber-400" + : "text-gray-600 dark:text-gray-300"; + const ctx = contextHtml + ? `

    ${contextHtml}

    ` + : ""; + return `
    +

    ${title}${infoKey ? _readinessInfoIcon(infoKey) : ""}

    + ${ctx} +
      ${listHtml}
    +
    `; +} + +/** Wrap needs-attention blocks in the standard amber/green panel. */ +function _renderReadinessNeedsAttentionPanel(naItems, emptyMessage) { + if (naItems.length) { + return `
    +

    Needs attention

    +
    ${naItems.join("")}
    +
    `; + } + return `
    +

    ${emptyMessage}

    +
    `; +} + /** Readiness metric definitions shown in info-icon tooltips. */ const _READINESS_METRIC_INFO = { feature_profile: @@ -2729,7 +2787,11 @@ function renderReadinessImpact(container, impact) { const fmtScore = (s) => (typeof s === "number" ? s.toFixed(2) : s); const fmtPair = (p) => - `
  • ${p.a} \u2194 ${p.b}${fmtScore(p.score)}
    • `; + _readinessNaRow( + `${_escapeHtml(p.a)} ↔ ${_escapeHtml(p.b)}`, + "Correlated feature pair", + `|score| ${fmtScore(p.score)}`, + ); const selectedCols = colCrit.selected || []; const selectedPreview = @@ -2791,11 +2853,15 @@ function renderReadinessImpact(container, impact) { leakage.length > 6 ? `
  • +${leakage.length - 6} more
    • ` : ""; - naItems.push(` -
    -

    Leakage risk (|score| ≥ 0.95)${_readinessInfoIcon("leakage_risk_pairs")}

    -
      ${items}${more}
    -
    `); + naItems.push( + _readinessNaBlock( + `Leakage risk (|score| ≥ 0.95) (${leakage.length})`, + "leakage_risk_pairs", + null, + items + more, + "red", + ), + ); } if (redundant.length) { const items = redundant.slice(0, 6).map(fmtPair).join(""); @@ -2803,40 +2869,47 @@ function renderReadinessImpact(container, impact) { redundant.length > 6 ? `
  • +${redundant.length - 6} more
    • ` : ""; - naItems.push(` -
    -

    Redundant pairs (|score| ≥ 0.8)${_readinessInfoIcon("redundant_pairs")}

    -
      ${items}${more}
    -
    `); + naItems.push( + _readinessNaBlock( + `Redundant pairs (|score| ≥ 0.8) (${redundant.length})`, + "redundant_pairs", + null, + items + more, + "amber", + ), + ); } if (isolated.length) { const items = isolated .slice(0, 10) - .map((f) => `
  • ${typeof f === "string" ? f : f.feature || f}
    • `) + .map((f) => { + const name = typeof f === "string" ? f : f.feature || f; + return _readinessNaRow( + `${_escapeHtml(name)}`, + "No strong correlation to other analyzed features", + null, + ); + }) .join(""); const more = isolated.length > 10 ? `
  • +${isolated.length - 10} more
    • ` : ""; - naItems.push(` -
    -

    Isolated features (no strong relationships)${_readinessInfoIcon("isolated_features")}

    -
      ${items}${more}
    -
    `); + naItems.push( + _readinessNaBlock( + `Isolated features (${isolated.length})`, + "isolated_features", + null, + items + more, + "amber", + ), + ); } - if (naItems.length) { - html += ` -
    -

    Needs attention

    -
    ${naItems.join("")}
    -
    `; - } else { - html += ` -
    -

    No redundancy, leakage risk, or isolated features detected.

    -
    `; - } + html += _renderReadinessNeedsAttentionPanel( + naItems, + "No redundancy, leakage risk, or isolated features detected.", + ); // --- Collapsible details --- const det = impact.details || {}; @@ -2983,69 +3056,106 @@ function renderReadinessFairness(container, fb) { if (repImbalance.length) { const items = repImbalance .slice(0, 5) - .map( - (s) => - `
  • ${s.column}max ratio ${s.max_ratio}
    • `, - ) + .map((s) => { + const pairHint = + s.flagged_pairs && s.flagged_pairs.length + ? `Worst pair: ${s.flagged_pairs[0].pair} (ratio ${s.flagged_pairs[0].ratio})` + : ""; + return _readinessNaRow( + `${_escapeHtml(s.column)}`, + pairHint, + `max ratio ${s.max_ratio}`, + ); + }) .join(""); - naItems.push(` -
    -

    Representation imbalance (${repImbalance.length})${_readinessInfoIcon("representation_imbalance")}

    -
      ${items}
    -
    `); + naItems.push( + _readinessNaBlock( + `Representation imbalance (${repImbalance.length})`, + "representation_imbalance", + "Sensitive attributes with extreme category probability ratios", + items, + "amber", + ), + ); } + const minorities = na.minority_classes || []; if (minorities.length) { + const targetCol = + minorities[0].target_column || targetCrit.selected || "target"; const items = minorities - .map( - (m) => - `
  • ${m.class}${_pct(m.share)} share
    • `, + .map((m) => + _readinessNaRow( + `${_escapeHtml(m.class)}`, + `Class in ${_escapeHtml(targetCol)}`, + `${_pct(m.share)} share`, + ), ) .join(""); - naItems.push(` -
    -

    Minority classes (${minorities.length})${_readinessInfoIcon("minority_classes")}

    -
      ${items}
    -
    `); + naItems.push( + _readinessNaBlock( + `Minority classes (${minorities.length})`, + "minority_classes", + `Target column: ${_escapeHtml(targetCol)}`, + items, + "amber", + ), + ); } + const outcomeDisp = na.outcome_disparities || []; if (outcomeDisp.length) { + const sensCol = outcomeDisp[0].sensitive_column || sel.primary_sensitive || "—"; + const tgtCol = outcomeDisp[0].target_column || targetCrit.selected || "—"; const items = outcomeDisp - .map( - (d) => - `
  • ${d.class}TSD ${d.tsd}
    • `, + .map((d) => + _readinessNaRow( + `${_escapeHtml(d.target_column || tgtCol)} = ${_escapeHtml(d.class)}`, + `Outcome rates vary by sensitive ${_escapeHtml(d.sensitive_column || sensCol)}`, + `TSD ${d.tsd}`, + ), ) .join(""); - naItems.push(` -
    -

    Outcome-rate disparities (${outcomeDisp.length})${_readinessInfoIcon("outcome_disparities")}

    -
      ${items}
    -
    `); + naItems.push( + _readinessNaBlock( + `Outcome-rate disparities (${outcomeDisp.length})`, + "outcome_disparities", + `Sensitive ${_escapeHtml(sensCol)} × target ${_escapeHtml(tgtCol)}`, + items, + "amber", + ), + ); } + const cddDisp = na.cdd_disparities || []; if (cddDisp.length) { + const sensCol = cddDisp[0].sensitive_column || sel.primary_sensitive || "—"; + const tgtCol = cddDisp[0].target_column || targetCrit.selected || "—"; + const posClass = cddDisp[0].positive_class || posCrit.selected || "—"; const items = cddDisp - .map((d) => `
  • ${d.group}
    • `) + .map((d) => + _readinessNaRow( + `${_escapeHtml(d.sensitive_column || sensCol)} = ${_escapeHtml(d.group)}`, + `CDD vs target ${_escapeHtml(d.target_column || tgtCol)} (positive: ${_escapeHtml(String(d.positive_class ?? posClass))})`, + null, + ), + ) .join(""); - naItems.push(` -
    -

    CDD flagged groups (${cddDisp.length})${_readinessInfoIcon("cdd_disparities")}

    -
      ${items}
    -
    `); + naItems.push( + _readinessNaBlock( + `CDD flagged groups (${cddDisp.length})`, + "cdd_disparities", + `Sensitive ${_escapeHtml(sensCol)} × target ${_escapeHtml(tgtCol)}`, + items, + "red", + ), + ); } - if (naItems.length) { - html += ` -
    -

    Needs attention

    -
    ${naItems.join("")}
    -
    `; - } else { - html += ` -
    -

    No fairness issues detected under the automated thresholds.

    -
    `; - } + html += _renderReadinessNeedsAttentionPanel( + naItems, + "No fairness issues detected under the automated thresholds.", + ); // --- Collapsible details (charts) --- const det = fb.details || {}; @@ -3217,74 +3327,122 @@ function renderReadinessGovernance(container, gov) { const naItems = []; const lowAnon = na.low_anonymity || []; if (lowAnon.length) { - const items = lowAnon - .map( - (x) => - `
  • ${x.metric}${x.value}
    • `, - ) - .join(""); - naItems.push(` -
    -

    Low anonymity (${lowAnon.length})${_readinessInfoIcon("low_anonymity")}

    -
      ${items}
    -
    `); + let items = ""; + lowAnon.forEach((x) => { + const qiList = (x.quasi_identifiers || []).join(", "); + items += _readinessNaRow( + `${_escapeHtml(x.metric)}: k = ${x.value}`, + x.detail || (qiList ? `Quasi-identifiers: ${qiList}` : null), + x.singleton_count != null ? `${x.singleton_count} singleton group(s)` : null, + ); + (x.worst_groups || []).slice(0, 3).forEach((g) => { + items += _readinessNaRow( + `Smallest group (size ${g.size})`, + _formatQiValues(g.qi_values), + null, + ); + }); + if (x.worst_single_qi) { + items += _readinessNaRow( + `Highest single-QI risk: ${_escapeHtml(x.worst_single_qi.feature)}`, + "May contribute to low k when combined with other quasi-identifiers", + `risk ${Number(x.worst_single_qi.mean_risk).toFixed(2)}`, + ); + } + }); + naItems.push( + _readinessNaBlock( + `Low anonymity (${lowAnon.length})`, + "low_anonymity", + lowAnon[0].quasi_identifiers?.length + ? `Quasi-identifiers: ${_escapeHtml(lowAnon[0].quasi_identifiers.join(", "))}` + : null, + items, + "red", + ), + ); } + const hipaaPhi = na.hipaa_phi || []; if (hipaaPhi.length) { const items = hipaaPhi .slice(0, 6) - .map( - (x) => - `
  • ${x.column}${(x.types || []).join(", ") || x.total_flags + " flags"}
    • `, + .map((x) => + _readinessNaRow( + `${_escapeHtml(x.column)}`, + (x.types || []).join(", ") || "Pattern match", + `${x.total_flags} flag(s)`, + ), ) .join(""); - naItems.push(` -
    -

    HIPAA pattern matches (${hipaaPhi.length})${_readinessInfoIcon("hipaa_phi")}

    -
      ${items}
    -
    `); + naItems.push( + _readinessNaBlock( + `HIPAA pattern matches (${hipaaPhi.length})`, + "hipaa_phi", + "Scanned text-like columns for HIPAA-style identifier patterns", + items, + "red", + ), + ); } + const linkage = na.high_linkage_risk || []; if (linkage.length) { const items = linkage .map((x) => { - const label = x.feature || (x.features || []).join(", "); - return `
  • ${x.metric}: ${label}${x.mean_risk}
    • `; + const qis = x.quasi_identifiers || x.features || []; + const featLabel = x.feature + ? `${_escapeHtml(x.feature)}` + : `${_escapeHtml(qis.join(", "))}`; + return _readinessNaRow( + `${_escapeHtml(x.metric)}: ${featLabel}`, + x.detail || (qis.length ? `Quasi-identifiers: ${qis.join(", ")}` : null), + `risk ${x.mean_risk}`, + ); }) .join(""); - naItems.push(` -
    -

    High linkage risk (${linkage.length})${_readinessInfoIcon("high_linkage_risk")}

    -
      ${items}
    -
    `); + naItems.push( + _readinessNaBlock( + `High linkage risk (${linkage.length})`, + "high_linkage_risk", + null, + items, + "amber", + ), + ); } + const attrDisc = na.attribute_disclosure || []; if (attrDisc.length) { const items = attrDisc - .map( - (x) => - `
  • ${x.metric}${x.value}
    • `, - ) + .map((x) => { + const qiList = (x.quasi_identifiers || []).join(", "); + const sens = x.sensitive_attribute || "—"; + return _readinessNaRow( + `${_escapeHtml(x.metric)} = ${x.value}`, + x.detail || + `Sensitive ${_escapeHtml(sens)} within groups of (${qiList})`, + null, + ); + }) .join(""); - naItems.push(` -
    -

    Attribute disclosure risk (${attrDisc.length})${_readinessInfoIcon("attribute_disclosure")}

    -
      ${items}
    -
    `); + naItems.push( + _readinessNaBlock( + `Attribute disclosure risk (${attrDisc.length})`, + "attribute_disclosure", + attrDisc[0].sensitive_attribute + ? `Sensitive: ${_escapeHtml(attrDisc[0].sensitive_attribute)}` + : null, + items, + "amber", + ), + ); } - if (naItems.length) { - html += ` -
    -

    Needs attention

    -
    ${naItems.join("")}
    -
    `; - } else { - html += ` -
    -

    No governance issues detected under the automated thresholds.

    -
    `; - } + html += _renderReadinessNeedsAttentionPanel( + naItems, + "No governance issues detected under the automated thresholds.", + ); const det = gov.details || {}; let detailsInner = ""; From 23925e22c342eb1c442ad69e8b801eb893519bb3 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 24 Jun 2026 15:31:46 -0400 Subject: [PATCH 08/23] Standardize readiness report number formatting with group-aware precision Use shared formatters so floats default to two decimal places and automatically gain precision when values would otherwise show as 0.00 (e.g. minority class shares). Apply consistent decimals within each metric group across all report sections; return raw numerical summary values from the backend for client-side formatting. --- web/routes/metrics.py | 9 +- web/static/js/inspector.js | 230 +++++++++++++++++++++++++++++-------- 2 files changed, 184 insertions(+), 55 deletions(-) diff --git a/web/routes/metrics.py b/web/routes/metrics.py index bedc3ec5..ae6e4484 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -311,14 +311,15 @@ def _build_dataset_overview_section(file_info): numerical_summary = {} num_df = df.select_dtypes(include="number") if not num_df.empty: - numerical_summary = num_df.describe().map( - lambda x: round(x, 2) if x == 0 or abs(x) >= 0.001 else f"{x:.2e}" - ).to_dict() + numerical_summary = num_df.describe().to_dict() for v in numerical_summary.values(): for old_key in list(v.keys()): if old_key in ["25%", "50%", "75%"]: new_key = old_key.replace("%", "th percentile") - v[new_key] = v.pop(old_key) + v[new_key] = float(v.pop(old_key)) + for stat_key, stat_val in list(v.items()): + if stat_val is not None and not isinstance(stat_val, str): + v[stat_key] = float(stat_val) return { "file_metadata": { diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 74c62bfb..bc4675d3 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2183,10 +2183,69 @@ function _dqStatusClasses(status) { } } -/** Format a 0–1 score as a whole percentage, or "N/A" if missing. */ -function _pct(value) { - if (value === null || value === undefined || isNaN(value)) return "N/A"; - return `${Math.round(value * 100)}%`; +/** Coerce readiness metric values to finite numbers. */ +function _readinessNums(values) { + return (values || []) + .map((v) => (typeof v === "number" ? v : parseFloat(v))) + .filter((v) => !Number.isNaN(v)); +} + +/** + * Decimal places for a group: default 2; if every value rounds to 0.00, use more + * (up to maxDecimals) so small non-zero values remain visible. Same precision + * is used for every value in the group. + */ +function _readinessDecimalPlaces(values, { minDecimals = 2, maxDecimals = 6 } = {}) { + const nums = _readinessNums(values); + if (!nums.length) return minDecimals; + if (nums.every((v) => v === 0)) return minDecimals; + + for (let d = minDecimals; d <= maxDecimals; d++) { + const allRoundedZero = nums.every((v) => Number(v.toFixed(d)) === 0); + if (!allRoundedZero) return d; + } + return maxDecimals; +} + +/** Decimal places for 0–1 ratios shown as percentages (value × 100). */ +function _readinessPctDecimals(values, options) { + return _readinessDecimalPlaces( + _readinessNums(values).map((v) => v * 100), + options, + ); +} + +/** Format a number with fixed decimals; integers omit the fractional part. */ +function _readinessNum(value, decimals = 2) { + if (value === null || value === undefined || Number.isNaN(value)) return "N/A"; + if (typeof value !== "number") return String(value); + if (Number.isInteger(value)) return value.toLocaleString(); + return value.toFixed(decimals); +} + +/** Build a formatter that uses one decimal precision for every value in *values*. */ +function _readinessNumFormatter(values, options) { + const nums = _readinessNums(values); + if (nums.length && nums.every((v) => Number.isInteger(v))) { + return (value) => { + if (value === null || value === undefined || Number.isNaN(value)) return "N/A"; + return Number(value).toLocaleString(); + }; + } + const decimals = _readinessDecimalPlaces(nums, options); + return (value) => _readinessNum(value, decimals); +} + +/** Build a percentage formatter (0–1 input) with group-consistent decimals. */ +function _readinessPctFormatter(values, options) { + const decimals = _readinessPctDecimals(values, options); + return (value) => _pct(value, decimals); +} + +/** Format a 0–1 ratio as a percentage, or "N/A" if missing. */ +function _pct(value, decimals = 2) { + if (value === null || value === undefined || Number.isNaN(value)) return "N/A"; + return `${(value * 100).toFixed(decimals)}%`; } /** Show an error message inside one readiness section container. */ @@ -2525,15 +2584,20 @@ function renderReadinessDatasetOverview(container, overview) { html += `Summary`; html += ``; + const profilePctValues = profiles.flatMap((p) => + [p.pct_missing, p.pct_dominant].filter((v) => v != null), + ); + const fmtProfilePct = _readinessPctFormatter(profilePctValues); + profiles.forEach((p, i) => { const stripe = i % 2 === 0 ? "bg-white dark:bg-gray-800" : "bg-gray-50 dark:bg-gray-700/50"; html += ``; html += `${p.feature}`; html += `${p.type}`; html += `${p.dtype}`; - html += `${_pct(p.pct_missing)}`; + html += `${fmtProfilePct(p.pct_missing)}`; html += `${p.n_unique}`; - html += `${p.pct_dominant != null ? _pct(p.pct_dominant) : "—"}`; + html += `${p.pct_dominant != null ? fmtProfilePct(p.pct_dominant) : "—"}`; html += `${_profileStatusBadge(p.status)}`; html += `${p.summary || "—"}`; html += ``; @@ -2555,6 +2619,12 @@ function renderReadinessDatasetOverview(container, overview) { .filter((s) => allStats.includes(s)) .concat(allStats.filter((s) => !preferredOrder.includes(s))); + const statFormatters = {}; + statKeys.forEach((s) => { + const colVals = numFeatures.map((feat) => numSummary[feat][s]); + statFormatters[s] = _readinessNumFormatter(colVals); + }); + detailsInner += `

    Numerical summary statistics

    `; detailsInner += `
    `; detailsInner += ``; @@ -2566,7 +2636,14 @@ function renderReadinessDatasetOverview(container, overview) { const stripe = i % 2 === 0 ? "bg-white dark:bg-gray-800" : "bg-gray-50 dark:bg-gray-700/50"; detailsInner += ``; statKeys.forEach((s) => { - detailsInner += ``; + const raw = numSummary[feat][s]; + const display = + raw === null || raw === undefined + ? "—" + : typeof raw === "number" + ? statFormatters[s](raw) + : raw; + detailsInner += ``; }); detailsInner += ``; }); @@ -2623,6 +2700,8 @@ function renderReadinessDataQuality(container, dq) { const gradeCls = _dqStatusClasses(dq.grade_status); const scopeCrit = (dq.auto_selection || {}).selection_criteria?.analysis_scope || {}; + const dqPctValues = [dq.grade, ...kpis.map((k) => k.value)].filter((v) => v != null); + const fmtDqPct = _readinessPctFormatter(dqPctValues); // --- Analysis scope (no column auto-selection required) --- let html = ` @@ -2638,7 +2717,7 @@ function renderReadinessDataQuality(container, dq) { html += `
    Overall data quality grade${_readinessInfoIcon("overall_dq_grade")} - ${_pct(dq.grade)} + ${fmtDqPct(dq.grade)}
    `; @@ -2652,7 +2731,7 @@ function renderReadinessDataQuality(container, dq) {
    ${k.label}${_readinessInfoIcon(k.id)} - ${_pct(k.value)} + ${fmtDqPct(k.value)}
    @@ -2667,6 +2746,12 @@ function renderReadinessDataQuality(container, dq) { const incomplete = na.incomplete_features || []; const outlierFeats = na.outlier_features || []; const dupRows = na.duplicate_rows || 0; + const naPctValues = [ + ...incomplete.map((f) => f.completeness), + ...outlierFeats.map((f) => f.outlier_proportion), + dupRows > 0 ? dupRows : null, + ].filter((v) => v != null); + const fmtNaPct = _readinessPctFormatter(naPctValues); const naItems = []; if (incomplete.length) { @@ -2674,7 +2759,7 @@ function renderReadinessDataQuality(container, dq) { .slice(0, 6) .map( (f) => - `
  • ${f.feature}${_pct(f.completeness)} complete
    • `, + `
  • ${f.feature}${fmtNaPct(f.completeness)} complete
    • `, ) .join(""); const more = @@ -2692,7 +2777,7 @@ function renderReadinessDataQuality(container, dq) { .slice(0, 6) .map( (f) => - `
  • ${f.feature}${_pct(f.outlier_proportion)} outliers
    • `, + `
  • ${f.feature}${fmtNaPct(f.outlier_proportion)} outliers
    • `, ) .join(""); const more = @@ -2709,7 +2794,7 @@ function renderReadinessDataQuality(container, dq) { naItems.push(`

    Duplicate rows${_readinessInfoIcon("uniqueness")}

    -

    ${_pct(dupRows)} of rows are exact duplicates.

    +

    ${fmtNaPct(dupRows)} of rows are exact duplicates.

    `); } @@ -2785,7 +2870,14 @@ function renderReadinessImpact(container, impact) { const dropped = colCrit.excluded || impact.columns_dropped || []; const analyzed = impact.columns_analyzed || (colCrit.selected || []).length; - const fmtScore = (s) => (typeof s === "number" ? s.toFixed(2) : s); + const impactScoreValues = [ + ...leakage.map((p) => p.score), + ...redundant.map((p) => p.score), + ...topPairs.map((p) => p.score), + ]; + const fmtScore = _readinessNumFormatter(impactScoreValues); + const impactPctValues = [impact.grade, ...kpis.map((k) => k.value)].filter((v) => v != null); + const fmtImpactPct = _readinessPctFormatter(impactPctValues); const fmtPair = (p) => _readinessNaRow( `${_escapeHtml(p.a)} ↔ ${_escapeHtml(p.b)}`, @@ -2808,9 +2900,9 @@ function renderReadinessImpact(container, impact) { ${colCrit.rule || ""}
  • Excluded columns: ${dropped.length}
  • Thresholds: - redundant |score| ≥ ${thresholds.redundant_threshold ?? "—"}, - leakage |score| ≥ ${thresholds.leakage_threshold ?? "—"}, - isolated max |score| < ${thresholds.isolated_threshold ?? "—"} + redundant |score| ≥ ${_readinessNum(thresholds.redundant_threshold)}, + leakage |score| ≥ ${_readinessNum(thresholds.leakage_threshold)}, + isolated max |score| < ${_readinessNum(thresholds.isolated_threshold)}
    • `; @@ -2819,14 +2911,14 @@ function renderReadinessImpact(container, impact) { html += `
    Overall impact grade${_readinessInfoIcon("overall_impact_grade")} - ${_pct(impact.grade)} + ${fmtImpactPct(impact.grade)}
    `; kpis.forEach((k) => { const cls = _dqStatusClasses(k.status); const displayVal = - k.raw_count != null ? `${k.raw_count} flagged` : _pct(k.value); + k.raw_count != null ? `${k.raw_count} flagged` : fmtImpactPct(k.value); const widthPct = k.value === null || k.value === undefined ? 0 @@ -2996,6 +3088,15 @@ function renderReadinessFairness(container, fb) { const kpis = fb.kpis || []; const na = fb.needs_attention || {}; const gradeCls = _dqStatusClasses(fb.grade_status); + const fairnessPctValues = [fb.grade, ...kpis.map((k) => k.value)].filter((v) => v != null); + const fmtFairnessPct = _readinessPctFormatter(fairnessPctValues); + const fmtThresholdPct = _readinessPctFormatter( + thresholds.minority_class_share != null ? [thresholds.minority_class_share] : [], + ); + const imbalanceVals = kpis + .filter((k) => k.id === "label_balance" && k.raw_imbalance_degree != null) + .map((k) => k.raw_imbalance_degree); + const fmtImbalance = _readinessNumFormatter(imbalanceVals); // --- Auto-selection criteria (transparent) --- let html = ` @@ -3010,10 +3111,10 @@ function renderReadinessFairness(container, fb) { ${posCrit.rule || ""}
  • Primary sensitive (statistical rate & CDD): ${sel.primary_sensitive || "none"}
  • Flags: - representation ratio ≥ ${thresholds.representation_ratio_flag ?? "—"}, - minority class < ${_pct(thresholds.minority_class_share)}, - TSD ≥ ${thresholds.tsd_disparity_flag ?? "—"}, - imbalance degree good/warning < ${thresholds.imbalance_degree_good ?? "—"} / ${thresholds.imbalance_degree_warning ?? "—"} + representation ratio ≥ ${_readinessNum(thresholds.representation_ratio_flag)}, + minority class < ${fmtThresholdPct(thresholds.minority_class_share)}, + TSD ≥ ${_readinessNum(thresholds.tsd_disparity_flag)}, + imbalance degree good/warning < ${_readinessNum(thresholds.imbalance_degree_good)} / ${_readinessNum(thresholds.imbalance_degree_warning)}
    • `; @@ -3022,7 +3123,7 @@ function renderReadinessFairness(container, fb) { html += `
    Overall fairness grade${_readinessInfoIcon("overall_fairness_grade")} - ${_pct(fb.grade)} + ${fmtFairnessPct(fb.grade)}
    `; @@ -3030,8 +3131,8 @@ function renderReadinessFairness(container, fb) { const cls = _dqStatusClasses(k.status); const displayVal = k.id === "label_balance" && k.raw_imbalance_degree != null - ? `ID ${Number(k.raw_imbalance_degree).toFixed(2)}` - : _pct(k.value); + ? `ID ${fmtImbalance(k.raw_imbalance_degree)}` + : fmtFairnessPct(k.value); const widthPct = k.value === null || k.value === undefined ? 0 @@ -3054,17 +3155,22 @@ function renderReadinessFairness(container, fb) { const naItems = []; const repImbalance = na.representation_imbalance || []; if (repImbalance.length) { + const ratioValues = repImbalance.flatMap((s) => [ + s.max_ratio, + ...(s.flagged_pairs || []).map((p) => p.ratio), + ]); + const fmtRatio = _readinessNumFormatter(ratioValues); const items = repImbalance .slice(0, 5) .map((s) => { const pairHint = s.flagged_pairs && s.flagged_pairs.length - ? `Worst pair: ${s.flagged_pairs[0].pair} (ratio ${s.flagged_pairs[0].ratio})` + ? `Worst pair: ${s.flagged_pairs[0].pair} (ratio ${fmtRatio(s.flagged_pairs[0].ratio)})` : ""; return _readinessNaRow( `${_escapeHtml(s.column)}`, pairHint, - `max ratio ${s.max_ratio}`, + `max ratio ${fmtRatio(s.max_ratio)}`, ); }) .join(""); @@ -3083,12 +3189,13 @@ function renderReadinessFairness(container, fb) { if (minorities.length) { const targetCol = minorities[0].target_column || targetCrit.selected || "target"; + const fmtMinorityShare = _readinessPctFormatter(minorities.map((m) => m.share)); const items = minorities .map((m) => _readinessNaRow( `${_escapeHtml(m.class)}`, `Class in ${_escapeHtml(targetCol)}`, - `${_pct(m.share)} share`, + `${fmtMinorityShare(m.share)} share`, ), ) .join(""); @@ -3107,12 +3214,13 @@ function renderReadinessFairness(container, fb) { if (outcomeDisp.length) { const sensCol = outcomeDisp[0].sensitive_column || sel.primary_sensitive || "—"; const tgtCol = outcomeDisp[0].target_column || targetCrit.selected || "—"; + const fmtTsd = _readinessNumFormatter(outcomeDisp.map((d) => d.tsd)); const items = outcomeDisp .map((d) => _readinessNaRow( `${_escapeHtml(d.target_column || tgtCol)} = ${_escapeHtml(d.class)}`, `Outcome rates vary by sensitive ${_escapeHtml(d.sensitive_column || sensCol)}`, - `TSD ${d.tsd}`, + `TSD ${fmtTsd(d.tsd)}`, ), ) .join(""); @@ -3258,7 +3366,28 @@ function renderReadinessGovernance(container, gov) { const thresholds = criteria.thresholds || {}; const kpis = gov.kpis || []; const na = gov.needs_attention || {}; + const det = gov.details || {}; + const singleRisk = det.single_attribute_risk?.by_quasi_identifier || {}; const gradeCls = _dqStatusClasses(gov.grade_status); + const lowAnon = na.low_anonymity || []; + const linkage = na.high_linkage_risk || []; + const attrDisc = na.attribute_disclosure || []; + const govPctValues = [gov.grade, ...kpis.map((k) => k.value)].filter((v) => v != null); + const fmtGovPct = _readinessPctFormatter(govPctValues); + const govRiskValues = [ + ...kpis.map((k) => k.raw_worst_mean ?? k.raw_mean).filter((v) => v != null), + ...linkage.map((x) => x.mean_risk), + ...lowAnon.flatMap((x) => + x.worst_single_qi ? [x.worst_single_qi.mean_risk] : [], + ), + ...Object.values(singleRisk).map((v) => v.mean_risk).filter((v) => v != null), + ]; + const govMetricValues = [ + ...kpis.filter((k) => k.id === "distribution_t" && k.raw_t != null).map((k) => k.raw_t), + ...attrDisc.map((x) => x.value), + ]; + const fmtGovRisk = _readinessNumFormatter(govRiskValues); + const fmtGovMetric = _readinessNumFormatter(govMetricValues); let html = `
    @@ -3289,20 +3418,21 @@ function renderReadinessGovernance(container, gov) { html += `
    Overall governance grade${_readinessInfoIcon("overall_governance_grade")} - ${_pct(gov.grade)} + ${fmtGovPct(gov.grade)}
    `; kpis.forEach((k) => { const cls = _dqStatusClasses(k.status); - let displayVal = _pct(k.value); + let displayVal = fmtGovPct(k.value); if (k.id === "anonymity_k" && k.raw_k != null) displayVal = `k=${k.raw_k}`; else if (k.id === "diversity_l" && k.raw_l != null) displayVal = `l=${k.raw_l}`; - else if (k.id === "distribution_t" && k.raw_t != null) displayVal = `t=${Number(k.raw_t).toFixed(3)}`; + else if (k.id === "distribution_t" && k.raw_t != null) + displayVal = `t=${fmtGovMetric(k.raw_t)}`; else if (k.id === "single_linkage_risk" && k.raw_worst_mean != null) - displayVal = `${Number(k.raw_worst_mean).toFixed(2)} risk`; + displayVal = `${fmtGovRisk(k.raw_worst_mean)} risk`; else if (k.id === "linkage_risk" && k.raw_mean != null) - displayVal = `${Number(k.raw_mean).toFixed(2)} risk`; + displayVal = `${fmtGovRisk(k.raw_mean)} risk`; else if (k.id === "phi_exposure" && k.columns_flagged != null) displayVal = k.columns_flagged === 0 ? "None" : `${k.columns_flagged} col(s)`; @@ -3325,7 +3455,7 @@ function renderReadinessGovernance(container, gov) { html += "
    "; const naItems = []; - const lowAnon = na.low_anonymity || []; + if (lowAnon.length) { let items = ""; lowAnon.forEach((x) => { @@ -3346,7 +3476,7 @@ function renderReadinessGovernance(container, gov) { items += _readinessNaRow( `Highest single-QI risk: ${_escapeHtml(x.worst_single_qi.feature)}`, "May contribute to low k when combined with other quasi-identifiers", - `risk ${Number(x.worst_single_qi.mean_risk).toFixed(2)}`, + `risk ${fmtGovRisk(x.worst_single_qi.mean_risk)}`, ); } }); @@ -3386,9 +3516,9 @@ function renderReadinessGovernance(container, gov) { ); } - const linkage = na.high_linkage_risk || []; - if (linkage.length) { - const items = linkage + const linkageNa = na.high_linkage_risk || []; + if (linkageNa.length) { + const items = linkageNa .map((x) => { const qis = x.quasi_identifiers || x.features || []; const featLabel = x.feature @@ -3397,13 +3527,13 @@ function renderReadinessGovernance(container, gov) { return _readinessNaRow( `${_escapeHtml(x.metric)}: ${featLabel}`, x.detail || (qis.length ? `Quasi-identifiers: ${qis.join(", ")}` : null), - `risk ${x.mean_risk}`, + `risk ${fmtGovRisk(x.mean_risk)}`, ); }) .join(""); naItems.push( _readinessNaBlock( - `High linkage risk (${linkage.length})`, + `High linkage risk (${linkageNa.length})`, "high_linkage_risk", null, items, @@ -3412,14 +3542,14 @@ function renderReadinessGovernance(container, gov) { ); } - const attrDisc = na.attribute_disclosure || []; - if (attrDisc.length) { - const items = attrDisc + const attrDiscNa = na.attribute_disclosure || []; + if (attrDiscNa.length) { + const items = attrDiscNa .map((x) => { const qiList = (x.quasi_identifiers || []).join(", "); const sens = x.sensitive_attribute || "—"; return _readinessNaRow( - `${_escapeHtml(x.metric)} = ${x.value}`, + `${_escapeHtml(x.metric)} = ${fmtGovMetric(x.value)}`, x.detail || `Sensitive ${_escapeHtml(sens)} within groups of (${qiList})`, null, @@ -3428,10 +3558,10 @@ function renderReadinessGovernance(container, gov) { .join(""); naItems.push( _readinessNaBlock( - `Attribute disclosure risk (${attrDisc.length})`, + `Attribute disclosure risk (${attrDiscNa.length})`, "attribute_disclosure", - attrDisc[0].sensitive_attribute - ? `Sensitive: ${_escapeHtml(attrDisc[0].sensitive_attribute)}` + attrDiscNa[0].sensitive_attribute + ? `Sensitive: ${_escapeHtml(attrDiscNa[0].sensitive_attribute)}` : null, items, "amber", @@ -3444,7 +3574,6 @@ function renderReadinessGovernance(container, gov) { "No governance issues detected under the automated thresholds.", ); - const det = gov.details || {}; let detailsInner = ""; const chartMetrics = [ @@ -3468,12 +3597,11 @@ function renderReadinessGovernance(container, gov) { } }); - const singleRisk = det.single_attribute_risk?.by_quasi_identifier || {}; const singleRows = Object.entries(singleRisk) .filter(([, v]) => v.mean_risk != null) .map( ([q, v]) => - `
    `, + ``, ) .join(""); if (singleRows) { From caf44ce6c2f484c0a43ffd31e7d7acb41534da86 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 24 Jun 2026 15:44:38 -0400 Subject: [PATCH 09/23] Show per-section build time at the bottom of readiness report scorecards Expose build_time_seconds from section API responses and render a footer (e.g. "Prepared in 55.01 seconds") after each scorecard loads, matching the timing already logged on the server. --- web/routes/metrics.py | 16 +++++++++++++--- web/static/js/inspector.js | 28 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/web/routes/metrics.py b/web/routes/metrics.py index ae6e4484..2c7d04cb 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -1920,15 +1920,17 @@ def readiness_report_section(section): start_time = time.time() data = _build_readiness_section(section, file_info) + build_time_seconds = round(time.time() - start_time, 2) metric_time_log.info( "Readiness report section %s built in %.2f seconds", section, - time.time() - start_time, + build_time_seconds, ) return jsonify(ensure_json_serializable({ "success": True, "section": section, "data": data, + "build_time_seconds": build_time_seconds, })) @@ -1943,9 +1945,17 @@ def readiness_report(): try: response = {"success": True} for slug in _READINESS_SECTION_BUILDERS: - response[_READINESS_SECTION_RESPONSE_KEYS[slug]] = _build_readiness_section( - slug, file_info + section_start = time.time() + section_data = _build_readiness_section(slug, file_info) + section_elapsed = round(time.time() - section_start, 2) + if isinstance(section_data, dict): + section_data = {**section_data, "build_time_seconds": section_elapsed} + metric_time_log.info( + "Readiness report section %s built in %.2f seconds", + slug, + section_elapsed, ) + response[_READINESS_SECTION_RESPONSE_KEYS[slug]] = section_data metric_time_log.info("Readiness report built in %.2f seconds", time.time() - start_time) return jsonify(ensure_json_serializable(response)) diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index bc4675d3..4299d9a4 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2255,6 +2255,27 @@ function _readinessSectionError(container, message) { container.innerHTML = `

    ${message}

    `; } +/** Human-readable section build duration (matches server log precision). */ +function _formatReadinessBuildTime(seconds) { + if (seconds === null || seconds === undefined || Number.isNaN(Number(seconds))) { + return ""; + } + return `Prepared in ${Number(seconds).toFixed(2)} seconds`; +} + +/** Append build-time footer at the bottom of a readiness section container. */ +function _appendReadinessBuildTimeFooter(container, seconds) { + if (!container) return; + const label = _formatReadinessBuildTime(seconds); + if (!label) return; + container.querySelector(".readiness-build-time")?.remove(); + const el = document.createElement("p"); + el.className = + "readiness-build-time text-xs text-gray-400 dark:text-gray-500 mt-4 pt-3 border-t border-gray-200 dark:border-gray-700 text-right"; + el.textContent = label; + container.appendChild(el); +} + /** * Fetch one readiness-report section and render it when ready. * @param {string} section - URL slug (e.g. "data-quality") @@ -2274,7 +2295,12 @@ function _fetchReadinessSection(section, container, renderFn) { ); return; } - renderFn(container, resp.data || {}); + const data = resp.data || {}; + renderFn(container, data); + _appendReadinessBuildTimeFooter( + container, + resp.build_time_seconds ?? data.build_time_seconds, + ); }) .catch((err) => { _readinessSectionError(container, `Error loading section: ${err.message}`); From a7fe1c1a0abb719da892671f31a67c41677e0851 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 24 Jun 2026 15:52:57 -0400 Subject: [PATCH 10/23] Fix raw HTML showing in readiness needs-attention secondary lines Stop escaping the secondary line in _readinessNaRow so intentional markup (e.g. monospace column names) renders correctly instead of appearing as literal span tags. --- web/static/js/inspector.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 4299d9a4..110e7290 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2376,14 +2376,16 @@ function _formatQiValues(qiValues) { /** * One needs-attention list row with optional secondary line (context/detail). + * *primary* and *secondary* may contain safe HTML built by callers; user text + * must be escaped before passing in. */ function _readinessNaRow(primary, secondary, value) { const valHtml = value - ? `${value}` + ? `${_escapeHtml(value)}` : ""; const sub = secondary - ? `

    ${_escapeHtml(secondary)}

    ` + ? `

    ${secondary}

    ` : ""; return `
  • From d164356af25322d33b4dd317474424f3b80efef3 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 25 Jun 2026 01:02:09 -0400 Subject: [PATCH 11/23] Defer readiness report charts to on-demand loading for faster initial scorecards. Add include_visualization flags to shared metric functions (default True so inspector tabs are unchanged), skip chart generation in readiness section builders, expose GET /readiness-report/
    /visualizations, and load charts from inspector.js when users expand detail panels. --- aidrin/structured_data_metrics/add_noise.py | 73 ++--- .../structured_data_metrics/completeness.py | 83 ++--- .../correlation_score.py | 182 +++++------ aidrin/structured_data_metrics/outliers.py | 4 +- .../privacy_measure.py | 238 +++++++------- .../statistical_rate.py | 124 ++++---- web/routes/metrics.py | 297 +++++++++++++++--- web/static/js/inspector.js | 289 ++++++++++++++--- 8 files changed, 862 insertions(+), 428 deletions(-) diff --git a/aidrin/structured_data_metrics/add_noise.py b/aidrin/structured_data_metrics/add_noise.py index 90b88bf6..ac339494 100644 --- a/aidrin/structured_data_metrics/add_noise.py +++ b/aidrin/structured_data_metrics/add_noise.py @@ -17,7 +17,7 @@ def add_laplace_noise(data, epsilon): raise Exception("Epsilon cannot be 0") -def return_noisy_stats(add_noise_columns, epsilon, file_info, save_output=True): +def return_noisy_stats(add_noise_columns, epsilon, file_info, save_output=True, include_visualization=True): # Convert JSON back to DataFrame if needed, otherwise use DataFrame directly import pandas as pd @@ -43,18 +43,20 @@ def return_noisy_stats(add_noise_columns, epsilon, file_info, save_output=True): num_cols = min(num_columns, max_columns_per_row) # Create subplots for the box plots - fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 8)) + if include_visualization: + fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 8)) for i, column in enumerate(add_noise_columns): - if num_rows == 1 and num_cols == 1: - current_ax = axes - elif num_rows == 1: - current_ax = axes[i % num_cols] - elif num_cols == 1: - current_ax = axes[i % num_rows, 0] - else: - row, col = divmod(i, num_cols) - current_ax = axes[row, col] + if include_visualization: + if num_rows == 1 and num_cols == 1: + current_ax = axes + elif num_rows == 1: + current_ax = axes[i % num_cols] + elif num_cols == 1: + current_ax = axes[i % num_rows, 0] + else: + row, col = divmod(i, num_cols) + current_ax = axes[row, col] noisy_feature = add_laplace_noise(df_drop_na[column], epsilon) @@ -85,28 +87,31 @@ def return_noisy_stats(add_noise_columns, epsilon, file_info, save_output=True): ) df_drop_na[f'noisy_{column}'] = noisy_feature - # Box plot for the normal feature - current_ax.boxplot( - df_drop_na[column], positions=[0], widths=0.6, showfliers=False - ) - current_ax.set_title(f"Normal vs Noisy representations: Feature {column}") - current_ax.set_ylabel("Value") - - # Box plot for the noisy feature - current_ax.boxplot(noisy_feature, positions=[1], widths=0.6, showfliers=False) - current_ax.set_ylabel("Value") - - # Adjust the spacing between subplots - plt.tight_layout() - - # Save the chart as BytesIO - img_buf = BytesIO() - plt.savefig(img_buf, format="png") - img_buf.seek(0) - - # Encode the combined image as base64 - combined_image_base64 = base64.b64encode(img_buf.getvalue()).decode("utf-8") - img_buf.close() + if include_visualization: + # Box plot for the normal feature + current_ax.boxplot( + df_drop_na[column], positions=[0], widths=0.6, showfliers=False + ) + current_ax.set_title(f"Normal vs Noisy representations: Feature {column}") + current_ax.set_ylabel("Value") + + # Box plot for the noisy feature + current_ax.boxplot(noisy_feature, positions=[1], widths=0.6, showfliers=False) + current_ax.set_ylabel("Value") + + if include_visualization: + # Adjust the spacing between subplots + plt.tight_layout() + + # Save the chart as BytesIO + img_buf = BytesIO() + plt.savefig(img_buf, format="png") + img_buf.seek(0) + + # Encode the combined image as base64 + combined_image_base64 = base64.b64encode(img_buf.getvalue()).decode("utf-8") + img_buf.close() + stat_dict["DP Statistics Visualization"] = combined_image_base64 if save_output: try: os.makedirs("noisy", exist_ok=True) @@ -117,6 +122,4 @@ def return_noisy_stats(add_noise_columns, epsilon, file_info, save_output=True): else: stat_dict["Noisy file saved"] = "Skipped (readiness report preview only)" - stat_dict["DP Statistics Visualization"] = combined_image_base64 - return stat_dict diff --git a/aidrin/structured_data_metrics/completeness.py b/aidrin/structured_data_metrics/completeness.py index 3f27e259..efca48e1 100644 --- a/aidrin/structured_data_metrics/completeness.py +++ b/aidrin/structured_data_metrics/completeness.py @@ -12,7 +12,7 @@ @shared_task(bind=True, ignore_result=False) -def completeness(self: Task, file_info): +def completeness(self: Task, file_info, include_visualization=True): """Compute per-column and overall completeness (non-missing rate) for a dataset. Reads the file, calculates the proportion of non-null values for every @@ -64,46 +64,47 @@ def completeness(self: Task, file_info): result_dict["Completeness scores"] = completeness_scores result_dict["Overall Completeness"] = overall_completeness - # Horizontal bar chart — grows vertically with feature count - labels = list(completeness_scores.keys()) - values = list(completeness_scores.values()) - n = len(labels) - text_color = "#6b7280" - - fig_height = max(3, n * 0.3) - fig, ax = plt.subplots(figsize=(8, fig_height)) - fig.patch.set_alpha(0) - ax.set_facecolor("none") - - bars = ax.barh(range(n), values, color="#4485F4", height=0.7) - ax.set_xlabel("Completeness Score", fontsize=10, color=text_color) - ax.set_yticks(range(n)) - ax.set_yticklabels(labels, fontsize=9, color=text_color) - ax.tick_params(axis="x", colors=text_color, labelsize=8) - ax.set_xlim(0, 1.12) - ax.invert_yaxis() - ax.set_ylim(n - 0.5, -0.5) - - for spine in ax.spines.values(): - spine.set_color(text_color) - - for bar, val in zip(bars, values): - if val > 0.15: - ax.text(val - 0.01, bar.get_y() + bar.get_height() / 2, - f'{val:.2f}', ha='right', va='center', fontsize=8, color='white', fontweight='bold') - else: - ax.text(val + 0.01, bar.get_y() + bar.get_height() / 2, - f'{val:.2f}', ha='left', va='center', fontsize=8, color=text_color) - - fig.tight_layout(pad=0.5) - - img_buf = io.BytesIO() - fig.savefig(img_buf, format="png", dpi=150, transparent=True) - img_buf.seek(0) - - img_base64 = base64.b64encode(img_buf.read()).decode("utf-8") - result_dict["Completeness Visualization"] = img_base64 - plt.close(fig) + if include_visualization: + # Horizontal bar chart — grows vertically with feature count + labels = list(completeness_scores.keys()) + values = list(completeness_scores.values()) + n = len(labels) + text_color = "#6b7280" + + fig_height = max(3, n * 0.3) + fig, ax = plt.subplots(figsize=(8, fig_height)) + fig.patch.set_alpha(0) + ax.set_facecolor("none") + + bars = ax.barh(range(n), values, color="#4485F4", height=0.7) + ax.set_xlabel("Completeness Score", fontsize=10, color=text_color) + ax.set_yticks(range(n)) + ax.set_yticklabels(labels, fontsize=9, color=text_color) + ax.tick_params(axis="x", colors=text_color, labelsize=8) + ax.set_xlim(0, 1.12) + ax.invert_yaxis() + ax.set_ylim(n - 0.5, -0.5) + + for spine in ax.spines.values(): + spine.set_color(text_color) + + for bar, val in zip(bars, values): + if val > 0.15: + ax.text(val - 0.01, bar.get_y() + bar.get_height() / 2, + f'{val:.2f}', ha='right', va='center', fontsize=8, color='white', fontweight='bold') + else: + ax.text(val + 0.01, bar.get_y() + bar.get_height() / 2, + f'{val:.2f}', ha='left', va='center', fontsize=8, color=text_color) + + fig.tight_layout(pad=0.5) + + img_buf = io.BytesIO() + fig.savefig(img_buf, format="png", dpi=150, transparent=True) + img_buf.seek(0) + + img_base64 = base64.b64encode(img_buf.read()).decode("utf-8") + result_dict["Completeness Visualization"] = img_base64 + plt.close(fig) logger.info("Completeness task completed: %d columns, overall=%.4f", len(completeness_scores), overall_completeness) return result_dict diff --git a/aidrin/structured_data_metrics/correlation_score.py b/aidrin/structured_data_metrics/correlation_score.py index 17051f0f..de1b3669 100644 --- a/aidrin/structured_data_metrics/correlation_score.py +++ b/aidrin/structured_data_metrics/correlation_score.py @@ -56,7 +56,7 @@ def _is_column_normal(series: pd.Series) -> bool: @shared_task(bind=True, ignore_result=False) -def calc_correlations(self: Task, columns: List[str], file_info): +def calc_correlations(self: Task, columns: List[str], file_info, include_visualization=True): df = read_file(file_info) try: # Separate categorical and numerical columns @@ -77,51 +77,52 @@ def calc_correlations(self: Task, columns: List[str], file_info): ) logger.debug("Categorical correlation matrix computed:\n%s", categorical_correlation["corr"]) - corr_matrix = categorical_correlation["corr"] - n = len(corr_matrix.columns) - fig_size = max(6, n * 0.7) - text_color = "#6b7280" - - fig, ax = plt.subplots(figsize=(fig_size, fig_size)) - fig.patch.set_alpha(0) - ax.set_facecolor("none") - - annot_size = max(7, min(10, 80 // max(n, 1))) - _ = sns.heatmap( - corr_matrix, annot=True, cmap="coolwarm", fmt=".2f", ax=ax, - annot_kws={"size": annot_size}, - linewidths=0.5, linecolor="#e5e7eb", - cbar=False, - ) - - # Truncate long labels - x_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_xticklabels()] - y_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_yticklabels()] - ax.set_xticklabels(x_labels, rotation=45, ha="right", fontsize=9, color=text_color) - ax.set_yticklabels(y_labels, rotation=0, fontsize=9, color=text_color) - - fig.tight_layout(pad=0.5) - - # Save the plot to a BytesIO object - image_stream_cat = BytesIO() - fig.savefig(image_stream_cat, format="png", dpi=150, transparent=True) - plt.close(fig) - - # Convert the plot to base64 - base64_image_cat = base64.b64encode(image_stream_cat.getvalue()).decode( - "utf-8" - ) - - # Close the BytesIO stream - image_stream_cat.close() - - result_dict["Correlations Analysis Categorical"][ - "Correlations Analysis Categorical Visualization" - ] = base64_image_cat - result_dict["Correlations Analysis Categorical"]["Description"] = ( - "Categorical correlations are calculated using Theil's U, with values ranging from 0 to 1. " - "A value of 1 indicates a perfect correlation, while a value of 0 indicates no correlation" - ) + if include_visualization: + corr_matrix = categorical_correlation["corr"] + n = len(corr_matrix.columns) + fig_size = max(6, n * 0.7) + text_color = "#6b7280" + + fig, ax = plt.subplots(figsize=(fig_size, fig_size)) + fig.patch.set_alpha(0) + ax.set_facecolor("none") + + annot_size = max(7, min(10, 80 // max(n, 1))) + _ = sns.heatmap( + corr_matrix, annot=True, cmap="coolwarm", fmt=".2f", ax=ax, + annot_kws={"size": annot_size}, + linewidths=0.5, linecolor="#e5e7eb", + cbar=False, + ) + + # Truncate long labels + x_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_xticklabels()] + y_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_yticklabels()] + ax.set_xticklabels(x_labels, rotation=45, ha="right", fontsize=9, color=text_color) + ax.set_yticklabels(y_labels, rotation=0, fontsize=9, color=text_color) + + fig.tight_layout(pad=0.5) + + # Save the plot to a BytesIO object + image_stream_cat = BytesIO() + fig.savefig(image_stream_cat, format="png", dpi=150, transparent=True) + plt.close(fig) + + # Convert the plot to base64 + base64_image_cat = base64.b64encode(image_stream_cat.getvalue()).decode( + "utf-8" + ) + + # Close the BytesIO stream + image_stream_cat.close() + + result_dict["Correlations Analysis Categorical"][ + "Correlations Analysis Categorical Visualization" + ] = base64_image_cat + result_dict["Correlations Analysis Categorical"]["Description"] = ( + "Categorical correlations are calculated using Theil's U, with values ranging from 0 to 1. " + "A value of 1 indicates a perfect correlation, while a value of 0 indicates no correlation" + ) # Check if there are numerical features if not numerical_columns.empty: @@ -135,50 +136,51 @@ def calc_correlations(self: Task, columns: List[str], file_info): # Numerical-numerical correlations are computed dynamically based on normality. numerical_correlation = numerical_df.corr(method=corr_method) - n = len(numerical_correlation.columns) - fig_size = max(6, n * 0.7) - text_color = "#6b7280" - - fig, ax = plt.subplots(figsize=(fig_size, fig_size)) - fig.patch.set_alpha(0) - ax.set_facecolor("none") - - annot_size = max(7, min(10, 80 // max(n, 1))) - _ = sns.heatmap( - numerical_correlation, annot=True, cmap="coolwarm", fmt=".2f", ax=ax, - annot_kws={"size": annot_size}, - linewidths=0.5, linecolor="#e5e7eb", - cbar=False, - ) - - x_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_xticklabels()] - y_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_yticklabels()] - ax.set_xticklabels(x_labels, rotation=45, ha="right", fontsize=9, color=text_color) - ax.set_yticklabels(y_labels, rotation=0, fontsize=9, color=text_color) - - fig.tight_layout(pad=0.5) - - # Save the plot to a BytesIO object - image_stream_num = BytesIO() - fig.savefig(image_stream_num, format="png", dpi=150, transparent=True) - plt.close(fig) - - # Convert the plot to base64 - base64_image_num = base64.b64encode(image_stream_num.getvalue()).decode( - "utf-8" - ) - - # Close the BytesIO stream - image_stream_num.close() - - result_dict["Correlations Analysis Numerical"][ - "Correlations Analysis Numerical Visualization" - ] = base64_image_num - result_dict["Correlations Analysis Numerical"]["Description"] = ( - f"Numerical correlations are calculated using {corr_method.title()}'s correlation coefficient, with values " - "ranging from -1 to 1. A value of 1 indicates a perfect positive correlation, -1 indicates a perfect " - "negative correlation, and 0 indicates no correlation" - ) + if include_visualization: + n = len(numerical_correlation.columns) + fig_size = max(6, n * 0.7) + text_color = "#6b7280" + + fig, ax = plt.subplots(figsize=(fig_size, fig_size)) + fig.patch.set_alpha(0) + ax.set_facecolor("none") + + annot_size = max(7, min(10, 80 // max(n, 1))) + _ = sns.heatmap( + numerical_correlation, annot=True, cmap="coolwarm", fmt=".2f", ax=ax, + annot_kws={"size": annot_size}, + linewidths=0.5, linecolor="#e5e7eb", + cbar=False, + ) + + x_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_xticklabels()] + y_labels = [t.get_text()[:12] + "..." if len(t.get_text()) > 12 else t.get_text() for t in ax.get_yticklabels()] + ax.set_xticklabels(x_labels, rotation=45, ha="right", fontsize=9, color=text_color) + ax.set_yticklabels(y_labels, rotation=0, fontsize=9, color=text_color) + + fig.tight_layout(pad=0.5) + + # Save the plot to a BytesIO object + image_stream_num = BytesIO() + fig.savefig(image_stream_num, format="png", dpi=150, transparent=True) + plt.close(fig) + + # Convert the plot to base64 + base64_image_num = base64.b64encode(image_stream_num.getvalue()).decode( + "utf-8" + ) + + # Close the BytesIO stream + image_stream_num.close() + + result_dict["Correlations Analysis Numerical"][ + "Correlations Analysis Numerical Visualization" + ] = base64_image_num + result_dict["Correlations Analysis Numerical"]["Description"] = ( + f"Numerical correlations are calculated using {corr_method.title()}'s correlation coefficient, with values " + "ranging from -1 to 1. A value of 1 indicates a perfect positive correlation, -1 indicates a perfect " + "negative correlation, and 0 indicates no correlation" + ) result_dict["Correlations Analysis Numerical"]["Method"] = ( corr_method.title() ) diff --git a/aidrin/structured_data_metrics/outliers.py b/aidrin/structured_data_metrics/outliers.py index 5f8a128f..2c09ba4e 100644 --- a/aidrin/structured_data_metrics/outliers.py +++ b/aidrin/structured_data_metrics/outliers.py @@ -13,7 +13,7 @@ @shared_task(bind=True, ignore_result=False) -def outliers(self: Task, file_info): +def outliers(self: Task, file_info, include_visualization=True): """Detect outliers in numerical columns using the IQR method. For each numerical column, computes the inter-quartile range (IQR) and @@ -92,7 +92,7 @@ def outliers(self: Task, file_info): k: v for k, v in proportions_dict.items() if k != "Overall outlier score" } - if feature_scores: # only plot if there are valid features + if feature_scores and include_visualization: # only plot if there are valid features labels = list(feature_scores.keys()) values = list(feature_scores.values()) n = len(labels) diff --git a/aidrin/structured_data_metrics/privacy_measure.py b/aidrin/structured_data_metrics/privacy_measure.py index 47d488a9..7e6c7b9d 100644 --- a/aidrin/structured_data_metrics/privacy_measure.py +++ b/aidrin/structured_data_metrics/privacy_measure.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -def generate_single_attribute_MM_risk_scores(df, id_col, eval_cols, task=None): +def generate_single_attribute_MM_risk_scores(df, id_col, eval_cols, task=None, include_visualization=True): """Compute Marketer/Prosecutor model re-identification risk for each quasi-identifier. For every column in *eval_cols*, calculates the proportion of individuals @@ -176,32 +176,34 @@ def generate_single_attribute_MM_risk_scores(df, id_col, eval_cols, task=None): descriptive_stats_dict[key] = stats_dict # Stage 4: Generate visualization (85-100%) - if task: - task.update_state( - state='PROGRESS', - meta={'current': 90, 'total': 100, 'status': 'Generating visualization...'} - ) + if include_visualization: + if task: + task.update_state( + state='PROGRESS', + meta={'current': 90, 'total': 100, 'status': 'Generating visualization...'} + ) - # Create a box plot - plt.figure(figsize=(8, 8)) - plt.boxplot(list(sing_res.values()), tick_labels=list(sing_res.keys())) - plt.title("Box plot of single feature risk scores") - plt.xlabel("Feature") - plt.ylabel("Risk Score") + # Create a box plot + plt.figure(figsize=(8, 8)) + plt.boxplot(list(sing_res.values()), tick_labels=list(sing_res.keys())) + plt.title("Box plot of single feature risk scores") + plt.xlabel("Feature") + plt.ylabel("Risk Score") - # Save the plot as a PNG image in memory - image_stream = io.BytesIO() - plt.tight_layout() - plt.savefig(image_stream, format="png", bbox_inches='tight', dpi=300) - plt.close() + # Save the plot as a PNG image in memory + image_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(image_stream, format="png", bbox_inches='tight', dpi=300) + plt.close() - # Convert the image to a base64 string - image_stream.seek(0) - base64_image = base64.b64encode(image_stream.read()).decode("utf-8") - image_stream.close() + # Convert the image to a base64 string + image_stream.seek(0) + base64_image = base64.b64encode(image_stream.read()).decode("utf-8") + image_stream.close() result_dict["Descriptive statistics of the risk scores"] = descriptive_stats_dict - result_dict["Single attribute risk scoring Visualization"] = base64_image + if include_visualization: + result_dict["Single attribute risk scoring Visualization"] = base64_image result_dict["Description"] = ( "This metric quantifies the re-identification risk for each " "quasi-identifier. Lower values are preferred, indicating " @@ -233,7 +235,7 @@ def generate_single_attribute_MM_risk_scores(df, id_col, eval_cols, task=None): return result_dict -def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None): +def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None, include_visualization=True): """Compute combined Marketer/Prosecutor re-identification risk across all quasi-identifiers. Unlike :func:`generate_single_attribute_MM_risk_scores`, this function @@ -434,31 +436,32 @@ def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None) } # Stage 5: Generate visualization (90-100%) - if task: - task.update_state( - state='PROGRESS', - meta={'current': 95, 'total': 100, 'status': 'Generating visualization...'} - ) + if include_visualization: + if task: + task.update_state( + state='PROGRESS', + meta={'current': 95, 'total': 100, 'status': 'Generating visualization...'} + ) - x_label = ",".join(eval_cols) - # Create a box plot - plt.figure(figsize=(8, 8)) - plt.boxplot(risk_scores, orientation="vertical") - plt.title('Box Plot of Multiple Attribute Risk Scores') - plt.ylabel('Risk Score') - plt.xlabel('Feature Combination') - plt.xticks([1], [x_label]) - - # Save the plot as a PNG image in memory - image_stream = io.BytesIO() - plt.tight_layout() - plt.savefig(image_stream, format='png', bbox_inches='tight', dpi=300) - plt.close() - - # Convert the image to a base64 string - image_stream.seek(0) - base64_image = base64.b64encode(image_stream.read()).decode('utf-8') - image_stream.close() + x_label = ",".join(eval_cols) + # Create a box plot + plt.figure(figsize=(8, 8)) + plt.boxplot(risk_scores, orientation="vertical") + plt.title('Box Plot of Multiple Attribute Risk Scores') + plt.ylabel('Risk Score') + plt.xlabel('Feature Combination') + plt.xticks([1], [x_label]) + + # Save the plot as a PNG image in memory + image_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(image_stream, format='png', bbox_inches='tight', dpi=300) + plt.close() + + # Convert the image to a base64 string + image_stream.seek(0) + base64_image = base64.b64encode(image_stream.read()).decode('utf-8') + image_stream.close() result_dict["Description"] = ( "This metric evaluates the joint risk posed by combinations of " @@ -470,7 +473,8 @@ def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None) "The box plot shows the distribution of combined risk scores. A distribution concentrated at lower values indicates better privacy." ) result_dict["Descriptive statistics of the risk scores"] = stats_dict - result_dict["Multiple attribute risk scoring Visualization"] = base64_image + if include_visualization: + result_dict["Multiple attribute risk scoring Visualization"] = base64_image result_dict['Dataset Risk Score'] = normalized_distance except SoftTimeLimitExceeded: @@ -493,7 +497,7 @@ def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None) return result_dict -def compute_k_anonymity(quasi_identifiers: List[str], file_info): +def compute_k_anonymity(quasi_identifiers: List[str], file_info, include_visualization=True): """Measure k-anonymity for the given quasi-identifier columns. Groups records by the combination of *quasi_identifiers* and returns the @@ -557,27 +561,27 @@ def compute_k_anonymity(quasi_identifiers: List[str], file_info): # Histogram of equivalence class sizes hist_data = counts.value_counts().sort_index().to_dict() - plt.figure(figsize=(8, 5)) - plt.bar(hist_data.keys(), hist_data.values(), color="skyblue") - plt.xlabel("Equivalence Class Size (k)") - plt.ylabel("Number of Equivalence Classes") - plt.title("Distribution of Equivalence Class Sizes") - plt.grid(axis="y", alpha=0.75) - # Save histogram to base64 - img_stream = io.BytesIO() - plt.tight_layout() - plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) - plt.close() - img_stream.seek(0) - base64_image = base64.b64encode(img_stream.read()).decode("utf-8") - img_stream.close() + if include_visualization: + plt.figure(figsize=(8, 5)) + plt.bar(hist_data.keys(), hist_data.values(), color="skyblue") + plt.xlabel("Equivalence Class Size (k)") + plt.ylabel("Number of Equivalence Classes") + plt.title("Distribution of Equivalence Class Sizes") + plt.grid(axis="y", alpha=0.75) + # Save histogram to base64 + img_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) + plt.close() + img_stream.seek(0) + base64_image = base64.b64encode(img_stream.read()).decode("utf-8") + img_stream.close() # Final result result_dict = { "k-Value": k_anonymity, "descriptive_statistics": desc_stats, "histogram_data": hist_data, - "k-Anonymity Visualization": base64_image, "Description": ( "k-anonymity measures the minimum group size sharing the same quasi-identifier values. " "Higher k values are preferred, as they indicate stronger anonymity." @@ -587,6 +591,8 @@ def compute_k_anonymity(quasi_identifiers: List[str], file_info): "class sizes (higher k) is desirable for privacy." ), } + if include_visualization: + result_dict["k-Anonymity Visualization"] = base64_image except SoftTimeLimitExceeded: raise Exception("K anonymity task timed out.") except ValueError as ve: @@ -609,6 +615,7 @@ def compute_l_diversity( quasi_identifiers: list, sensitive_column: str, file_info, + include_visualization=True, ): """Quantify l-diversity within groups defined by quasi-identifiers. @@ -685,29 +692,29 @@ def compute_l_diversity( # or use: (l_diversities / 2).round() * 2 for bin size of 2 binned_l_diversities = l_diversities.round() hist_data = binned_l_diversities.value_counts().sort_index() - plt.figure(figsize=(8, 8)) - plt.bar(hist_data.index, hist_data.values, color="skyblue") - plt.xlabel("Number of Distinct Sensitive Values (l)") - plt.ylabel("Number of Equivalence Classes") - plt.title("Distribution of l-Diversity Across Equivalence Classes") - plt.xticks(sorted(hist_data.index)) - plt.grid(axis="y", alpha=0.75) - - # Save plot to base64 string - img_stream = io.BytesIO() - plt.tight_layout() - plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) - plt.close() - img_stream.seek(0) - base64_image = base64.b64encode(img_stream.read()).decode("utf-8") - img_stream.close() + if include_visualization: + plt.figure(figsize=(8, 8)) + plt.bar(hist_data.index, hist_data.values, color="skyblue") + plt.xlabel("Number of Distinct Sensitive Values (l)") + plt.ylabel("Number of Equivalence Classes") + plt.title("Distribution of l-Diversity Across Equivalence Classes") + plt.xticks(sorted(hist_data.index)) + plt.grid(axis="y", alpha=0.75) + + # Save plot to base64 string + img_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) + plt.close() + img_stream.seek(0) + base64_image = base64.b64encode(img_stream.read()).decode("utf-8") + img_stream.close() # Compose result dictionary result_dict = { "l-Value": min_l_diversity, "descriptive_statistics": desc_stats, "histogram_data": hist_data.to_dict(), - "l-Diversity Visualization": base64_image, "Description": ( "l-diversity quantifies the diversity of sensitive attributes within each group. " "Higher l values are preferred, indicating less risk of attribute disclosure." @@ -716,6 +723,8 @@ def compute_l_diversity( "The histogram displays the spread of l-diversity values. A distribution concentrated at higher l values is optimal." ), } + if include_visualization: + result_dict["l-Diversity Visualization"] = base64_image except SoftTimeLimitExceeded: raise Exception("L Diversity task timed out.") except ValueError as ve: @@ -738,6 +747,7 @@ def compute_t_closeness( quasi_identifiers: List[str], sensitive_column: str, file_info, + include_visualization=True, ): """Measure t-closeness between each group's and the global sensitive attribute distribution. @@ -825,26 +835,26 @@ def tvd(p, q): # Histogram plot hist_data = t_series.round(2).value_counts().sort_index() - plt.figure(figsize=(8, 5)) - plt.bar(hist_data.index, hist_data.values, color="salmon") - plt.xlabel("t-Closeness Value (TVD)") - plt.ylabel("Number of Equivalence Classes") - plt.title("Distribution of T-Closeness Across Equivalence Classes") - plt.grid(axis="y", alpha=0.75) - - img_stream = io.BytesIO() - plt.tight_layout() - plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) - plt.close() - img_stream.seek(0) - base64_image = base64.b64encode(img_stream.read()).decode("utf-8") - img_stream.close() + if include_visualization: + plt.figure(figsize=(8, 5)) + plt.bar(hist_data.index, hist_data.values, color="salmon") + plt.xlabel("t-Closeness Value (TVD)") + plt.ylabel("Number of Equivalence Classes") + plt.title("Distribution of T-Closeness Across Equivalence Classes") + plt.grid(axis="y", alpha=0.75) + + img_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) + plt.close() + img_stream.seek(0) + base64_image = base64.b64encode(img_stream.read()).decode("utf-8") + img_stream.close() result_dict = { "t-Value": max_t, "descriptive_statistics": desc_stats, "histogram_data": hist_data.to_dict(), - "t-Closeness Visualization": base64_image, "Description": ( "t-closeness measures the distance between the distribution of sensitive attributes " "in a group and the overall distribution. Lower t values are preferred, indicating less information leakage." @@ -853,6 +863,8 @@ def tvd(p, q): "The histogram shows the distribution of t values. Lower t values across groups indicate stronger privacy." ), } + if include_visualization: + result_dict["t-Closeness Visualization"] = base64_image except SoftTimeLimitExceeded: raise Exception("T Closeness task timed out.") except ValueError as ve: @@ -871,7 +883,7 @@ def tvd(p, q): return result_dict -def compute_entropy_risk(quasi_identifiers, file_info): +def compute_entropy_risk(quasi_identifiers, file_info, include_visualization=True): """Calculate entropy-based re-identification risk for quasi-identifier columns. Groups records by the combination of *quasi_identifiers* and computes the @@ -938,20 +950,21 @@ def compute_entropy_risk(quasi_identifiers, file_info): # Histogram plot of entropy values hist_data = entropy_series.round(2).value_counts().sort_index() - plt.figure(figsize=(8, 5)) - plt.bar(hist_data.index, hist_data.values, color="royalblue") - plt.xlabel("Entropy Value") - plt.ylabel("Number of Equivalence Classes") - plt.title("Distribution of Entropy Across Equivalence Classes") - plt.grid(axis="y", alpha=0.75) - - img_stream = io.BytesIO() - plt.tight_layout() - plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) - plt.close() - img_stream.seek(0) - base64_image = base64.b64encode(img_stream.read()).decode("utf-8") - img_stream.close() + if include_visualization: + plt.figure(figsize=(8, 5)) + plt.bar(hist_data.index, hist_data.values, color="royalblue") + plt.xlabel("Entropy Value") + plt.ylabel("Number of Equivalence Classes") + plt.title("Distribution of Entropy Across Equivalence Classes") + plt.grid(axis="y", alpha=0.75) + + img_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(img_stream, format="png", bbox_inches='tight', dpi=300) + plt.close() + img_stream.seek(0) + base64_image = base64.b64encode(img_stream.read()).decode("utf-8") + img_stream.close() desc_stats = { "min": round(entropy_series.min(), 4), @@ -964,7 +977,6 @@ def compute_entropy_risk(quasi_identifiers, file_info): "Entropy-Value": rounded_entropy, "descriptive_statistics": desc_stats, "histogram_data": hist_data.to_dict(), - "Entropy Risk Visualization": base64_image, "Description": ( "Entropy risk quantifies the uncertainty in identifying individuals within equivalence classes. " "Higher entropy values are preferred, indicating greater anonymity and lower re-identification risk." @@ -974,6 +986,8 @@ def compute_entropy_risk(quasi_identifiers, file_info): "indicate better privacy; left-skewed distributions suggest higher risk." ), } + if include_visualization: + result_dict["Entropy Risk Visualization"] = base64_image except SoftTimeLimitExceeded: raise Exception("Entropy Risk task timed out.") except ValueError as ve: diff --git a/aidrin/structured_data_metrics/statistical_rate.py b/aidrin/structured_data_metrics/statistical_rate.py index b01d9359..1393d2d8 100644 --- a/aidrin/structured_data_metrics/statistical_rate.py +++ b/aidrin/structured_data_metrics/statistical_rate.py @@ -20,7 +20,7 @@ @shared_task(bind=True, ignore_result=False) def calculate_statistical_rates( - self: Task, y_true_column, sensitive_attribute_column, file_info + self: Task, y_true_column, sensitive_attribute_column, file_info, include_visualization=True ): try: logger.info("Statistical Rate task started: target=%r, sensitive=%r", y_true_column, sensitive_attribute_column) @@ -71,59 +71,60 @@ def calculate_statistical_rates( for class_label, proportion in tsd.items(): tsd[class_label] = np.std(proportion) - # Set up the plot - fig, ax = plt.subplots(figsize=(8, 8)) + if include_visualization: + # Set up the plot + fig, ax = plt.subplots(figsize=(8, 8)) - # Calculate the total number of classes and sensitive attribute values - num_classes = len(unique_class_labels) - num_sensitive_values = len(unique_sensitive_values) + # Calculate the total number of classes and sensitive attribute values + num_classes = len(unique_class_labels) + num_sensitive_values = len(unique_sensitive_values) - # Calculate the width of each bar and the total width of each group - bar_width = 0.1 - group_width = bar_width * num_classes + # Calculate the width of each bar and the total width of each group + bar_width = 0.1 + group_width = bar_width * num_classes - # Calculate the offset for each bar within a group - bar_offset = np.arange(num_sensitive_values) * group_width - ( - group_width * (num_classes - 1) / 2 - ) - - # Iterate through each unique class label - for i, class_label in enumerate(unique_class_labels): - # Extract proportions for the current class label - proportions = [ - class_proportions[sensitive_value].get(class_label, 0) - for sensitive_value in unique_sensitive_values - ] - - # Plot the bars for each sensitive attribute value with the adjusted position - bar_positions = bar_offset + i * bar_width - ax.bar( - bar_positions, - proportions, - width=bar_width, - label=f"Class: {class_label}", + # Calculate the offset for each bar within a group + bar_offset = np.arange(num_sensitive_values) * group_width - ( + group_width * (num_classes - 1) / 2 ) - # Set up labels and title - ax.set_xticks(bar_offset + (num_classes - 1) * bar_width / 2) - # Adjust fontsize and rotation - ax.set_xticklabels(unique_sensitive_values, rotation=30, ha="right", fontsize=8) - ax.set_xlabel("Sensitive Attribute") - ax.set_ylabel("Proportion") - ax.set_title("Class Proportions for Each Sensitive Attribute") - ax.legend() - - # Adjust the bottom margin to avoid xticks being cropped - plt.subplots_adjust(bottom=0.25) - - # Save the plot as a base64 string - buffer = io.BytesIO() - plt.savefig(buffer, format="png") - buffer.seek(0) - base64_plot = base64.b64encode(buffer.read()).decode("utf-8") - # Close the figure and BytesIO stream to free memory - plt.close(fig) - buffer.close() + # Iterate through each unique class label + for i, class_label in enumerate(unique_class_labels): + # Extract proportions for the current class label + proportions = [ + class_proportions[sensitive_value].get(class_label, 0) + for sensitive_value in unique_sensitive_values + ] + + # Plot the bars for each sensitive attribute value with the adjusted position + bar_positions = bar_offset + i * bar_width + ax.bar( + bar_positions, + proportions, + width=bar_width, + label=f"Class: {class_label}", + ) + + # Set up labels and title + ax.set_xticks(bar_offset + (num_classes - 1) * bar_width / 2) + # Adjust fontsize and rotation + ax.set_xticklabels(unique_sensitive_values, rotation=30, ha="right", fontsize=8) + ax.set_xlabel("Sensitive Attribute") + ax.set_ylabel("Proportion") + ax.set_title("Class Proportions for Each Sensitive Attribute") + ax.legend() + + # Adjust the bottom margin to avoid xticks being cropped + plt.subplots_adjust(bottom=0.25) + + # Save the plot as a base64 string + buffer = io.BytesIO() + plt.savefig(buffer, format="png") + buffer.seek(0) + base64_plot = base64.b64encode(buffer.read()).decode("utf-8") + # Close the figure and BytesIO stream to free memory + plt.close(fig) + buffer.close() # Full disclosure: This workaround is from stackoverflow. # Recasts all numpy types to their native Python types so Celery can pass the data correctly. @@ -137,23 +138,24 @@ def to_serializable(obj): else: return obj - cleaned_payload = to_serializable( - { - "Statistical Rates": class_proportions, - "TSD scores": tsd, - "Description": "The TSD values are calculated by getting the standard deviation of the " - "proportions of each group across the different classes...", - "Statistical Rate Visualization": base64_plot, - } - ) + serializable_payload = { + "Statistical Rates": class_proportions, + "TSD scores": tsd, + "Description": "The TSD values are calculated by getting the standard deviation of the " + "proportions of each group across the different classes...", + } + if include_visualization: + serializable_payload["Statistical Rate Visualization"] = base64_plot + cleaned_payload = to_serializable(serializable_payload) result = { "Statistical Rates": cleaned_payload["Statistical Rates"], "TSD scores": cleaned_payload["TSD scores"], "Description": cleaned_payload["Description"], - "Statistical Rate Visualization": cleaned_payload[ - "Statistical Rate Visualization" - ], } + if include_visualization: + result["Statistical Rate Visualization"] = cleaned_payload[ + "Statistical Rate Visualization" + ] logger.info("Statistical Rate task completed: %d sensitive groups, %d classes", len(unique_sensitive_values), len(unique_class_labels)) return result except SoftTimeLimitExceeded: diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 2c7d04cb..f557fdf1 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -285,7 +285,7 @@ def _build_categorical_distributions(df, top_n=_OVERVIEW_CAT_TOP_N): return distributions -def _build_dataset_overview_section(file_info): +def _build_dataset_overview_section(file_info, include_visualizations=False): """Build the dataset-overview portion of the readiness report. Returns file metadata, per-feature readiness profiles, numerical describe() @@ -321,7 +321,7 @@ def _build_dataset_overview_section(file_info): if stat_val is not None and not isinstance(stat_val, str): v[stat_key] = float(stat_val) - return { + overview = { "file_metadata": { "file_name": file_name, "file_type": file_type, @@ -337,8 +337,6 @@ def _build_dataset_overview_section(file_info): "feature_profiles": profiles, "numerical_summary": numerical_summary, "categorical_distributions": _build_categorical_distributions(df), - "categorical_charts": categorical_distribution_charts(df), - "histograms": summary_histograms(df, figsize=(7, 4.5)), "profile_thresholds": { "missing_warning": _OVERVIEW_MISSING_WARNING, "missing_poor": _OVERVIEW_MISSING_POOR, @@ -346,10 +344,15 @@ def _build_dataset_overview_section(file_info): "high_cardinality": _OVERVIEW_HIGH_CARDINALITY, "id_unique_ratio": _OVERVIEW_ID_UNIQUE_RATIO, }, + "visualizations_deferred": not include_visualizations, } + if include_visualizations: + overview["categorical_charts"] = categorical_distribution_charts(df) + overview["histograms"] = summary_histograms(df, figsize=(7, 4.5)) + return overview -def _build_data_quality_section(file_info): +def _build_data_quality_section(file_info, include_visualizations=False): """Compute the data-quality portion of the readiness report. Runs completeness, outliers, and duplicity (the same functions backing the @@ -361,12 +364,12 @@ def _build_data_quality_section(file_info): section = {} # --- Completeness ----------------------------------------------------- - compl = completeness(file_info) + compl = completeness(file_info, include_visualization=include_visualizations) compl_scores = compl.get("Completeness scores", {}) or {} overall_completeness = compl.get("Overall Completeness") # --- Outliers --------------------------------------------------------- - out = outliers(file_info) + out = outliers(file_info, include_visualization=include_visualizations) out_scores_raw = out.get("Outlier scores", {}) if isinstance(out, dict) else {} overall_outlier = None out_scores = {} @@ -462,16 +465,19 @@ def _build_data_quality_section(file_info): "completeness": { "overall": overall_completeness, "scores": compl_scores, - "visualization": compl.get("Completeness Visualization"), + "visualization": compl.get("Completeness Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, }, "outliers": { "overall": overall_outlier, "scores": out_scores, - "visualization": out.get("Outliers Visualization") if isinstance(out, dict) else None, + "visualization": out.get("Outliers Visualization") if include_visualizations and isinstance(out, dict) else None, + "visualization_deferred": not include_visualizations, "error": outliers_error, }, "duplicity": {"overall": overall_duplicity}, }, + "visualizations_deferred": not include_visualizations, } return section @@ -589,7 +595,7 @@ def _pairwise_signals(scores): } -def _build_impact_on_ai_section(file_info): +def _build_impact_on_ai_section(file_info, include_visualizations=False): """Compute the Impact-on-AI portion of the readiness report. Runs an automated, non-interactive all-pairs correlation analysis (numerical @@ -610,7 +616,7 @@ def _build_impact_on_ai_section(file_info): "columns_dropped": dropped, } - corr = calc_correlations(kept, file_info) + corr = calc_correlations(kept, file_info, include_visualization=include_visualizations) if isinstance(corr, dict) and "Message" in corr: return {"error": corr["Message"], "columns_dropped": dropped} @@ -708,12 +714,14 @@ def _build_impact_on_ai_section(file_info): "details": { "categorical_visualization": cat.get( "Correlations Analysis Categorical Visualization" - ), + ) if include_visualizations else None, "numerical_visualization": num.get( "Correlations Analysis Numerical Visualization" - ), + ) if include_visualizations else None, "numerical_method": num.get("Method"), + "visualizations_deferred": not include_visualizations, }, + "visualizations_deferred": not include_visualizations, } @@ -906,7 +914,7 @@ def _imbalance_status(id_score): return "poor" -def _build_fairness_bias_section(file_info): +def _build_fairness_bias_section(file_info, include_visualizations=False): """Compute the Fairness & Bias portion of the readiness report. Auto-selects sensitive attributes and a target, then runs representation @@ -942,17 +950,19 @@ def _build_fairness_bias_section(file_info): s for s in rep_summaries if s["max_ratio"] >= _FAIRNESS_REP_RATIO_FLAG ] rep_visualizations = {} - for col in sensitive_cols: - try: - vis = create_representation_rate_vis([col], file_info) - if isinstance(vis, str): - rep_visualizations[col] = vis - except Exception: - pass + if include_visualizations: + for col in sensitive_cols: + try: + vis = create_representation_rate_vis([col], file_info) + if isinstance(vis, str): + rep_visualizations[col] = vis + except Exception: + pass details["representation_rate"] = { "ratios": ratios if not rep_error else None, "summaries": rep_summaries, "visualizations": rep_visualizations, + "visualizations_deferred": not include_visualizations, "error": rep_error, } else: @@ -972,14 +982,17 @@ def _build_fairness_bias_section(file_info): imbalance_degree = None ci_error = None if target_col: - ci_dict = _compute_class_imbalance(df, target_col, "EU") + ci_dict = _compute_class_imbalance( + df, target_col, "EU", include_visualization=include_visualizations + ) if "Error" in ci_dict: ci_error = ci_dict["Error"] else: imb = ci_dict.get("Imbalance degree") or {} imbalance_degree = imb.get("Imbalance Degree score") details["class_imbalance"] = { - "visualization": ci_dict.get("Class Imbalance Visualization"), + "visualization": ci_dict.get("Class Imbalance Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, "imbalance_degree": imbalance_degree, } # Minority classes @@ -1014,7 +1027,10 @@ def _build_fairness_bias_section(file_info): # --- Statistical Rate (primary sensitive + target) ---------------------- disparity_kpi = None if primary_sensitive and target_col: - sr = calculate_statistical_rates(target_col, primary_sensitive, file_info) + sr = calculate_statistical_rates( + target_col, primary_sensitive, file_info, + include_visualization=include_visualizations, + ) if isinstance(sr, dict) and "Error" in sr: details["statistical_rate"] = {"error": sr["Error"]} else: @@ -1040,7 +1056,8 @@ def _build_fairness_bias_section(file_info): "sensitive": primary_sensitive, "target": target_col, "tsd_scores": tsd_scores, - "visualization": sr.get("Statistical Rate Visualization"), + "visualization": sr.get("Statistical Rate Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, } else: details["statistical_rate"] = { @@ -1105,6 +1122,7 @@ def _build_fairness_bias_section(file_info): "kpis": kpis, "needs_attention": needs_attention, "details": details, + "visualizations_deferred": not include_visualizations, } @@ -1508,7 +1526,7 @@ def _auto_select_governance_columns(df, fairness_target=None): } -def _build_data_governance_section(file_info): +def _build_data_governance_section(file_info, include_visualizations=False): """Compute the Data Governance portion of the readiness report.""" df = read_file(file_info) if hasattr(df, "columns"): @@ -1543,7 +1561,7 @@ def _build_data_governance_section(file_info): # --- k-Anonymity --------------------------------------------------------- k_val = None if qi: - k_res = compute_k_anonymity(qi, work_df) + k_res = compute_k_anonymity(qi, work_df, include_visualization=include_visualizations) if "Error" not in k_res: k_val = k_res.get("k-Value") if k_val is not None and k_val < _GOV_K_WARNING: @@ -1563,7 +1581,8 @@ def _build_data_governance_section(file_info): "quasi_identifiers": qi, "k_value": k_val, "descriptive_statistics": k_res.get("descriptive_statistics"), - "visualization": k_res.get("k-Anonymity Visualization"), + "visualization": k_res.get("k-Anonymity Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, } else: details["k_anonymity"] = {"error": k_res.get("Error")} @@ -1583,7 +1602,7 @@ def _build_data_governance_section(file_info): # --- l-Diversity --------------------------------------------------------- l_val = None if qi and sensitive: - l_res = compute_l_diversity(qi, sensitive, work_df) + l_res = compute_l_diversity(qi, sensitive, work_df, include_visualization=include_visualizations) if "Error" not in l_res: l_val = l_res.get("l-Value") if l_val is not None and l_val < _GOV_L_WARNING: @@ -1602,7 +1621,8 @@ def _build_data_governance_section(file_info): "sensitive_attribute": sensitive, "l_value": l_val, "descriptive_statistics": l_res.get("descriptive_statistics"), - "visualization": l_res.get("l-Diversity Visualization"), + "visualization": l_res.get("l-Diversity Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, } else: details["l_diversity"] = {"error": l_res.get("Error")} @@ -1624,7 +1644,7 @@ def _build_data_governance_section(file_info): # --- t-Closeness --------------------------------------------------------- t_val = None if qi and sensitive: - t_res = compute_t_closeness(qi, sensitive, work_df) + t_res = compute_t_closeness(qi, sensitive, work_df, include_visualization=include_visualizations) if "Error" not in t_res: t_val = t_res.get("t-Value") if t_val is not None and t_val > _GOV_T_WARNING: @@ -1643,7 +1663,8 @@ def _build_data_governance_section(file_info): "sensitive_attribute": sensitive, "t_value": t_val, "descriptive_statistics": t_res.get("descriptive_statistics"), - "visualization": t_res.get("t-Closeness Visualization"), + "visualization": t_res.get("t-Closeness Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, } else: details["t_closeness"] = {"error": t_res.get("Error")} @@ -1664,13 +1685,14 @@ def _build_data_governance_section(file_info): # --- Entropy risk -------------------------------------------------------- if qi: - e_res = compute_entropy_risk(qi, work_df) + e_res = compute_entropy_risk(qi, work_df, include_visualization=include_visualizations) if "Error" not in e_res: details["entropy_risk"] = { "quasi_identifiers": qi, "entropy_value": e_res.get("Entropy-Value"), "descriptive_statistics": e_res.get("descriptive_statistics"), - "visualization": e_res.get("Entropy Risk Visualization"), + "visualization": e_res.get("Entropy Risk Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, } else: details["entropy_risk"] = {"error": e_res.get("Error")} @@ -1683,7 +1705,9 @@ def _build_data_governance_section(file_info): if mm_qis: for q in mm_qis: try: - s_res = generate_single_attribute_MM_risk_scores(work_df, id_col, [q]) + s_res = generate_single_attribute_MM_risk_scores( + work_df, id_col, [q], include_visualization=include_visualizations + ) if "Error" in s_res: single_by_qi[q] = {"error": s_res["Error"]} continue @@ -1743,7 +1767,9 @@ def _build_data_governance_section(file_info): multi_mean = None if mm_qis: try: - m_res = generate_multiple_attribute_MM_risk_scores(work_df, id_col, mm_qis) + m_res = generate_multiple_attribute_MM_risk_scores( + work_df, id_col, mm_qis, include_visualization=include_visualizations + ) if "Error" not in m_res: m_stats = m_res.get("Descriptive statistics of the risk scores") or {} multi_mean = m_stats.get("mean") @@ -1766,7 +1792,8 @@ def _build_data_governance_section(file_info): "mean_risk": round(multi_mean, 4) if multi_mean is not None else None, "dataset_risk_score": m_res.get("Dataset Risk Score"), "stats": m_stats, - "visualization": m_res.get("Multiple attribute risk scoring Visualization"), + "visualization": m_res.get("Multiple attribute risk scoring Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, } else: details["multiple_attribute_risk"] = {"error": m_res.get("Error")} @@ -1829,14 +1856,16 @@ def _build_data_governance_section(file_info): if dp_features: try: dp_res = return_noisy_stats( - dp_features, _GOV_DP_EPSILON, work_df, save_output=False + dp_features, _GOV_DP_EPSILON, work_df, + save_output=False, include_visualization=include_visualizations, ) if "Error" not in dp_res: details["differential_privacy"] = { "features": dp_features, "epsilon": _GOV_DP_EPSILON, "illustrative": True, - "visualization": dp_res.get("DP Statistics Visualization"), + "visualization": dp_res.get("DP Statistics Visualization") if include_visualizations else None, + "visualization_deferred": not include_visualizations, "summary": { k: v for k, v in dp_res.items() if k.endswith("(before noise)") or k.endswith("(after noise)") @@ -1862,9 +1891,154 @@ def _build_data_governance_section(file_info): "kpis": kpis, "needs_attention": needs_attention, "details": details, + "visualizations_deferred": not include_visualizations, + } + + +def _build_dataset_overview_visualizations(file_info): + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + return { + "categorical_charts": categorical_distribution_charts(df), + "histograms": summary_histograms(df, figsize=(7, 4.5)), } +def _build_data_quality_visualizations(file_info): + compl = completeness(file_info, include_visualization=True) + out = outliers(file_info, include_visualization=True) + viz = {} + if compl.get("Completeness Visualization"): + viz["completeness"] = compl["Completeness Visualization"] + if isinstance(out, dict) and out.get("Outliers Visualization"): + viz["outliers"] = out["Outliers Visualization"] + return viz + + +def _build_impact_on_ai_visualizations(file_info): + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + kept, _ = _prune_columns_for_corr(df) + if len(kept) < 2: + return {} + corr = calc_correlations(kept, file_info, include_visualization=True) + if isinstance(corr, dict) and "Message" in corr: + return {} + cat = corr.get("Correlations Analysis Categorical", {}) or {} + num = corr.get("Correlations Analysis Numerical", {}) or {} + viz = {} + cat_img = cat.get("Correlations Analysis Categorical Visualization") + num_img = num.get("Correlations Analysis Numerical Visualization") + if cat_img: + viz["categorical_correlation"] = cat_img + if num_img: + viz["numerical_correlation"] = num_img + return viz + + +def _build_fairness_bias_visualizations(file_info): + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + selection = _auto_select_fairness_columns(df) + sensitive_cols = selection["sensitive_columns"] + target_col = selection["target_column"] + primary_sensitive = selection["primary_sensitive"] + viz = {} + for col in sensitive_cols: + try: + vis = create_representation_rate_vis([col], file_info) + if isinstance(vis, str): + viz[f"representation_rate.{col}"] = vis + except Exception: + pass + if target_col: + ci_dict = _compute_class_imbalance(df, target_col, "EU", include_visualization=True) + img = ci_dict.get("Class Imbalance Visualization") + if img: + viz["class_imbalance"] = img + if primary_sensitive and target_col: + sr = calculate_statistical_rates( + target_col, primary_sensitive, file_info, include_visualization=True, + ) + if isinstance(sr, dict) and sr.get("Statistical Rate Visualization"): + viz["statistical_rate"] = sr["Statistical Rate Visualization"] + return viz + + +def _build_data_governance_visualizations(file_info): + """Build only chart payloads for governance (metrics already computed on initial load).""" + df = read_file(file_info) + if hasattr(df, "columns"): + df.columns = [str(c) for c in df.columns] + + fairness_sel = _auto_select_fairness_columns(df) + selection = _auto_select_governance_columns( + df, fairness_target=fairness_sel.get("target_column") + ) + qi = selection["quasi_identifiers"] + mm_qis = selection["mm_quasi_identifiers"] + sensitive = selection["sensitive_attribute"] + id_col = selection["id_column"] + id_synthetic = selection["id_synthetic"] + dp_features = selection["dp_features"] + + work_df = df.copy() + if id_synthetic: + work_df[_SYNTHETIC_ID_COL] = range(len(work_df)) + id_col = _SYNTHETIC_ID_COL + + viz = {} + if qi: + k_res = compute_k_anonymity(qi, work_df, include_visualization=True) + img = k_res.get("k-Anonymity Visualization") + if img: + viz["k_anonymity"] = img + e_res = compute_entropy_risk(qi, work_df, include_visualization=True) + img = e_res.get("Entropy Risk Visualization") + if img: + viz["entropy_risk"] = img + if qi and sensitive: + l_res = compute_l_diversity(qi, sensitive, work_df, include_visualization=True) + img = l_res.get("l-Diversity Visualization") + if img: + viz["l_diversity"] = img + t_res = compute_t_closeness(qi, sensitive, work_df, include_visualization=True) + img = t_res.get("t-Closeness Visualization") + if img: + viz["t_closeness"] = img + if mm_qis: + m_res = generate_multiple_attribute_MM_risk_scores( + work_df, id_col, mm_qis, include_visualization=True + ) + img = m_res.get("Multiple attribute risk scoring Visualization") + if img: + viz["multiple_attribute_risk"] = img + if dp_features: + try: + dp_res = return_noisy_stats( + dp_features, _GOV_DP_EPSILON, work_df, + save_output=False, include_visualization=True, + ) + img = dp_res.get("DP Statistics Visualization") + if img: + viz["differential_privacy"] = img + except Exception: + pass + return viz + + +_READINESS_VIZ_BUILDERS = { + "dataset-overview": _build_dataset_overview_visualizations, + "data-quality": _build_data_quality_visualizations, + "impact-on-ai": _build_impact_on_ai_visualizations, + "fairness-bias": _build_fairness_bias_visualizations, + "data-governance": _build_data_governance_visualizations, +} + + _READINESS_SECTION_BUILDERS = { "dataset-overview": _build_dataset_overview_section, "data-quality": _build_data_quality_section, @@ -1894,13 +2068,13 @@ def _readiness_file_info(): ) -def _build_readiness_section(section, file_info): +def _build_readiness_section(section, file_info, include_visualizations=False): """Build one readiness-report section; return error dict on failure.""" builder = _READINESS_SECTION_BUILDERS.get(section) if builder is None: return None try: - return builder(file_info) + return builder(file_info, include_visualizations=include_visualizations) except Exception as e: metric_time_log.error( "Readiness report — %s error: %s", section, e, exc_info=True @@ -1908,6 +2082,42 @@ def _build_readiness_section(section, file_info): return {"error": f"{type(e).__name__}: {e}"} +@metrics_bp.route("/readiness-report/
    /visualizations", methods=["GET"]) +def readiness_report_visualizations(section): + """Return on-demand chart images for a readiness-report section.""" + if section not in _READINESS_VIZ_BUILDERS: + return jsonify({"success": False, "message": f"Unknown section: {section}"}), 404 + + file_info = _readiness_file_info() + if file_info is None: + return jsonify({"success": False, "message": "No file uploaded"}), 200 + + start_time = time.time() + try: + visualizations = _READINESS_VIZ_BUILDERS[section](file_info) + except Exception as e: + metric_time_log.error( + "Readiness report visualizations — %s error: %s", section, e, exc_info=True + ) + return jsonify({ + "success": False, + "message": f"{type(e).__name__}: {e}", + }), 200 + + build_time_seconds = round(time.time() - start_time, 2) + metric_time_log.info( + "Readiness report section %s visualizations built in %.2f seconds", + section, + build_time_seconds, + ) + return jsonify(ensure_json_serializable({ + "success": True, + "section": section, + "visualizations": visualizations, + "build_time_seconds": build_time_seconds, + })) + + @metrics_bp.route("/readiness-report/
    ", methods=["GET"]) def readiness_report_section(section): """Return a single readiness-report section as JSON (for progressive UI loading).""" @@ -2302,10 +2512,11 @@ def class_imbalance(): return get_result_or_default("metrics.class_imbalance", file_path, file_name) -def _compute_class_imbalance(file, classes, dist_metric): +def _compute_class_imbalance(file, classes, dist_metric, include_visualization=True): ci_dict = {} try: - ci_dict["Class Imbalance Visualization"] = class_distribution_plot(file, classes) + if include_visualization: + ci_dict["Class Imbalance Visualization"] = class_distribution_plot(file, classes) ci_dict["Description"] = ( "The chart displays the distribution of classes within the " "specified feature, providing a visual representation of the " diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 110e7290..2fe562b6 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2276,6 +2276,114 @@ function _appendReadinessBuildTimeFooter(container, seconds) { container.appendChild(el); } +const _readinessVizCache = {}; + +/** Placeholder for a chart that loads when the details panel is opened. */ +function _readinessVizSlot(section, vizKey, title) { + return `
    +

    ${title}

    +
    Open this section to load chart…
    +
    `; +} + +function _readinessVizSpinnerHtml() { + return `
    + +
    `; +} + +function _applyReadinessVisualizations(section, root, vizMap) { + if (!root || !vizMap) return; + + if (section === "dataset-overview") { + if (vizMap.categorical_charts) { + const catHost = root.querySelector("#readiness-categorical-charts"); + if (catHost) { + renderCategoricalPieCharts(vizMap.categorical_charts, "readiness-categorical-charts"); + } + } + if (vizMap.histograms) { + const histHost = root.querySelector("#readiness-histograms-inner"); + if (histHost) { + renderWorkspaceHistograms( + vizMap.histograms, + "readiness-histograms-inner", + true, + "large", + ); + } + } + return; + } + + root.querySelectorAll(".readiness-viz-slot").forEach((slot) => { + const key = slot.dataset.readinessViz; + const target = slot.querySelector(".readiness-viz-content"); + if (!target || !key) return; + const b64 = vizMap[key]; + if (b64) { + target.innerHTML = `${_escapeHtml(key)}`; + } else { + target.innerHTML = `

    Chart unavailable.

    `; + } + }); +} + +function _loadReadinessSectionVisualizations(section, root) { + if (!root) return Promise.resolve(); + const slots = root.querySelectorAll(".readiness-viz-slot"); + if (!section || (!slots.length && section !== "dataset-overview")) { + return Promise.resolve(); + } + + if (_readinessVizCache[section]) { + _applyReadinessVisualizations(section, root, _readinessVizCache[section]); + return Promise.resolve(); + } + + slots.forEach((slot) => { + const target = slot.querySelector(".readiness-viz-content"); + if (target) target.innerHTML = _readinessVizSpinnerHtml(); + }); + const catHost = root.querySelector("#readiness-categorical-charts"); + const histHost = root.querySelector("#readiness-histograms-inner"); + if (catHost && !catHost.childElementCount) catHost.innerHTML = _readinessVizSpinnerHtml(); + if (histHost && !histHost.childElementCount) histHost.innerHTML = _readinessVizSpinnerHtml(); + + return fetch(`/readiness-report/${section}/visualizations`) + .then((r) => r.json()) + .then((resp) => { + if (!resp.success) { + const msg = resp.message || "Could not load charts"; + slots.forEach((slot) => { + const target = slot.querySelector(".readiness-viz-content"); + if (target) target.innerHTML = `

    ${_escapeHtml(msg)}

    `; + }); + return; + } + _readinessVizCache[section] = resp.visualizations || {}; + _applyReadinessVisualizations(section, root, _readinessVizCache[section]); + }) + .catch((err) => { + slots.forEach((slot) => { + const target = slot.querySelector(".readiness-viz-content"); + if (target) { + target.innerHTML = `

    ${_escapeHtml(err.message)}

    `; + } + }); + }); +} + +/** Fetch charts the first time a readiness details panel is expanded. */ +function _wireReadinessDetailsViz(detailsEl, section) { + if (!detailsEl || detailsEl.dataset.vizWired === "1") return; + detailsEl.dataset.vizWired = "1"; + detailsEl.addEventListener("toggle", () => { + if (!detailsEl.open) return; + _loadReadinessSectionVisualizations(section, detailsEl); + }); +} + /** * Fetch one readiness-report section and render it when ready. * @param {string} section - URL slug (e.g. "data-quality") @@ -2680,12 +2788,19 @@ function renderReadinessDatasetOverview(container, overview) { const catCharts = overview.categorical_charts || {}; const catChartCols = Object.keys(catCharts); - if (catChartCols.length > 0) { + const hasHistograms = + overview.histograms && Object.keys(overview.histograms).length > 0; + const vizDeferred = overview.visualizations_deferred; + const showCatCharts = catChartCols.length > 0 || (vizDeferred && (overview.categorical_distributions || {}).length); + const showHistograms = + hasHistograms || (vizDeferred && numFeatures.length > 0); + + if (showCatCharts) { detailsInner += `

    Categorical value distributions

    `; detailsInner += `
    `; } - if (overview.histograms && Object.keys(overview.histograms).length > 0) { + if (showHistograms) { detailsInner += `

    Feature distributions (numerical)

    `; detailsInner += `
    `; } @@ -2703,16 +2818,23 @@ function renderReadinessDatasetOverview(container, overview) { container.classList.remove("text-center", "py-8"); container.innerHTML = html; - if (catChartCols.length > 0) { - renderCategoricalPieCharts(overview.categorical_charts, "readiness-categorical-charts"); - } - if (overview.histograms) { - renderWorkspaceHistograms( - overview.histograms, - "readiness-histograms-inner", - true, - "large", - ); + if (vizDeferred) { + const detailsEl = container.querySelector("details"); + if (detailsEl && (showCatCharts || showHistograms)) { + _wireReadinessDetailsViz(detailsEl, "dataset-overview"); + } + } else { + if (catChartCols.length > 0) { + renderCategoricalPieCharts(overview.categorical_charts, "readiness-categorical-charts"); + } + if (hasHistograms) { + renderWorkspaceHistograms( + overview.histograms, + "readiness-histograms-inner", + true, + "large", + ); + } } } @@ -2841,36 +2963,50 @@ function renderReadinessDataQuality(container, dq) { // --- Collapsible details (original charts) --- const det = dq.details || {}; - let detailsInner = ""; - if (det.completeness && det.completeness.visualization) { - detailsInner += ` -
    -

    Completeness by feature

    - Completeness chart -
    `; - } - if (det.outliers && det.outliers.visualization) { - detailsInner += ` -
    -

    Outliers by feature

    - Outliers chart -
    `; - } else if (det.outliers && det.outliers.error) { - detailsInner += `

    Outliers: ${det.outliers.error}

    `; - } + const vizDeferred = dq.visualizations_deferred; + let detailsInner = ""; + if (det.completeness) { + if (det.completeness.visualization) { + detailsInner += ` +
    +

    Completeness by feature

    + Completeness chart +
    `; + } else if (vizDeferred || det.completeness.visualization_deferred) { + detailsInner += _readinessVizSlot("data-quality", "completeness", "Completeness by feature"); + } + } + if (det.outliers) { + if (det.outliers.visualization) { + detailsInner += ` +
    +

    Outliers by feature

    + Outliers chart +
    `; + } else if (det.outliers.error) { + detailsInner += `

    Outliers: ${det.outliers.error}

    `; + } else if (vizDeferred || det.outliers.visualization_deferred) { + detailsInner += _readinessVizSlot("data-quality", "outliers", "Outliers by feature"); + } + } - if (detailsInner) { - html += ` -
    - - Show detailed charts - -
    ${detailsInner}
    -
    `; - } + if (detailsInner) { + html += ` +
    + + Show detailed charts + +
    ${detailsInner}
    +
    `; + } container.classList.remove("text-center", "py-8"); container.innerHTML = html; + + if (vizDeferred) { + const detailsEl = container.querySelector("details"); + if (detailsEl) _wireReadinessDetailsViz(detailsEl, "data-quality"); + } } /** @@ -3033,6 +3169,7 @@ function renderReadinessImpact(container, impact) { // --- Collapsible details --- const det = impact.details || {}; + const vizDeferred = impact.visualizations_deferred || det.visualizations_deferred; let detailsInner = ""; if (topPairs.length) { @@ -3061,6 +3198,13 @@ function renderReadinessImpact(container, impact) {

    Numerical correlation${method}

    Numerical correlation heatmap
    `; + } else if (vizDeferred && analyzed >= 2) { + const method = det.numerical_method ? ` (${det.numerical_method})` : ""; + detailsInner += _readinessVizSlot( + "impact-on-ai", + "numerical_correlation", + `Numerical correlation${method}`, + ); } if (det.categorical_visualization) { detailsInner += ` @@ -3068,6 +3212,12 @@ function renderReadinessImpact(container, impact) {

    Categorical correlation (Theil's U)

    Categorical correlation heatmap `; + } else if (vizDeferred && analyzed >= 2) { + detailsInner += _readinessVizSlot( + "impact-on-ai", + "categorical_correlation", + "Categorical correlation (Theil's U)", + ); } if (dropped.length) { const items = dropped @@ -3095,6 +3245,11 @@ function renderReadinessImpact(container, impact) { container.classList.remove("text-center", "py-8"); container.innerHTML = html; + + if (vizDeferred) { + const detailsEl = container.querySelector("details"); + if (detailsEl) _wireReadinessDetailsViz(detailsEl, "impact-on-ai"); + } } /** @@ -3295,15 +3450,29 @@ function renderReadinessFairness(container, fb) { // --- Collapsible details (charts) --- const det = fb.details || {}; + const vizDeferred = + fb.visualizations_deferred || + det.representation_rate?.visualizations_deferred; let detailsInner = ""; const repVis = det.representation_rate?.visualizations || {}; - for (const [col, b64] of Object.entries(repVis)) { - detailsInner += ` -
    -

    Representation rate — ${col}

    - Representation ${col} -
    `; + const sensCols = sensCrit.selected || []; + if (vizDeferred && sensCols.length && !det.representation_rate?.error) { + sensCols.forEach((col) => { + detailsInner += _readinessVizSlot( + "fairness-bias", + `representation_rate.${col}`, + `Representation rate — ${col}`, + ); + }); + } else { + for (const [col, b64] of Object.entries(repVis)) { + detailsInner += ` +
    +

    Representation rate — ${col}

    + Representation ${col} +
    `; + } } if (det.representation_rate?.error && !Object.keys(repVis).length) { detailsInner += `

    Representation rate: ${det.representation_rate.error}

    `; @@ -3317,6 +3486,15 @@ function renderReadinessFairness(container, fb) { `; } else if (det.class_imbalance?.error) { detailsInner += `

    Class imbalance: ${det.class_imbalance.error}

    `; + } else if ( + (vizDeferred || det.class_imbalance?.visualization_deferred) && + targetCrit.selected + ) { + detailsInner += _readinessVizSlot( + "fairness-bias", + "class_imbalance", + `Class imbalance — ${targetCrit.selected || "target"}`, + ); } if (det.statistical_rate?.visualization) { @@ -3327,6 +3505,16 @@ function renderReadinessFairness(container, fb) { `; } else if (det.statistical_rate?.error) { detailsInner += `

    Statistical rate: ${det.statistical_rate.error}

    `; + } else if ( + (vizDeferred || det.statistical_rate?.visualization_deferred) && + sel.primary_sensitive && + targetCrit.selected + ) { + detailsInner += _readinessVizSlot( + "fairness-bias", + "statistical_rate", + `Statistical rate — ${sel.primary_sensitive} × ${targetCrit.selected}`, + ); } if (det.cdd?.disparities && !det.cdd.error) { @@ -3372,6 +3560,11 @@ function renderReadinessFairness(container, fb) { container.classList.remove("text-center", "py-8"); container.innerHTML = html; + + if (vizDeferred) { + const detailsEl = container.querySelector("details"); + if (detailsEl) _wireReadinessDetailsViz(detailsEl, "fairness-bias"); + } } /** @@ -3603,6 +3796,7 @@ function renderReadinessGovernance(container, gov) { ); let detailsInner = ""; + const vizDeferred = gov.visualizations_deferred; const chartMetrics = [ ["k_anonymity", "k-Anonymity", "visualization"], @@ -3622,6 +3816,8 @@ function renderReadinessGovernance(container, gov) { `; } else if (block?.error) { detailsInner += `

    ${title}: ${block.error}

    `; + } else if (block && (vizDeferred || block.visualization_deferred)) { + detailsInner += _readinessVizSlot("data-governance", key, title); } }); @@ -3686,6 +3882,11 @@ function renderReadinessGovernance(container, gov) { container.classList.remove("text-center", "py-8"); container.innerHTML = html; + + if (vizDeferred) { + const detailsEl = container.querySelector("details"); + if (detailsEl) _wireReadinessDetailsViz(detailsEl, "data-governance"); + } } // ==================== Workspace Init ==================== From ecb81ff640d1ab5e28b84201d262c30a093fd528 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 25 Jun 2026 01:48:15 -0400 Subject: [PATCH 12/23] Speed up readiness governance MM risk with groupby-based scoring path. Add generate_*_MM_risk_scores_groupby implementations that match legacy outputs, wire them only into the readiness report governance builders, and add parity tests against the original privacy-tab functions. --- .../privacy_measure.py | 356 ++++++++++++++++++ tests/unit/test_privacy.py | 146 +++++++ web/routes/metrics.py | 10 +- 3 files changed, 507 insertions(+), 5 deletions(-) diff --git a/aidrin/structured_data_metrics/privacy_measure.py b/aidrin/structured_data_metrics/privacy_measure.py index 7e6c7b9d..107e279d 100644 --- a/aidrin/structured_data_metrics/privacy_measure.py +++ b/aidrin/structured_data_metrics/privacy_measure.py @@ -497,6 +497,362 @@ def generate_multiple_attribute_MM_risk_scores(df, id_col, eval_cols, task=None, return result_dict +def _mm_descriptive_stats_array(values): + """Summary stats for a 1-D risk score array (used by groupby MM helpers).""" + return { + "mean": np.mean(values), + "std": np.std(values), + "min": np.min(values), + "25%": np.percentile(values, 25), + "50%": np.median(values), + "75%": np.percentile(values, 75), + "max": np.max(values), + } + + +def _vectorized_mm_risk_for_column(series, n_rows): + """Per-row MM risk for one quasi-identifier (unique ID rows).""" + attr1_tot = series.groupby(series, dropna=False).transform("count") + if (attr1_tot == 0).any(): + raise ValueError( + f"Column '{series.name}' has unexpected data structure causing division by zero." + ) + priv_prob_mm = (attr1_tot / n_rows) * (1.0 - 1.0 / attr1_tot) + return np.round(1 - priv_prob_mm, 2).to_numpy() + + +def _vectorized_mm_multi_risk(selected_df, eval_cols): + """Per-row combined MM risk via groupby counts (unique ID rows).""" + n_rows = len(selected_df) + priv_prob = np.ones(n_rows, dtype=float) + + if len(eval_cols) == 1: + col = eval_cols[0] + attr1_tot = selected_df.groupby(col, dropna=False)[col].transform("count") + if (attr1_tot == 0).any(): + raise ValueError( + f"Column '{col}' has unexpected data structure causing division by zero." + ) + priv_prob *= ((attr1_tot / n_rows) * (1.0 - 1.0 / attr1_tot)).to_numpy() + else: + for idx in range(1, len(eval_cols)): + col_a = eval_cols[idx - 1] + col_b = eval_cols[idx] + attr1_tot = selected_df.groupby(col_a, dropna=False)[col_a].transform("count") + if (attr1_tot == 0).any(): + raise ValueError( + f"Column '{col_a}' has unexpected data structure causing division by zero." + ) + joint_pair = selected_df.groupby( + [col_a, col_b], dropna=False + )[col_a].transform("count") + attr2_tot = selected_df.groupby(col_b, dropna=False)[col_b].transform("count") + if (attr2_tot == 0).any(): + raise ValueError( + f"Column '{col_b}' has unexpected data structure causing division by zero." + ) + step = ( + (attr1_tot / n_rows) + * (1.0 - 1.0 / attr1_tot) + * (joint_pair / attr1_tot) + * (1.0 - 1.0 / attr2_tot) + ) + priv_prob *= step.to_numpy() + + return np.round(1 - priv_prob, 2) + + +def generate_single_attribute_MM_risk_scores_groupby( + df, id_col, eval_cols, task=None, include_visualization=True +): + """Groupby-accelerated MM single-attribute risk (readiness report path). + + Same return contract as :func:`generate_single_attribute_MM_risk_scores`. + """ + result_dict = {} + + try: + if task: + task.update_state( + state='PROGRESS', + meta={'current': 5, 'total': 100, 'status': 'Data validation & preprocessing...'} + ) + + if df.empty: + raise ValueError("Dataset is empty. Please upload a dataset with data.") + + if isinstance(eval_cols, str): + eval_cols = [col.strip() for col in eval_cols.split(",") if col.strip()] + elif isinstance(eval_cols, list): + eval_cols = [col.strip() for col in eval_cols if col.strip()] + else: + raise ValueError("Quasi-identifiers must be provided as a string or list.") + + if not eval_cols: + raise ValueError("No valid quasi-identifiers provided.") + + missing_cols = [col for col in eval_cols if col not in df.columns] + if missing_cols: + raise ValueError(f"Quasi-identifier columns not found in dataset: {', '.join(missing_cols)}") + + if not id_col or id_col not in df.columns: + raise ValueError(f"ID column '{id_col}' not found in dataset.") + + if df[id_col].nunique() != len(df): + raise ValueError(f"ID column '{id_col}' must contain unique values for each row.") + + selected_columns = [id_col] + eval_cols + selected_df = df[selected_columns].dropna() + rows_after_dropna = len(selected_df) + logger.debug("Rows remaining after dropna: %d", rows_after_dropna) + if rows_after_dropna == 0: + raise ValueError( + "After removing missing values, no data remains. Please check your data quality or select different columns." + ) + + non_categorical_cols = [] + for col in eval_cols: + if pd.api.types.is_numeric_dtype(df[col]) and df[col].nunique() > 100: + non_categorical_cols.append(col) + + if non_categorical_cols: + raise ValueError( + f"Columns {', '.join(non_categorical_cols)} appear to be numerical with too many unique values." + "Quasi-identifiers should be categorical." + ) + + if task: + task.update_state( + state='PROGRESS', + meta={'current': 15, 'total': 100, 'status': 'Calculating risk scores...'} + ) + + sing_res = {} + total_columns = len(eval_cols) + n_rows = len(selected_df) + + for col_idx, col in enumerate(eval_cols): + if task: + progress = 15 + (col_idx / total_columns) * 55 + task.update_state( + state='PROGRESS', + meta={ + 'current': int(progress), + 'total': 100, + 'status': f'Calculating risk scores for {col}... ({col_idx + 1}/{total_columns})', + }, + ) + + if selected_df[col].nunique() <= 1: + raise ValueError( + f"Column '{col}' has only one unique value, making risk assessment meaningless." + ) + + sing_res[col] = _vectorized_mm_risk_for_column(selected_df[col], n_rows) + + if task: + task.update_state( + state='PROGRESS', + meta={'current': 75, 'total': 100, 'status': 'Calculating descriptive statistics...'} + ) + + descriptive_stats_dict = { + key: _mm_descriptive_stats_array(value) for key, value in sing_res.items() + } + + if include_visualization: + if task: + task.update_state( + state='PROGRESS', + meta={'current': 90, 'total': 100, 'status': 'Generating visualization...'} + ) + + plt.figure(figsize=(8, 8)) + plt.boxplot(list(sing_res.values()), tick_labels=list(sing_res.keys())) + plt.title("Box plot of single feature risk scores") + plt.xlabel("Feature") + plt.ylabel("Risk Score") + + image_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(image_stream, format="png", bbox_inches='tight', dpi=300) + plt.close() + + image_stream.seek(0) + base64_image = base64.b64encode(image_stream.read()).decode("utf-8") + image_stream.close() + + result_dict["Descriptive statistics of the risk scores"] = descriptive_stats_dict + if include_visualization: + result_dict["Single attribute risk scoring Visualization"] = base64_image + result_dict["Description"] = ( + "This metric quantifies the re-identification risk for each " + "quasi-identifier. Lower values are preferred, indicating " + "features that are less likely to uniquely identify individuals. " + "High-risk features may require further anonymization or removal." + ) + result_dict["Graph interpretation"] = ( + "The box plot displays the distribution of risk scores for each feature. Features with " + "higher medians or more outliers indicate greater privacy risk. A compact, lower box is desirable." + ) + + except SoftTimeLimitExceeded: + raise Exception("Single Attribute Risk task timed out. The dataset may be too large or complex.") + except ValueError as ve: + result_dict["Error"] = str(ve) + result_dict["Single attribute risk scoring Visualization"] = "" + result_dict["Description"] = f"Validation Error: {str(ve)}" + result_dict["Graph interpretation"] = "No visualization available due to validation error." + result_dict["ErrorType"] = "Validation Error" + except Exception as e: + result_dict["Error"] = f"Processing error: {str(e)}" + result_dict["Single attribute risk scoring Visualization"] = "" + result_dict["Description"] = f"Processing Error: {str(e)}" + result_dict["Graph interpretation"] = "No visualization available due to processing error." + result_dict["ErrorType"] = "Processing Error" + + return result_dict + + +def generate_multiple_attribute_MM_risk_scores_groupby( + df, id_col, eval_cols, task=None, include_visualization=True +): + """Groupby-accelerated MM multi-attribute risk (readiness report path). + + Same return contract as :func:`generate_multiple_attribute_MM_risk_scores`. + """ + result_dict = {} + + try: + if task: + task.update_state( + state='PROGRESS', + meta={'current': 5, 'total': 100, 'status': 'Data validation & preprocessing...'} + ) + + if df.empty: + raise ValueError("Input DataFrame is empty.") + + if isinstance(eval_cols, str): + eval_cols = [col.strip() for col in eval_cols.split(',') if col.strip()] + elif isinstance(eval_cols, list): + eval_cols = [col.strip() for col in eval_cols if col.strip()] + else: + raise ValueError("eval_cols must be a string or list") + + if not eval_cols: + raise ValueError("No valid columns provided in eval_cols after processing") + + missing_cols = [col for col in eval_cols if col not in df.columns] + if missing_cols: + raise ValueError(f"Columns not found in dataset: {missing_cols}") + + if not id_col or id_col not in df.columns: + raise ValueError(f"ID column '{id_col}' not found in dataset") + + selected_columns = [id_col] + eval_cols + selected_df = df[selected_columns].dropna() + rows_after_dropna = len(selected_df) + + if rows_after_dropna == 0: + logger.debug("No data remains after dropna — raising ValueError") + raise ValueError( + "After removing missing values, no data remains. Please check your data quality or select different columns." + ) + + for col in eval_cols: + if col in df.columns and df[col].nunique() <= 1: + raise ValueError( + f"Column '{col}' has only one unique value, making risk assessment meaningless." + ) + + if df[id_col].nunique() != len(df): + raise ValueError(f"ID column '{id_col}' must contain unique values for each row.") + + if task: + task.update_state( + state='PROGRESS', + meta={'current': 15, 'total': 100, 'status': 'Starting risk score calculations...'} + ) + + risk_scores = _vectorized_mm_multi_risk(selected_df, eval_cols) + + if task: + task.update_state( + state='PROGRESS', + meta={'current': 75, 'total': 100, 'status': 'Calculating dataset privacy level...'} + ) + + min_risk_scores = np.zeros(len(risk_scores)) + euclidean_distance = np.linalg.norm(risk_scores - min_risk_scores) + max_risk_scores = np.ones(len(risk_scores)) + max_euclidean_distance = np.linalg.norm(max_risk_scores - min_risk_scores) + normalized_distance = euclidean_distance / max_euclidean_distance + + if task: + task.update_state( + state='PROGRESS', + meta={'current': 85, 'total': 100, 'status': 'Calculating descriptive statistics...'} + ) + + stats_dict = _mm_descriptive_stats_array(risk_scores) + + if include_visualization: + if task: + task.update_state( + state='PROGRESS', + meta={'current': 95, 'total': 100, 'status': 'Generating visualization...'} + ) + + x_label = ",".join(eval_cols) + plt.figure(figsize=(8, 8)) + plt.boxplot(risk_scores, orientation="vertical") + plt.title('Box Plot of Multiple Attribute Risk Scores') + plt.ylabel('Risk Score') + plt.xlabel('Feature Combination') + plt.xticks([1], [x_label]) + + image_stream = io.BytesIO() + plt.tight_layout() + plt.savefig(image_stream, format='png', bbox_inches='tight', dpi=300) + plt.close() + + image_stream.seek(0) + base64_image = base64.b64encode(image_stream.read()).decode('utf-8') + image_stream.close() + + result_dict["Description"] = ( + "This metric evaluates the joint risk posed by combinations of " + "quasi-identifiers. Lower values are preferred, as they indicate " + "that the selected set of features does not easily allow " + "re-identification." + ) + result_dict["Graph interpretation"] = ( + "The box plot shows the distribution of combined risk scores. A distribution concentrated at lower values indicates better privacy." + ) + result_dict["Descriptive statistics of the risk scores"] = stats_dict + if include_visualization: + result_dict["Multiple attribute risk scoring Visualization"] = base64_image + result_dict['Dataset Risk Score'] = normalized_distance + + except SoftTimeLimitExceeded: + raise Exception("Multiple Attribute Risk task timed out. The dataset may be too large or complex.") + except ValueError as ve: + result_dict["Error"] = str(ve) + result_dict["Multiple attribute risk scoring Visualization"] = "" + result_dict["Description"] = f"Validation Error: {str(ve)}" + result_dict["Graph interpretation"] = "No visualization available due to validation error." + result_dict["ErrorType"] = "Validation Error" + except Exception as e: + result_dict["Error"] = f"Processing error: {str(e)}" + result_dict["Multiple attribute risk scoring Visualization"] = "" + result_dict["Description"] = f"Processing Error: {str(e)}" + result_dict["Graph interpretation"] = "No visualization available due to processing error." + result_dict["ErrorType"] = "Processing Error" + + return result_dict + + def compute_k_anonymity(quasi_identifiers: List[str], file_info, include_visualization=True): """Measure k-anonymity for the given quasi-identifier columns. diff --git a/tests/unit/test_privacy.py b/tests/unit/test_privacy.py index 05907ea5..7da04f38 100644 --- a/tests/unit/test_privacy.py +++ b/tests/unit/test_privacy.py @@ -31,7 +31,9 @@ class _FakeDist: from aidrin.structured_data_metrics.privacy_measure import ( # noqa: E402 generate_single_attribute_MM_risk_scores, + generate_single_attribute_MM_risk_scores_groupby, generate_multiple_attribute_MM_risk_scores, + generate_multiple_attribute_MM_risk_scores_groupby, compute_k_anonymity, compute_l_diversity, compute_t_closeness, @@ -51,6 +53,29 @@ def _make_df(n_rows=20, n_categories=4): return pd.DataFrame({"id": ids, "qi": qi}) +def _make_multi_qi_df(n_rows=500, seed=42): + """Larger frame with three quasi-identifiers for parity checks.""" + rng = np.random.default_rng(seed) + return pd.DataFrame({ + "id": np.arange(n_rows), + "qi1": rng.choice(["A", "B", "C", "D"], size=n_rows), + "qi2": rng.choice(["X", "Y", "Z"], size=n_rows), + "qi3": rng.choice(["P", "Q"], size=n_rows), + }) + + +def _assert_stats_equal(test_case, original_stats, groupby_stats, context): + """Assert descriptive-stat dicts from legacy vs groupby MM paths match.""" + test_case.assertEqual(set(original_stats.keys()), set(groupby_stats.keys()), context) + for key in original_stats: + test_case.assertAlmostEqual( + original_stats[key], + groupby_stats[key], + places=9, + msg=f"{context}: stat {key}", + ) + + # =========================================================================== # generate_single_attribute_MM_risk_scores # =========================================================================== @@ -242,6 +267,127 @@ def test_visualization_is_base64(self): base64.b64decode(vis) +# =========================================================================== +# groupby MM risk parity (readiness-report fast path vs legacy) +# =========================================================================== + + +class TestMMRiskGroupbyParity(unittest.TestCase): + """Groupby implementations must match legacy MM risk outputs.""" + + def test_single_attribute_matches_legacy_one_qi(self): + df = _make_df(n_rows=80, n_categories=5) + cols = ["qi"] + self._assert_single_parity(df, cols) + + def test_single_attribute_matches_legacy_multiple_qis(self): + df = _make_multi_qi_df() + for cols in (["qi1"], ["qi1", "qi2"], ["qi1", "qi2", "qi3"]): + with self.subTest(cols=cols): + self._assert_single_parity(df, cols) + + def test_single_attribute_string_eval_cols_parity(self): + df = _make_multi_qi_df(n_rows=120, seed=7) + legacy = generate_single_attribute_MM_risk_scores( + df, "id", "qi1, qi2", include_visualization=False + ) + groupby = generate_single_attribute_MM_risk_scores_groupby( + df, "id", "qi1, qi2", include_visualization=False + ) + self.assertNotIn("Error", legacy) + self.assertNotIn("Error", groupby) + for col in ("qi1", "qi2"): + _assert_stats_equal( + self, + legacy["Descriptive statistics of the risk scores"][col], + groupby["Descriptive statistics of the risk scores"][col], + f"single {col}", + ) + + def test_multiple_attribute_matches_legacy(self): + df = _make_multi_qi_df() + for cols in (["qi1"], ["qi1", "qi2"], ["qi1", "qi2", "qi3"]): + with self.subTest(cols=cols): + self._assert_multi_parity(df, cols) + + def test_multiple_attribute_string_eval_cols_parity(self): + df = _make_multi_qi_df(n_rows=200, seed=11) + legacy = generate_multiple_attribute_MM_risk_scores( + df, "id", "qi1,qi2", include_visualization=False + ) + groupby = generate_multiple_attribute_MM_risk_scores_groupby( + df, "id", "qi1,qi2", include_visualization=False + ) + self.assertNotIn("Error", legacy) + self.assertNotIn("Error", groupby) + _assert_stats_equal( + self, + legacy["Descriptive statistics of the risk scores"], + groupby["Descriptive statistics of the risk scores"], + "multi qi1,qi2", + ) + self.assertAlmostEqual( + legacy["Dataset Risk Score"], + groupby["Dataset Risk Score"], + places=9, + ) + + def test_error_parity_non_unique_id(self): + df = pd.DataFrame({ + "id": [1, 1, 2, 3], + "qi1": ["A", "B", "A", "C"], + "qi2": ["X", "Y", "X", "Z"], + }) + legacy = generate_multiple_attribute_MM_risk_scores( + df, "id", ["qi1", "qi2"], include_visualization=False + ) + groupby = generate_multiple_attribute_MM_risk_scores_groupby( + df, "id", ["qi1", "qi2"], include_visualization=False + ) + self.assertIn("Error", legacy) + self.assertIn("Error", groupby) + self.assertEqual(legacy["Error"], groupby["Error"]) + + def _assert_single_parity(self, df, cols): + legacy = generate_single_attribute_MM_risk_scores( + df, "id", cols, include_visualization=False + ) + groupby = generate_single_attribute_MM_risk_scores_groupby( + df, "id", cols, include_visualization=False + ) + self.assertNotIn("Error", legacy, legacy) + self.assertNotIn("Error", groupby, groupby) + for col in cols: + _assert_stats_equal( + self, + legacy["Descriptive statistics of the risk scores"][col], + groupby["Descriptive statistics of the risk scores"][col], + f"single {col}", + ) + + def _assert_multi_parity(self, df, cols): + legacy = generate_multiple_attribute_MM_risk_scores( + df, "id", cols, include_visualization=False + ) + groupby = generate_multiple_attribute_MM_risk_scores_groupby( + df, "id", cols, include_visualization=False + ) + self.assertNotIn("Error", legacy, legacy) + self.assertNotIn("Error", groupby, groupby) + _assert_stats_equal( + self, + legacy["Descriptive statistics of the risk scores"], + groupby["Descriptive statistics of the risk scores"], + f"multi {cols}", + ) + self.assertAlmostEqual( + legacy["Dataset Risk Score"], + groupby["Dataset Risk Score"], + places=9, + msg=f"Dataset Risk Score for {cols}", + ) + + # =========================================================================== # compute_k_anonymity # =========================================================================== diff --git a/web/routes/metrics.py b/web/routes/metrics.py index f557fdf1..7a970ee2 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -47,8 +47,8 @@ compute_k_anonymity, compute_l_diversity, compute_t_closeness, - generate_multiple_attribute_MM_risk_scores, - generate_single_attribute_MM_risk_scores, + generate_multiple_attribute_MM_risk_scores_groupby, + generate_single_attribute_MM_risk_scores_groupby, ) from aidrin.structured_data_metrics.representation_rate import ( calculate_representation_rate, @@ -1705,7 +1705,7 @@ def _build_data_governance_section(file_info, include_visualizations=False): if mm_qis: for q in mm_qis: try: - s_res = generate_single_attribute_MM_risk_scores( + s_res = generate_single_attribute_MM_risk_scores_groupby( work_df, id_col, [q], include_visualization=include_visualizations ) if "Error" in s_res: @@ -1767,7 +1767,7 @@ def _build_data_governance_section(file_info, include_visualizations=False): multi_mean = None if mm_qis: try: - m_res = generate_multiple_attribute_MM_risk_scores( + m_res = generate_multiple_attribute_MM_risk_scores_groupby( work_df, id_col, mm_qis, include_visualization=include_visualizations ) if "Error" not in m_res: @@ -2010,7 +2010,7 @@ def _build_data_governance_visualizations(file_info): if img: viz["t_closeness"] = img if mm_qis: - m_res = generate_multiple_attribute_MM_risk_scores( + m_res = generate_multiple_attribute_MM_risk_scores_groupby( work_df, id_col, mm_qis, include_visualization=True ) img = m_res.get("Multiple attribute risk scoring Visualization") From c0bdd22f3117d48b9a6759bc0f46cd7fc01e8bef Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 25 Jun 2026 02:11:47 -0400 Subject: [PATCH 13/23] Add client-side PDF export for readiness report scorecards. Expose a download button after all five sections load, then build the PDF from the rendered scorecard HTML so metrics are not recomputed. Omit detail charts and normalize dark-mode/animation styles so html2canvas captures crisp output. --- web/static/css/inspector.css | 20 ++ web/static/js/inspector.js | 187 ++++++++++++++++++- web/templates/_panels/_readiness_report.html | 14 +- 3 files changed, 219 insertions(+), 2 deletions(-) diff --git a/web/static/css/inspector.css b/web/static/css/inspector.css index 0ee0d18e..9919efea 100644 --- a/web/static/css/inspector.css +++ b/web/static/css/inspector.css @@ -95,3 +95,23 @@ button.is-submitting::after { transform: rotate(360deg); } } + +/* Readiness PDF export: html2canvas must not inherit fade-in or dark-mode opacity */ +.readiness-pdf-export, +.readiness-pdf-export * { + animation: none !important; + transition: none !important; + opacity: 1 !important; + transform: none !important; +} + +.readiness-pdf-export { + background: #ffffff !important; + color: #111827 !important; +} + +html.dark .readiness-pdf-export, +html.dark .readiness-pdf-export * { + --tw-bg-opacity: 1 !important; + --tw-text-opacity: 1 !important; +} diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 2fe562b6..8bb017be 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -12,6 +12,15 @@ let codeMirrorEditor = null; let lastMetricResult = null; // Store last result for JSON download let _readinessReportLoaded = false; // Lazy-load guard for the Readiness Report panel +const _READINESS_REPORT_SECTIONS = [ + "dataset-overview", + "data-quality", + "impact-on-ai", + "fairness-bias", + "data-governance", +]; +const _readinessSectionStatus = {}; + /** * Show a metric panel by ID, hiding all others. * @param {string} panelId - The panel name (e.g., 'data-quality', 'fairness') @@ -2392,11 +2401,16 @@ function _wireReadinessDetailsViz(detailsEl, section) { * @returns {Promise} */ function _fetchReadinessSection(section, container, renderFn) { - if (!container) return Promise.resolve(); + if (!container) { + _readinessSectionStatus[section] = "error"; + _updateReadinessExportButton(); + return Promise.resolve(); + } return fetch(`/readiness-report/${section}`) .then((r) => r.json()) .then((resp) => { if (!resp.success) { + _readinessSectionStatus[section] = "error"; _readinessSectionError( container, `Could not load ${section.replace(/-/g, " ")}: ${resp.message || "unknown error"}`, @@ -2404,6 +2418,12 @@ function _fetchReadinessSection(section, container, renderFn) { return; } const data = resp.data || {}; + if (data.error) { + _readinessSectionStatus[section] = "error"; + _readinessSectionError(container, data.error); + return; + } + _readinessSectionStatus[section] = "ok"; renderFn(container, data); _appendReadinessBuildTimeFooter( container, @@ -2411,7 +2431,11 @@ function _fetchReadinessSection(section, container, renderFn) { ); }) .catch((err) => { + _readinessSectionStatus[section] = "error"; _readinessSectionError(container, `Error loading section: ${err.message}`); + }) + .finally(() => { + _updateReadinessExportButton(); }); } @@ -2420,6 +2444,12 @@ function _fetchReadinessSection(section, container, renderFn) { * dataset overview first, then remaining sections in parallel. */ function loadReadinessReport() { + _wireReadinessExportButton(); + _READINESS_REPORT_SECTIONS.forEach((section) => { + _readinessSectionStatus[section] = "pending"; + }); + _updateReadinessExportButton(); + const overviewContainer = document.getElementById("readiness-summary"); const parallelSections = [ @@ -2466,6 +2496,161 @@ function loadReadinessReport() { ).finally(loadParallelSections); } +function _readinessAllSectionsReady() { + return _READINESS_REPORT_SECTIONS.every( + (section) => _readinessSectionStatus[section] === "ok", + ); +} + +function _updateReadinessExportButton() { + const bar = document.getElementById("readiness-export-bar"); + const btn = document.getElementById("readiness-export-pdf-btn"); + if (!bar || !btn) return; + bar.classList.toggle("hidden", !_readinessAllSectionsReady()); + if (!btn.hasAttribute("aria-busy")) { + btn.disabled = false; + } +} + +function _wireReadinessExportButton() { + const btn = document.getElementById("readiness-export-pdf-btn"); + if (!btn || btn.dataset.wired === "1") return; + btn.dataset.wired = "1"; + btn.addEventListener("click", () => { + _exportReadinessReportPdf(); + }); +} + +function _loadHtml2PdfLib() { + if (window.html2pdf) return Promise.resolve(window.html2pdf); + return new Promise((resolve, reject) => { + const existing = document.getElementById("html2pdf-script"); + if (existing) { + existing.addEventListener("load", () => resolve(window.html2pdf)); + existing.addEventListener("error", () => reject(new Error("Could not load PDF library"))); + return; + } + const script = document.createElement("script"); + script.id = "html2pdf-script"; + script.src = + "https://cdnjs.cloudflare.com/ajax/libs/html2pdf.js/0.10.1/html2pdf.bundle.min.js"; + script.async = true; + script.onload = () => resolve(window.html2pdf); + script.onerror = () => reject(new Error("Could not load PDF library")); + document.head.appendChild(script); + }); +} + +function _stripReadinessPdfUnsafeClasses(root) { + const isUnsafe = (cls) => + cls.startsWith("dark:") || cls.includes("/") || cls === "animate-spin"; + const strip = (el) => { + [...el.classList].filter(isUnsafe).forEach((c) => el.classList.remove(c)); + }; + strip(root); + root.querySelectorAll("*").forEach(strip); +} + +function _prepareReadinessPanelClone(source) { + const clone = source.cloneNode(true); + clone.removeAttribute("id"); + clone.classList.remove("metric-panel", "hidden"); + clone.classList.add("readiness-pdf-export"); + clone.querySelector("#readiness-export-bar")?.remove(); + clone.querySelectorAll("details").forEach((el) => el.remove()); + clone.querySelectorAll(".readiness-viz-slot").forEach((el) => el.remove()); + clone.querySelectorAll(".info-icon").forEach((el) => el.remove()); + clone + .querySelectorAll("#readiness-categorical-charts, #readiness-histograms-inner") + .forEach((el) => el.remove()); + _stripReadinessPdfUnsafeClasses(clone); + clone.style.cssText = + "background:#ffffff;color:#111827;padding:0;opacity:1;animation:none;"; + clone.querySelectorAll(".bg-white, .bg-gray-50").forEach((el) => { + el.style.backgroundColor = el.classList.contains("bg-gray-50") ? "#f9fafb" : "#ffffff"; + el.style.color = "#111827"; + }); + clone + .querySelectorAll( + ".bg-blue-50, .bg-amber-50, .bg-green-50, .bg-red-50, .bg-green-100, .bg-amber-100, .bg-red-100", + ) + .forEach((el) => { + if (el.classList.contains("bg-blue-50")) el.style.backgroundColor = "#eff6ff"; + else if (el.classList.contains("bg-amber-50")) el.style.backgroundColor = "#fffbeb"; + else if (el.classList.contains("bg-green-50")) el.style.backgroundColor = "#f0fdf4"; + else if (el.classList.contains("bg-red-50")) el.style.backgroundColor = "#fef2f2"; + else if (el.classList.contains("bg-green-100")) el.style.backgroundColor = "#dcfce7"; + else if (el.classList.contains("bg-amber-100")) el.style.backgroundColor = "#fef3c7"; + else if (el.classList.contains("bg-red-100")) el.style.backgroundColor = "#fee2e2"; + }); + return clone; +} + +function _readinessPdfFilename() { + const panel = document.getElementById("panel-readiness-report"); + const raw = panel?.dataset?.datasetName || "dataset"; + const stem = String(raw).replace(/\.[^.]+$/, "").replace(/[^\w.-]+/g, "_"); + const date = new Date().toISOString().slice(0, 10); + return `readiness-report-${stem || "dataset"}-${date}.pdf`; +} + +async function _exportReadinessReportPdf() { + const panel = document.getElementById("panel-readiness-report"); + const btn = document.getElementById("readiness-export-pdf-btn"); + const labelEl = document.getElementById("readiness-export-pdf-label"); + if (!panel || !_readinessAllSectionsReady()) return; + + if (btn) { + btn.disabled = true; + btn.setAttribute("aria-busy", "true"); + } + if (labelEl) labelEl.textContent = "Preparing PDF…"; + + let host = null; + try { + const html2pdf = await _loadHtml2PdfLib(); + const clone = _prepareReadinessPanelClone(panel); + host = document.createElement("div"); + host.style.cssText = + "position:fixed;left:-10000px;top:0;width:900px;background:#fff;padding:0;margin:0;"; + host.appendChild(clone); + document.body.appendChild(host); + + await new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(resolve)), + ); + + await html2pdf() + .set({ + margin: [8, 8, 8, 8], + filename: _readinessPdfFilename(), + image: { type: "jpeg", quality: 0.95 }, + html2canvas: { + scale: 2, + useCORS: true, + logging: false, + backgroundColor: "#ffffff", + scrollX: 0, + scrollY: 0, + }, + jsPDF: { unit: "mm", format: "a4", orientation: "portrait" }, + pagebreak: { mode: ["css", "legacy"] }, + }) + .from(clone) + .save(); + } catch (err) { + console.error("Readiness PDF export failed:", err); + alert(err.message || "Could not generate PDF."); + } finally { + if (host && host.parentNode) host.parentNode.removeChild(host); + if (btn) { + btn.disabled = false; + btn.removeAttribute("aria-busy"); + } + if (labelEl) labelEl.textContent = "Download PDF report"; + } +} + /** Escape text for safe inclusion in readiness info tooltips. */ function _escapeHtml(s) { return String(s) diff --git a/web/templates/_panels/_readiness_report.html b/web/templates/_panels/_readiness_report.html index f3ddc474..8ec5593d 100644 --- a/web/templates/_panels/_readiness_report.html +++ b/web/templates/_panels/_readiness_report.html @@ -1,5 +1,17 @@ -
    • Feature
    ${feat}${numSummary[feat][s] ?? "—"}${display}
    ${q}${v.mean_risk}
    ${q}${fmtGovRisk(v.mean_risk)}
    `; html += ``; From ed2110a67d7b48fcd758262421409a3376854188 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Mon, 29 Jun 2026 22:43:36 -0400 Subject: [PATCH 15/23] Speed up readiness report PDF export on wide datasets. Cap overview detail payloads (numerical summary, categorical distributions, excluded-column lists) and mirror those limits in the UI so collapsed panels do not build huge DOM trees. Export PDFs from scorecard sections only instead of cloning the full panel, and use html2canvas scale 1 to cut rasterization cost. Readiness scores are unchanged. --- tests/unit/test_readiness_overview.py | 42 ++++++++- web/routes/metrics.py | 122 ++++++++++++++++++++----- web/static/js/inspector.js | 126 +++++++++++++++++--------- 3 files changed, 223 insertions(+), 67 deletions(-) diff --git a/tests/unit/test_readiness_overview.py b/tests/unit/test_readiness_overview.py index 4fc8f4b3..a5cf7b15 100644 --- a/tests/unit/test_readiness_overview.py +++ b/tests/unit/test_readiness_overview.py @@ -2,7 +2,47 @@ import unittest -from web.routes.metrics import _prepare_feature_profiles_for_display +import pandas as pd + +from web.routes.metrics import ( + _build_categorical_distributions, + _build_numerical_summary_for_overview, + _cap_detail_list, + _prepare_feature_profiles_for_display, +) + + +class TestCapDetailList(unittest.TestCase): + def test_no_truncation_when_under_cap(self): + items = [{"feature": f"c{i}"} for i in range(10)] + capped, meta = _cap_detail_list(items, max_items=50) + self.assertEqual(len(capped), 10) + self.assertFalse(meta["truncated"]) + + def test_truncation_when_over_cap(self): + items = [{"feature": f"c{i}"} for i in range(100)] + capped, meta = _cap_detail_list(items, max_items=50) + self.assertEqual(len(capped), 50) + self.assertTrue(meta["truncated"]) + self.assertEqual(meta["total"], 100) + + +class TestBuildNumericalSummaryForOverview(unittest.TestCase): + def test_caps_wide_numerical_summary(self): + df = pd.DataFrame({f"n{i}": range(5) for i in range(600)}) + summary, meta = _build_numerical_summary_for_overview(df) + self.assertEqual(len(summary), 500) + self.assertTrue(meta["truncated"]) + self.assertEqual(meta["total"], 600) + + +class TestBuildCategoricalDistributions(unittest.TestCase): + def test_caps_wide_categorical_distributions(self): + df = pd.DataFrame({f"c{i}": ["a", "b", "a"] for i in range(80)}) + dists, meta = _build_categorical_distributions(df, max_columns=50) + self.assertEqual(len(dists), 50) + self.assertTrue(meta["truncated"]) + self.assertEqual(meta["total"], 80) class TestPrepareFeatureProfilesForDisplay(unittest.TestCase): diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 6603d30b..6cfd25c0 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -167,9 +167,52 @@ def _grade_label(score): _OVERVIEW_ID_UNIQUE_RATIO = 0.9 _OVERVIEW_CAT_TOP_N = 5 _OVERVIEW_MAX_FEATURE_PROFILES = 500 +_OVERVIEW_MAX_NUMERICAL_SUMMARY_COLS = 500 +_OVERVIEW_MAX_CATEGORICAL_DIST_COLS = 50 +_READINESS_MAX_DETAIL_LIST_ITEMS = 50 _PROFILE_STATUS_RANK = {"poor": 0, "warning": 1, "good": 2} +def _cap_detail_list(items, max_items): + """Return a capped list plus metadata for UI/PDF truncation notes.""" + total = len(items) + if total <= max_items: + return items, {"total": total, "shown": total, "truncated": False} + return items[:max_items], { + "total": total, + "shown": max_items, + "truncated": True, + } + + +def _build_numerical_summary_for_overview(num_df): + """Build describe() summary for overview details, capped on wide datasets.""" + if num_df.empty: + return {}, {"total": 0, "shown": 0, "truncated": False} + + total = len(num_df.columns) + cols = num_df.columns + truncated = total > _OVERVIEW_MAX_NUMERICAL_SUMMARY_COLS + if truncated: + cols = cols[:_OVERVIEW_MAX_NUMERICAL_SUMMARY_COLS] + + numerical_summary = num_df[cols].describe().to_dict() + for v in numerical_summary.values(): + for old_key in list(v.keys()): + if old_key in ["25%", "50%", "75%"]: + new_key = old_key.replace("%", "th percentile") + v[new_key] = float(v.pop(old_key)) + for stat_key, stat_val in list(v.items()): + if stat_val is not None and not isinstance(stat_val, str): + v[stat_key] = float(stat_val) + + return numerical_summary, { + "total": total, + "shown": len(cols), + "truncated": truncated, + } + + def _classify_feature_type(series): """Map a pandas Series to a coarse feature type label.""" if pd.api.types.is_bool_dtype(series): @@ -298,12 +341,18 @@ def _prepare_feature_profiles_for_display( return [profile for _, profile in ranked[:max_profiles]], meta -def _build_categorical_distributions(df, top_n=_OVERVIEW_CAT_TOP_N): +def _build_categorical_distributions(df, top_n=_OVERVIEW_CAT_TOP_N, max_columns=None): """Top-*n* value counts (with percentages) for each categorical column.""" distributions = {} - for col in df.columns: - if _classify_feature_type(df[col]) != "categorical": - continue + cat_cols = [ + col for col in df.columns if _classify_feature_type(df[col]) == "categorical" + ] + total_cat = len(cat_cols) + truncated = False + if max_columns is not None and total_cat > max_columns: + cat_cols = cat_cols[:max_columns] + truncated = True + for col in cat_cols: vc = df[col].value_counts(dropna=True) total = len(df[col].dropna()) if total == 0: @@ -317,7 +366,12 @@ def _build_categorical_distributions(df, top_n=_OVERVIEW_CAT_TOP_N): "pct": round(float(count) / total, 4), }) distributions[str(col)] = entries - return distributions + meta = { + "total": total_cat, + "shown": len(cat_cols), + "truncated": truncated, + } + return distributions, meta def _build_dataset_overview_section(file_info, include_visualizations=False): @@ -344,18 +398,14 @@ def _build_dataset_overview_section(file_info, include_visualizations=False): memory_bytes = int(df.memory_usage(deep=True).sum()) - numerical_summary = {} - num_df = df.select_dtypes(include="number") - if not num_df.empty: - numerical_summary = num_df.describe().to_dict() - for v in numerical_summary.values(): - for old_key in list(v.keys()): - if old_key in ["25%", "50%", "75%"]: - new_key = old_key.replace("%", "th percentile") - v[new_key] = float(v.pop(old_key)) - for stat_key, stat_val in list(v.items()): - if stat_val is not None and not isinstance(stat_val, str): - v[stat_key] = float(stat_val) + numerical_summary, numerical_summary_meta = _build_numerical_summary_for_overview( + df.select_dtypes(include="number") + ) + categorical_distributions, categorical_distributions_meta = ( + _build_categorical_distributions( + df, max_columns=_OVERVIEW_MAX_CATEGORICAL_DIST_COLS + ) + ) overview = { "file_metadata": { @@ -373,7 +423,9 @@ def _build_dataset_overview_section(file_info, include_visualizations=False): "feature_profiles": display_profiles, "feature_profiles_meta": profile_meta, "numerical_summary": numerical_summary, - "categorical_distributions": _build_categorical_distributions(df), + "numerical_summary_meta": numerical_summary_meta, + "categorical_distributions": categorical_distributions, + "categorical_distributions_meta": categorical_distributions_meta, "profile_thresholds": { "missing_warning": _OVERVIEW_MISSING_WARNING, "missing_poor": _OVERVIEW_MISSING_POOR, @@ -646,16 +698,19 @@ def _build_impact_on_ai_section(file_info, include_visualizations=False): df.columns = [str(c) for c in df.columns] kept, dropped = _prune_columns_for_corr(df) + dropped_capped, dropped_meta = _cap_detail_list( + dropped, _READINESS_MAX_DETAIL_LIST_ITEMS + ) if len(kept) < 2: return { "error": "Not enough usable columns for correlation analysis after pruning.", "columns_analyzed": len(kept), - "columns_dropped": dropped, + "columns_dropped": dropped_capped, } corr = calc_correlations(kept, file_info, include_visualization=include_visualizations) if isinstance(corr, dict) and "Message" in corr: - return {"error": corr["Message"], "columns_dropped": dropped} + return {"error": corr["Message"], "columns_dropped": dropped_capped} scores = corr.get("Correlation Scores", {}) if isinstance(corr, dict) else {} signals = _pairwise_signals(scores) @@ -728,7 +783,8 @@ def _build_impact_on_ai_section(file_info, include_visualizations=False): "columns (numerical prioritized)." ), "selected": kept, - "excluded": dropped, + "excluded": dropped_capped, + "excluded_meta": dropped_meta, }, "thresholds": { "redundant_threshold": _CORR_REDUNDANT_THRESHOLD, @@ -744,7 +800,7 @@ def _build_impact_on_ai_section(file_info, include_visualizations=False): "isolated_features": isolated_features, }, "top_pairs": top_pairs, - "columns_dropped": dropped, + "columns_dropped": dropped_capped, "redundant_pairs": redundant_pairs, "leakage_pairs": leakage_pairs, "isolated_features": isolated_features, @@ -867,6 +923,10 @@ def _auto_select_fairness_columns(df): primary_sensitive = selected_sensitive[0] if selected_sensitive else None + excluded_capped, excluded_meta = _cap_detail_list( + excluded_sensitive, _READINESS_MAX_DETAIL_LIST_ITEMS + ) + return { "sensitive_columns": selected_sensitive, "primary_sensitive": primary_sensitive, @@ -883,7 +943,8 @@ def _auto_select_fairness_columns(df): ), "name_hints": list(_FAIRNESS_SENSITIVE_NAME_HINTS), "selected": selected_sensitive, - "excluded": excluded_sensitive, + "excluded": excluded_capped, + "excluded_meta": excluded_meta, }, "target_column": { "rule": ( @@ -1490,6 +1551,15 @@ def _auto_select_governance_columns(df, fairness_target=None): ) dp_features = [c["feature"] for c in dp_candidates[:_GOV_DP_MAX_FEATURES]] + qi_excluded_raw = [e for e in excluded if e["role"] == "quasi-identifier"] + sens_excluded_raw = [e for e in excluded if e["role"] == "sensitive"] + qi_excluded, qi_excluded_meta = _cap_detail_list( + qi_excluded_raw, _READINESS_MAX_DETAIL_LIST_ITEMS + ) + sens_excluded, sens_excluded_meta = _cap_detail_list( + sens_excluded_raw, _READINESS_MAX_DETAIL_LIST_ITEMS + ) + return { "quasi_identifiers": quasi_identifiers, "mm_quasi_identifiers": mm_quasi_identifiers, @@ -1510,7 +1580,8 @@ def _auto_select_governance_columns(df, fairness_target=None): ), "name_hints": list(_GOV_QI_NAME_HINTS), "selected": quasi_identifiers, - "excluded": [e for e in excluded if e["role"] == "quasi-identifier"], + "excluded": qi_excluded, + "excluded_meta": qi_excluded_meta, }, "sensitive_attribute": { "rule": ( @@ -1520,7 +1591,8 @@ def _auto_select_governance_columns(df, fairness_target=None): ), "name_hints": list(_GOV_SENSITIVE_NAME_HINTS), "selected": sensitive_col, - "excluded": [e for e in excluded if e["role"] == "sensitive"], + "excluded": sens_excluded, + "excluded_meta": sens_excluded_meta, }, "id_column": { "rule": ( diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 12464cb2..6c6a26ca 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -20,6 +20,20 @@ const _READINESS_REPORT_SECTIONS = [ "data-governance", ]; const _readinessSectionStatus = {}; +const _READINESS_MAX_DETAIL_LIST_ITEMS = 50; +const _READINESS_MAX_DETAIL_TABLE_ROWS = 500; + +/** Render up to *maxItems* list entries plus a “+N more” tail when truncated. */ +function _readinessTruncatedListItems(items, maxItems, renderItem) { + const total = items.length; + const shown = items.slice(0, maxItems); + let html = shown.map(renderItem).join(""); + const more = total - shown.length; + if (more > 0) { + html += `
  • +${more} more not shown
    • `; + } + return { html, total, shownCount: shown.length }; +} /** * Show a metric panel by ID, hiding all others. @@ -2551,26 +2565,27 @@ function _stripReadinessPdfUnsafeClasses(root) { root.querySelectorAll("*").forEach(strip); } -function _prepareReadinessPanelClone(source) { - const clone = source.cloneNode(true); - clone.removeAttribute("id"); - clone.classList.remove("metric-panel", "hidden"); - clone.classList.add("readiness-pdf-export"); - clone.querySelector("#readiness-export-bar")?.remove(); - clone.querySelectorAll("details").forEach((el) => el.remove()); - clone.querySelectorAll(".readiness-viz-slot").forEach((el) => el.remove()); - clone.querySelectorAll(".info-icon").forEach((el) => el.remove()); - clone +function _pruneReadinessNodeForPdf(node) { + node.querySelectorAll("details").forEach((el) => el.remove()); + node.querySelectorAll(".readiness-viz-slot").forEach((el) => el.remove()); + node.querySelectorAll(".info-icon").forEach((el) => el.remove()); + node.querySelectorAll(".readiness-build-time").forEach((el) => el.remove()); + node .querySelectorAll("#readiness-categorical-charts, #readiness-histograms-inner") .forEach((el) => el.remove()); - _stripReadinessPdfUnsafeClasses(clone); - clone.style.cssText = + return node; +} + +function _normalizeReadinessPdfRoot(root) { + root.classList.add("readiness-pdf-export"); + _stripReadinessPdfUnsafeClasses(root); + root.style.cssText = "background:#ffffff;color:#111827;padding:0;opacity:1;animation:none;"; - clone.querySelectorAll(".bg-white, .bg-gray-50").forEach((el) => { + root.querySelectorAll(".bg-white, .bg-gray-50").forEach((el) => { el.style.backgroundColor = el.classList.contains("bg-gray-50") ? "#f9fafb" : "#ffffff"; el.style.color = "#111827"; }); - clone + root .querySelectorAll( ".bg-blue-50, .bg-amber-50, .bg-green-50, .bg-red-50, .bg-green-100, .bg-amber-100, .bg-red-100", ) @@ -2583,7 +2598,16 @@ function _prepareReadinessPanelClone(source) { else if (el.classList.contains("bg-amber-100")) el.style.backgroundColor = "#fef3c7"; else if (el.classList.contains("bg-red-100")) el.style.backgroundColor = "#fee2e2"; }); - return clone; + return root; +} + +/** Clone only scorecard sections for PDF (avoids copying huge collapsed detail trees). */ +function _buildSlimReadinessPanelForPdf(panel) { + const root = document.createElement("div"); + panel.querySelectorAll(":scope > div.rounded-lg.shadow-sm").forEach((card) => { + root.appendChild(_pruneReadinessNodeForPdf(card.cloneNode(true))); + }); + return _normalizeReadinessPdfRoot(root); } function _readinessPdfFilename() { @@ -2609,7 +2633,7 @@ async function _exportReadinessReportPdf() { let host = null; try { const html2pdf = await _loadHtml2PdfLib(); - const clone = _prepareReadinessPanelClone(panel); + const clone = _buildSlimReadinessPanelForPdf(panel); host = document.createElement("div"); host.style.cssText = "position:fixed;left:-10000px;top:0;width:900px;background:#fff;padding:0;margin:0;"; @@ -2626,7 +2650,7 @@ async function _exportReadinessReportPdf() { filename: _readinessPdfFilename(), image: { type: "jpeg", quality: 0.95 }, html2canvas: { - scale: 2, + scale: 1, useCORS: true, logging: false, backgroundColor: "#ffffff", @@ -2936,7 +2960,9 @@ function renderReadinessDatasetOverview(container, overview) { let detailsInner = ""; const numSummary = overview.numerical_summary || {}; - const numFeatures = Object.keys(numSummary); + const numMeta = overview.numerical_summary_meta || {}; + const allNumFeatures = Object.keys(numSummary); + const numFeatures = allNumFeatures.slice(0, _READINESS_MAX_DETAIL_TABLE_ROWS); if (numFeatures.length > 0) { const allStats = Object.keys(numSummary[numFeatures[0]] || {}); const preferredOrder = [ @@ -2976,19 +3002,31 @@ function renderReadinessDatasetOverview(container, overview) { detailsInner += `    `; }); detailsInner += `
    `; + if (numMeta.truncated || allNumFeatures.length > numFeatures.length) { + const total = numMeta.total || allNumFeatures.length; + const shown = numMeta.shown || numFeatures.length; + detailsInner += `

    Showing first ${shown} of ${total} numerical features. Profile table lists prioritized features.

    `; + } } const catCharts = overview.categorical_charts || {}; + const catDistMeta = overview.categorical_distributions_meta || {}; const catChartCols = Object.keys(catCharts); const hasHistograms = overview.histograms && Object.keys(overview.histograms).length > 0; const vizDeferred = overview.visualizations_deferred; - const showCatCharts = catChartCols.length > 0 || (vizDeferred && (overview.categorical_distributions || {}).length); + const catDists = overview.categorical_distributions || {}; + const showCatCharts = + catChartCols.length > 0 || + (vizDeferred && Object.keys(catDists).length > 0); const showHistograms = - hasHistograms || (vizDeferred && numFeatures.length > 0); + hasHistograms || (vizDeferred && (allNumFeatures.length > 0 || numMeta.total > 0)); if (showCatCharts) { detailsInner += `

    Categorical value distributions

    `; + if (catDistMeta.truncated) { + detailsInner += `

    Showing first ${catDistMeta.shown} of ${catDistMeta.total} categorical features.

    `; + } detailsInner += `
    `; } @@ -3412,15 +3450,17 @@ function renderReadinessImpact(container, impact) { ); } if (dropped.length) { - const items = dropped - .map( - (d) => - `
  • ${d.feature}${d.reason}
    • `, - ) - .join(""); + const excludedMeta = colCrit.excluded_meta || {}; + const excludedTotal = excludedMeta.total || dropped.length; + const { html: items } = _readinessTruncatedListItems( + dropped, + _READINESS_MAX_DETAIL_LIST_ITEMS, + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ); detailsInner += `
    -

    Excluded columns (${dropped.length})

    +

    Excluded columns (${excludedTotal})

      ${items}
    `; } @@ -3727,15 +3767,17 @@ function renderReadinessFairness(container, fb) { const excluded = sensCrit.excluded || []; if (excluded.length) { - const items = excluded - .map( - (d) => - `
  • ${d.feature}${d.reason}
    • `, - ) - .join(""); + const excludedMeta = sensCrit.excluded_meta || {}; + const excludedTotal = excludedMeta.total || excluded.length; + const { html: items } = _readinessTruncatedListItems( + excluded, + _READINESS_MAX_DETAIL_LIST_ITEMS, + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ); detailsInner += `
    -

    Excluded sensitive candidates (${excluded.length})

    +

    Excluded sensitive candidates (${excludedTotal})

      ${items}
    `; } @@ -4045,15 +4087,17 @@ function renderReadinessGovernance(container, gov) { const qiExcluded = qiCrit.excluded || []; if (qiExcluded.length) { - const items = qiExcluded - .map( - (d) => - `
  • ${d.feature}${d.reason}
    • `, - ) - .join(""); + const excludedMeta = qiCrit.excluded_meta || {}; + const excludedTotal = excludedMeta.total || qiExcluded.length; + const { html: items } = _readinessTruncatedListItems( + qiExcluded, + _READINESS_MAX_DETAIL_LIST_ITEMS, + (d) => + `
  • ${d.feature}${d.reason}
    • `, + ); detailsInner += `
    -

    Excluded quasi-identifier candidates (${qiExcluded.length})

    +

    Excluded quasi-identifier candidates (${excludedTotal})

      ${items}
    `; } From ac0c0d0ddc579b2feeb2f0d57293dc7f20827488 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 30 Jun 2026 11:57:19 -0400 Subject: [PATCH 16/23] Limit readiness overview detail charts to capped profile features. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate histograms and categorical pie charts only for the features shown in the profile table (up to 500, poor → warning → good), including on-demand chart loads. Update the details panel to reflect this scope. --- tests/unit/test_readiness_overview.py | 14 ++++++++++++++ web/routes/metrics.py | 20 ++++++++++++++++---- web/static/js/inspector.js | 15 ++++++++------- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_readiness_overview.py b/tests/unit/test_readiness_overview.py index a5cf7b15..75300850 100644 --- a/tests/unit/test_readiness_overview.py +++ b/tests/unit/test_readiness_overview.py @@ -8,6 +8,7 @@ _build_categorical_distributions, _build_numerical_summary_for_overview, _cap_detail_list, + _dataframe_for_overview_detail_charts, _prepare_feature_profiles_for_display, ) @@ -45,6 +46,19 @@ def test_caps_wide_categorical_distributions(self): self.assertEqual(meta["total"], 80) +class TestDataframeForOverviewDetailCharts(unittest.TestCase): + def test_uses_capped_profile_features_only(self): + profiles = ( + [{"feature": f"p{i}", "status": "poor", "type": "numerical"} for i in range(3)] + + [{"feature": f"g{i}", "status": "good", "type": "numerical"} for i in range(10)] + ) + df = pd.DataFrame({p["feature"]: range(5) for p in profiles}) + display, meta = _prepare_feature_profiles_for_display(profiles, max_profiles=5) + self.assertTrue(meta["truncated"]) + chart_df = _dataframe_for_overview_detail_charts(df, display) + self.assertEqual(list(chart_df.columns), [p["feature"] for p in display]) + + class TestPrepareFeatureProfilesForDisplay(unittest.TestCase): def test_no_truncation_when_under_cap(self): profiles = [ diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 6cfd25c0..def96242 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -341,6 +341,14 @@ def _prepare_feature_profiles_for_display( return [profile for _, profile in ranked[:max_profiles]], meta +def _dataframe_for_overview_detail_charts(df, display_profiles): + """Subset *df* to capped profile-table features for overview detail charts.""" + cols = [p["feature"] for p in display_profiles if p["feature"] in df.columns] + if not cols: + return df.iloc[:, :0] + return df[cols] + + def _build_categorical_distributions(df, top_n=_OVERVIEW_CAT_TOP_N, max_columns=None): """Top-*n* value counts (with percentages) for each categorical column.""" distributions = {} @@ -436,8 +444,9 @@ def _build_dataset_overview_section(file_info, include_visualizations=False): "visualizations_deferred": not include_visualizations, } if include_visualizations: - overview["categorical_charts"] = categorical_distribution_charts(df) - overview["histograms"] = summary_histograms(df, figsize=(7, 4.5)) + chart_df = _dataframe_for_overview_detail_charts(df, display_profiles) + overview["categorical_charts"] = categorical_distribution_charts(chart_df) + overview["histograms"] = summary_histograms(chart_df, figsize=(7, 4.5)) return overview @@ -2008,9 +2017,12 @@ def _build_dataset_overview_visualizations(file_info): df = read_file(file_info) if hasattr(df, "columns"): df.columns = [str(c) for c in df.columns] + profiles, _ = _build_feature_profiles(df) + display_profiles, _ = _prepare_feature_profiles_for_display(profiles) + chart_df = _dataframe_for_overview_detail_charts(df, display_profiles) return { - "categorical_charts": categorical_distribution_charts(df), - "histograms": summary_histograms(df, figsize=(7, 4.5)), + "categorical_charts": categorical_distribution_charts(chart_df), + "histograms": summary_histograms(chart_df, figsize=(7, 4.5)), } diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 6c6a26ca..da9f0879 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -3010,23 +3010,24 @@ function renderReadinessDatasetOverview(container, overview) { } const catCharts = overview.categorical_charts || {}; - const catDistMeta = overview.categorical_distributions_meta || {}; const catChartCols = Object.keys(catCharts); const hasHistograms = overview.histograms && Object.keys(overview.histograms).length > 0; const vizDeferred = overview.visualizations_deferred; - const catDists = overview.categorical_distributions || {}; + const profileNumericalCount = profiles.filter((p) => p.type === "numerical").length; + const profileCategoricalCount = profiles.filter((p) => p.type === "categorical").length; const showCatCharts = catChartCols.length > 0 || - (vizDeferred && Object.keys(catDists).length > 0); + (vizDeferred && profileCategoricalCount > 0); const showHistograms = - hasHistograms || (vizDeferred && (allNumFeatures.length > 0 || numMeta.total > 0)); + hasHistograms || (vizDeferred && profileNumericalCount > 0); + + if ((showCatCharts || showHistograms) && profileMeta.truncated) { + detailsInner += `

    Distribution charts use the same ${profileMeta.shown.toLocaleString()} features shown in the profile table above (prioritized: poor → warning → good).

    `; + } if (showCatCharts) { detailsInner += `

    Categorical value distributions

    `; - if (catDistMeta.truncated) { - detailsInner += `

    Showing first ${catDistMeta.shown} of ${catDistMeta.total} categorical features.

    `; - } detailsInner += `
    `; } From 83a79613b35bd731bd5b3b0f0bb2b75ab9b3779e Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 30 Jun 2026 12:27:59 -0400 Subject: [PATCH 17/23] Add optional FAIR Compliance section to readiness report. Append a metadata-driven FAIR card at the bottom of the report that reuses POST /fair-assessment and shared render helpers. Automated sections still gate PDF export; FAIR is included only after a successful evaluation. --- web/static/js/inspector.js | 343 ++++++++++++------- web/templates/_panels/_readiness_report.html | 41 +++ 2 files changed, 254 insertions(+), 130 deletions(-) diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index da9f0879..df6c3c0d 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -20,6 +20,8 @@ const _READINESS_REPORT_SECTIONS = [ "data-governance", ]; const _readinessSectionStatus = {}; +/** Optional FAIR Compliance state for the readiness report appendix. */ +let _readinessFairCompliance = { status: "idle", data: null }; const _READINESS_MAX_DETAIL_LIST_ITEMS = 50; const _READINESS_MAX_DETAIL_TABLE_ROWS = 500; @@ -1735,125 +1737,216 @@ function escapeHtml(str) { // ==================== FAIR Assessment ==================== -function submitFairAssessment() { - const form = document.getElementById("form-fair-assessment"); - if (!form) return; - - const formData = new FormData(form); - const resultContainer = document.getElementById("fair-result-container"); - if (resultContainer) - resultContainer.innerHTML = '

    Processing...

    '; +function _wireFairFileInput(fileInput, labelEl, iconEl) { + if (!fileInput || !labelEl) return; + fileInput.addEventListener("change", () => { + if (fileInput.files.length) { + labelEl.textContent = fileInput.files[0].name; + if (iconEl) { + iconEl.innerHTML = + ''; + iconEl.classList.remove("text-gray-400"); + iconEl.classList.add("text-green-500"); + } + } else { + labelEl.textContent = "JSON metadata file"; + if (iconEl) { + iconEl.innerHTML = + ''; + iconEl.classList.remove("text-green-500"); + iconEl.classList.add("text-gray-400"); + } + } + }); +} - // Return the promise so withSubmitGuard re-enables the button when it settles. - return fetch("/fair-assessment", { method: "POST", body: formData }) - .then((response) => response.json()) - .then((data) => { - if (!resultContainer) return; +/** Build FAIR assessment result HTML (shared by panel and readiness report). */ +function buildFairAssessmentResultHtml(data) { + const checks = data["FAIR Compliance Checks"] || {}; + const totalCheck = checks["Total Checks"] || ""; + const totalMatch = totalCheck.match(/(\d+)\/(\d+)/); + const totalPassed = totalMatch ? parseInt(totalMatch[1], 10) : 0; + const totalExpected = totalMatch ? parseInt(totalMatch[2], 10) : 1; + const totalPct = Math.round((totalPassed / totalExpected) * 100); + + let html = `
    +
    +

    FAIR Compliance

    + ${totalPassed}/${totalExpected} checks passed +
    +
    +
    +
    +
    `; + + const fairKeys = ["Findable", "Accessible", "Interoperable", "Reusable"]; + fairKeys.forEach((k) => { + const checkStr = checks[`${k} Checks`] || "0/0"; + const m = checkStr.match(/(\d+)\/(\d+)/); + const passed = m ? parseInt(m[1], 10) : 0; + const total = m ? parseInt(m[2], 10) : 1; + const pct = Math.round((passed / total) * 100); + html += `
    +
    ${k}
    +
    ${passed}/${total}
    +
    +
    +
    +
    `; + }); + html += "
    "; + + html += '
    '; + fairKeys.forEach((k) => { + let val = "—"; + const checkStr = checks[`${k} Checks`] || ""; + if (data[k] !== undefined && typeof data[k] === "object") { + val = renderFairValue(data[k]); + } else if (data[k] !== undefined) { + val = `
    ${data[k]}
    `; + } + html += `
    + + ${k} + ${checkStr} + +
    ${val}
    +
    `; + }); + html += "
    "; - // Check for error response - if (data.error) { - resultContainer.innerHTML = ``; - return; + const extraKeys = Object.keys(data).filter( + (k) => !fairKeys.includes(k) && k !== "Pie chart", + ); + if (extraKeys.length > 0) { + html += + '
    '; + html += + '

    Detailed Results

    '; + extraKeys.forEach((k) => { + const val = data[k]; + if (typeof val === "object" && val !== null) { + html += `
    + + ${k} + + +
    + 
    ${escapeHtml(JSON.stringify(val, null, 2))}
    +
    +
    `; + } else { + html += `
    + ${k} + ${val ?? "—"} +
    `; } + }); + html += "
    "; + } - let html = ""; + return html; +} - // Compliance summary bar — extract from FAIR Compliance Checks - const checks = data["FAIR Compliance Checks"] || {}; - const totalCheck = checks["Total Checks"] || ""; - const totalMatch = totalCheck.match(/(\d+)\/(\d+)/); - const totalPassed = totalMatch ? parseInt(totalMatch[1]) : 0; - const totalExpected = totalMatch ? parseInt(totalMatch[2]) : 1; - const totalPct = Math.round((totalPassed / totalExpected) * 100); +function renderFairAssessmentResult(data, resultContainer) { + if (!resultContainer) return false; + if (data.error) { + resultContainer.innerHTML = ``; + return false; + } + resultContainer.innerHTML = buildFairAssessmentResultHtml(data); + return true; +} - html += `
    -
    -

    FAIR Compliance

    - ${totalPassed}/${totalExpected} checks passed -
    -
    -
    -
    -
    `; - - // Per-principle mini bars - const fairKeys = ["Findable", "Accessible", "Interoperable", "Reusable"]; - fairKeys.forEach((k) => { - const checkStr = checks[`${k} Checks`] || "0/0"; - const m = checkStr.match(/(\d+)\/(\d+)/); - const passed = m ? parseInt(m[1]) : 0; - const total = m ? parseInt(m[2]) : 1; - const pct = Math.round((passed / total) * 100); - html += `
    -
    ${k}
    -
    ${passed}/${total}
    -
    -
    -
    -
    `; - }); - html += "
    "; - - // FAIR principle details as collapsible accordions - html += '
    '; - fairKeys.forEach((k) => { - let val = "—"; - let checkStr = checks[`${k} Checks`] || ""; - if (data[k] !== undefined && typeof data[k] === "object") { - val = renderFairValue(data[k]); - } else if (data[k] !== undefined) { - val = `
    ${data[k]}
    `; - } - html += `
    - - ${k} - ${checkStr} - -
    ${val}
    -
    `; - }); - html += "
    "; +function submitFairAssessmentForm(form, resultContainer, callbacks) { + if (!form) return Promise.resolve(); - // Other data (FAIR Compliance Checks, Other, Original Metadata) - const extraKeys = Object.keys(data).filter( - (k) => !fairKeys.includes(k) && k !== "Pie chart", - ); - if (extraKeys.length > 0) { - html += - '
    '; - html += - '

    Detailed Results

    '; - extraKeys.forEach((k) => { - const val = data[k]; - if (typeof val === "object" && val !== null) { - html += `
    - - ${k} - - -
    - 
    ${escapeHtml(JSON.stringify(val, null, 2))}
    -
    -
    `; - } else { - html += `
    - ${k} - ${val ?? "—"} -
    `; - } - }); - html += "
    "; - } + const formData = new FormData(form); + if (resultContainer) { + resultContainer.classList.remove("hidden"); + resultContainer.innerHTML = '

    Processing…

    '; + } - resultContainer.innerHTML = html; + return fetch("/fair-assessment", { method: "POST", body: formData }) + .then((response) => response.json()) + .then((data) => { + const ok = renderFairAssessmentResult(data, resultContainer); + if (ok) { + callbacks?.onSuccess?.(data); + } else { + callbacks?.onError?.(data); + } }) .catch((error) => { console.error("Error:", error); - if (resultContainer) - resultContainer.innerHTML = ``; + if (resultContainer) { + resultContainer.innerHTML = ``; + } + callbacks?.onError?.(error); }); } +function submitFairAssessment() { + return submitFairAssessmentForm( + document.getElementById("form-fair-assessment"), + document.getElementById("fair-result-container"), + ); +} + +function submitReadinessFairAssessment() { + _readinessFairCompliance = { status: "computing", data: null }; + const uploadEl = document.getElementById("readiness-fair-upload"); + const resultsEl = document.getElementById("readiness-fair-results"); + if (resultsEl) resultsEl.classList.remove("hidden"); + + return submitFairAssessmentForm( + document.getElementById("form-readiness-fair"), + resultsEl, + { + onSuccess(data) { + _readinessFairCompliance = { status: "ok", data }; + uploadEl?.classList.add("hidden"); + }, + onError() { + _readinessFairCompliance = { status: "error", data: null }; + }, + }, + ); +} + +function _resetReadinessFairSection() { + _readinessFairCompliance = { status: "idle", data: null }; + const uploadEl = document.getElementById("readiness-fair-upload"); + const resultsEl = document.getElementById("readiness-fair-results"); + const form = document.getElementById("form-readiness-fair"); + uploadEl?.classList.remove("hidden"); + if (resultsEl) { + resultsEl.classList.add("hidden"); + resultsEl.innerHTML = ""; + } + form?.reset(); + const label = document.getElementById("readinessFairFileLabel"); + const icon = document.getElementById("readinessFairUploadIcon"); + if (label) label.textContent = "JSON metadata file"; + if (icon) { + icon.innerHTML = + ''; + icon.classList.remove("text-green-500"); + icon.classList.add("text-gray-400"); + } +} + +function initReadinessFairSection() { + const form = document.getElementById("form-readiness-fair"); + if (!form || form.dataset.wired === "1") return; + form.dataset.wired = "1"; + _wireFairFileInput( + document.getElementById("readiness-fair-file"), + document.getElementById("readinessFairFileLabel"), + document.getElementById("readinessFairUploadIcon"), + ); +} + /** Render a FAIR value object as readable HTML with pass/fail badges */ function renderFairValue(obj) { if (typeof obj !== "object" || obj === null) return String(obj ?? "—"); @@ -2459,6 +2552,8 @@ function _fetchReadinessSection(section, container, renderFn) { */ function loadReadinessReport() { _wireReadinessExportButton(); + _resetReadinessFairSection(); + initReadinessFairSection(); _READINESS_REPORT_SECTIONS.forEach((section) => { _readinessSectionStatus[section] = "pending"; }); @@ -2570,6 +2665,7 @@ function _pruneReadinessNodeForPdf(node) { node.querySelectorAll(".readiness-viz-slot").forEach((el) => el.remove()); node.querySelectorAll(".info-icon").forEach((el) => el.remove()); node.querySelectorAll(".readiness-build-time").forEach((el) => el.remove()); + node.querySelectorAll("#readiness-fair-upload").forEach((el) => el.remove()); node .querySelectorAll("#readiness-categorical-charts, #readiness-histograms-inner") .forEach((el) => el.remove()); @@ -2603,9 +2699,15 @@ function _normalizeReadinessPdfRoot(root) { /** Clone only scorecard sections for PDF (avoids copying huge collapsed detail trees). */ function _buildSlimReadinessPanelForPdf(panel) { + const includeFair = _readinessFairCompliance?.status === "ok"; const root = document.createElement("div"); panel.querySelectorAll(":scope > div.rounded-lg.shadow-sm").forEach((card) => { - root.appendChild(_pruneReadinessNodeForPdf(card.cloneNode(true))); + if (card.dataset.readinessPdf === "conditional" && !includeFair) return; + const clone = _pruneReadinessNodeForPdf(card.cloneNode(true)); + clone.querySelector("#readiness-fair-upload")?.remove(); + const fairResults = clone.querySelector("#readiness-fair-results"); + if (fairResults) fairResults.classList.remove("hidden"); + root.appendChild(clone); }); return _normalizeReadinessPdfRoot(root); } @@ -4182,30 +4284,11 @@ function initWorkspace() { } // Handle FAIR assessment file input UI - const fairFile = document.getElementById("fair-file"); - const fairLabel = document.getElementById("fairFileLabel"); - const fairIcon = document.getElementById("fairUploadIcon"); - if (fairFile && fairLabel) { - fairFile.addEventListener("change", () => { - if (fairFile.files.length) { - fairLabel.textContent = fairFile.files[0].name; - if (fairIcon) { - fairIcon.innerHTML = - ''; - fairIcon.classList.remove("text-gray-400"); - fairIcon.classList.add("text-green-500"); - } - } else { - fairLabel.textContent = "JSON metadata file"; - if (fairIcon) { - fairIcon.innerHTML = - ''; - fairIcon.classList.remove("text-green-500"); - fairIcon.classList.add("text-gray-400"); - } - } - }); - } + _wireFairFileInput( + document.getElementById("fair-file"), + document.getElementById("fairFileLabel"), + document.getElementById("fairUploadIcon"), + ); } /** diff --git a/web/templates/_panels/_readiness_report.html b/web/templates/_panels/_readiness_report.html index 8ec5593d..fee8ab90 100644 --- a/web/templates/_panels/_readiness_report.html +++ b/web/templates/_panels/_readiness_report.html @@ -78,4 +78,45 @@

    Data Gove

    Computing governance metrics...

    + + +
    +
    +

    FAIR Compliance

    + Optional · requires metadata +
    +

    + Upload a JSON metadata file (DCAT or Datacite) to evaluate FAIR compliance. This section is not computed automatically. +

    +
    +
    +
    + + +
    +
    + +
    +
    + +
    +
    +
    + +
    From c8f6094222ab1d21f4164d1606a618ab3d2143fc Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 21 Jul 2026 14:57:17 -0400 Subject: [PATCH 18/23] Cache readiness report sections server-side to avoid recomputation on reload Store each readiness section and its on-demand visualizations in TEMP_RESULTS_CACHE under user/file-scoped keys with a 30-minute TTL. Reuse existing upload invalidation via clear_all_user_cache; expose cached status in section, visualization, and full-report API responses. Add unit tests for key format, store/retrieve, error skip, expiry, and upload-style clearing. --- tests/unit/test_readiness_cache.py | 100 ++++++++++++++++++++++ web/routes/metrics.py | 130 +++++++++++++++++++++++------ 2 files changed, 206 insertions(+), 24 deletions(-) create mode 100644 tests/unit/test_readiness_cache.py diff --git a/tests/unit/test_readiness_cache.py b/tests/unit/test_readiness_cache.py new file mode 100644 index 00000000..867646b1 --- /dev/null +++ b/tests/unit/test_readiness_cache.py @@ -0,0 +1,100 @@ +"""Unit tests for readiness report server-side section cache.""" + +import time +import unittest +from unittest.mock import MagicMock, patch + +from flask import Flask + +from web.routes.metrics import ( + _cache_readiness_payload, + _get_cached_readiness_payload, + _readiness_section_cache_key, +) + + +class TestReadinessSectionCacheKey(unittest.TestCase): + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + def test_section_key_matches_cached_result_pattern(self, _mock_user): + key = _readiness_section_cache_key("data.csv", "data-quality") + self.assertEqual(key, "user:user-1:file:data.csv:readiness_report:data-quality") + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + def test_visualization_key_is_per_section(self, _mock_user): + key = _readiness_section_cache_key("data.csv", "dataset-overview", viz=True) + self.assertEqual( + key, + "user:user-1:file:data.csv:readiness_report:dataset-overview:visualizations", + ) + + +class TestReadinessSectionCacheStore(unittest.TestCase): + def setUp(self): + self.app = Flask(__name__) + self.app.TEMP_RESULTS_CACHE = {} + self.app_context = self.app.app_context() + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_store_and_retrieve_section_payload(self, mock_current_app, _mock_user): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + payload = {"grade": 0.9, "kpis": []} + _cache_readiness_payload("data.csv", "data-quality", payload, 1.23) + + cached, build_time = _get_cached_readiness_payload("data.csv", "data-quality") + self.assertEqual(cached, payload) + self.assertEqual(build_time, 1.23) + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_error_sections_are_not_cached(self, mock_current_app, _mock_user): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + _cache_readiness_payload( + "data.csv", "impact-on-ai", {"error": "failed"}, 0.5 + ) + cached, _ = _get_cached_readiness_payload("data.csv", "impact-on-ai") + self.assertIsNone(cached) + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_expired_entries_are_evicted(self, mock_current_app, _mock_user): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + key = _readiness_section_cache_key("data.csv", "data-quality") + self.app.TEMP_RESULTS_CACHE[key] = { + "data": {"grade": 1.0}, + "timestamp": time.time() - 7200, + "expires_at": time.time() - 1, + "build_time_seconds": 2.0, + } + cached, _ = _get_cached_readiness_payload("data.csv", "data-quality") + self.assertIsNone(cached) + self.assertNotIn(key, self.app.TEMP_RESULTS_CACHE) + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_upload_style_clear_removes_user_keys(self, mock_current_app, _mock_user): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + _cache_readiness_payload("data.csv", "data-quality", {"grade": 0.8}, 1.0) + _cache_readiness_payload( + "data.csv", "dataset-overview", {"histograms": {}}, 2.0, viz=True + ) + self.assertEqual(len(self.app.TEMP_RESULTS_CACHE), 2) + + keys_to_remove = [ + key + for key in self.app.TEMP_RESULTS_CACHE + if key.startswith("user:user-1") + ] + for key in keys_to_remove: + self.app.TEMP_RESULTS_CACHE.pop(key, None) + + cached, _ = _get_cached_readiness_payload("data.csv", "data-quality") + self.assertIsNone(cached) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/routes/metrics.py b/web/routes/metrics.py index def96242..9fb46c4f 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -59,6 +59,7 @@ ensure_json_serializable, format_dict_values, generate_metric_cache_key, + get_current_user_id, get_result_or_default, is_metric_cache_valid, store_result, @@ -2176,6 +2177,81 @@ def _build_data_governance_visualizations(file_info): "data-governance": "data_governance", } +_READINESS_CACHE_TTL_SECONDS = 30 * 60 + + +def _readiness_section_cache_key(file_name, section, *, viz=False): + """User/file-scoped cache key (same colon pattern as ``/cached-result``).""" + user_id = get_current_user_id() + if viz: + return f"user:{user_id}:file:{file_name}:readiness_report:{section}:visualizations" + return f"user:{user_id}:file:{file_name}:readiness_report:{section}" + + +def _get_cached_readiness_payload(file_name, section, *, viz=False): + """Return cached section or visualization payload, or *(None, None)*.""" + key = _readiness_section_cache_key(file_name, section, viz=viz) + entry = current_app.TEMP_RESULTS_CACHE.get(key) + if entry and is_metric_cache_valid(entry): + return entry.get("data"), entry.get("build_time_seconds") + if entry: + current_app.TEMP_RESULTS_CACHE.pop(key, None) + return None, None + + +def _cache_readiness_payload(file_name, section, data, build_time_seconds, *, viz=False): + """Store a successful readiness section or visualization payload.""" + if data is None: + return + if not viz and isinstance(data, dict) and data.get("error"): + return + key = _readiness_section_cache_key(file_name, section, viz=viz) + current_app.TEMP_RESULTS_CACHE[key] = { + "data": ensure_json_serializable(data), + "timestamp": time.time(), + "expires_at": time.time() + _READINESS_CACHE_TTL_SECONDS, + "build_time_seconds": build_time_seconds, + } + + +def _get_or_build_readiness_section(section, file_info, include_visualizations=False): + """Return section data from cache or build, store on miss.""" + file_name = file_info[1] + cached, build_time = _get_cached_readiness_payload(file_name, section) + if cached is not None: + metric_time_log.info("Readiness report section %s cache hit", section) + return cached, build_time, True + + start_time = time.time() + data = _build_readiness_section( + section, file_info, include_visualizations=include_visualizations + ) + build_time_seconds = round(time.time() - start_time, 2) + _cache_readiness_payload(file_name, section, data, build_time_seconds) + return data, build_time_seconds, False + + +def _get_or_build_readiness_visualizations(section, file_info): + """Return visualization payload from cache or build, store on miss.""" + file_name = file_info[1] + cached, build_time = _get_cached_readiness_payload( + file_name, section, viz=True + ) + if cached is not None: + metric_time_log.info( + "Readiness report section %s visualizations cache hit", section + ) + return cached, build_time, True + + builder = _READINESS_VIZ_BUILDERS.get(section) + start_time = time.time() + visualizations = builder(file_info) if builder else {} + build_time_seconds = round(time.time() - start_time, 2) + _cache_readiness_payload( + file_name, section, visualizations, build_time_seconds, viz=True + ) + return visualizations, build_time_seconds, False + def _readiness_file_info(): """Return ``(file_path, file_name, file_type)`` from session, or *None*.""" @@ -2213,9 +2289,10 @@ def readiness_report_visualizations(section): if file_info is None: return jsonify({"success": False, "message": "No file uploaded"}), 200 - start_time = time.time() try: - visualizations = _READINESS_VIZ_BUILDERS[section](file_info) + visualizations, build_time_seconds, from_cache = ( + _get_or_build_readiness_visualizations(section, file_info) + ) except Exception as e: metric_time_log.error( "Readiness report visualizations — %s error: %s", section, e, exc_info=True @@ -2225,17 +2302,18 @@ def readiness_report_visualizations(section): "message": f"{type(e).__name__}: {e}", }), 200 - build_time_seconds = round(time.time() - start_time, 2) - metric_time_log.info( - "Readiness report section %s visualizations built in %.2f seconds", - section, - build_time_seconds, - ) + if not from_cache: + metric_time_log.info( + "Readiness report section %s visualizations built in %.2f seconds", + section, + build_time_seconds, + ) return jsonify(ensure_json_serializable({ "success": True, "section": section, "visualizations": visualizations, "build_time_seconds": build_time_seconds, + "cached": from_cache, })) @@ -2249,19 +2327,21 @@ def readiness_report_section(section): if file_info is None: return jsonify({"success": False, "message": "No file uploaded"}), 200 - start_time = time.time() - data = _build_readiness_section(section, file_info) - build_time_seconds = round(time.time() - start_time, 2) - metric_time_log.info( - "Readiness report section %s built in %.2f seconds", - section, - build_time_seconds, + data, build_time_seconds, from_cache = _get_or_build_readiness_section( + section, file_info ) + if not from_cache: + metric_time_log.info( + "Readiness report section %s built in %.2f seconds", + section, + build_time_seconds, + ) return jsonify(ensure_json_serializable({ "success": True, "section": section, "data": data, "build_time_seconds": build_time_seconds, + "cached": from_cache, })) @@ -2274,18 +2354,20 @@ def readiness_report(): start_time = time.time() try: - response = {"success": True} + response = {"success": True, "sections_cached": {}} for slug in _READINESS_SECTION_BUILDERS: - section_start = time.time() - section_data = _build_readiness_section(slug, file_info) - section_elapsed = round(time.time() - section_start, 2) + section_data, section_elapsed, from_cache = ( + _get_or_build_readiness_section(slug, file_info) + ) if isinstance(section_data, dict): section_data = {**section_data, "build_time_seconds": section_elapsed} - metric_time_log.info( - "Readiness report section %s built in %.2f seconds", - slug, - section_elapsed, - ) + if not from_cache: + metric_time_log.info( + "Readiness report section %s built in %.2f seconds", + slug, + section_elapsed, + ) + response["sections_cached"][slug] = from_cache response[_READINESS_SECTION_RESPONSE_KEYS[slug]] = section_data metric_time_log.info("Readiness report built in %.2f seconds", time.time() - start_time) From 32d617188ba5736a8e0f54bd0b8fe6430ae400bb Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 21 Jul 2026 15:07:22 -0400 Subject: [PATCH 19/23] Restore readiness report from aggregated server cache on panel open. Expose GET /cached-result/readiness_report to return all cached sections in one response when every section is available. On first panel open, try that endpoint before progressive section fetches so reloads skip spinners and multiple round trips. Add unit tests for aggregated cache lookup. --- tests/unit/test_readiness_cache.py | 46 ++++++++ web/routes/core.py | 8 ++ web/routes/metrics.py | 20 ++++ web/static/js/inspector.js | 170 +++++++++++++++++++++-------- 4 files changed, 197 insertions(+), 47 deletions(-) diff --git a/tests/unit/test_readiness_cache.py b/tests/unit/test_readiness_cache.py index 867646b1..500ab38a 100644 --- a/tests/unit/test_readiness_cache.py +++ b/tests/unit/test_readiness_cache.py @@ -10,6 +10,7 @@ _cache_readiness_payload, _get_cached_readiness_payload, _readiness_section_cache_key, + get_cached_readiness_report, ) @@ -96,5 +97,50 @@ def test_upload_style_clear_removes_user_keys(self, mock_current_app, _mock_user self.assertIsNone(cached) +class TestReadinessAggregatedCache(unittest.TestCase): + def setUp(self): + self.app = Flask(__name__) + self.app.TEMP_RESULTS_CACHE = {} + self.app_context = self.app.app_context() + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_get_cached_readiness_report_returns_all_sections( + self, mock_current_app, _mock_user + ): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + sections = { + "dataset-overview": {"rows": 10}, + "data-quality": {"grade": 0.9}, + "impact-on-ai": {"grade": 0.8}, + "fairness-bias": {"grade": 0.7}, + "data-governance": {"grade": 0.6}, + } + for slug, payload in sections.items(): + _cache_readiness_payload("data.csv", slug, payload, 1.0) + + result = get_cached_readiness_report("data.csv") + self.assertIsNotNone(result) + self.assertTrue(result["cached"]) + self.assertEqual(set(result["sections"].keys()), set(sections.keys())) + self.assertEqual(result["sections"]["data-quality"]["grade"], 0.9) + self.assertEqual(result["sections"]["data-quality"]["build_time_seconds"], 1.0) + self.assertTrue(all(result["sections_cached"].values())) + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_get_cached_readiness_report_requires_every_section( + self, mock_current_app, _mock_user + ): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + _cache_readiness_payload("data.csv", "data-quality", {"grade": 0.9}, 1.0) + + self.assertIsNone(get_cached_readiness_report("data.csv")) + + if __name__ == "__main__": unittest.main() diff --git a/web/routes/core.py b/web/routes/core.py index c5b18b6e..d74e9570 100644 --- a/web/routes/core.py +++ b/web/routes/core.py @@ -300,6 +300,14 @@ def cached_result(metric_name): if not file_name: return jsonify({"cached": False}) + if metric_name == "readiness_report": + from web.routes.metrics import get_cached_readiness_report + + cached_report = get_cached_readiness_report(file_name) + if cached_report: + return jsonify(ensure_json_serializable(cached_report)) + return jsonify({"cached": False}) + cache_key = f"user:{user_id}:file:{file_name}:{metric_name}" entry = current_app.TEMP_RESULTS_CACHE.get(cache_key) if entry and entry.get("data"): diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 9fb46c4f..8965d3ac 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -2231,6 +2231,26 @@ def _get_or_build_readiness_section(section, file_info, include_visualizations=F return data, build_time_seconds, False +def get_cached_readiness_report(file_name): + """Return all cached readiness sections for ``/cached-result/readiness_report``. + + Returns a JSON-serializable dict when every section is cached, else *None*. + """ + sections = {} + sections_cached = {} + for slug in _READINESS_SECTION_BUILDERS: + cached, build_time = _get_cached_readiness_payload(file_name, slug) + if cached is None: + return None + sections[slug] = {**cached, "build_time_seconds": build_time} + sections_cached[slug] = True + return { + "cached": True, + "sections": sections, + "sections_cached": sections_cached, + } + + def _get_or_build_readiness_visualizations(section, file_info): """Return visualization payload from cache or build, store on miss.""" file_name = file_info[1] diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index df6c3c0d..6bc8e1d3 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -88,11 +88,14 @@ function showPanel(panelId, pushHistory) { initCodeMirror(); } - // Lazy load the data overview + data quality into the Readiness Report - // panel on first open + // Lazy load the readiness report on first open; restore from server cache when available if (panelId === "readiness-report" && !_readinessReportLoaded) { _readinessReportLoaded = true; - loadReadinessReport(); + _restoreCachedReadinessReport().then((restored) => { + if (!restored && activePanel === "readiness-report") { + loadReadinessReport(); + } + }); } // Close mobile sidebar after selection @@ -2507,6 +2510,66 @@ function _wireReadinessDetailsViz(detailsEl, section) { * @param {Function} renderFn - (container, data) => void * @returns {Promise} */ +function _getReadinessSectionRenderers() { + return [ + { + section: "dataset-overview", + container: document.getElementById("readiness-summary"), + render: renderReadinessDatasetOverview, + }, + { + section: "data-quality", + container: document.getElementById("readiness-data-quality"), + render: renderReadinessDataQuality, + }, + { + section: "impact-on-ai", + container: document.getElementById("readiness-impact"), + render: renderReadinessImpact, + }, + { + section: "fairness-bias", + container: document.getElementById("readiness-fairness"), + render: renderReadinessFairness, + }, + { + section: "data-governance", + container: document.getElementById("readiness-governance"), + render: renderReadinessGovernance, + }, + ]; +} + +function _initReadinessReportShell() { + _wireReadinessExportButton(); + _resetReadinessFairSection(); + initReadinessFairSection(); + _READINESS_REPORT_SECTIONS.forEach((section) => { + _readinessSectionStatus[section] = "pending"; + }); + _updateReadinessExportButton(); +} + +function _applyReadinessSection(container, section, data, renderFn, buildTimeSeconds) { + if (!container) { + _readinessSectionStatus[section] = "error"; + return false; + } + if (data.error) { + _readinessSectionStatus[section] = "error"; + _readinessSectionError(container, data.error); + return false; + } + _readinessSectionStatus[section] = "ok"; + container.classList.remove("text-center", "py-8"); + renderFn(container, data); + _appendReadinessBuildTimeFooter( + container, + buildTimeSeconds ?? data.build_time_seconds, + ); + return true; +} + function _fetchReadinessSection(section, container, renderFn) { if (!container) { _readinessSectionStatus[section] = "error"; @@ -2525,16 +2588,12 @@ function _fetchReadinessSection(section, container, renderFn) { return; } const data = resp.data || {}; - if (data.error) { - _readinessSectionStatus[section] = "error"; - _readinessSectionError(container, data.error); - return; - } - _readinessSectionStatus[section] = "ok"; - renderFn(container, data); - _appendReadinessBuildTimeFooter( + _applyReadinessSection( container, - resp.build_time_seconds ?? data.build_time_seconds, + section, + data, + renderFn, + resp.build_time_seconds, ); }) .catch((err) => { @@ -2546,43 +2605,60 @@ function _fetchReadinessSection(section, container, renderFn) { }); } +/** + * Restore the readiness report from the aggregated server cache (one request). + * @returns {Promise} true when all sections were restored from cache + */ +function _restoreCachedReadinessReport() { + return fetch("/cached-result/readiness_report") + .then((r) => r.json()) + .then((resp) => { + if (!resp.cached || !resp.sections || activePanel !== "readiness-report") { + return false; + } + _initReadinessReportShell(); + let allOk = true; + _getReadinessSectionRenderers().forEach(({ section, container, render }) => { + const data = resp.sections[section]; + if (!data) { + _readinessSectionStatus[section] = "error"; + allOk = false; + return; + } + if ( + !_applyReadinessSection( + container, + section, + data, + render, + data.build_time_seconds, + ) + ) { + allOk = false; + } + }); + _updateReadinessExportButton(); + return allOk; + }) + .catch((err) => { + debugLog("Readiness cache restore error:", err); + return false; + }); +} + /** * Load the readiness report with hybrid progressive rendering: * dataset overview first, then remaining sections in parallel. */ function loadReadinessReport() { - _wireReadinessExportButton(); - _resetReadinessFairSection(); - initReadinessFairSection(); - _READINESS_REPORT_SECTIONS.forEach((section) => { - _readinessSectionStatus[section] = "pending"; - }); - _updateReadinessExportButton(); - - const overviewContainer = document.getElementById("readiness-summary"); + _initReadinessReportShell(); - const parallelSections = [ - { - section: "data-quality", - container: document.getElementById("readiness-data-quality"), - render: renderReadinessDataQuality, - }, - { - section: "impact-on-ai", - container: document.getElementById("readiness-impact"), - render: renderReadinessImpact, - }, - { - section: "fairness-bias", - container: document.getElementById("readiness-fairness"), - render: renderReadinessFairness, - }, - { - section: "data-governance", - container: document.getElementById("readiness-governance"), - render: renderReadinessGovernance, - }, - ]; + const overviewEntry = _getReadinessSectionRenderers().find( + ({ section }) => section === "dataset-overview", + ); + const parallelSections = _getReadinessSectionRenderers().filter( + ({ section }) => section !== "dataset-overview", + ); const loadParallelSections = () => { Promise.all( @@ -2592,16 +2668,16 @@ function loadReadinessReport() { ); }; - if (!overviewContainer) { + if (!overviewEntry?.container) { loadParallelSections(); return; } // Hybrid: overview first, then parallel for the rest (spinners stay until each resolves). _fetchReadinessSection( - "dataset-overview", - overviewContainer, - renderReadinessDatasetOverview, + overviewEntry.section, + overviewEntry.container, + overviewEntry.render, ).finally(loadParallelSections); } From 6f311003c3badd720ec52df31427808d5a55285e Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 21 Jul 2026 15:14:09 -0400 Subject: [PATCH 20/23] Cache readiness report FAIR compliance server-side and restore on reload. Store optional FAIR assessment results under user/file-scoped keys when submitted from the readiness report, keyed by metadata fingerprint so re-uploading the same JSON is instant. Expose GET /cached-result/readiness_report_fair and include fair_compliance in the aggregated readiness cache response. Restore cached FAIR on panel open without requiring metadata re-upload after page reload. --- tests/unit/test_readiness_cache.py | 97 ++++++++++++ web/routes/core.py | 8 + web/routes/metrics.py | 152 ++++++++++++++++--- web/static/js/inspector.js | 58 ++++++- web/templates/_panels/_readiness_report.html | 1 + 5 files changed, 297 insertions(+), 19 deletions(-) diff --git a/tests/unit/test_readiness_cache.py b/tests/unit/test_readiness_cache.py index 500ab38a..2db5f411 100644 --- a/tests/unit/test_readiness_cache.py +++ b/tests/unit/test_readiness_cache.py @@ -7,9 +7,14 @@ from flask import Flask from web.routes.metrics import ( + _cache_readiness_fair_compliance, _cache_readiness_payload, + _fair_metadata_fingerprint, + _get_cached_readiness_fair_compliance, _get_cached_readiness_payload, + _readiness_fair_cache_key, _readiness_section_cache_key, + get_cached_readiness_fair_report, get_cached_readiness_report, ) @@ -142,5 +147,97 @@ def test_get_cached_readiness_report_requires_every_section( self.assertIsNone(get_cached_readiness_report("data.csv")) +class TestReadinessFairCache(unittest.TestCase): + def setUp(self): + self.app = Flask(__name__) + self.app.TEMP_RESULTS_CACHE = {} + self.app_context = self.app.app_context() + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + def test_fair_cache_key_matches_pattern(self, _mock_user): + key = _readiness_fair_cache_key("data.csv") + self.assertEqual(key, "user:user-1:file:data.csv:readiness_report:fair") + + def test_metadata_fingerprint_changes_with_type(self): + payload = b'{"title": "example"}' + dcat = _fair_metadata_fingerprint(payload, "DCAT") + datacite = _fair_metadata_fingerprint(payload, "Datacite") + self.assertNotEqual(dcat, datacite) + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_store_and_retrieve_fair_compliance(self, mock_current_app, _mock_user): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + payload = {"FAIR Compliance Checks": {"Total Checks": "10/19"}} + _cache_readiness_fair_compliance( + "data.csv", + payload, + metadata_type="DCAT", + metadata_filename="meta.json", + metadata_fingerprint="abc123", + build_time_seconds=0.75, + ) + + cached = _get_cached_readiness_fair_compliance("data.csv") + self.assertIsNotNone(cached) + self.assertEqual(cached["data"], payload) + self.assertEqual(cached["metadata_type"], "DCAT") + self.assertEqual(cached["metadata_filename"], "meta.json") + self.assertEqual(cached["metadata_fingerprint"], "abc123") + self.assertEqual(cached["build_time_seconds"], 0.75) + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_get_cached_readiness_fair_report(self, mock_current_app, _mock_user): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + _cache_readiness_fair_compliance( + "data.csv", + {"Findable": {}}, + metadata_type="Datacite", + metadata_filename="dc.json", + metadata_fingerprint="fp1", + build_time_seconds=1.0, + ) + + result = get_cached_readiness_fair_report("data.csv") + self.assertIsNotNone(result) + self.assertTrue(result["cached"]) + self.assertEqual(result["fair_compliance"]["metadata_filename"], "dc.json") + + @patch("web.routes.metrics.get_current_user_id", return_value="user-1") + @patch("web.routes.metrics.current_app") + def test_aggregated_report_includes_optional_fair_compliance( + self, mock_current_app, _mock_user + ): + mock_current_app.TEMP_RESULTS_CACHE = self.app.TEMP_RESULTS_CACHE + for slug in ( + "dataset-overview", + "data-quality", + "impact-on-ai", + "fairness-bias", + "data-governance", + ): + _cache_readiness_payload("data.csv", slug, {"grade": 0.5}, 1.0) + _cache_readiness_fair_compliance( + "data.csv", + {"FAIR Compliance Checks": {"Total Checks": "5/19"}}, + metadata_type="DCAT", + metadata_filename="meta.json", + metadata_fingerprint="fp2", + build_time_seconds=0.5, + ) + + result = get_cached_readiness_report("data.csv") + self.assertIn("fair_compliance", result) + self.assertEqual( + result["fair_compliance"]["data"]["FAIR Compliance Checks"]["Total Checks"], + "5/19", + ) + + if __name__ == "__main__": unittest.main() diff --git a/web/routes/core.py b/web/routes/core.py index d74e9570..c0da2a2d 100644 --- a/web/routes/core.py +++ b/web/routes/core.py @@ -308,6 +308,14 @@ def cached_result(metric_name): return jsonify(ensure_json_serializable(cached_report)) return jsonify({"cached": False}) + if metric_name == "readiness_report_fair": + from web.routes.metrics import get_cached_readiness_fair_report + + cached_fair = get_cached_readiness_fair_report(file_name) + if cached_fair: + return jsonify(ensure_json_serializable(cached_fair)) + return jsonify({"cached": False}) + cache_key = f"user:{user_id}:file:{file_name}:{metric_name}" entry = current_app.TEMP_RESULTS_CACHE.get(cache_key) if entry and entry.get("data"): diff --git a/web/routes/metrics.py b/web/routes/metrics.py index 8965d3ac..fe7687a4 100644 --- a/web/routes/metrics.py +++ b/web/routes/metrics.py @@ -1,3 +1,4 @@ +import hashlib import json import logging import math @@ -2235,6 +2236,7 @@ def get_cached_readiness_report(file_name): """Return all cached readiness sections for ``/cached-result/readiness_report``. Returns a JSON-serializable dict when every section is cached, else *None*. + Optionally includes ``fair_compliance`` when a readiness FAIR result is cached. """ sections = {} sections_cached = {} @@ -2244,11 +2246,94 @@ def get_cached_readiness_report(file_name): return None sections[slug] = {**cached, "build_time_seconds": build_time} sections_cached[slug] = True - return { + payload = { "cached": True, "sections": sections, "sections_cached": sections_cached, } + fair_compliance = _get_cached_readiness_fair_compliance(file_name) + if fair_compliance is not None: + payload["fair_compliance"] = fair_compliance + return payload + + +def _readiness_fair_cache_key(file_name): + """Cache key for optional readiness-report FAIR compliance.""" + user_id = get_current_user_id() + return f"user:{user_id}:file:{file_name}:readiness_report:fair" + + +def _fair_metadata_fingerprint(json_bytes, metadata_type): + """Stable fingerprint for readiness FAIR cache invalidation.""" + digest = hashlib.sha256() + digest.update(json_bytes) + digest.update(b"|") + digest.update(metadata_type.encode("utf-8")) + return digest.hexdigest() + + +def _get_cached_readiness_fair_compliance(file_name): + """Return cached readiness FAIR payload metadata, or *None*.""" + key = _readiness_fair_cache_key(file_name) + entry = current_app.TEMP_RESULTS_CACHE.get(key) + if entry and is_metric_cache_valid(entry): + data = entry.get("data") + if data is None or (isinstance(data, dict) and data.get("error")): + return None + return { + "data": data, + "metadata_type": entry.get("metadata_type"), + "metadata_filename": entry.get("metadata_filename"), + "metadata_fingerprint": entry.get("metadata_fingerprint"), + "build_time_seconds": entry.get("build_time_seconds"), + "cached": True, + } + if entry: + current_app.TEMP_RESULTS_CACHE.pop(key, None) + return None + + +def get_cached_readiness_fair_report(file_name): + """Return readiness FAIR cache for ``/cached-result/readiness_report_fair``.""" + fair_compliance = _get_cached_readiness_fair_compliance(file_name) + if fair_compliance is None: + return None + return {"cached": True, "fair_compliance": fair_compliance} + + +def _cache_readiness_fair_compliance( + file_name, + data, + *, + metadata_type, + metadata_filename, + metadata_fingerprint, + build_time_seconds, +): + """Store a successful readiness-report FAIR assessment.""" + if data is None or (isinstance(data, dict) and data.get("error")): + return + key = _readiness_fair_cache_key(file_name) + current_app.TEMP_RESULTS_CACHE[key] = { + "data": ensure_json_serializable(data), + "timestamp": time.time(), + "expires_at": time.time() + _READINESS_CACHE_TTL_SECONDS, + "build_time_seconds": build_time_seconds, + "metadata_type": metadata_type, + "metadata_filename": metadata_filename, + "metadata_fingerprint": metadata_fingerprint, + } + + +def _run_fair_assessment(data_dict, metadata_type): + """Run FAIR assessment for DCAT or Datacite metadata.""" + if metadata_type == "DCAT": + extracted_json = extract_keys_and_values(data_dict) + fair_dict = categorize_metadata(extracted_json, data_dict) + return format_dict_values(fair_dict) + if metadata_type == "Datacite": + return categorize_keys_fair(data_dict) + raise ValueError("Unknown metadata type") def _get_or_build_readiness_visualizations(section, file_info): @@ -3032,24 +3117,48 @@ def fair_assessment(): return jsonify({"error": "Invalid file format. Please upload a JSON file."}), 400 json_data = file.read() - data_dict = json.loads(json_data.decode("utf-8")) - metadata_type = request.form.get("metadata type", "") + readiness_context = request.form.get("readiness_context") == "1" + dataset_file_name = ( + session.get("uploaded_file_name") + or session.get("globus_file_name") + or "" + ) + metadata_fingerprint = _fair_metadata_fingerprint(json_data, metadata_type) + + if readiness_context and dataset_file_name: + cached_fair = _get_cached_readiness_fair_compliance(dataset_file_name) + if ( + cached_fair is not None + and cached_fair.get("metadata_fingerprint") == metadata_fingerprint + ): + metric_time_log.info( + "Readiness report FAIR compliance cache hit for %s", + dataset_file_name, + ) + return jsonify( + ensure_json_serializable( + { + **cached_fair["data"], + "cached": True, + "build_time_seconds": cached_fair.get( + "build_time_seconds" + ), + } + ) + ) - if metadata_type == "DCAT": - try: - extracted_json = extract_keys_and_values(data_dict) - fair_dict = categorize_metadata(extracted_json, data_dict) - result = format_dict_values(fair_dict) - except json.JSONDecodeError as e: - return jsonify({"error": f"Error parsing JSON: {str(e)}"}), 400 - elif metadata_type == "Datacite": - try: - result = categorize_keys_fair(data_dict) - except json.JSONDecodeError as e: - return jsonify({"error": f"Error parsing JSON: {str(e)}"}), 400 - else: + try: + data_dict = json.loads(json_data.decode("utf-8")) + except json.JSONDecodeError as e: + return jsonify({"error": f"Error parsing JSON: {str(e)}"}), 400 + + try: + result = _run_fair_assessment(data_dict, metadata_type) + except ValueError: return jsonify({"error": "Unknown metadata type"}), 400 + except json.JSONDecodeError as e: + return jsonify({"error": f"Error parsing JSON: {str(e)}"}), 400 duration = time.time() - start_time metric_time_log.info("FAIR Assessment completed in %.2f seconds", duration) @@ -3058,7 +3167,16 @@ def fair_assessment(): span.set_attribute("metadata.type", metadata_type) result = ensure_json_serializable(result) - return jsonify(result) + if readiness_context and dataset_file_name: + _cache_readiness_fair_compliance( + dataset_file_name, + result, + metadata_type=metadata_type, + metadata_filename=file.filename, + metadata_fingerprint=metadata_fingerprint, + build_time_seconds=round(duration, 2), + ) + return jsonify({**result, "cached": False, "build_time_seconds": round(duration, 2)}) else: results_id = request.args.get("results_id") diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 6bc8e1d3..d36609ec 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -1873,9 +1873,12 @@ function submitFairAssessmentForm(form, resultContainer, callbacks) { return fetch("/fair-assessment", { method: "POST", body: formData }) .then((response) => response.json()) .then((data) => { - const ok = renderFairAssessmentResult(data, resultContainer); + const resultData = { ...data }; + delete resultData.cached; + delete resultData.build_time_seconds; + const ok = renderFairAssessmentResult(resultData, resultContainer); if (ok) { - callbacks?.onSuccess?.(data); + callbacks?.onSuccess?.(resultData, data); } else { callbacks?.onError?.(data); } @@ -1917,6 +1920,53 @@ function submitReadinessFairAssessment() { ); } +function _restoreReadinessFairFromCache(fairPayload) { + const data = fairPayload?.data; + if (!data || data.error) return false; + + const uploadEl = document.getElementById("readiness-fair-upload"); + const resultsEl = document.getElementById("readiness-fair-results"); + if (!resultsEl) return false; + + _readinessFairCompliance = { status: "ok", data }; + uploadEl?.classList.add("hidden"); + resultsEl.classList.remove("hidden"); + renderFairAssessmentResult(data, resultsEl); + + const metadataType = fairPayload.metadata_type; + if (metadataType) { + const select = document.getElementById("readiness-fair-metadata-type"); + if (select) select.value = metadataType; + } + const metadataFilename = fairPayload.metadata_filename; + if (metadataFilename) { + const label = document.getElementById("readinessFairFileLabel"); + const icon = document.getElementById("readinessFairUploadIcon"); + if (label) label.textContent = metadataFilename; + if (icon) { + icon.classList.remove("text-gray-400"); + icon.classList.add("text-green-500"); + } + } + return true; +} + +/** Restore optional FAIR compliance from server cache (separate lightweight fetch). */ +function _tryRestoreCachedReadinessFair() { + return fetch("/cached-result/readiness_report_fair") + .then((r) => r.json()) + .then((resp) => { + if (!resp.cached || !resp.fair_compliance || activePanel !== "readiness-report") { + return false; + } + return _restoreReadinessFairFromCache(resp.fair_compliance); + }) + .catch((err) => { + debugLog("Readiness FAIR cache restore error:", err); + return false; + }); +} + function _resetReadinessFairSection() { _readinessFairCompliance = { status: "idle", data: null }; const uploadEl = document.getElementById("readiness-fair-upload"); @@ -2638,6 +2688,9 @@ function _restoreCachedReadinessReport() { } }); _updateReadinessExportButton(); + if (resp.fair_compliance) { + _restoreReadinessFairFromCache(resp.fair_compliance); + } return allOk; }) .catch((err) => { @@ -2652,6 +2705,7 @@ function _restoreCachedReadinessReport() { */ function loadReadinessReport() { _initReadinessReportShell(); + _tryRestoreCachedReadinessFair(); const overviewEntry = _getReadinessSectionRenderers().find( ({ section }) => section === "dataset-overview", diff --git a/web/templates/_panels/_readiness_report.html b/web/templates/_panels/_readiness_report.html index fee8ab90..57b19bc3 100644 --- a/web/templates/_panels/_readiness_report.html +++ b/web/templates/_panels/_readiness_report.html @@ -90,6 +90,7 @@

    FAIR Complianc

    +