Feature: Readiness Report

.gitignore

@@ -160,7 +160,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 # datasets
 metadata/

aidrin/structured_data_metrics/add_noise.py

@@ -17,7 +17,7 @@ def add_laplace_noise(data, epsilon):
         raise Exception("Epsilon cannot be 0")
 
 
-def return_noisy_stats(add_noise_columns, epsilon, file_info):
+def return_noisy_stats(add_noise_columns, epsilon, file_info, save_output=True, include_visualization=True):
     # Convert JSON back to DataFrame if needed, otherwise use DataFrame directly
     import pandas as pd
 
@@ -43,18 +43,20 @@ def return_noisy_stats(add_noise_columns, epsilon, file_info):
     num_cols = min(num_columns, max_columns_per_row)
 
     # Create subplots for the box plots
-    fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 8))
+    if include_visualization:
+        fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 8))
 
     for i, column in enumerate(add_noise_columns):
-        if num_rows == 1 and num_cols == 1:
-            current_ax = axes
-        elif num_rows == 1:
-            current_ax = axes[i % num_cols]
-        elif num_cols == 1:
-            current_ax = axes[i % num_rows, 0]
-        else:
-            row, col = divmod(i, num_cols)
-            current_ax = axes[row, col]
+        if include_visualization:
+            if num_rows == 1 and num_cols == 1:
+                current_ax = axes
+            elif num_rows == 1:
+                current_ax = axes[i % num_cols]
+            elif num_cols == 1:
+                current_ax = axes[i % num_rows, 0]
+            else:
+                row, col = divmod(i, num_cols)
+                current_ax = axes[row, col]
 
         noisy_feature = add_laplace_noise(df_drop_na[column], epsilon)
 
@@ -85,36 +87,39 @@ def return_noisy_stats(add_noise_columns, epsilon, file_info):
         )
         df_drop_na[f'noisy_{column}'] = noisy_feature
 
-        # Box plot for the normal feature
-        current_ax.boxplot(
-            df_drop_na[column], positions=[0], widths=0.6, showfliers=False
-        )
-        current_ax.set_title(f"Normal vs Noisy representations: Feature {column}")
-        current_ax.set_ylabel("Value")
-
-        # Box plot for the noisy feature
-        current_ax.boxplot(noisy_feature, positions=[1], widths=0.6, showfliers=False)
-        current_ax.set_ylabel("Value")
-
-    # Adjust the spacing between subplots
-    plt.tight_layout()
-
-    # Save the chart as BytesIO
-    img_buf = BytesIO()
-    plt.savefig(img_buf, format="png")
-    img_buf.seek(0)
-
-    # Encode the combined image as base64
-    combined_image_base64 = base64.b64encode(img_buf.getvalue()).decode("utf-8")
-    img_buf.close()
-    try:
-        # Create the new directory
-        os.makedirs("noisy", exist_ok=True)
-        df_drop_na.to_csv("noisy/noisy_data.csv", index=False)
-        stat_dict["Noisy file saved"] = "Successful"
-    except Exception:
-        stat_dict["Noisy file saved"] = "Error"
-
-    stat_dict["DP Statistics Visualization"] = combined_image_base64
+        if include_visualization:
+            # Box plot for the normal feature
+            current_ax.boxplot(
+                df_drop_na[column], positions=[0], widths=0.6, showfliers=False
+            )
+            current_ax.set_title(f"Normal vs Noisy representations: Feature {column}")
+            current_ax.set_ylabel("Value")
+
+            # Box plot for the noisy feature
+            current_ax.boxplot(noisy_feature, positions=[1], widths=0.6, showfliers=False)
+            current_ax.set_ylabel("Value")
+
+    if include_visualization:
+        # Adjust the spacing between subplots
+        plt.tight_layout()
+
+        # Save the chart as BytesIO
+        img_buf = BytesIO()
+        plt.savefig(img_buf, format="png")
+        img_buf.seek(0)
+
+        # Encode the combined image as base64
+        combined_image_base64 = base64.b64encode(img_buf.getvalue()).decode("utf-8")
+        img_buf.close()
+        stat_dict["DP Statistics Visualization"] = combined_image_base64
+    if save_output:
+        try:
+            os.makedirs("noisy", exist_ok=True)
+            df_drop_na.to_csv("noisy/noisy_data.csv", index=False)
+            stat_dict["Noisy file saved"] = "Successful"
+        except Exception:
+            stat_dict["Noisy file saved"] = "Error"
+    else:
+        stat_dict["Noisy file saved"] = "Skipped (readiness report preview only)"
 
     return stat_dict

aidrin/structured_data_metrics/completeness.py

@@ -12,7 +12,7 @@
 
 
 @shared_task(bind=True, ignore_result=False)
-def completeness(self: Task, file_info):
+def completeness(self: Task, file_info, include_visualization=True):
     """Compute per-column and overall completeness (non-missing rate) for a dataset.
 
     Reads the file, calculates the proportion of non-null values for every
@@ -64,46 +64,47 @@ def completeness(self: Task, file_info):
         result_dict["Completeness scores"] = completeness_scores
         result_dict["Overall Completeness"] = overall_completeness
 
-        # Horizontal bar chart — grows vertically with feature count
-        labels = list(completeness_scores.keys())
-        values = list(completeness_scores.values())
-        n = len(labels)
-        text_color = "#6b7280"
-
-        fig_height = max(3, n * 0.3)
-        fig, ax = plt.subplots(figsize=(8, fig_height))
-        fig.patch.set_alpha(0)
-        ax.set_facecolor("none")
-
-        bars = ax.barh(range(n), values, color="#4485F4", height=0.7)
-        ax.set_xlabel("Completeness Score", fontsize=10, color=text_color)
-        ax.set_yticks(range(n))
-        ax.set_yticklabels(labels, fontsize=9, color=text_color)
-        ax.tick_params(axis="x", colors=text_color, labelsize=8)
-        ax.set_xlim(0, 1.12)
-        ax.invert_yaxis()
-        ax.set_ylim(n - 0.5, -0.5)
-
-        for spine in ax.spines.values():
-            spine.set_color(text_color)
-
-        for bar, val in zip(bars, values):
-            if val > 0.15:
-                ax.text(val - 0.01, bar.get_y() + bar.get_height() / 2,
-                        f'{val:.2f}', ha='right', va='center', fontsize=8, color='white', fontweight='bold')
-            else:
-                ax.text(val + 0.01, bar.get_y() + bar.get_height() / 2,
-                        f'{val:.2f}', ha='left', va='center', fontsize=8, color=text_color)
-
-        fig.tight_layout(pad=0.5)
-
-        img_buf = io.BytesIO()
-        fig.savefig(img_buf, format="png", dpi=150, transparent=True)
-        img_buf.seek(0)
-
-        img_base64 = base64.b64encode(img_buf.read()).decode("utf-8")
-        result_dict["Completeness Visualization"] = img_base64
-        plt.close(fig)
+        if include_visualization:
+            # Horizontal bar chart — grows vertically with feature count
+            labels = list(completeness_scores.keys())
+            values = list(completeness_scores.values())
+            n = len(labels)
+            text_color = "#6b7280"
+
+            fig_height = max(3, n * 0.3)
+            fig, ax = plt.subplots(figsize=(8, fig_height))
+            fig.patch.set_alpha(0)
+            ax.set_facecolor("none")
+
+            bars = ax.barh(range(n), values, color="#4485F4", height=0.7)
+            ax.set_xlabel("Completeness Score", fontsize=10, color=text_color)
+            ax.set_yticks(range(n))
+            ax.set_yticklabels(labels, fontsize=9, color=text_color)
+            ax.tick_params(axis="x", colors=text_color, labelsize=8)
+            ax.set_xlim(0, 1.12)
+            ax.invert_yaxis()
+            ax.set_ylim(n - 0.5, -0.5)
+
+            for spine in ax.spines.values():
+                spine.set_color(text_color)
+
+            for bar, val in zip(bars, values):
+                if val > 0.15:
+                    ax.text(val - 0.01, bar.get_y() + bar.get_height() / 2,
+                            f'{val:.2f}', ha='right', va='center', fontsize=8, color='white', fontweight='bold')
+                else:
+                    ax.text(val + 0.01, bar.get_y() + bar.get_height() / 2,
+                            f'{val:.2f}', ha='left', va='center', fontsize=8, color=text_color)
+
+            fig.tight_layout(pad=0.5)
+
+            img_buf = io.BytesIO()
+            fig.savefig(img_buf, format="png", dpi=150, transparent=True)
+            img_buf.seek(0)
+
+            img_base64 = base64.b64encode(img_buf.read()).decode("utf-8")
+            result_dict["Completeness Visualization"] = img_base64
+            plt.close(fig)
 
         logger.info("Completeness task completed: %d columns, overall=%.4f", len(completeness_scores), overall_completeness)
         return result_dict