diff --git a/.gitignore b/.gitignore index a6868a12..8b4424cc 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,14 @@ examples/*_vector_store/ dump.rdb celerybeat-schedule.db + +# Runtime data (uploads, persisted file lists, scratch test data) +data/ + +# Runtime-generated custom metrics & remedy data (package keeps __init__.py + base_dr.py) +aidrin/custom_metrics/customDR_*.py +aidrin/custom_metrics/remedy_data/ + +# session/local artifacts +AIDRIN-gtag/ +claude.session diff --git a/aidrin/__init__.py b/aidrin/__init__.py index 4a637a87..c9676768 100644 --- a/aidrin/__init__.py +++ b/aidrin/__init__.py @@ -334,8 +334,20 @@ def compute_entropy_risk(quasi_identifiers, file_info): return _fn(quasi_identifiers, file_info) +# --------------------------------------------------------------------------- +# Batch / Structural Overview +# --------------------------------------------------------------------------- + +def summarize_files(file_infos): + """Per-file structural overview + totals for a batch of files (no metrics).""" + from aidrin.batch import summarize_files as _fn + return _fn(file_infos) + + __all__ = [ "__version__", + # Batch / Structural Overview + "summarize_files", # Data Quality "calculate_completeness", "calculate_duplicates", diff --git a/aidrin/agentic/retriever.py b/aidrin/agentic/retriever.py index a6c9a54c..cd3ca8d6 100644 --- a/aidrin/agentic/retriever.py +++ b/aidrin/agentic/retriever.py @@ -304,7 +304,7 @@ def _compress_one(item: dict[str, Any]) -> str: context_texts = [item.get("full_text", "") for item in retrieved] prompt_context = "\n\n".join( - f"[Source: {item.get('source','unknown')}]\n{self._sanitize(text)}" + f"[Source: {item.get('source', 'unknown')}]\n{self._sanitize(text)}" for item, text in zip(retrieved, context_texts) ) diff --git a/aidrin/batch.py b/aidrin/batch.py new file mode 100644 index 00000000..a4a3105e --- /dev/null +++ b/aidrin/batch.py @@ -0,0 +1,71 @@ +"""Session-free batch helpers shared by the web UI, CLI, and library users.""" + +import os + +import pandas as pd + +from aidrin.file_handling.file_parser import read_file + + +def _summarize_one(file_info): + path, name, file_type = file_info + if not path: + return { + "name": name, "type": file_type, + "records": None, "features": None, + "numerical": None, "categorical": None, + "size_bytes": None, "status": "error", + "error": "No file path provided.", + } + size = None + try: + if path and os.path.exists(path): + size = os.path.getsize(path) + except OSError: + size = None + + result = read_file(file_info) # DataFrame | None | str + if isinstance(result, pd.DataFrame): + # Same dtype convention as the Data Overview panel: numeric vs string. + numerical = int(sum(pd.api.types.is_numeric_dtype(d) for d in result.dtypes)) + categorical = int(sum(pd.api.types.is_string_dtype(d) for d in result.dtypes)) + return { + "name": name, "type": file_type, + "records": int(len(result)), + "features": int(len(result.columns)), + "numerical": numerical, "categorical": categorical, + "size_bytes": size, "status": "ok", "error": None, + } + + message = result if isinstance(result, str) else ( + "Could not read the file. The format may be unsupported or the file " + "may be corrupted." + ) + return { + "name": name, "type": file_type, + "records": None, "features": None, + "numerical": None, "categorical": None, + "size_bytes": size, "status": "error", "error": message, + } + + +def summarize_files(file_infos): + """Return {"files": [per_file, ...], "totals": {...}} for a list of files. + + Computes structural facts only (records, features, size, load status) — no + metrics. A file that fails to load becomes a status:"error" row and never + aborts the batch. ``file_infos`` is a list of (path, name, type) tuples. + """ + files = [_summarize_one(fi) for fi in file_infos] + by_type = {} + for f in files: + by_type[f["type"]] = by_type.get(f["type"], 0) + 1 + totals = { + "file_count": len(files), + "ok_count": sum(1 for f in files if f["status"] == "ok"), + "error_count": sum(1 for f in files if f["status"] == "error"), + "total_records": sum(f["records"] or 0 for f in files), + "total_size_bytes": sum(f["size_bytes"] or 0 for f in files), + "by_type": by_type, + } + return {"files": files, "totals": totals} diff --git a/aidrin/file_handling/file_parser.py b/aidrin/file_handling/file_parser.py index a51e9cb2..68cce7ec 100644 --- a/aidrin/file_handling/file_parser.py +++ b/aidrin/file_handling/file_parser.py @@ -38,6 +38,37 @@ # (file_type,file_type_name) ] +# Map a real file extension to the READER_MAP key that handles it. +# Excel uses a single combined reader key, so all its extensions point at it. +_EXCEL_KEY = ".xls, .xlsb, .xlsx, .xlsm" +EXTENSION_MAP = { + ".csv": ".csv", + ".json": ".json", + ".npz": ".npz", + ".h5": ".h5", + ".parquet": ".parquet", # resolves only once a parquet reader is registered in READER_MAP + ".xls": _EXCEL_KEY, + ".xlsb": _EXCEL_KEY, + ".xlsx": _EXCEL_KEY, + ".xlsm": _EXCEL_KEY, +} + + +def file_extension(filename): + """Return the lowercased real extension (e.g. ``.csv``), or ``""``.""" + return os.path.splitext(filename or "")[1].lower() + + +def infer_file_type(filename): + """Return the READER_MAP key for a filename's extension, or None. + + Only returns a key that is actually registered in READER_MAP on this + install (e.g. ``.parquet`` resolves only if the parquet reader exists). + """ + key = EXTENSION_MAP.get(file_extension(filename)) + return key if key in READER_MAP else None + + # logger config file_upload_time_log = logging.getLogger("file_upload") diff --git a/aidrin/structured_data_metrics/privacy_measure.py b/aidrin/structured_data_metrics/privacy_measure.py index 47d488a9..e5cb96d8 100644 --- a/aidrin/structured_data_metrics/privacy_measure.py +++ b/aidrin/structured_data_metrics/privacy_measure.py @@ -517,6 +517,8 @@ def compute_k_anonymity(quasi_identifiers: List[str], file_info): "k-Anonymity Visualization": base64_str}`` or ``{"Error": str}`` on validation failure. """ + if not quasi_identifiers: + return {"Error": "Please select at least one quasi-identifier."} # Handle both DataFrame and tuple inputs if isinstance(file_info, tuple): data = read_file(file_info) @@ -634,6 +636,8 @@ def compute_l_diversity( "l-Diversity Visualization": base64_str}`` or ``{"Error": str}`` on validation failure. """ + if not quasi_identifiers: + return {"Error": "Please select at least one quasi-identifier."} # Handle both DataFrame and tuple inputs if isinstance(file_info, tuple): data = read_file(file_info) @@ -765,6 +769,8 @@ def compute_t_closeness( where ``t-Value`` is in ``[0, 1]``, or ``{"Error": str}`` on validation failure. """ + if not quasi_identifiers: + return {"Error": "Please select at least one quasi-identifier."} # Handle both DataFrame and tuple inputs if isinstance(file_info, tuple): data = read_file(file_info) @@ -895,6 +901,8 @@ def compute_entropy_risk(quasi_identifiers, file_info): where ``Entropy-Value >= 0``, or ``{"Error": str}`` on validation failure. """ + if not quasi_identifiers: + return {"Error": "Please select at least one quasi-identifier."} # Handle both DataFrame and tuple inputs if isinstance(file_info, tuple): data = read_file(file_info) diff --git a/docs/superpowers/plans/2026-06-04-multi-file-batch-analysis.md b/docs/superpowers/plans/2026-06-04-multi-file-batch-analysis.md new file mode 100644 index 00000000..c7e99c32 --- /dev/null +++ b/docs/superpowers/plans/2026-06-04-multi-file-batch-analysis.md @@ -0,0 +1,1327 @@ +# Multi-File Batch Analysis Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let AIDRIN analyze many independent files at once — an active-file switcher plus a "Batch Overview" landing summary — fed by both local multi-upload and Globus. + +**Architecture:** Two layers. (1) A session-free **core** primitive in `aidrin` (`summarize_files`, `infer_file_type`). (2) A **web** active-file shim: a server-side `uploaded_files` list + `active_file_id`; `set_active_file` mirrors the active file's identity into the existing `uploaded_file_*` (and, for Globus, `globus_file_*`) session keys so every existing route keeps working unchanged. Execution stays dispatched by source (local routes vs. remote `remote_metric_runner`). + +**Tech Stack:** Python 3.10+, Flask, pandas, pytest (unit + integration via `tests/integration/conftest.py`), vanilla JS + Tailwind. + +**Spec:** `docs/superpowers/specs/2026-06-04-multi-file-batch-analysis-design.md` + +--- + +## Prerequisites & Cross-Branch Notes + +- This branch (`multi-file-analysis`) is off `develop`. It does **not** have + `MAX_CONTENT_LENGTH` (on `upload-size-limit`) or `load_dataframe` (on + `parquet-support`). This plan adds the small pieces it needs directly + (Milestone 4 adds the size cap + 413; `summarize_files` handles `read_file`'s + tri-state itself), so it is self-contained. +- Run tests with the project venv: `.venv/bin/python -m pytest`. +- Follow TDD: write the failing test, watch it fail, implement minimally, watch it pass, commit. + +## File Structure + +**Create** +- `aidrin/batch.py` — `summarize_files(file_infos)` (session-free, no Celery). +- `tests/unit/test_infer_file_type.py` +- `tests/unit/test_summarize_files.py` +- `web/routes/files.py` — file-management blueprint (`/files`, activate, remove, summary). +- `web/templates/_components/file_switcher.html` — sidebar file list. +- `web/templates/_panels/_batch_overview.html` — batch overview panel. +- `tests/integration/test_files_routes.py` +- `tests/integration/test_multi_upload.py` + +**Modify** +- `aidrin/file_handling/file_parser.py` — add `EXTENSION_MAP` + `infer_file_type`. +- `aidrin/__init__.py` — export `summarize_files`. +- `web/routes/utils.py` — file-list store + `set_active_file`; cache key → file_id. +- `web/routes/core.py` — multi-file `/inspector`; Globus-aware stale check; land on overview. +- `web/routes/__init__.py` — register the `files` blueprint. +- `web/routes/globus.py` — append selections into the shared list; fold `globus_file_*`. +- `web/__init__.py` — `MAX_CONTENT_LENGTH`, `AIDRIN_MAX_UPLOAD_FILES`, 413 handler. +- `web/templates/inspector.html` — include switcher + batch overview; default panel. +- `web/templates/_components/upload_panel.html` — `multiple`; remove type `` + +**Files:** +- Modify: `web/templates/_components/upload_panel.html`, `web/static/js/main.js` + +- [ ] **Step 1: Edit the template** — add `multiple` and drop the select. + +In `web/templates/_components/upload_panel.html`: delete the `