diff --git a/aidrin/compute/executor.py b/aidrin/compute/executor.py index d892ac23..0d6c69ad 100644 --- a/aidrin/compute/executor.py +++ b/aidrin/compute/executor.py @@ -15,7 +15,7 @@ """ from dataclasses import asdict, is_dataclass -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional from aidrin.compute import client from aidrin.compute.profiles import RemoteTarget @@ -138,11 +138,16 @@ def summarize_dataset( file_path: str, file_type: Optional[str] = None, max_features: Optional[int] = None, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: - return self._call( - "summarize", - {"file_path": file_path, "file_type": file_type, "max_features": max_features}, - ) + payload: Dict[str, Any] = { + "file_path": file_path, + "file_type": file_type, + "max_features": max_features, + } + if selected_keys: + payload["selected_keys"] = selected_keys + return self._call("summarize", payload) def run_data_quality( self, @@ -151,17 +156,18 @@ def run_data_quality( file_name: Optional[str] = None, verbose: bool = False, strip_visualizations: bool = True, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: - return self._call( - "data_quality", - { - "file_path": file_path, - "file_type": file_type, - "file_name": file_name, - "verbose": verbose, - "strip_visualizations": strip_visualizations, - }, - ) + payload: Dict[str, Any] = { + "file_path": file_path, + "file_type": file_type, + "file_name": file_name, + "verbose": verbose, + "strip_visualizations": strip_visualizations, + } + if selected_keys: + payload["selected_keys"] = selected_keys + return self._call("data_quality", payload) def run_batch_metrics( self, diff --git a/aidrin/file_handling/file_parser.py b/aidrin/file_handling/file_parser.py index 7c3ecf6a..dbe1afe4 100644 --- a/aidrin/file_handling/file_parser.py +++ b/aidrin/file_handling/file_parser.py @@ -8,6 +8,7 @@ from aidrin.file_handling.readers.json_reader import jsonReader from aidrin.file_handling.readers.npz_reader import npzReader from aidrin.file_handling.readers.parquet_reader import parquetReader +from aidrin.file_handling.readers.zarr_reader import zarrReader # Notes: # To add support for new file types: @@ -15,6 +16,8 @@ # (and optionally .parse(), .filter()) to 'file_readers'. # - Register the class in READER_MAP. # - Add a display name and extension to SUPPORTED_FILE_TYPES for the front end. +# - Formats that are CLI/library/Globus-only (e.g. Zarr directories) go in +# GLOBUS_FILE_TYPES but not SUPPORTED_FILE_TYPES (local upload dropdown). # Reader Map. Used to create file type specific parsing READER_MAP = { @@ -24,10 +27,11 @@ ".json": jsonReader, ".h5": hdf5Reader, ".parquet": parquetReader, + ".zarr": zarrReader, # Add additional file types here } -# Supported file types. Read on front end to create select features. +# Supported file types. Read on front end to create local upload select features. SUPPORTED_FILE_TYPES = [ (".csv", "CSV"), (".xls, .xlsb, .xlsx, .xlsm", "Excel"), @@ -39,6 +43,15 @@ # (file_type,file_type_name) ] +# Globus (and other path-based surfaces) may include directory-shaped formats +# that are not offered in the local browser upload dropdown. +GLOBUS_FILE_TYPES = SUPPORTED_FILE_TYPES + [ + (".zarr", "Zarr"), +] + +# File types that accept selected_keys for multi-array / multi-dataset selection. +_SELECTION_FILE_TYPES = {".h5", ".zarr"} + # logger config file_upload_time_log = logging.getLogger("file_upload") @@ -204,6 +217,7 @@ def read_file(file_info, columns=None): -file_path: str, relative or absolute path of the file. -file_name: str, file name. -file_type: str, file format. Passed from front end select value. + - optional selected_keys (4th element) for HDF5/Zarr path selection. columns: list of str, optional When given, only these columns are returned. Returns @@ -250,7 +264,7 @@ def read_file(file_info, columns=None): # Slow path: parse the source once. reader_cls = READER_MAP[file_type] - if file_type == ".h5": + if file_type in _SELECTION_FILE_TYPES: df = reader_cls( file_path, file_upload_time_log, selected_keys=selected_keys ).read() diff --git a/aidrin/file_handling/readers/root_reader.py b/aidrin/file_handling/readers/root_reader.py new file mode 100644 index 00000000..576640e4 --- /dev/null +++ b/aidrin/file_handling/readers/root_reader.py @@ -0,0 +1,17 @@ +from aidrin.file_handling.readers.structured import ( + INVENTORY_UNSUPPORTED, + InventoryResult, + StructuredFileReader, + make_inventory, +) + + +class rootReader(StructuredFileReader): + """ROOT file reader (scaffold). v1 will support TTrees only via uproot.""" + + def inventory(self) -> InventoryResult: + return make_inventory(INVENTORY_UNSUPPORTED) + + def read(self): + self.logger.warning("ROOT read is not implemented yet.") + return None diff --git a/aidrin/file_handling/readers/structured.py b/aidrin/file_handling/readers/structured.py new file mode 100644 index 00000000..bd86b8ce --- /dev/null +++ b/aidrin/file_handling/readers/structured.py @@ -0,0 +1,81 @@ +""" +Shared contract for structured file readers (HDF5, Zarr, ROOT). + +Structured formats may hold many arrays or tables under paths. Readers classify +the on-disk layout via ``inventory()`` before ``read()`` loads tabular data. + +Inventory ``type`` values +------------------------- +empty + No readable datasets or arrays. +single_dataset + One dataset; auto-read without a picker. +multi_dataset + Incompatible or grouped layout; caller must select paths. +legacy + Internal auto-read path for compatible multi-array layouts (and HDF5 + PyTables stores). Never surface this label in API or UI responses. +unsupported + Reader stub or format not implemented yet. Never treat as auto-read. +""" + +from __future__ import annotations + +from typing import Any, TypedDict + +from aidrin.file_handling.readers.base_reader import BaseFileReader + +INVENTORY_EMPTY = "empty" +INVENTORY_SINGLE = "single_dataset" +INVENTORY_MULTI = "multi_dataset" +INVENTORY_LEGACY = "legacy" +INVENTORY_UNSUPPORTED = "unsupported" + +# Types that may be shown to users when driving picker or error messages. +USER_FACING_INVENTORY_TYPES = frozenset( + {INVENTORY_EMPTY, INVENTORY_SINGLE, INVENTORY_MULTI} +) + + +class DatasetEntry(TypedDict): + path: str + shape: tuple[int, ...] + ndim: int + dtype: str + size: int + + +class PickerGroup(TypedDict, total=False): + name: str + paths: list[str] + prefix: str + + +class InventoryResult(TypedDict): + type: str + datasets: list[DatasetEntry] + groups: list[dict[str, Any]] + + +def make_inventory( + inv_type: str, + datasets: list[DatasetEntry] | None = None, + groups: list[dict[str, Any]] | None = None, +) -> InventoryResult: + """Build a normalized inventory dict.""" + return { + "type": inv_type, + "datasets": datasets or [], + "groups": groups or [], + } + + +class StructuredFileReader(BaseFileReader): + """Base for hierarchical array/tree formats with inventory and metadata hooks.""" + + def inventory(self) -> InventoryResult: + return make_inventory(INVENTORY_UNSUPPORTED) + + def get_metadata(self) -> dict[str, Any]: + """Embedded file metadata for FAIR assessment (``.zattrs``, ROOT headers, etc.).""" + return {} diff --git a/aidrin/file_handling/readers/zarr_reader.py b/aidrin/file_handling/readers/zarr_reader.py new file mode 100644 index 00000000..19a28cb9 --- /dev/null +++ b/aidrin/file_handling/readers/zarr_reader.py @@ -0,0 +1,390 @@ +"""Zarr store reader for directory paths (CLI, library, Globus). + +Local browser upload of Zarr stores is intentionally not registered in +``SUPPORTED_FILE_TYPES``; use a directory path with CLI/library or Globus. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd + +from aidrin.file_handling.readers.structured import ( + INVENTORY_EMPTY, + INVENTORY_LEGACY, + INVENTORY_MULTI, + INVENTORY_SINGLE, + DatasetEntry, + InventoryResult, + StructuredFileReader, + make_inventory, +) + + +def _require_zarr(): + try: + import zarr + except ImportError as exc: + raise ImportError( + "Zarr support requires the 'zarr' package on Python >=3.11. " + "Install with: pip install 'aidrin[zarr]' or pip install 'zarr>=3.0.8,<3.2'" + ) from exc + return zarr + + +class zarrReader(StructuredFileReader): + """Read Zarr directory stores into pandas DataFrames. + + Same selection model as HDF5: pass ``selected_keys`` to choose array paths. + Only tabular-friendly arrays (0D/1D, or a single 2D array) are converted; + higher-dimensional arrays are refused so metrics see raw values, not aggregates. + """ + + def __init__(self, file_path: str, logger, selected_keys=None): + super().__init__(file_path, logger) + self._explicit_selected_keys = selected_keys + + def _open_store(self): + zarr = _require_zarr() + return zarr.open(self.file_path, mode="r") + + def _list_group_paths(self): + zarr = _require_zarr() + root = self._open_store() + if isinstance(root, zarr.Array): + return [] + + groups = [] + + def walk(group, prefix=""): + for key in group.keys(): + child = group[key] + path = f"{prefix}/{key}" if prefix else key + if isinstance(child, zarr.Group): + groups.append(path) + walk(child, path) + + walk(root) + return groups + + def _list_datasets(self) -> list[DatasetEntry]: + zarr = _require_zarr() + root = self._open_store() + datasets: list[DatasetEntry] = [] + + if isinstance(root, zarr.Array): + return [ + { + "path": "", + "shape": tuple(int(s) for s in root.shape), + "ndim": int(root.ndim), + "dtype": str(root.dtype), + "size": int(root.size), + } + ] + + def walk(group, prefix=""): + for key in group.keys(): + child = group[key] + path = f"{prefix}/{key}" if prefix else key + if isinstance(child, zarr.Array): + datasets.append( + { + "path": path, + "shape": tuple(int(s) for s in child.shape), + "ndim": int(child.ndim), + "dtype": str(child.dtype), + "size": int(child.size), + } + ) + elif isinstance(child, zarr.Group): + walk(child, path) + + walk(root) + return datasets + + def _build_picker_groups(self, datasets: list[DatasetEntry]): + paths = {ds["path"] for ds in datasets if ds["path"]} + assigned = set() + groups = {} + + for group_path in sorted(self._list_group_paths(), key=len, reverse=True): + prefix = f"{group_path}/" + members = sorted(p for p in paths if p.startswith(prefix) and p not in assigned) + if len(members) >= 2: + groups[group_path] = { + "id": group_path, + "label": group_path, + "type": "zarr_group", + "dataset_paths": members, + } + assigned.update(members) + + return sorted(groups.values(), key=lambda group: group["label"].lower()) + + def _is_incompatible_root_layout(self, datasets: list[DatasetEntry]): + root = [ds for ds in datasets if "/" not in ds["path"] and ds["path"] != ""] + # Root-store single array uses path "" + if any(ds["path"] == "" for ds in datasets): + return False + if len(root) < 2: + return False + if not all(ds["ndim"] == 1 for ds in root): + return False + lengths = {ds["shape"][0] for ds in root if ds["shape"]} + return len(lengths) > 1 + + def _is_grouped_hierarchical_layout(self, datasets: list[DatasetEntry]): + nested = [ds for ds in datasets if "/" in ds["path"]] + if len(nested) < 4: + return False + + groups = self._build_picker_groups(datasets) + if len(groups) < 2: + return False + + lengths = set() + for ds in nested: + if ds["ndim"] == 1 and ds["shape"]: + lengths.add(ds["shape"][0]) + return len(lengths) > 1 + + def _needs_dataset_selection(self, datasets: list[DatasetEntry]): + return self._is_incompatible_root_layout(datasets) or self._is_grouped_hierarchical_layout( + datasets + ) + + def inventory(self) -> InventoryResult: + try: + datasets = self._list_datasets() + except Exception as e: + self.logger.error("Failed to inventory Zarr store: %s", e, exc_info=True) + return make_inventory(INVENTORY_EMPTY) + + if not datasets: + layout = INVENTORY_EMPTY + elif len(datasets) == 1: + layout = INVENTORY_SINGLE + elif self._needs_dataset_selection(datasets): + layout = INVENTORY_MULTI + else: + layout = INVENTORY_LEGACY + + groups = self._build_picker_groups(datasets) if layout == INVENTORY_MULTI else [] + return make_inventory(layout, datasets, groups) + + def parse(self): + return [ds["path"] or "(root)" for ds in self._list_datasets()] + + def get_metadata(self) -> dict[str, Any]: + zarr = _require_zarr() + root = self._open_store() + metadata: dict[str, Any] = {} + + def attrs_to_dict(attrs): + try: + return dict(attrs) + except Exception: + return {} + + if isinstance(root, zarr.Array): + metadata["(root)"] = attrs_to_dict(root.attrs) + return metadata + + metadata["(root)"] = attrs_to_dict(root.attrs) + + def walk(group, prefix=""): + for key in group.keys(): + child = group[key] + path = f"{prefix}/{key}" if prefix else key + child_attrs = attrs_to_dict(child.attrs) + if child_attrs: + metadata[path] = child_attrs + if isinstance(child, zarr.Group): + walk(child, path) + + walk(root) + return metadata + + def _normalize_selected_keys(self, keys): + if isinstance(keys, str): + keys = [key.strip() for key in keys.split(",") if key.strip()] + return [str(key) for key in keys if key] + + def _get_selected_dataset_keys(self): + if self._explicit_selected_keys is not None: + return self._normalize_selected_keys(self._explicit_selected_keys) + try: + from flask import session + + keys = session.get("selected_keys") or [] + return self._normalize_selected_keys(keys) + except RuntimeError: + return [] + + def _resolve_array(self, root, path: str): + zarr = _require_zarr() + if path in ("", "(root)"): + if isinstance(root, zarr.Array): + return root + self.logger.warning("Zarr root path requested but store is a group") + return None + try: + obj = root[path] + except Exception: + self.logger.warning("Zarr array path not found: %s", path) + return None + if not isinstance(obj, zarr.Array): + self.logger.warning("Zarr path is not an array: %s", path) + return None + return obj + + def _prepare_array_data(self, arr): + """Load array data; refuse ndim >= 3 (no averaging / reshape for metrics).""" + data = np.asarray(arr[:]) + if getattr(data, "ndim", 0) >= 3: + self.logger.warning( + "Zarr array has ndim=%s; refusing to aggregate or flatten. " + "Select 1D (or a single 2D) arrays via selected_keys.", + data.ndim, + ) + return None + return data + + def _column_name_from_path(self, path: str, used_names): + full = path.strip("/") if path not in ("", "(root)") else "value" + if not full: + full = "value" + if full not in used_names: + return full + short = path.split("/")[-1] or full + if short not in used_names: + return short + dotted = full.replace("/", ".") + if dotted not in used_names: + return dotted + suffix = 2 + while f"{full}_{suffix}" in used_names: + suffix += 1 + return f"{full}_{suffix}" + + def _array_to_frame(self, path: str, arr): + data = self._prepare_array_data(arr) + if data is None: + return None + + col_name = self._column_name_from_path(path, set()) + if getattr(data, "ndim", 0) == 0: + df = pd.DataFrame({col_name: [data]}) + elif data.ndim == 1: + df = pd.DataFrame({col_name: data}) + elif data.ndim == 2: + try: + df = pd.DataFrame(data) + except Exception: + df = pd.DataFrame(data.tolist()) + df.columns = [f"{col_name}_{i}" for i in range(df.shape[1])] + else: + self.logger.warning( + "Cannot convert ndim=%s Zarr array '%s' to a DataFrame", + getattr(data, "ndim", None), + path, + ) + return None + df.columns = [str(col) for col in df.columns] + return df if not df.empty else None + + def _read_array_path(self, path: str): + root = self._open_store() + arr = self._resolve_array(root, path) + if arr is None: + return None + return self._array_to_frame(path, arr) + + def _read_compatible_array_paths(self, paths): + if not paths: + return None + + columns = {} + expected_len = None + root = self._open_store() + + for path in paths: + arr = self._resolve_array(root, path) + if arr is None: + return None + data = self._prepare_array_data(arr) + if data is None: + return None + data = np.asarray(data) + if data.ndim != 1: + self.logger.warning( + "Zarr multi-select requires 1D arrays; '%s' has ndim=%s", + path, + data.ndim, + ) + return None + length = int(data.shape[0]) + if expected_len is None: + expected_len = length + elif length != expected_len: + self.logger.warning( + "Zarr selected arrays have incompatible lengths (%d vs %d)", + expected_len, + length, + ) + return None + name = self._column_name_from_path(path, set(columns)) + columns[name] = data + + df = pd.DataFrame(columns) + return df if not df.empty else None + + def _auto_read_all(self, datasets: list[DatasetEntry]): + """Auto-read when layout does not require explicit selection.""" + if len(datasets) == 1: + return self._read_array_path(datasets[0]["path"] or "") + + ones = [ds for ds in datasets if ds["ndim"] == 1] + if ones and len(ones) == len(datasets): + lengths = {ds["shape"][0] for ds in ones if ds["shape"]} + if len(lengths) == 1: + return self._read_compatible_array_paths([ds["path"] for ds in ones]) + + self.logger.warning( + "Zarr store has %d arrays that cannot be auto-merged; " + "pass selected_keys to read specific paths.", + len(datasets), + ) + return None + + def read(self): + try: + inv = self.inventory() + if inv["type"] == INVENTORY_EMPTY: + self.logger.warning("No arrays found in Zarr store") + return None + + # Honor explicit selection for any non-empty layout (CLI/library/Globus). + selected = self._get_selected_dataset_keys() + if selected: + if len(selected) == 1: + return self._read_array_path(selected[0]) + return self._read_compatible_array_paths(selected) + + if inv["type"] == INVENTORY_MULTI: + self.logger.warning( + "Zarr store has %d arrays in an incompatible layout; " + "refusing to flatten. Pass selected_keys to choose paths.", + len(inv["datasets"]), + ) + return None + + return self._auto_read_all(inv["datasets"]) + except ImportError: + raise + except Exception as e: + self.logger.error("Error reading Zarr store: %s", e, exc_info=True) + return None diff --git a/aidrin/headless/api.py b/aidrin/headless/api.py index 06172134..59fd3f00 100644 --- a/aidrin/headless/api.py +++ b/aidrin/headless/api.py @@ -29,6 +29,7 @@ run_feature_relevance, run_hipaa_compliance, run_k_anonymity, + using_selected_keys, run_l_diversity, run_multiple_attribute_risk, run_null_count_trend, @@ -361,14 +362,26 @@ def summarize_dataset( file_path: str, file_type: Optional[str] = None, max_features: Optional[int] = None, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: """Return shape, per-column descriptive stats, and missing counts for a dataset.""" - from pathlib import Path from aidrin.file_handling.file_parser import read_file + from .runners import _build_file_info - path = Path(file_path) - ext = f".{file_type}" if file_type else path.suffix.lower() - df = read_file((file_path, path.name, ext)) + file_info = _build_file_info( + file_path, + file_type, + None, + selected_keys=_normalize_list(selected_keys), + ) + df = read_file(file_info) + + if df is None or isinstance(df, str): + detail = df if isinstance(df, str) else ( + "Unable to build a table from this file. For multi-array Zarr/HDF5 stores, " + "pass selected_keys with compatible 1D paths." + ) + raise ValueError(detail) num_cols = df.select_dtypes(include="number").columns.tolist() cat_cols = df.select_dtypes(include="object").columns.tolist() @@ -506,6 +519,31 @@ def run_metric( verbose: bool = False, strip_visualizations: bool = False, **kwargs: Any, +) -> Dict[str, Any]: + with using_selected_keys(_normalize_list(kwargs.get("selected_keys"))): + return _run_metric_impl( + metric_name, + file_path, + file_type=file_type, + file_name=file_name, + save_images=save_images, + image_dir=image_dir, + verbose=verbose, + strip_visualizations=strip_visualizations, + **kwargs, + ) + + +def _run_metric_impl( + metric_name: str, + file_path: str, + file_type: Optional[str] = None, + file_name: Optional[str] = None, + save_images: bool = True, + image_dir: Optional[str] = None, + verbose: bool = False, + strip_visualizations: bool = False, + **kwargs: Any, ) -> Dict[str, Any]: metric_key = metric_name.strip().lower().replace("-", "_") metric = METRIC_REGISTRY.get(metric_key) @@ -757,6 +795,7 @@ def run_batch_metrics( "timestamp_column": config_obj.timestamp_column, "batch_column": config_obj.batch_column, "target_columns": config_obj.target_columns, + "selected_keys": config_obj.selected_keys, "path_targets": config_obj.path_targets, "base_dir": config_obj.base_dir, "max_results": config_obj.max_results, @@ -786,6 +825,7 @@ def run_data_quality( file_name: Optional[str] = None, verbose: bool = False, strip_visualizations: bool = True, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: """Run fast data quality metrics (completeness, duplicity, outliers). @@ -798,6 +838,7 @@ def run_data_quality( file_type=file_type, file_name=file_name, metrics=["completeness", "duplicity", "outliers"], + selected_keys=selected_keys or [], save_images=False, ), verbose=verbose, diff --git a/aidrin/headless/cli.py b/aidrin/headless/cli.py index f0a5a243..70c2df23 100644 --- a/aidrin/headless/cli.py +++ b/aidrin/headless/cli.py @@ -307,6 +307,7 @@ def _build_run_kwargs(args: argparse.Namespace) -> dict: "timestamp_column": getattr(args, "timestamp_column", None), "batch_column": getattr(args, "batch_column", None), "target_columns": _parse_list(getattr(args, "target_columns", None)), + "selected_keys": _parse_list(getattr(args, "selected_keys", None)), "path_targets": _parse_path_targets(getattr(args, "path_targets", None), target_match), "base_dir": getattr(args, "base_dir", None), "max_results": getattr(args, "max_results", 100), @@ -329,6 +330,12 @@ def _build_run_kwargs(args: argparse.Namespace) -> dict: def _configure_common_run_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--file-type", dest="file_type", default=None, help="Input file type override") + parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read (same idea as web selected_keys)", + ) parser.add_argument("--save-images", dest="save_images", action="store_true", help="Save visualizations to disk") parser.add_argument("--no-save-images", dest="save_images", action="store_false", help="Do not save visualizations") parser.set_defaults(save_images=True) @@ -340,6 +347,13 @@ def _configure_common_run_args(parser: argparse.ArgumentParser) -> None: def _configure_minimal_run_args(parser: argparse.ArgumentParser) -> None: """Lightweight args for top-level metric shortcuts.""" + parser.add_argument("--file-type", dest="file_type", default=None, help="Input file type override") + parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read", + ) parser.add_argument("-v", "--verbose", action="store_true", help="Show progress output") @@ -848,13 +862,30 @@ def main() -> None: dq_parser = subparsers.add_parser("data-quality", help="Run fast data quality metrics (completeness, duplicity, outliers)") dq_parser.add_argument("file_path") dq_parser.add_argument("--file-type", dest="file_type", default=None) + dq_parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read", + ) dq_parser.add_argument("-v", "--verbose", action="store_true", help="Show progress output") dq_parser.add_argument("--detail", action="store_true", help="Output full per-feature JSON instead of summary") # Dataset summary command summarize_parser = subparsers.add_parser("summarize", help="Describe numerical and categorical features of a dataset") summarize_parser.add_argument("file_path", help="Path to the dataset") - summarize_parser.add_argument("--file-type", dest="file_type", default=None, help="File type override (csv, parquet, xlsx, hdf5, json, npz)") + summarize_parser.add_argument( + "--file-type", + dest="file_type", + default=None, + help="File type override (csv, parquet, xlsx, hdf5, json, npz, zarr)", + ) + summarize_parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read", + ) summarize_parser.add_argument( "--max-features", dest="max_features", type=int, default=None, help="Limit stats to N features (split evenly between numerical and categorical)" @@ -1011,6 +1042,7 @@ def _note_submission(task_id: str) -> None: args.file_path, file_type=args.file_type, max_features=args.max_features, + selected_keys=_parse_list(getattr(args, "selected_keys", None)), ) _fail_on_remote_error(result, remote_opts) if args.human_readable: @@ -1025,6 +1057,7 @@ def _note_submission(task_id: str) -> None: file_type=args.file_type, verbose=args.verbose, strip_visualizations=True, + selected_keys=_parse_list(getattr(args, "selected_keys", None)), ) _fail_on_remote_error(result, remote_opts) if args.detail: diff --git a/aidrin/headless/config.py b/aidrin/headless/config.py index e909fee2..1ebff2c8 100644 --- a/aidrin/headless/config.py +++ b/aidrin/headless/config.py @@ -46,6 +46,7 @@ class HeadlessConfig: timestamp_column: Optional[str] = None batch_column: Optional[str] = None target_columns: Optional[List[str]] = field(default_factory=list) + selected_keys: Optional[List[str]] = field(default_factory=list) path_targets: Optional[List[str]] = field(default_factory=list) base_dir: Optional[str] = None max_results: Optional[int] = 100 @@ -85,6 +86,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "HeadlessConfig": "file-name": "file_name", "image-dir": "image_dir", "save-images": "save_images", + "selected-keys": "selected_keys", } normalized: Dict[str, Any] = {} @@ -107,6 +109,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "HeadlessConfig": "required_columns", "duplicate_columns", "target_columns", + "selected_keys", ): if key in normalized: normalized[key] = _normalize_list(normalized[key]) diff --git a/aidrin/headless/runners.py b/aidrin/headless/runners.py index 6fdfffde..95570fcc 100644 --- a/aidrin/headless/runners.py +++ b/aidrin/headless/runners.py @@ -1,7 +1,9 @@ import os import json +from contextlib import contextmanager +from contextvars import ContextVar from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterator, List, Optional from aidrin.file_handling.file_parser import read_file from aidrin.structured_data_metrics.add_noise import return_noisy_stats @@ -49,6 +51,22 @@ _EXCEL_TYPES = {".xls", ".xlsx", ".xlsm", ".xlsb"} _EXCEL_KEY = ".xls, .xlsb, .xlsx, .xlsm" +# Optional HDF5/Zarr path selection for the current headless call (CLI/API/batch). +_selected_keys_ctx: ContextVar[Optional[List[str]]] = ContextVar( + "aidrin_selected_keys", default=None +) + + +@contextmanager +def using_selected_keys(selected_keys: Optional[List[str]] = None) -> Iterator[None]: + """Apply ``selected_keys`` to all ``_build_file_info`` calls in this context.""" + keys = [str(k).strip() for k in (selected_keys or []) if str(k).strip()] or None + token = _selected_keys_ctx.set(keys) + try: + yield + finally: + _selected_keys_ctx.reset(token) + class NullTask: def update_state(self, *args: Any, **kwargs: Any) -> None: @@ -69,10 +87,16 @@ def _normalize_file_type(file_type: Optional[str], file_path: str) -> Optional[s def _build_file_info( - file_path: str, file_type: Optional[str], file_name: Optional[str] + file_path: str, + file_type: Optional[str], + file_name: Optional[str], + selected_keys: Optional[List[str]] = None, ) -> tuple: normalized_type = _normalize_file_type(file_type, file_path) final_name = file_name or os.path.basename(file_path) + keys = selected_keys if selected_keys is not None else _selected_keys_ctx.get() + if keys: + return (file_path, final_name, normalized_type, list(keys)) return (file_path, final_name, normalized_type) diff --git a/docs/source/cli_usage.rst b/docs/source/cli_usage.rst index 2e5a4471..e96533ec 100644 --- a/docs/source/cli_usage.rst +++ b/docs/source/cli_usage.rst @@ -20,6 +20,14 @@ Quick Start # Run a batch of metrics from a YAML config aidrin batch /path/to/my_project/batch_config.yaml + # HDF5 / Zarr: pick compatible 1D arrays (comma-separated paths) + aidrin run completeness /path/to/store.zarr --selected-keys age,income + aidrin summarize /path/to/file.h5 --selected-keys S1/X,S1/Y + +Install Zarr support with ``pip install 'aidrin[zarr]'`` (Python >= 3.11). Local web upload +does not accept ``.zarr`` directories; use the CLI or library. Multi-dimensional grids are not +auto-flattened — select 1D (or a single 2D) arrays only. + ---- Sample Dataset @@ -252,6 +260,9 @@ Results are printed as JSON to stdout. Redirect to a file to save: target-column: approved +For HDF5/Zarr multi-array stores, add ``selected-keys`` (list or comma-separated string) +with compatible 1D paths, for example ``selected-keys: [age, income]``. + For a file manifest, batch configuration accepts dashed or underscored forms and list or comma-separated targets: diff --git a/pyproject.toml b/pyproject.toml index 5f93931e..238c2cf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,10 @@ globus = [ "globus-compute-sdk>=2.3.0", "globus-sdk>=3.20.0", ] +# Zarr 3.x needs Python >=3.11 (3.2+ needs >=3.12). Pin <3.2 so 3.11 stays resolvable. +zarr = [ + "zarr>=3.0.8,<3.2; python_version >= '3.11'", +] telemetry = [ "opentelemetry-api>=1.20", "opentelemetry-sdk>=1.20", @@ -126,6 +130,9 @@ version = {attr = "aidrin._version.__version__"} # --------------------------------------------------------------------------- [dependency-groups] +zarr = [ + "zarr>=3.0.8,<3.2; python_version >= '3.11'", +] dev = [ "pytest>=7.0", "pytest-flask", diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index de67224b..12de4480 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -725,5 +725,65 @@ def test_no_command_exits_nonzero(self): self.assertNotEqual(code, 0) +# =========================================================================== +# selected-keys (HDF5 / Zarr) +# =========================================================================== + + +class TestSelectedKeysCLI(unittest.TestCase): + """CLI --selected-keys for multi-array Zarr stores.""" + + @classmethod + def setUpClass(cls): + pytest = __import__("pytest") + zarr = pytest.importorskip("zarr") + cls._tmpdir = tempfile.TemporaryDirectory() + store = os.path.join(cls._tmpdir.name, "pick.zarr") + root = zarr.open_group(store, mode="w") + for name, length in (("age", 20), ("income", 20), ("meta", 1)): + arr = root.create_array(name, shape=(length,), dtype="f8") + arr[:] = np.arange(length, dtype=np.float64) + cls.store = store + + @classmethod + def tearDownClass(cls): + cls._tmpdir.cleanup() + + def test_run_completeness_with_selected_keys(self): + stdout, stderr, code = _run_cli( + "run", + "completeness", + self.store, + "--selected-keys", + "age,income", + ) + self.assertEqual(code, 0, msg=stderr) + payload = json.loads(stdout) + scores = payload.get("Completeness scores", {}) + self.assertIn("age", scores) + self.assertIn("income", scores) + self.assertNotIn("meta", scores) + + def test_summarize_with_selected_keys(self): + stdout, stderr, code = _run_cli( + "summarize", + self.store, + "--selected-keys", + "age", + ) + self.assertEqual(code, 0, msg=stderr) + payload = json.loads(stdout) + self.assertEqual(payload["shape"]["columns"], 1) + self.assertIn("age", payload.get("numerical", {})) + + def test_build_file_info_includes_selected_keys(self): + from aidrin.headless.runners import _build_file_info, using_selected_keys + + with using_selected_keys(["S1/X", "S1/Y"]): + info = _build_file_info(self.store, None, None) + self.assertEqual(info[2], ".zarr") + self.assertEqual(info[3], ["S1/X", "S1/Y"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_structured_readers.py b/tests/unit/test_structured_readers.py new file mode 100644 index 00000000..ab72e594 --- /dev/null +++ b/tests/unit/test_structured_readers.py @@ -0,0 +1,214 @@ +"""Unit tests for structured reader scaffold and Zarr reader.""" + +import logging +import sys +import types + +import numpy as np +import pytest + +# Compatibility shim: pkg_resources for clean Python 3.12+ environments. +if "pkg_resources" not in sys.modules: + _pkg_resources = types.ModuleType("pkg_resources") + + class _Dist: + def __init__(self): + self.version = "0.0.0" + + _pkg_resources.get_distribution = lambda _name: _Dist() + sys.modules["pkg_resources"] = _pkg_resources + +from aidrin.file_handling.file_parser import ( + GLOBUS_FILE_TYPES, + READER_MAP, + SUPPORTED_FILE_TYPES, + read_file, +) +from aidrin.file_handling.readers.root_reader import rootReader +from aidrin.file_handling.readers.structured import ( + INVENTORY_EMPTY, + INVENTORY_LEGACY, + INVENTORY_MULTI, + INVENTORY_SINGLE, + INVENTORY_UNSUPPORTED, + USER_FACING_INVENTORY_TYPES, + make_inventory, +) +from aidrin.file_handling.readers.zarr_reader import zarrReader + +zarr = pytest.importorskip("zarr") + + +@pytest.fixture +def logger(): + return logging.getLogger("test_structured_readers") + + +@pytest.mark.parametrize("reader_cls", [rootReader]) +def test_root_stub_inventory_contract(tmp_path, logger, reader_cls): + path = tmp_path / "placeholder" + path.write_text("not used", encoding="utf-8") + inv = reader_cls(str(path), logger).inventory() + + assert set(inv.keys()) == {"type", "datasets", "groups"} + assert inv["type"] == INVENTORY_UNSUPPORTED + assert inv["datasets"] == [] + assert inv["groups"] == [] + assert inv["type"] not in USER_FACING_INVENTORY_TYPES + + +def test_make_inventory_helper(): + inv = make_inventory(INVENTORY_UNSUPPORTED) + assert inv == {"type": INVENTORY_UNSUPPORTED, "datasets": [], "groups": []} + + +def test_zarr_registered_for_cli_not_local_upload(): + assert ".zarr" in READER_MAP + local_exts = [ext for ext, _ in SUPPORTED_FILE_TYPES] + globus_exts = [ext for ext, _ in GLOBUS_FILE_TYPES] + assert ".zarr" not in local_exts + assert ".zarr" in globus_exts + + +def _write_single_array_store(path): + arr = zarr.open(str(path), mode="w", shape=(5,), dtype="f8") + arr[:] = np.arange(5, dtype=np.float64) + arr.attrs["unit"] = "m" + + +def _write_group_store(path): + root = zarr.open_group(str(path), mode="w") + root.attrs["license"] = "MIT" + a = root.create_array("temp", shape=(4,), dtype="f8") + a[:] = np.arange(4, dtype=np.float64) + a.attrs["unit"] = "C" + g = root.create_group("station") + b = g.create_array("x", shape=(4,), dtype="i4") + b[:] = np.arange(4, dtype=np.int32) + + +def _write_incompatible_root_store(path): + root = zarr.open_group(str(path), mode="w") + a = root.create_array("short", shape=(3,), dtype="f8") + a[:] = [1.0, 2.0, 3.0] + b = root.create_array("long", shape=(5,), dtype="f8") + b[:] = np.arange(5, dtype=np.float64) + + +def _write_grouped_hierarchical_store(path): + root = zarr.open_group(str(path), mode="w") + for station in ("S1", "S2"): + g = root.create_group(station) + for axis, length in (("X", 10), ("Y", 10), ("meta", 1)): + arr = g.create_array(axis, shape=(length,), dtype="f8") + arr[:] = np.arange(length, dtype=np.float64) + + +def test_zarr_single_array_inventory_and_read(tmp_path, logger): + store = tmp_path / "single.zarr" + _write_single_array_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_SINGLE + assert len(inv["datasets"]) == 1 + + df = reader.read() + assert df is not None + assert list(df.columns) == ["value"] + assert len(df) == 5 + assert list(df["value"]) == [0.0, 1.0, 2.0, 3.0, 4.0] + + +def test_zarr_compatible_group_auto_read(tmp_path, logger): + store = tmp_path / "group.zarr" + _write_group_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_LEGACY + assert inv["type"] not in USER_FACING_INVENTORY_TYPES + + df = reader.read() + assert df is not None + assert set(df.columns) == {"temp", "station/x"} + assert len(df) == 4 + + +def test_zarr_incompatible_root_needs_selection(tmp_path, logger): + store = tmp_path / "bad.zarr" + _write_incompatible_root_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_MULTI + assert reader.read() is None + + df = zarrReader(str(store), logger, selected_keys=["short"]).read() + assert df is not None + assert list(df.columns) == ["short"] + assert len(df) == 3 + + +def test_zarr_grouped_hierarchical_selection(tmp_path, logger): + store = tmp_path / "stations.zarr" + _write_grouped_hierarchical_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_MULTI + assert len(inv["groups"]) >= 2 + + df = zarrReader( + str(store), logger, selected_keys=["S1/X", "S1/Y"] + ).read() + assert df is not None + assert set(df.columns) == {"S1/X", "S1/Y"} + assert len(df) == 10 + + +def test_zarr_get_metadata(tmp_path, logger): + store = tmp_path / "meta.zarr" + _write_group_store(store) + meta = zarrReader(str(store), logger).get_metadata() + assert meta["(root)"]["license"] == "MIT" + assert meta["temp"]["unit"] == "C" + + +def test_zarr_read_file_integration(tmp_path): + store = tmp_path / "cli.zarr" + _write_single_array_store(store) + df = read_file((str(store), "cli.zarr", ".zarr")) + assert df is not None + assert len(df) == 5 + + +def test_zarr_empty_store(tmp_path, logger): + store = tmp_path / "empty.zarr" + zarr.open_group(str(store), mode="w") + inv = zarrReader(str(store), logger).inventory() + assert inv["type"] == INVENTORY_EMPTY + assert zarrReader(str(store), logger).read() is None + + +def _write_multidim_store(path): + """3D array shaped like (time, lat, lon) — not tabular without aggregation.""" + root = zarr.open_group(str(path), mode="w") + data = np.zeros((5, 4, 3), dtype=np.float64) + for t in range(5): + data[t, :, :] = float(t) + arr = root.create_array("tmax_grid", shape=data.shape, dtype="f8") + arr[:] = data + + +def test_zarr_multidim_refused(tmp_path, logger): + """ndim >= 3 must not be averaged or flattened for metrics.""" + store = tmp_path / "grid.zarr" + _write_multidim_store(store) + df = zarrReader(str(store), logger, selected_keys=["tmax_grid"]).read() + assert df is None + + +def test_zarr_read_file_selected_keys(tmp_path): + store = tmp_path / "pick.zarr" + _write_grouped_hierarchical_store(store) + df = read_file((str(store), "pick.zarr", ".zarr", ["S1/X", "S1/Y"])) + assert df is not None + assert list(df.columns) == ["S1/X", "S1/Y"] + assert len(df) == 10 diff --git a/uv.lock b/uv.lock index 50c8924e..31608110 100644 --- a/uv.lock +++ b/uv.lock @@ -104,6 +104,9 @@ telemetry = [ { name = "opentelemetry-instrumentation-flask" }, { name = "opentelemetry-sdk" }, ] +zarr = [ + { name = "zarr", marker = "python_full_version >= '3.11'" }, +] [package.dev-dependencies] agentic = [ @@ -144,6 +147,9 @@ mcp = [ { name = "pymupdf" }, { name = "python-dotenv" }, ] +zarr = [ + { name = "zarr", marker = "python_full_version >= '3.11'" }, +] [package.metadata] requires-dist = [ @@ -201,8 +207,9 @@ requires-dist = [ { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = "==3.1.0" }, { name = "sphinxemoji", marker = "extra == 'docs'" }, { name = "tables", specifier = ">=3.10.0" }, + { name = "zarr", marker = "python_full_version >= '3.11' and extra == 'zarr'", specifier = ">=3.0.8,<3.2" }, ] -provides-extras = ["globus", "telemetry", "llm", "mcp", "agentic", "dev", "docs"] +provides-extras = ["globus", "zarr", "telemetry", "llm", "mcp", "agentic", "dev", "docs"] [package.metadata.requires-dev] agentic = [ @@ -241,6 +248,7 @@ mcp = [ { name = "pymupdf", specifier = ">=1.24.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, ] +zarr = [{ name = "zarr", marker = "python_full_version >= '3.11'", specifier = ">=3.0.8,<3.2" }] [[package]] name = "aiohappyeyeballs" @@ -1298,6 +1306,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, ] +[[package]] +name = "donfig" +version = "0.8.1.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, +] + [[package]] name = "dython" version = "0.7.12" @@ -1638,6 +1658,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/bb/2b865d79c5bdc574b428e5502a24e7517cdaeca2569f6baa051106f9fa7e/globus_sdk-4.8.1-py3-none-any.whl", hash = "sha256:48b57862902a26d8606e73374c354415724cfa9d99301af7e9e406659dafe78d", size = 442599, upload-time = "2026-06-16T16:42:16.226Z" }, ] +[[package]] +name = "google-crc32c" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/ac/6f7bc93886a823ab545948c2dd48143027b2355ad1944c7cf852b338dc91/google_crc32c-1.8.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff", size = 31296, upload-time = "2025-12-16T00:19:07.261Z" }, + { url = "https://files.pythonhosted.org/packages/f7/97/a5accde175dee985311d949cfcb1249dcbb290f5ec83c994ea733311948f/google_crc32c-1.8.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288", size = 30870, upload-time = "2025-12-16T00:29:17.669Z" }, + { url = "https://files.pythonhosted.org/packages/3d/63/bec827e70b7a0d4094e7476f863c0dbd6b5f0f1f91d9c9b32b76dcdfeb4e/google_crc32c-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d", size = 33214, upload-time = "2025-12-16T00:40:19.618Z" }, + { url = "https://files.pythonhosted.org/packages/63/bc/11b70614df04c289128d782efc084b9035ef8466b3d0a8757c1b6f5cf7ac/google_crc32c-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092", size = 33589, upload-time = "2025-12-16T00:40:20.7Z" }, + { url = "https://files.pythonhosted.org/packages/3e/00/a08a4bc24f1261cc5b0f47312d8aebfbe4b53c2e6307f1b595605eed246b/google_crc32c-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733", size = 34437, upload-time = "2025-12-16T00:35:19.437Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" }, + { url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" }, + { url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" }, + { url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, + { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, + { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, + { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, + { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, + { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, + { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, + { url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.75.0" @@ -3041,6 +3096,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/f2/dca53d50b8f2289dd01954ace9da261e0487d5b74b188b4304e4ecc3492c/numba-0.66.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d426178fb991a85714c43112a8ea7b9d9579ea856ad8dcdb9c1c3941903ba5be", size = 2804772, upload-time = "2026-07-01T23:12:44.399Z" }, ] +[[package]] +name = "numcodecs" +version = "0.16.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/85/1ac101a40ead81eaa1c7dc49a8827a30e2e436211b43ebdc63c590eb1347/numcodecs-0.16.5-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:78382dcea50622f2ef1e6e7a71dbe7f861d8fe376b27b7c297c26907304fef1e", size = 1621795, upload-time = "2025-11-21T02:49:17.418Z" }, + { url = "https://files.pythonhosted.org/packages/0e/cc/0d97ef55dda48cb0f93d7b92d761208e7a99bd2eea6b0e859426e6a99a21/numcodecs-0.16.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2d04a19cb57a3c519b4127ac377cca6471aee1990d7c18f5b1e3a4fe1306689", size = 1153030, upload-time = "2025-11-21T02:49:19.089Z" }, + { url = "https://files.pythonhosted.org/packages/5e/41/e120ee1b390730ac5987cde2afd82e2b8442cec315ab40b94b0373e93e73/numcodecs-0.16.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c043af648eb280cd61785c99c22ff5c3c3460f906eb51a8511327c4f5111b283", size = 8510503, upload-time = "2025-11-21T02:49:20.324Z" }, + { url = "https://files.pythonhosted.org/packages/54/4b/195ac84cc8f6077b4f0f421e8daee21b7f1bd88cb7716414234379fe68ec/numcodecs-0.16.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c398919ef2eb0e56b8e97456f622640bfd3deed06de3acc976989cbcb22628a3", size = 9123428, upload-time = "2025-11-21T02:49:22.328Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5b/af02c417954f46e5c7bd5163ac251f535877d909fce54861c99ae197f6f6/numcodecs-0.16.5-cp311-cp311-win_amd64.whl", hash = "sha256:3820860ed302d4d84a1c66e70981ff959d5eb712555be4e7d8ced49888594773", size = 801542, upload-time = "2025-11-21T02:49:24.265Z" }, + { url = "https://files.pythonhosted.org/packages/75/cc/55420f3641a67f78392dc0bc5d02cb9eb0a9dcebf2848d1ac77253ca61fa/numcodecs-0.16.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:24e675dc8d1550cd976a99479b87d872cb142632c75cc402fea04c08c4898523", size = 1656287, upload-time = "2025-11-21T02:49:25.755Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6c/86644987505dcb90ba6d627d6989c27bafb0699f9fd00187e06d05ea8594/numcodecs-0.16.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:94ddfa4341d1a3ab99989d13b01b5134abb687d3dab2ead54b450aefe4ad5bd6", size = 1148899, upload-time = "2025-11-21T02:49:26.87Z" }, + { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, + { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, + { url = "https://files.pythonhosted.org/packages/1c/20/2fdec87fc7f8cec950d2b0bea603c12dc9f05b4966dc5924ba5a36a61bf6/numcodecs-0.16.5-cp312-cp312-win_amd64.whl", hash = "sha256:845a9857886ffe4a3172ba1c537ae5bcc01e65068c31cf1fce1a844bd1da050f", size = 801412, upload-time = "2025-11-21T02:49:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/38/38/071ced5a5fd1c85ba0e14ba721b66b053823e5176298c2f707e50bed11d9/numcodecs-0.16.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25be3a516ab677dad890760d357cfe081a371d9c0a2e9a204562318ac5969de3", size = 1654359, upload-time = "2025-11-21T02:49:33.673Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c0/5f84ba7525577c1b9909fc2d06ef11314825fc4ad4378f61d0e4c9883b4a/numcodecs-0.16.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0107e839ef75b854e969cb577e140b1aadb9847893937636582d23a2a4c6ce50", size = 1144237, upload-time = "2025-11-21T02:49:35.294Z" }, + { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, + { url = "https://files.pythonhosted.org/packages/27/72/6663cc0382ddbb866136c255c837bcb96cc7ce5e83562efec55e1b995941/numcodecs-0.16.5-cp313-cp313-win_amd64.whl", hash = "sha256:5088145502ad1ebf677ec47d00eb6f0fd600658217db3e0c070c321c85d6cf3d", size = 799275, upload-time = "2025-11-21T02:49:39.558Z" }, + { url = "https://files.pythonhosted.org/packages/3c/9e/38e7ca8184c958b51f45d56a4aeceb1134ecde2d8bd157efadc98502cc42/numcodecs-0.16.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b05647b8b769e6bc8016e9fd4843c823ce5c9f2337c089fb5c9c4da05e5275de", size = 1654721, upload-time = "2025-11-21T02:49:40.602Z" }, + { url = "https://files.pythonhosted.org/packages/a1/37/260fa42e7b2b08e6e00ad632f8dd620961a60a459426c26cea390f8c68d0/numcodecs-0.16.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3832bd1b5af8bb3e413076b7d93318c8e7d7b68935006b9fa36ca057d1725a8f", size = 1146887, upload-time = "2025-11-21T02:49:41.721Z" }, + { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, + { url = "https://files.pythonhosted.org/packages/31/a5/a0425af36c20d55a3ea884db4b4efca25a43bea9214ba69ca7932dd997b4/numcodecs-0.16.5-cp314-cp314-win_amd64.whl", hash = "sha256:b16a14303800e9fb88abc39463ab4706c037647ac17e49e297faa5f7d7dbbf1d", size = 819022, upload-time = "2025-11-21T02:49:47.39Z" }, +] + [[package]] name = "numexpr" version = "2.14.1" @@ -6581,6 +6668,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, ] +[[package]] +name = "zarr" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "donfig" }, + { name = "google-crc32c" }, + { name = "numcodecs" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/5a/b8a0cf39a14c770c30bd1f2d120c54000c8cd9e84e8e79f38d9a7ce58071/zarr-3.1.6.tar.gz", hash = "sha256:d95e72cbea4b90e9a70679468b8266400331756232576ae2b43400ac5108d0eb", size = 386531, upload-time = "2026-03-23T17:25:18.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/7c/ba8ca8cbe9dbef8e83a95fc208fed8e6686c98b4719aaa0aa7f3d31fe390/zarr-3.1.6-py3-none-any.whl", hash = "sha256:b5a82c5079d1c3d4ee8f06746fa3b9a98a7d804300fa3f4be154362a33e1207e", size = 295655, upload-time = "2026-03-23T17:25:17.189Z" }, +] + [[package]] name = "zstandard" version = "0.25.0" diff --git a/web/routes/core.py b/web/routes/core.py index 466ee6f4..e0e81386 100644 --- a/web/routes/core.py +++ b/web/routes/core.py @@ -17,7 +17,11 @@ url_for, ) from werkzeug.utils import secure_filename -from aidrin.file_handling.file_parser import SUPPORTED_FILE_TYPES, READER_MAP +from aidrin.file_handling.file_parser import ( + GLOBUS_FILE_TYPES, + READER_MAP, + SUPPORTED_FILE_TYPES, +) from aidrin.file_handling.hashable_utils import hashable_series, safe_nunique from aidrin.file_handling.readers.hdf5_reader import hdf5Reader from web.routes.utils import ( @@ -156,6 +160,7 @@ def inspector(): uploaded_file_name=effective_file_name or "", file_type=effective_file_type or "", supported_file_types=SUPPORTED_FILE_TYPES, + globus_file_types=GLOBUS_FILE_TYPES, file_preview=file_preview if file_preview is not None else [], current_checked_keys=current_checked_keys if current_checked_keys is not None diff --git a/web/templates/_components/globus_panel.html b/web/templates/_components/globus_panel.html index 865e41e3..038b8caa 100644 --- a/web/templates/_components/globus_panel.html +++ b/web/templates/_components/globus_panel.html @@ -38,7 +38,7 @@

Connect t
-
@@ -47,7 +47,7 @@

Connect t