From e4989782c73aaa3a5b08e84d283f162a45af2420 Mon Sep 17 00:00:00 2001 From: Dhanushkumar Jv Date: Tue, 21 Jul 2026 22:37:24 +0530 Subject: [PATCH 1/5] Add Zarr reader for CLI, library, and Globus. --- aidrin/file_handling/file_parser.py | 17 +- aidrin/file_handling/readers/root_reader.py | 17 + aidrin/file_handling/readers/structured.py | 81 +++++ aidrin/file_handling/readers/zarr_reader.py | 344 ++++++++++++++++++++ pyproject.toml | 6 + tests/unit/test_structured_readers.py | 187 +++++++++++ web/routes/core.py | 7 +- web/templates/_components/globus_panel.html | 4 +- 8 files changed, 658 insertions(+), 5 deletions(-) create mode 100644 aidrin/file_handling/readers/root_reader.py create mode 100644 aidrin/file_handling/readers/structured.py create mode 100644 aidrin/file_handling/readers/zarr_reader.py create mode 100644 tests/unit/test_structured_readers.py diff --git a/aidrin/file_handling/file_parser.py b/aidrin/file_handling/file_parser.py index 2657f9dc..8d8461b5 100644 --- a/aidrin/file_handling/file_parser.py +++ b/aidrin/file_handling/file_parser.py @@ -8,6 +8,7 @@ from aidrin.file_handling.readers.json_reader import jsonReader from aidrin.file_handling.readers.npz_reader import npzReader from aidrin.file_handling.readers.parquet_reader import parquetReader +from aidrin.file_handling.readers.zarr_reader import zarrReader # Notes: # To add support for new file types: @@ -15,6 +16,8 @@ # (and optionally .parse(), .filter()) to 'file_readers'. # - Register the class in READER_MAP. # - Add a display name and extension to SUPPORTED_FILE_TYPES for the front end. +# - Formats that are CLI/library/Globus-only (e.g. Zarr directories) go in +# GLOBUS_FILE_TYPES but not SUPPORTED_FILE_TYPES (local upload dropdown). # Reader Map. Used to create file type specific parsing READER_MAP = { @@ -24,10 +27,11 @@ ".json": jsonReader, ".h5": hdf5Reader, ".parquet": parquetReader, + ".zarr": zarrReader, # Add additional file types here } -# Supported file types. Read on front end to create select features. +# Supported file types. Read on front end to create local upload select features. SUPPORTED_FILE_TYPES = [ (".csv", "CSV"), (".xls, .xlsb, .xlsx, .xlsm", "Excel"), @@ -39,6 +43,15 @@ # (file_type,file_type_name) ] +# Globus (and other path-based surfaces) may include directory-shaped formats +# that are not offered in the local browser upload dropdown. +GLOBUS_FILE_TYPES = SUPPORTED_FILE_TYPES + [ + (".zarr", "Zarr"), +] + +# File types that accept selected_keys for multi-array / multi-dataset selection. +_SELECTION_FILE_TYPES = {".h5", ".zarr"} + # logger config file_upload_time_log = logging.getLogger("file_upload") @@ -227,7 +240,7 @@ def read_file(file_info, columns=None): # Slow path: parse the source once. reader_cls = READER_MAP[file_type] - if file_type == ".h5": + if file_type in _SELECTION_FILE_TYPES: df = reader_cls( file_path, file_upload_time_log, selected_keys=selected_keys ).read() diff --git a/aidrin/file_handling/readers/root_reader.py b/aidrin/file_handling/readers/root_reader.py new file mode 100644 index 00000000..576640e4 --- /dev/null +++ b/aidrin/file_handling/readers/root_reader.py @@ -0,0 +1,17 @@ +from aidrin.file_handling.readers.structured import ( + INVENTORY_UNSUPPORTED, + InventoryResult, + StructuredFileReader, + make_inventory, +) + + +class rootReader(StructuredFileReader): + """ROOT file reader (scaffold). v1 will support TTrees only via uproot.""" + + def inventory(self) -> InventoryResult: + return make_inventory(INVENTORY_UNSUPPORTED) + + def read(self): + self.logger.warning("ROOT read is not implemented yet.") + return None diff --git a/aidrin/file_handling/readers/structured.py b/aidrin/file_handling/readers/structured.py new file mode 100644 index 00000000..bd86b8ce --- /dev/null +++ b/aidrin/file_handling/readers/structured.py @@ -0,0 +1,81 @@ +""" +Shared contract for structured file readers (HDF5, Zarr, ROOT). + +Structured formats may hold many arrays or tables under paths. Readers classify +the on-disk layout via ``inventory()`` before ``read()`` loads tabular data. + +Inventory ``type`` values +------------------------- +empty + No readable datasets or arrays. +single_dataset + One dataset; auto-read without a picker. +multi_dataset + Incompatible or grouped layout; caller must select paths. +legacy + Internal auto-read path for compatible multi-array layouts (and HDF5 + PyTables stores). Never surface this label in API or UI responses. +unsupported + Reader stub or format not implemented yet. Never treat as auto-read. +""" + +from __future__ import annotations + +from typing import Any, TypedDict + +from aidrin.file_handling.readers.base_reader import BaseFileReader + +INVENTORY_EMPTY = "empty" +INVENTORY_SINGLE = "single_dataset" +INVENTORY_MULTI = "multi_dataset" +INVENTORY_LEGACY = "legacy" +INVENTORY_UNSUPPORTED = "unsupported" + +# Types that may be shown to users when driving picker or error messages. +USER_FACING_INVENTORY_TYPES = frozenset( + {INVENTORY_EMPTY, INVENTORY_SINGLE, INVENTORY_MULTI} +) + + +class DatasetEntry(TypedDict): + path: str + shape: tuple[int, ...] + ndim: int + dtype: str + size: int + + +class PickerGroup(TypedDict, total=False): + name: str + paths: list[str] + prefix: str + + +class InventoryResult(TypedDict): + type: str + datasets: list[DatasetEntry] + groups: list[dict[str, Any]] + + +def make_inventory( + inv_type: str, + datasets: list[DatasetEntry] | None = None, + groups: list[dict[str, Any]] | None = None, +) -> InventoryResult: + """Build a normalized inventory dict.""" + return { + "type": inv_type, + "datasets": datasets or [], + "groups": groups or [], + } + + +class StructuredFileReader(BaseFileReader): + """Base for hierarchical array/tree formats with inventory and metadata hooks.""" + + def inventory(self) -> InventoryResult: + return make_inventory(INVENTORY_UNSUPPORTED) + + def get_metadata(self) -> dict[str, Any]: + """Embedded file metadata for FAIR assessment (``.zattrs``, ROOT headers, etc.).""" + return {} diff --git a/aidrin/file_handling/readers/zarr_reader.py b/aidrin/file_handling/readers/zarr_reader.py new file mode 100644 index 00000000..e7edf947 --- /dev/null +++ b/aidrin/file_handling/readers/zarr_reader.py @@ -0,0 +1,344 @@ +"""Zarr store reader for directory paths (CLI, library, Globus). + +Local browser upload of Zarr stores is intentionally not registered in +``SUPPORTED_FILE_TYPES``; use a directory path with CLI/library or Globus. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd + +from aidrin.file_handling.readers.structured import ( + INVENTORY_EMPTY, + INVENTORY_LEGACY, + INVENTORY_MULTI, + INVENTORY_SINGLE, + DatasetEntry, + InventoryResult, + StructuredFileReader, + make_inventory, +) + + +def _require_zarr(): + try: + import zarr + except ImportError as exc: + raise ImportError( + "Zarr support requires the 'zarr' package. " + "Install with: pip install 'aidrin[zarr]' or pip install zarr" + ) from exc + return zarr + + +class zarrReader(StructuredFileReader): + """Read Zarr directory stores into pandas DataFrames.""" + + def __init__(self, file_path: str, logger, selected_keys=None): + super().__init__(file_path, logger) + self._explicit_selected_keys = selected_keys + + def _open_store(self): + zarr = _require_zarr() + return zarr.open(self.file_path, mode="r") + + def _list_group_paths(self): + zarr = _require_zarr() + root = self._open_store() + if isinstance(root, zarr.Array): + return [] + + groups = [] + + def walk(group, prefix=""): + for key in group.keys(): + child = group[key] + path = f"{prefix}/{key}" if prefix else key + if isinstance(child, zarr.Group): + groups.append(path) + walk(child, path) + + walk(root) + return groups + + def _list_datasets(self) -> list[DatasetEntry]: + zarr = _require_zarr() + root = self._open_store() + datasets: list[DatasetEntry] = [] + + if isinstance(root, zarr.Array): + return [ + { + "path": "", + "shape": tuple(int(s) for s in root.shape), + "ndim": int(root.ndim), + "dtype": str(root.dtype), + "size": int(root.size), + } + ] + + def walk(group, prefix=""): + for key in group.keys(): + child = group[key] + path = f"{prefix}/{key}" if prefix else key + if isinstance(child, zarr.Array): + datasets.append( + { + "path": path, + "shape": tuple(int(s) for s in child.shape), + "ndim": int(child.ndim), + "dtype": str(child.dtype), + "size": int(child.size), + } + ) + elif isinstance(child, zarr.Group): + walk(child, path) + + walk(root) + return datasets + + def _build_picker_groups(self, datasets: list[DatasetEntry]): + paths = {ds["path"] for ds in datasets if ds["path"]} + assigned = set() + groups = {} + + for group_path in sorted(self._list_group_paths(), key=len, reverse=True): + prefix = f"{group_path}/" + members = sorted(p for p in paths if p.startswith(prefix) and p not in assigned) + if len(members) >= 2: + groups[group_path] = { + "id": group_path, + "label": group_path, + "type": "zarr_group", + "dataset_paths": members, + } + assigned.update(members) + + return sorted(groups.values(), key=lambda group: group["label"].lower()) + + def _is_incompatible_root_layout(self, datasets: list[DatasetEntry]): + root = [ds for ds in datasets if "/" not in ds["path"] and ds["path"] != ""] + # Root-store single array uses path "" + if any(ds["path"] == "" for ds in datasets): + return False + if len(root) < 2: + return False + if not all(ds["ndim"] == 1 for ds in root): + return False + lengths = {ds["shape"][0] for ds in root if ds["shape"]} + return len(lengths) > 1 + + def _is_grouped_hierarchical_layout(self, datasets: list[DatasetEntry]): + nested = [ds for ds in datasets if "/" in ds["path"]] + if len(nested) < 4: + return False + + groups = self._build_picker_groups(datasets) + if len(groups) < 2: + return False + + lengths = set() + for ds in nested: + if ds["ndim"] == 1 and ds["shape"]: + lengths.add(ds["shape"][0]) + return len(lengths) > 1 + + def _needs_dataset_selection(self, datasets: list[DatasetEntry]): + return self._is_incompatible_root_layout(datasets) or self._is_grouped_hierarchical_layout( + datasets + ) + + def inventory(self) -> InventoryResult: + try: + datasets = self._list_datasets() + except Exception as e: + self.logger.error("Failed to inventory Zarr store: %s", e, exc_info=True) + return make_inventory(INVENTORY_EMPTY) + + if not datasets: + layout = INVENTORY_EMPTY + elif len(datasets) == 1: + layout = INVENTORY_SINGLE + elif self._needs_dataset_selection(datasets): + layout = INVENTORY_MULTI + else: + layout = INVENTORY_LEGACY + + groups = self._build_picker_groups(datasets) if layout == INVENTORY_MULTI else [] + return make_inventory(layout, datasets, groups) + + def parse(self): + return [ds["path"] or "(root)" for ds in self._list_datasets()] + + def get_metadata(self) -> dict[str, Any]: + zarr = _require_zarr() + root = self._open_store() + metadata: dict[str, Any] = {} + + def attrs_to_dict(attrs): + try: + return dict(attrs) + except Exception: + return {} + + if isinstance(root, zarr.Array): + metadata["(root)"] = attrs_to_dict(root.attrs) + return metadata + + metadata["(root)"] = attrs_to_dict(root.attrs) + + def walk(group, prefix=""): + for key in group.keys(): + child = group[key] + path = f"{prefix}/{key}" if prefix else key + child_attrs = attrs_to_dict(child.attrs) + if child_attrs: + metadata[path] = child_attrs + if isinstance(child, zarr.Group): + walk(child, path) + + walk(root) + return metadata + + def _normalize_selected_keys(self, keys): + if isinstance(keys, str): + keys = [key.strip() for key in keys.split(",") if key.strip()] + return [str(key) for key in keys if key] + + def _get_selected_dataset_keys(self): + if self._explicit_selected_keys is not None: + return self._normalize_selected_keys(self._explicit_selected_keys) + try: + from flask import session + + keys = session.get("selected_keys") or [] + return self._normalize_selected_keys(keys) + except RuntimeError: + return [] + + def _resolve_array(self, root, path: str): + zarr = _require_zarr() + if path in ("", "(root)"): + if isinstance(root, zarr.Array): + return root + self.logger.warning("Zarr root path requested but store is a group") + return None + try: + obj = root[path] + except Exception: + self.logger.warning("Zarr array path not found: %s", path) + return None + if not isinstance(obj, zarr.Array): + self.logger.warning("Zarr path is not an array: %s", path) + return None + return obj + + def _array_to_frame(self, path: str, arr): + data = arr[:] + col_name = path.split("/")[-1] if path not in ("", "(root)") else "value" + if getattr(data, "ndim", 0) == 0: + df = pd.DataFrame({col_name: [data]}) + elif data.ndim == 1: + df = pd.DataFrame({col_name: data}) + else: + try: + df = pd.DataFrame(data) + except Exception: + df = pd.DataFrame(data.tolist()) + df.columns = [str(col) for col in df.columns] + df.columns = [str(col) for col in df.columns] + return df if not df.empty else None + + def _read_array_path(self, path: str): + root = self._open_store() + arr = self._resolve_array(root, path) + if arr is None: + return None + return self._array_to_frame(path, arr) + + def _read_compatible_array_paths(self, paths): + if not paths: + return None + + columns = {} + expected_len = None + root = self._open_store() + + for path in paths: + arr = self._resolve_array(root, path) + if arr is None: + return None + data = np.asarray(arr[:]) + if data.ndim != 1: + self.logger.warning( + "Zarr multi-select requires 1D arrays; '%s' has ndim=%s", + path, + data.ndim, + ) + return None + length = int(data.shape[0]) + if expected_len is None: + expected_len = length + elif length != expected_len: + self.logger.warning( + "Zarr selected arrays have incompatible lengths (%d vs %d)", + expected_len, + length, + ) + return None + short = path.split("/")[-1] or path + name = short if short not in columns else path.replace("/", ".") + columns[name] = data + + df = pd.DataFrame(columns) + return df if not df.empty else None + + def _auto_read_all(self, datasets: list[DatasetEntry]): + """Auto-read when layout does not require explicit selection.""" + if len(datasets) == 1: + return self._read_array_path(datasets[0]["path"] or "") + + ones = [ds for ds in datasets if ds["ndim"] == 1] + if ones and len(ones) == len(datasets): + lengths = {ds["shape"][0] for ds in ones if ds["shape"]} + if len(lengths) == 1: + return self._read_compatible_array_paths([ds["path"] for ds in ones]) + + self.logger.warning( + "Zarr store has %d arrays that cannot be auto-merged; " + "pass selected_keys to read specific paths.", + len(datasets), + ) + return None + + def read(self): + try: + inv = self.inventory() + if inv["type"] == INVENTORY_EMPTY: + self.logger.warning("No arrays found in Zarr store") + return None + + # Honor explicit selection for any non-empty layout (CLI/library/Globus). + selected = self._get_selected_dataset_keys() + if selected: + if len(selected) == 1: + return self._read_array_path(selected[0]) + return self._read_compatible_array_paths(selected) + + if inv["type"] == INVENTORY_MULTI: + self.logger.warning( + "Zarr store has %d arrays in an incompatible layout; " + "refusing to flatten. Pass selected_keys to choose paths.", + len(inv["datasets"]), + ) + return None + + return self._auto_read_all(inv["datasets"]) + except ImportError: + raise + except Exception as e: + self.logger.error("Error reading Zarr store: %s", e, exc_info=True) + return None diff --git a/pyproject.toml b/pyproject.toml index 9e39ed13..edfe35fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ globus = [ "globus-compute-sdk>=2.3.0", "globus-sdk>=3.20.0", ] +zarr = [ + "zarr>=3.0", +] telemetry = [ "opentelemetry-api>=1.20", "opentelemetry-sdk>=1.20", @@ -126,6 +129,9 @@ version = {attr = "aidrin._version.__version__"} # --------------------------------------------------------------------------- [dependency-groups] +zarr = [ + "zarr>=3.0", +] dev = [ "pytest>=7.0", "pytest-flask", diff --git a/tests/unit/test_structured_readers.py b/tests/unit/test_structured_readers.py new file mode 100644 index 00000000..d4cab412 --- /dev/null +++ b/tests/unit/test_structured_readers.py @@ -0,0 +1,187 @@ +"""Unit tests for structured reader scaffold and Zarr reader.""" + +import logging +import sys +import types + +import numpy as np +import pytest + +# Compatibility shim: pkg_resources for clean Python 3.12+ environments. +if "pkg_resources" not in sys.modules: + _pkg_resources = types.ModuleType("pkg_resources") + + class _Dist: + def __init__(self): + self.version = "0.0.0" + + _pkg_resources.get_distribution = lambda _name: _Dist() + sys.modules["pkg_resources"] = _pkg_resources + +from aidrin.file_handling.file_parser import ( + GLOBUS_FILE_TYPES, + READER_MAP, + SUPPORTED_FILE_TYPES, + read_file, +) +from aidrin.file_handling.readers.root_reader import rootReader +from aidrin.file_handling.readers.structured import ( + INVENTORY_EMPTY, + INVENTORY_LEGACY, + INVENTORY_MULTI, + INVENTORY_SINGLE, + INVENTORY_UNSUPPORTED, + USER_FACING_INVENTORY_TYPES, + make_inventory, +) +from aidrin.file_handling.readers.zarr_reader import zarrReader + +zarr = pytest.importorskip("zarr") + + +@pytest.fixture +def logger(): + return logging.getLogger("test_structured_readers") + + +@pytest.mark.parametrize("reader_cls", [rootReader]) +def test_root_stub_inventory_contract(tmp_path, logger, reader_cls): + path = tmp_path / "placeholder" + path.write_text("not used", encoding="utf-8") + inv = reader_cls(str(path), logger).inventory() + + assert set(inv.keys()) == {"type", "datasets", "groups"} + assert inv["type"] == INVENTORY_UNSUPPORTED + assert inv["datasets"] == [] + assert inv["groups"] == [] + assert inv["type"] not in USER_FACING_INVENTORY_TYPES + + +def test_make_inventory_helper(): + inv = make_inventory(INVENTORY_UNSUPPORTED) + assert inv == {"type": INVENTORY_UNSUPPORTED, "datasets": [], "groups": []} + + +def test_zarr_registered_for_cli_not_local_upload(): + assert ".zarr" in READER_MAP + local_exts = [ext for ext, _ in SUPPORTED_FILE_TYPES] + globus_exts = [ext for ext, _ in GLOBUS_FILE_TYPES] + assert ".zarr" not in local_exts + assert ".zarr" in globus_exts + + +def _write_single_array_store(path): + arr = zarr.open(str(path), mode="w", shape=(5,), dtype="f8") + arr[:] = np.arange(5, dtype=np.float64) + arr.attrs["unit"] = "m" + + +def _write_group_store(path): + root = zarr.open_group(str(path), mode="w") + root.attrs["license"] = "MIT" + a = root.create_array("temp", shape=(4,), dtype="f8") + a[:] = np.arange(4, dtype=np.float64) + a.attrs["unit"] = "C" + g = root.create_group("station") + b = g.create_array("x", shape=(4,), dtype="i4") + b[:] = np.arange(4, dtype=np.int32) + + +def _write_incompatible_root_store(path): + root = zarr.open_group(str(path), mode="w") + a = root.create_array("short", shape=(3,), dtype="f8") + a[:] = [1.0, 2.0, 3.0] + b = root.create_array("long", shape=(5,), dtype="f8") + b[:] = np.arange(5, dtype=np.float64) + + +def _write_grouped_hierarchical_store(path): + root = zarr.open_group(str(path), mode="w") + for station in ("S1", "S2"): + g = root.create_group(station) + for axis, length in (("X", 10), ("Y", 10), ("meta", 1)): + arr = g.create_array(axis, shape=(length,), dtype="f8") + arr[:] = np.arange(length, dtype=np.float64) + + +def test_zarr_single_array_inventory_and_read(tmp_path, logger): + store = tmp_path / "single.zarr" + _write_single_array_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_SINGLE + assert len(inv["datasets"]) == 1 + + df = reader.read() + assert df is not None + assert list(df.columns) == ["value"] + assert len(df) == 5 + assert list(df["value"]) == [0.0, 1.0, 2.0, 3.0, 4.0] + + +def test_zarr_compatible_group_auto_read(tmp_path, logger): + store = tmp_path / "group.zarr" + _write_group_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_LEGACY + assert inv["type"] not in USER_FACING_INVENTORY_TYPES + + df = reader.read() + assert df is not None + assert set(df.columns) == {"temp", "x"} + assert len(df) == 4 + + +def test_zarr_incompatible_root_needs_selection(tmp_path, logger): + store = tmp_path / "bad.zarr" + _write_incompatible_root_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_MULTI + assert reader.read() is None + + df = zarrReader(str(store), logger, selected_keys=["short"]).read() + assert df is not None + assert list(df.columns) == ["short"] + assert len(df) == 3 + + +def test_zarr_grouped_hierarchical_selection(tmp_path, logger): + store = tmp_path / "stations.zarr" + _write_grouped_hierarchical_store(store) + reader = zarrReader(str(store), logger) + inv = reader.inventory() + assert inv["type"] == INVENTORY_MULTI + assert len(inv["groups"]) >= 2 + + df = zarrReader( + str(store), logger, selected_keys=["S1/X", "S1/Y"] + ).read() + assert df is not None + assert set(df.columns) == {"X", "Y"} + assert len(df) == 10 + + +def test_zarr_get_metadata(tmp_path, logger): + store = tmp_path / "meta.zarr" + _write_group_store(store) + meta = zarrReader(str(store), logger).get_metadata() + assert meta["(root)"]["license"] == "MIT" + assert meta["temp"]["unit"] == "C" + + +def test_zarr_read_file_integration(tmp_path): + store = tmp_path / "cli.zarr" + _write_single_array_store(store) + df = read_file((str(store), "cli.zarr", ".zarr")) + assert df is not None + assert len(df) == 5 + + +def test_zarr_empty_store(tmp_path, logger): + store = tmp_path / "empty.zarr" + zarr.open_group(str(store), mode="w") + inv = zarrReader(str(store), logger).inventory() + assert inv["type"] == INVENTORY_EMPTY + assert zarrReader(str(store), logger).read() is None diff --git a/web/routes/core.py b/web/routes/core.py index 4b5ba87d..37b5eec8 100644 --- a/web/routes/core.py +++ b/web/routes/core.py @@ -16,7 +16,11 @@ url_for, ) from werkzeug.utils import secure_filename -from aidrin.file_handling.file_parser import SUPPORTED_FILE_TYPES, READER_MAP +from aidrin.file_handling.file_parser import ( + GLOBUS_FILE_TYPES, + READER_MAP, + SUPPORTED_FILE_TYPES, +) from aidrin.file_handling.readers.hdf5_reader import hdf5Reader from web.routes.utils import ( clear_all_user_cache, @@ -144,6 +148,7 @@ def inspector(): uploaded_file_name=effective_file_name or "", file_type=effective_file_type or "", supported_file_types=SUPPORTED_FILE_TYPES, + globus_file_types=GLOBUS_FILE_TYPES, file_preview=file_preview if file_preview is not None else [], current_checked_keys=current_checked_keys if current_checked_keys is not None diff --git a/web/templates/_components/globus_panel.html b/web/templates/_components/globus_panel.html index 865e41e3..038b8caa 100644 --- a/web/templates/_components/globus_panel.html +++ b/web/templates/_components/globus_panel.html @@ -38,7 +38,7 @@

Connect t
-
@@ -47,7 +47,7 @@

Connect t
From a8206072820e7c12c324d9ee99d990fa82132e5d Mon Sep 17 00:00:00 2001 From: Dhanushkumar Jv Date: Tue, 21 Jul 2026 23:43:14 +0530 Subject: [PATCH 2/5] Fix Zarr optional extra for uv lock and Python 3.10. --- aidrin/file_handling/readers/zarr_reader.py | 4 +- pyproject.toml | 5 +- uv.lock | 202 +++++++++++++++----- 3 files changed, 158 insertions(+), 53 deletions(-) diff --git a/aidrin/file_handling/readers/zarr_reader.py b/aidrin/file_handling/readers/zarr_reader.py index e7edf947..228209f5 100644 --- a/aidrin/file_handling/readers/zarr_reader.py +++ b/aidrin/file_handling/readers/zarr_reader.py @@ -28,8 +28,8 @@ def _require_zarr(): import zarr except ImportError as exc: raise ImportError( - "Zarr support requires the 'zarr' package. " - "Install with: pip install 'aidrin[zarr]' or pip install zarr" + "Zarr support requires the 'zarr' package on Python >=3.11. " + "Install with: pip install 'aidrin[zarr]' or pip install 'zarr>=3.0.8,<3.2'" ) from exc return zarr diff --git a/pyproject.toml b/pyproject.toml index edfe35fe..77f4de25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,9 @@ globus = [ "globus-compute-sdk>=2.3.0", "globus-sdk>=3.20.0", ] +# Zarr 3.x needs Python >=3.11 (3.2+ needs >=3.12). Pin <3.2 so 3.11 stays resolvable. zarr = [ - "zarr>=3.0", + "zarr>=3.0.8,<3.2; python_version >= '3.11'", ] telemetry = [ "opentelemetry-api>=1.20", @@ -130,7 +131,7 @@ version = {attr = "aidrin._version.__version__"} [dependency-groups] zarr = [ - "zarr>=3.0", + "zarr>=3.0.8,<3.2; python_version >= '3.11'", ] dev = [ "pytest>=7.0", diff --git a/uv.lock b/uv.lock index 0025d3ba..8845b871 100644 --- a/uv.lock +++ b/uv.lock @@ -104,6 +104,9 @@ telemetry = [ { name = "opentelemetry-instrumentation-flask" }, { name = "opentelemetry-sdk" }, ] +zarr = [ + { name = "zarr", marker = "python_full_version >= '3.11'" }, +] [package.dev-dependencies] agentic = [ @@ -144,6 +147,9 @@ mcp = [ { name = "pymupdf" }, { name = "python-dotenv" }, ] +zarr = [ + { name = "zarr", marker = "python_full_version >= '3.11'" }, +] [package.metadata] requires-dist = [ @@ -201,8 +207,9 @@ requires-dist = [ { name = "sphinx", marker = "extra == 'docs'", specifier = "==6.2.1" }, { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = "==1.2.2" }, { name = "sphinxemoji", marker = "extra == 'docs'" }, + { name = "zarr", marker = "python_full_version >= '3.11' and extra == 'zarr'", specifier = ">=3.0.8,<3.2" }, ] -provides-extras = ["globus", "telemetry", "llm", "mcp", "agentic", "dev", "docs"] +provides-extras = ["globus", "zarr", "telemetry", "llm", "mcp", "agentic", "dev", "docs"] [package.metadata.requires-dev] agentic = [ @@ -241,6 +248,7 @@ mcp = [ { name = "pymupdf", specifier = ">=1.24.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, ] +zarr = [{ name = "zarr", marker = "python_full_version >= '3.11'", specifier = ">=3.0.8,<3.2" }] [[package]] name = "aiohappyeyeballs" @@ -813,7 +821,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -898,7 +906,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'x86_64' and sys_platform == 'darwin') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -1189,6 +1197,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8d/14/69b4bad34e3f250afe29a854da03acb6747711f3df06c359fa053fae4e76/docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c", size = 570050, upload-time = "2021-11-23T17:49:38.556Z" }, ] +[[package]] +name = "donfig" +version = "0.8.1.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, +] + [[package]] name = "dython" version = "0.7.12" @@ -1527,6 +1547,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/f0/67d4b279d5a19324792e29499856160d3a478e864cfec3919d23ebc88268/globus_sdk-4.7.0-py3-none-any.whl", hash = "sha256:6f2a15cff130c93ca70ddc25a8156ae636865850d9a9c9dbb7ffc365e70930e2", size = 439273, upload-time = "2026-05-20T16:20:27.683Z" }, ] +[[package]] +name = "google-crc32c" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/ac/6f7bc93886a823ab545948c2dd48143027b2355ad1944c7cf852b338dc91/google_crc32c-1.8.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff", size = 31296, upload-time = "2025-12-16T00:19:07.261Z" }, + { url = "https://files.pythonhosted.org/packages/f7/97/a5accde175dee985311d949cfcb1249dcbb290f5ec83c994ea733311948f/google_crc32c-1.8.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288", size = 30870, upload-time = "2025-12-16T00:29:17.669Z" }, + { url = "https://files.pythonhosted.org/packages/3d/63/bec827e70b7a0d4094e7476f863c0dbd6b5f0f1f91d9c9b32b76dcdfeb4e/google_crc32c-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d", size = 33214, upload-time = "2025-12-16T00:40:19.618Z" }, + { url = "https://files.pythonhosted.org/packages/63/bc/11b70614df04c289128d782efc084b9035ef8466b3d0a8757c1b6f5cf7ac/google_crc32c-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092", size = 33589, upload-time = "2025-12-16T00:40:20.7Z" }, + { url = "https://files.pythonhosted.org/packages/3e/00/a08a4bc24f1261cc5b0f47312d8aebfbe4b53c2e6307f1b595605eed246b/google_crc32c-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733", size = 34437, upload-time = "2025-12-16T00:35:19.437Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" }, + { url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" }, + { url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" }, + { url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, + { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, + { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, + { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, + { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, + { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, + { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, + { url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.75.0" @@ -2748,6 +2803,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/36/8be7118ffd4c8440881046eac3d0982cc5ab42909508cf5d67024d62a2e4/numba-0.65.1-cp314-cp314t-win_amd64.whl", hash = "sha256:20609346e3bd75204950dcbbfe383a8d7dbf4902f442aedbf00f97fef4aa8f38", size = 2758237, upload-time = "2026-04-24T02:02:54.612Z" }, ] +[[package]] +name = "numcodecs" +version = "0.16.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/85/1ac101a40ead81eaa1c7dc49a8827a30e2e436211b43ebdc63c590eb1347/numcodecs-0.16.5-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:78382dcea50622f2ef1e6e7a71dbe7f861d8fe376b27b7c297c26907304fef1e", size = 1621795, upload-time = "2025-11-21T02:49:17.418Z" }, + { url = "https://files.pythonhosted.org/packages/0e/cc/0d97ef55dda48cb0f93d7b92d761208e7a99bd2eea6b0e859426e6a99a21/numcodecs-0.16.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2d04a19cb57a3c519b4127ac377cca6471aee1990d7c18f5b1e3a4fe1306689", size = 1153030, upload-time = "2025-11-21T02:49:19.089Z" }, + { url = "https://files.pythonhosted.org/packages/5e/41/e120ee1b390730ac5987cde2afd82e2b8442cec315ab40b94b0373e93e73/numcodecs-0.16.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c043af648eb280cd61785c99c22ff5c3c3460f906eb51a8511327c4f5111b283", size = 8510503, upload-time = "2025-11-21T02:49:20.324Z" }, + { url = "https://files.pythonhosted.org/packages/54/4b/195ac84cc8f6077b4f0f421e8daee21b7f1bd88cb7716414234379fe68ec/numcodecs-0.16.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c398919ef2eb0e56b8e97456f622640bfd3deed06de3acc976989cbcb22628a3", size = 9123428, upload-time = "2025-11-21T02:49:22.328Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5b/af02c417954f46e5c7bd5163ac251f535877d909fce54861c99ae197f6f6/numcodecs-0.16.5-cp311-cp311-win_amd64.whl", hash = "sha256:3820860ed302d4d84a1c66e70981ff959d5eb712555be4e7d8ced49888594773", size = 801542, upload-time = "2025-11-21T02:49:24.265Z" }, + { url = "https://files.pythonhosted.org/packages/75/cc/55420f3641a67f78392dc0bc5d02cb9eb0a9dcebf2848d1ac77253ca61fa/numcodecs-0.16.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:24e675dc8d1550cd976a99479b87d872cb142632c75cc402fea04c08c4898523", size = 1656287, upload-time = "2025-11-21T02:49:25.755Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6c/86644987505dcb90ba6d627d6989c27bafb0699f9fd00187e06d05ea8594/numcodecs-0.16.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:94ddfa4341d1a3ab99989d13b01b5134abb687d3dab2ead54b450aefe4ad5bd6", size = 1148899, upload-time = "2025-11-21T02:49:26.87Z" }, + { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, + { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, + { url = "https://files.pythonhosted.org/packages/1c/20/2fdec87fc7f8cec950d2b0bea603c12dc9f05b4966dc5924ba5a36a61bf6/numcodecs-0.16.5-cp312-cp312-win_amd64.whl", hash = "sha256:845a9857886ffe4a3172ba1c537ae5bcc01e65068c31cf1fce1a844bd1da050f", size = 801412, upload-time = "2025-11-21T02:49:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/38/38/071ced5a5fd1c85ba0e14ba721b66b053823e5176298c2f707e50bed11d9/numcodecs-0.16.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25be3a516ab677dad890760d357cfe081a371d9c0a2e9a204562318ac5969de3", size = 1654359, upload-time = "2025-11-21T02:49:33.673Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c0/5f84ba7525577c1b9909fc2d06ef11314825fc4ad4378f61d0e4c9883b4a/numcodecs-0.16.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0107e839ef75b854e969cb577e140b1aadb9847893937636582d23a2a4c6ce50", size = 1144237, upload-time = "2025-11-21T02:49:35.294Z" }, + { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, + { url = "https://files.pythonhosted.org/packages/27/72/6663cc0382ddbb866136c255c837bcb96cc7ce5e83562efec55e1b995941/numcodecs-0.16.5-cp313-cp313-win_amd64.whl", hash = "sha256:5088145502ad1ebf677ec47d00eb6f0fd600658217db3e0c070c321c85d6cf3d", size = 799275, upload-time = "2025-11-21T02:49:39.558Z" }, + { url = "https://files.pythonhosted.org/packages/3c/9e/38e7ca8184c958b51f45d56a4aeceb1134ecde2d8bd157efadc98502cc42/numcodecs-0.16.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b05647b8b769e6bc8016e9fd4843c823ce5c9f2337c089fb5c9c4da05e5275de", size = 1654721, upload-time = "2025-11-21T02:49:40.602Z" }, + { url = "https://files.pythonhosted.org/packages/a1/37/260fa42e7b2b08e6e00ad632f8dd620961a60a459426c26cea390f8c68d0/numcodecs-0.16.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3832bd1b5af8bb3e413076b7d93318c8e7d7b68935006b9fa36ca057d1725a8f", size = 1146887, upload-time = "2025-11-21T02:49:41.721Z" }, + { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, + { url = "https://files.pythonhosted.org/packages/31/a5/a0425af36c20d55a3ea884db4b4efca25a43bea9214ba69ca7932dd997b4/numcodecs-0.16.5-cp314-cp314-win_amd64.whl", hash = "sha256:b16a14303800e9fb88abc39463ab4706c037647ac17e49e297faa5f7d7dbbf1d", size = 819022, upload-time = "2025-11-21T02:49:47.39Z" }, +] + [[package]] name = "numpy" version = "2.2.6" @@ -3263,10 +3350,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -3342,9 +3429,9 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'x86_64' and sys_platform == 'darwin') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -4652,10 +4739,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, + { name = "joblib" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -4714,10 +4801,10 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'x86_64' and sys_platform == 'darwin') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, + { name = "joblib" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" } }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } wheels = [ @@ -4767,7 +4854,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -4841,7 +4928,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'x86_64' and sys_platform == 'darwin') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -4944,20 +5031,20 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "cloudpickle", marker = "(python_full_version < '3.11' and platform_machine != 'x86_64') or (python_full_version < '3.11' and sys_platform != 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')" }, - { name = "numba", marker = "(python_full_version < '3.11' and platform_machine != 'x86_64') or (python_full_version < '3.11' and sys_platform != 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')" }, + { name = "cloudpickle" }, + { name = "numba" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'darwin'" }, - { name = "packaging", marker = "(python_full_version < '3.11' and platform_machine != 'x86_64') or (python_full_version < '3.11' and sys_platform != 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')" }, + { name = "packaging" }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'darwin'" }, { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'darwin'" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'darwin'" }, - { name = "slicer", marker = "(python_full_version < '3.11' and platform_machine != 'x86_64') or (python_full_version < '3.11' and sys_platform != 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')" }, - { name = "tqdm", marker = "(python_full_version < '3.11' and platform_machine != 'x86_64') or (python_full_version < '3.11' and sys_platform != 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')" }, - { name = "typing-extensions", marker = "(python_full_version < '3.11' and platform_machine != 'x86_64') or (python_full_version < '3.11' and sys_platform != 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')" }, + { name = "slicer" }, + { name = "tqdm" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dc/c6/9823a7f483aa9f3179fc359c10d22da9e418b1a7a3fc99a42b705d05e82a/shap-0.49.1.tar.gz", hash = "sha256:1114ecd804fff29f50d522ce6031082fcf42fe4a32fb1b5da233b2415d784c8c", size = 4084725, upload-time = "2025-10-14T10:04:49.75Z" } wheels = [ @@ -4997,17 +5084,17 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'x86_64' and sys_platform == 'darwin') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] dependencies = [ - { name = "cloudpickle", marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "llvmlite", marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "numba", marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "packaging", marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "slicer", marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "tqdm", marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, - { name = "typing-extensions", marker = "(python_full_version == '3.11.*' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')" }, + { name = "cloudpickle" }, + { name = "llvmlite" }, + { name = "numba" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" } }, + { name = "slicer" }, + { name = "tqdm" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a4/0a/4a3ee4b1a3654f2a9ae038a64bb3e91a42af3da07577d69b65241f010970/shap-0.51.0.tar.gz", hash = "sha256:cfa17ff213657c9d50285aa923d79b0037a62e2ee1a31bc3eec7e196b00bdb59", size = 4108336, upload-time = "2026-03-04T09:18:19.985Z" } wheels = [ @@ -5060,16 +5147,16 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'x86_64' and sys_platform == 'darwin') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] dependencies = [ - { name = "cloudpickle", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "llvmlite", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "numba", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "packaging", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "slicer", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, - { name = "tqdm", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'darwin')" }, + { name = "cloudpickle" }, + { name = "llvmlite" }, + { name = "numba" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" } }, + { name = "slicer" }, + { name = "tqdm" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/0a/aa278f42c08cb47f2bb503085be0c521da2886929c6605b6105748a7590f/shap-0.52.0.tar.gz", hash = "sha256:81d4ae478f67f8122de1bb411dc4e3ddff0604cbc27dc9cb8ea66d5c73462fd2", size = 4192842, upload-time = "2026-05-28T14:17:49.011Z" } wheels = [ @@ -5671,8 +5758,8 @@ name = "uvicorn" version = "0.49.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, - { name = "h11", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, + { name = "click" }, + { name = "h11" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", size = 91284, upload-time = "2026-06-03T22:01:30.448Z" } @@ -6128,6 +6215,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, ] +[[package]] +name = "zarr" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "donfig" }, + { name = "google-crc32c" }, + { name = "numcodecs" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/5a/b8a0cf39a14c770c30bd1f2d120c54000c8cd9e84e8e79f38d9a7ce58071/zarr-3.1.6.tar.gz", hash = "sha256:d95e72cbea4b90e9a70679468b8266400331756232576ae2b43400ac5108d0eb", size = 386531, upload-time = "2026-03-23T17:25:18.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/7c/ba8ca8cbe9dbef8e83a95fc208fed8e6686c98b4719aaa0aa7f3d31fe390/zarr-3.1.6-py3-none-any.whl", hash = "sha256:b5a82c5079d1c3d4ee8f06746fa3b9a98a7d804300fa3f4be154362a33e1207e", size = 295655, upload-time = "2026-03-23T17:25:17.189Z" }, +] + [[package]] name = "zstandard" version = "0.25.0" From 0bdb0d3b8a8beb06d58f1952ec13c017e4e80176 Mon Sep 17 00:00:00 2001 From: Dhanushkumar Jv Date: Fri, 31 Jul 2026 22:42:09 +0530 Subject: [PATCH 3/5] Add Zarr subset/reduce and clearer summarize errors --- aidrin/file_handling/file_parser.py | 24 +++- aidrin/file_handling/readers/zarr_reader.py | 130 ++++++++++++++++++-- aidrin/headless/api.py | 7 ++ tests/unit/test_structured_readers.py | 66 +++++++++- 4 files changed, 211 insertions(+), 16 deletions(-) diff --git a/aidrin/file_handling/file_parser.py b/aidrin/file_handling/file_parser.py index 1a49431b..08d2f996 100644 --- a/aidrin/file_handling/file_parser.py +++ b/aidrin/file_handling/file_parser.py @@ -200,7 +200,7 @@ def filter_file(file_info, kept_keys): # Parses the uploaded file into a pandas database -def read_file(file_info, columns=None): +def read_file(file_info, columns=None, subset=None, reduce=None): """ Parses a given file into pandas Dataframe. @@ -217,8 +217,14 @@ def read_file(file_info, columns=None): -file_path: str, relative or absolute path of the file. -file_name: str, file name. -file_type: str, file format. Passed from front end select value. + - optional selected_keys (4th element) for HDF5/Zarr path selection. columns: list of str, optional When given, only these columns are returned. + subset: dict, optional + Zarr only. Map of axis index -> slice or int, applied before reduce. + reduce: str, optional + Zarr only. ``\"spatial_mean\"`` averages all axes except axis 0 so a + multi-dim array becomes a 1D column suitable for existing metrics. Returns ---------- pd.Dataframe, None, or str @@ -248,7 +254,9 @@ def read_file(file_info, columns=None): cache_path = _frame_cache_path(file_path) if cacheable else None # Fast path: reload a fresh cache without touching the source. - if cacheable and os.path.exists(cache_path): + # Skip cache when Zarr subset/reduce options are set (options change the frame). + use_cache = cacheable and subset is None and reduce is None + if use_cache and os.path.exists(cache_path): try: import pandas as pd @@ -263,7 +271,15 @@ def read_file(file_info, columns=None): # Slow path: parse the source once. reader_cls = READER_MAP[file_type] - if file_type in _SELECTION_FILE_TYPES: + if file_type == ".zarr": + df = reader_cls( + file_path, + file_upload_time_log, + selected_keys=selected_keys, + subset=subset, + reduce=reduce, + ).read() + elif file_type in _SELECTION_FILE_TYPES: df = reader_cls( file_path, file_upload_time_log, selected_keys=selected_keys ).read() @@ -274,7 +290,7 @@ def read_file(file_info, columns=None): # Best-effort cache for future calls. Failure is non-fatal: we still # return the freshly parsed frame below, so a read-only directory or an # unserialisable frame never turns a successful parse into an error. - if cacheable: + if use_cache: _write_frame_cache(cache_path, df) if columns is not None: diff --git a/aidrin/file_handling/readers/zarr_reader.py b/aidrin/file_handling/readers/zarr_reader.py index 228209f5..7f3c7967 100644 --- a/aidrin/file_handling/readers/zarr_reader.py +++ b/aidrin/file_handling/readers/zarr_reader.py @@ -34,12 +34,25 @@ def _require_zarr(): return zarr +# Reduce a multi-dim array to 1D by averaging all axes except axis 0 (time-first). +REDUCE_SPATIAL_MEAN = "spatial_mean" +_VALID_REDUCE = {None, "", REDUCE_SPATIAL_MEAN} + + class zarrReader(StructuredFileReader): - """Read Zarr directory stores into pandas DataFrames.""" + """Read Zarr directory stores into pandas DataFrames. + + Multi-dimensional arrays can be sliced and reduced before conversion:: - def __init__(self, file_path: str, logger, selected_keys=None): + zarrReader(path, logger, selected_keys=["tmax"], subset={0: slice(0, 14)}, + reduce="spatial_mean") + """ + + def __init__(self, file_path: str, logger, selected_keys=None, subset=None, reduce=None): super().__init__(file_path, logger) self._explicit_selected_keys = selected_keys + self._subset = subset + self._reduce = reduce def _open_store(self): zarr = _require_zarr() @@ -236,19 +249,113 @@ def _resolve_array(self, root, path: str): return None return obj + def _normalize_subset(self, subset, ndim: int): + """Build a numpy index tuple from ``{axis: slice|int}``.""" + if not subset: + return None + if not isinstance(subset, dict): + raise ValueError("subset must be a dict of axis -> slice or int") + + index = [slice(None)] * ndim + for axis, selector in subset.items(): + try: + axis_i = int(axis) + except (TypeError, ValueError) as exc: + raise ValueError(f"subset axis must be an int, got {axis!r}") from exc + if axis_i < 0: + axis_i += ndim + if axis_i < 0 or axis_i >= ndim: + raise ValueError(f"subset axis {axis} out of range for ndim={ndim}") + if not isinstance(selector, (slice, int, np.integer)): + raise ValueError( + f"subset[{axis}] must be slice or int, got {type(selector).__name__}" + ) + index[axis_i] = selector + return tuple(index) + + def _apply_subset(self, data, subset): + if subset is None: + return data + index = self._normalize_subset(subset, data.ndim) + if index is None: + return data + return data[index] + + def _apply_reduce(self, data, reduce): + """Reduce multi-dim data to 1D (or scalar) for tabular metrics.""" + mode = (reduce or "").strip().lower() or None + if mode not in _VALID_REDUCE: + raise ValueError( + f"Unsupported reduce={reduce!r}; use None or '{REDUCE_SPATIAL_MEAN}'" + ) + if mode is None: + return data + if data.ndim <= 1: + return data + # Keep axis 0 (typically time); average remaining spatial/member axes. + axes = tuple(range(1, data.ndim)) + return np.nanmean(data, axis=axes) + + def _prepare_array_data(self, arr): + """Load array, apply optional subset/reduce, return numpy data.""" + data = np.asarray(arr[:]) + try: + data = self._apply_subset(data, self._subset) + data = self._apply_reduce(data, self._reduce) + except ValueError as exc: + self.logger.warning("%s", exc) + return None + + if getattr(data, "ndim", 0) >= 3 and not self._reduce: + self.logger.warning( + "Zarr array has ndim=%s after subset; pass reduce='%s' " + "(mean over axes 1..) to build a 1D table column, or subset further.", + data.ndim, + REDUCE_SPATIAL_MEAN, + ) + return None + return data + + def _column_name_from_path(self, path: str, used_names): + full = path.strip("/") if path not in ("", "(root)") else "value" + if not full: + full = "value" + if full not in used_names: + return full + short = path.split("/")[-1] or full + if short not in used_names: + return short + dotted = full.replace("/", ".") + if dotted not in used_names: + return dotted + suffix = 2 + while f"{full}_{suffix}" in used_names: + suffix += 1 + return f"{full}_{suffix}" + def _array_to_frame(self, path: str, arr): - data = arr[:] - col_name = path.split("/")[-1] if path not in ("", "(root)") else "value" + data = self._prepare_array_data(arr) + if data is None: + return None + + col_name = self._column_name_from_path(path, set()) if getattr(data, "ndim", 0) == 0: df = pd.DataFrame({col_name: [data]}) elif data.ndim == 1: df = pd.DataFrame({col_name: data}) - else: + elif data.ndim == 2: try: df = pd.DataFrame(data) except Exception: df = pd.DataFrame(data.tolist()) - df.columns = [str(col) for col in df.columns] + df.columns = [f"{col_name}_{i}" for i in range(df.shape[1])] + else: + self.logger.warning( + "Cannot convert ndim=%s Zarr array '%s' to a DataFrame", + getattr(data, "ndim", None), + path, + ) + return None df.columns = [str(col) for col in df.columns] return df if not df.empty else None @@ -271,10 +378,14 @@ def _read_compatible_array_paths(self, paths): arr = self._resolve_array(root, path) if arr is None: return None - data = np.asarray(arr[:]) + data = self._prepare_array_data(arr) + if data is None: + return None + data = np.asarray(data) if data.ndim != 1: self.logger.warning( - "Zarr multi-select requires 1D arrays; '%s' has ndim=%s", + "Zarr multi-select requires 1D arrays after subset/reduce; " + "'%s' has ndim=%s", path, data.ndim, ) @@ -289,8 +400,7 @@ def _read_compatible_array_paths(self, paths): length, ) return None - short = path.split("/")[-1] or path - name = short if short not in columns else path.replace("/", ".") + name = self._column_name_from_path(path, set(columns)) columns[name] = data df = pd.DataFrame(columns) diff --git a/aidrin/headless/api.py b/aidrin/headless/api.py index dd22f180..c5a4d075 100644 --- a/aidrin/headless/api.py +++ b/aidrin/headless/api.py @@ -284,6 +284,13 @@ def summarize_dataset( ext = f".{file_type}" if file_type else path.suffix.lower() df = read_file((file_path, path.name, ext)) + if df is None or isinstance(df, str): + detail = df if isinstance(df, str) else ( + "Unable to build a table from this file. For multi-array Zarr/HDF5 stores, " + "select compatible paths (selected_keys) or reduce multi-dim arrays first." + ) + raise ValueError(detail) + num_cols = df.select_dtypes(include="number").columns.tolist() cat_cols = df.select_dtypes(include="object").columns.tolist() missing = df.isnull().sum() diff --git a/tests/unit/test_structured_readers.py b/tests/unit/test_structured_readers.py index d4cab412..f3aa0b3c 100644 --- a/tests/unit/test_structured_readers.py +++ b/tests/unit/test_structured_readers.py @@ -129,7 +129,7 @@ def test_zarr_compatible_group_auto_read(tmp_path, logger): df = reader.read() assert df is not None - assert set(df.columns) == {"temp", "x"} + assert set(df.columns) == {"temp", "station/x"} assert len(df) == 4 @@ -159,7 +159,7 @@ def test_zarr_grouped_hierarchical_selection(tmp_path, logger): str(store), logger, selected_keys=["S1/X", "S1/Y"] ).read() assert df is not None - assert set(df.columns) == {"X", "Y"} + assert set(df.columns) == {"S1/X", "S1/Y"} assert len(df) == 10 @@ -185,3 +185,65 @@ def test_zarr_empty_store(tmp_path, logger): inv = zarrReader(str(store), logger).inventory() assert inv["type"] == INVENTORY_EMPTY assert zarrReader(str(store), logger).read() is None + + +def _write_multidim_store(path): + """3D array shaped like (time, lat, lon) for subset/reduce tests.""" + root = zarr.open_group(str(path), mode="w") + # time=5, lat=4, lon=3 — values = time index for easy spatial-mean checks + data = np.zeros((5, 4, 3), dtype=np.float64) + for t in range(5): + data[t, :, :] = float(t) + arr = root.create_array("tmax_grid", shape=data.shape, dtype="f8") + arr[:] = data + + +def test_zarr_multidim_requires_reduce(tmp_path, logger): + store = tmp_path / "grid.zarr" + _write_multidim_store(store) + df = zarrReader(str(store), logger, selected_keys=["tmax_grid"]).read() + assert df is None + + +def test_zarr_spatial_mean_reduce(tmp_path, logger): + store = tmp_path / "grid.zarr" + _write_multidim_store(store) + df = zarrReader( + str(store), + logger, + selected_keys=["tmax_grid"], + reduce="spatial_mean", + ).read() + assert df is not None + assert list(df.columns) == ["tmax_grid"] + assert len(df) == 5 + assert list(df["tmax_grid"]) == [0.0, 1.0, 2.0, 3.0, 4.0] + + +def test_zarr_subset_then_spatial_mean(tmp_path, logger): + store = tmp_path / "grid.zarr" + _write_multidim_store(store) + # First 3 time steps, subset of lat/lon + df = zarrReader( + str(store), + logger, + selected_keys=["tmax_grid"], + subset={0: slice(0, 3), 1: slice(0, 2), 2: slice(0, 2)}, + reduce="spatial_mean", + ).read() + assert df is not None + assert len(df) == 3 + assert list(df["tmax_grid"]) == [0.0, 1.0, 2.0] + + +def test_zarr_read_file_passes_subset_reduce(tmp_path): + store = tmp_path / "grid.zarr" + _write_multidim_store(store) + df = read_file( + (str(store), "grid.zarr", ".zarr", ["tmax_grid"]), + reduce="spatial_mean", + subset={0: slice(1, 4)}, + ) + assert df is not None + assert len(df) == 3 + assert list(df["tmax_grid"]) == [1.0, 2.0, 3.0] From d592c7e2e8fc3f7792d0c54f64bd68409c14510b Mon Sep 17 00:00:00 2001 From: Dhanushkumar Jv Date: Tue, 4 Aug 2026 18:22:40 +0530 Subject: [PATCH 4/5] Drop Zarr mean/reduce; add CLI selected-keys --- aidrin/file_handling/file_parser.py | 23 +----- aidrin/file_handling/readers/zarr_reader.py | 82 +++------------------ aidrin/headless/api.py | 44 +++++++++-- aidrin/headless/cli.py | 35 ++++++++- aidrin/headless/config.py | 3 + aidrin/headless/runners.py | 28 ++++++- docs/source/cli_usage.rst | 11 +++ tests/unit/test_cli.py | 60 +++++++++++++++ tests/unit/test_structured_readers.py | 53 +++---------- 9 files changed, 195 insertions(+), 144 deletions(-) diff --git a/aidrin/file_handling/file_parser.py b/aidrin/file_handling/file_parser.py index 08d2f996..dbe1afe4 100644 --- a/aidrin/file_handling/file_parser.py +++ b/aidrin/file_handling/file_parser.py @@ -200,7 +200,7 @@ def filter_file(file_info, kept_keys): # Parses the uploaded file into a pandas database -def read_file(file_info, columns=None, subset=None, reduce=None): +def read_file(file_info, columns=None): """ Parses a given file into pandas Dataframe. @@ -220,11 +220,6 @@ def read_file(file_info, columns=None, subset=None, reduce=None): - optional selected_keys (4th element) for HDF5/Zarr path selection. columns: list of str, optional When given, only these columns are returned. - subset: dict, optional - Zarr only. Map of axis index -> slice or int, applied before reduce. - reduce: str, optional - Zarr only. ``\"spatial_mean\"`` averages all axes except axis 0 so a - multi-dim array becomes a 1D column suitable for existing metrics. Returns ---------- pd.Dataframe, None, or str @@ -254,9 +249,7 @@ def read_file(file_info, columns=None, subset=None, reduce=None): cache_path = _frame_cache_path(file_path) if cacheable else None # Fast path: reload a fresh cache without touching the source. - # Skip cache when Zarr subset/reduce options are set (options change the frame). - use_cache = cacheable and subset is None and reduce is None - if use_cache and os.path.exists(cache_path): + if cacheable and os.path.exists(cache_path): try: import pandas as pd @@ -271,15 +264,7 @@ def read_file(file_info, columns=None, subset=None, reduce=None): # Slow path: parse the source once. reader_cls = READER_MAP[file_type] - if file_type == ".zarr": - df = reader_cls( - file_path, - file_upload_time_log, - selected_keys=selected_keys, - subset=subset, - reduce=reduce, - ).read() - elif file_type in _SELECTION_FILE_TYPES: + if file_type in _SELECTION_FILE_TYPES: df = reader_cls( file_path, file_upload_time_log, selected_keys=selected_keys ).read() @@ -290,7 +275,7 @@ def read_file(file_info, columns=None, subset=None, reduce=None): # Best-effort cache for future calls. Failure is non-fatal: we still # return the freshly parsed frame below, so a read-only directory or an # unserialisable frame never turns a successful parse into an error. - if use_cache: + if cacheable: _write_frame_cache(cache_path, df) if columns is not None: diff --git a/aidrin/file_handling/readers/zarr_reader.py b/aidrin/file_handling/readers/zarr_reader.py index 7f3c7967..19a28cb9 100644 --- a/aidrin/file_handling/readers/zarr_reader.py +++ b/aidrin/file_handling/readers/zarr_reader.py @@ -34,25 +34,17 @@ def _require_zarr(): return zarr -# Reduce a multi-dim array to 1D by averaging all axes except axis 0 (time-first). -REDUCE_SPATIAL_MEAN = "spatial_mean" -_VALID_REDUCE = {None, "", REDUCE_SPATIAL_MEAN} - - class zarrReader(StructuredFileReader): """Read Zarr directory stores into pandas DataFrames. - Multi-dimensional arrays can be sliced and reduced before conversion:: - - zarrReader(path, logger, selected_keys=["tmax"], subset={0: slice(0, 14)}, - reduce="spatial_mean") + Same selection model as HDF5: pass ``selected_keys`` to choose array paths. + Only tabular-friendly arrays (0D/1D, or a single 2D array) are converted; + higher-dimensional arrays are refused so metrics see raw values, not aggregates. """ - def __init__(self, file_path: str, logger, selected_keys=None, subset=None, reduce=None): + def __init__(self, file_path: str, logger, selected_keys=None): super().__init__(file_path, logger) self._explicit_selected_keys = selected_keys - self._subset = subset - self._reduce = reduce def _open_store(self): zarr = _require_zarr() @@ -249,69 +241,14 @@ def _resolve_array(self, root, path: str): return None return obj - def _normalize_subset(self, subset, ndim: int): - """Build a numpy index tuple from ``{axis: slice|int}``.""" - if not subset: - return None - if not isinstance(subset, dict): - raise ValueError("subset must be a dict of axis -> slice or int") - - index = [slice(None)] * ndim - for axis, selector in subset.items(): - try: - axis_i = int(axis) - except (TypeError, ValueError) as exc: - raise ValueError(f"subset axis must be an int, got {axis!r}") from exc - if axis_i < 0: - axis_i += ndim - if axis_i < 0 or axis_i >= ndim: - raise ValueError(f"subset axis {axis} out of range for ndim={ndim}") - if not isinstance(selector, (slice, int, np.integer)): - raise ValueError( - f"subset[{axis}] must be slice or int, got {type(selector).__name__}" - ) - index[axis_i] = selector - return tuple(index) - - def _apply_subset(self, data, subset): - if subset is None: - return data - index = self._normalize_subset(subset, data.ndim) - if index is None: - return data - return data[index] - - def _apply_reduce(self, data, reduce): - """Reduce multi-dim data to 1D (or scalar) for tabular metrics.""" - mode = (reduce or "").strip().lower() or None - if mode not in _VALID_REDUCE: - raise ValueError( - f"Unsupported reduce={reduce!r}; use None or '{REDUCE_SPATIAL_MEAN}'" - ) - if mode is None: - return data - if data.ndim <= 1: - return data - # Keep axis 0 (typically time); average remaining spatial/member axes. - axes = tuple(range(1, data.ndim)) - return np.nanmean(data, axis=axes) - def _prepare_array_data(self, arr): - """Load array, apply optional subset/reduce, return numpy data.""" + """Load array data; refuse ndim >= 3 (no averaging / reshape for metrics).""" data = np.asarray(arr[:]) - try: - data = self._apply_subset(data, self._subset) - data = self._apply_reduce(data, self._reduce) - except ValueError as exc: - self.logger.warning("%s", exc) - return None - - if getattr(data, "ndim", 0) >= 3 and not self._reduce: + if getattr(data, "ndim", 0) >= 3: self.logger.warning( - "Zarr array has ndim=%s after subset; pass reduce='%s' " - "(mean over axes 1..) to build a 1D table column, or subset further.", + "Zarr array has ndim=%s; refusing to aggregate or flatten. " + "Select 1D (or a single 2D) arrays via selected_keys.", data.ndim, - REDUCE_SPATIAL_MEAN, ) return None return data @@ -384,8 +321,7 @@ def _read_compatible_array_paths(self, paths): data = np.asarray(data) if data.ndim != 1: self.logger.warning( - "Zarr multi-select requires 1D arrays after subset/reduce; " - "'%s' has ndim=%s", + "Zarr multi-select requires 1D arrays; '%s' has ndim=%s", path, data.ndim, ) diff --git a/aidrin/headless/api.py b/aidrin/headless/api.py index dff6882d..97f97287 100644 --- a/aidrin/headless/api.py +++ b/aidrin/headless/api.py @@ -28,6 +28,7 @@ run_feature_relevance, run_hipaa_compliance, run_k_anonymity, + using_selected_keys, run_l_diversity, run_multiple_attribute_risk, run_null_count_trend, @@ -354,19 +355,24 @@ def summarize_dataset( file_path: str, file_type: Optional[str] = None, max_features: Optional[int] = None, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: """Return shape, per-column descriptive stats, and missing counts for a dataset.""" - from pathlib import Path from aidrin.file_handling.file_parser import read_file + from .runners import _build_file_info - path = Path(file_path) - ext = f".{file_type}" if file_type else path.suffix.lower() - df = read_file((file_path, path.name, ext)) + file_info = _build_file_info( + file_path, + file_type, + None, + selected_keys=_normalize_list(selected_keys), + ) + df = read_file(file_info) if df is None or isinstance(df, str): detail = df if isinstance(df, str) else ( "Unable to build a table from this file. For multi-array Zarr/HDF5 stores, " - "select compatible paths (selected_keys) or reduce multi-dim arrays first." + "pass selected_keys with compatible 1D paths." ) raise ValueError(detail) @@ -506,6 +512,31 @@ def run_metric( verbose: bool = False, strip_visualizations: bool = False, **kwargs: Any, +) -> Dict[str, Any]: + with using_selected_keys(_normalize_list(kwargs.get("selected_keys"))): + return _run_metric_impl( + metric_name, + file_path, + file_type=file_type, + file_name=file_name, + save_images=save_images, + image_dir=image_dir, + verbose=verbose, + strip_visualizations=strip_visualizations, + **kwargs, + ) + + +def _run_metric_impl( + metric_name: str, + file_path: str, + file_type: Optional[str] = None, + file_name: Optional[str] = None, + save_images: bool = True, + image_dir: Optional[str] = None, + verbose: bool = False, + strip_visualizations: bool = False, + **kwargs: Any, ) -> Dict[str, Any]: metric_key = metric_name.strip().lower().replace("-", "_") metric = METRIC_REGISTRY.get(metric_key) @@ -741,6 +772,7 @@ def run_batch_metrics( "timestamp_column": config_obj.timestamp_column, "batch_column": config_obj.batch_column, "target_columns": config_obj.target_columns, + "selected_keys": config_obj.selected_keys, "save_images": bool(config_obj.save_images) if config_obj.save_images is not None else True, "image_dir": config_obj.image_dir, "verbose": verbose, @@ -765,6 +797,7 @@ def run_data_quality( file_name: Optional[str] = None, verbose: bool = False, strip_visualizations: bool = True, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: """Run fast data quality metrics (completeness, duplicity, outliers). @@ -777,6 +810,7 @@ def run_data_quality( file_type=file_type, file_name=file_name, metrics=["completeness", "duplicity", "outliers"], + selected_keys=selected_keys or [], save_images=False, ), verbose=verbose, diff --git a/aidrin/headless/cli.py b/aidrin/headless/cli.py index e4671c88..aaf37b29 100644 --- a/aidrin/headless/cli.py +++ b/aidrin/headless/cli.py @@ -269,6 +269,7 @@ def _build_run_kwargs(args: argparse.Namespace) -> dict: "timestamp_column": getattr(args, "timestamp_column", None), "batch_column": getattr(args, "batch_column", None), "target_columns": _parse_list(getattr(args, "target_columns", None)), + "selected_keys": _parse_list(getattr(args, "selected_keys", None)), "rules": parsed_rules, "rules_json": rules_json, "rules_file": rules_file, @@ -287,6 +288,12 @@ def _build_run_kwargs(args: argparse.Namespace) -> dict: def _configure_common_run_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--file-type", dest="file_type", default=None, help="Input file type override") + parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read (same idea as web selected_keys)", + ) parser.add_argument("--save-images", dest="save_images", action="store_true", help="Save visualizations to disk") parser.add_argument("--no-save-images", dest="save_images", action="store_false", help="Do not save visualizations") parser.set_defaults(save_images=True) @@ -298,6 +305,13 @@ def _configure_common_run_args(parser: argparse.ArgumentParser) -> None: def _configure_minimal_run_args(parser: argparse.ArgumentParser) -> None: """Lightweight args for top-level metric shortcuts.""" + parser.add_argument("--file-type", dest="file_type", default=None, help="Input file type override") + parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read", + ) parser.add_argument("-v", "--verbose", action="store_true", help="Show progress output") @@ -530,13 +544,30 @@ def main() -> None: dq_parser = subparsers.add_parser("data-quality", help="Run fast data quality metrics (completeness, duplicity, outliers)") dq_parser.add_argument("file_path") dq_parser.add_argument("--file-type", dest="file_type", default=None) + dq_parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read", + ) dq_parser.add_argument("-v", "--verbose", action="store_true", help="Show progress output") dq_parser.add_argument("--detail", action="store_true", help="Output full per-feature JSON instead of summary") # Dataset summary command summarize_parser = subparsers.add_parser("summarize", help="Describe numerical and categorical features of a dataset") summarize_parser.add_argument("file_path", help="Path to the dataset") - summarize_parser.add_argument("--file-type", dest="file_type", default=None, help="File type override (csv, parquet, xlsx, hdf5, json, npz)") + summarize_parser.add_argument( + "--file-type", + dest="file_type", + default=None, + help="File type override (csv, parquet, xlsx, hdf5, json, npz, zarr)", + ) + summarize_parser.add_argument( + "--selected-keys", + dest="selected_keys", + default=None, + help="Comma-separated HDF5/Zarr array paths to read", + ) summarize_parser.add_argument( "--max-features", dest="max_features", type=int, default=None, help="Limit stats to N features (split evenly between numerical and categorical)" @@ -669,6 +700,7 @@ def main() -> None: args.file_path, file_type=args.file_type, max_features=args.max_features, + selected_keys=_parse_list(getattr(args, "selected_keys", None)), ) if args.human_readable: _print_summary_table(result, args.file_path) @@ -682,6 +714,7 @@ def main() -> None: file_type=args.file_type, verbose=args.verbose, strip_visualizations=True, + selected_keys=_parse_list(getattr(args, "selected_keys", None)), ) if args.detail: _dump_result(_round_floats(result)) diff --git a/aidrin/headless/config.py b/aidrin/headless/config.py index 0e675599..61767d6e 100644 --- a/aidrin/headless/config.py +++ b/aidrin/headless/config.py @@ -39,6 +39,7 @@ class HeadlessConfig: timestamp_column: Optional[str] = None batch_column: Optional[str] = None target_columns: Optional[List[str]] = field(default_factory=list) + selected_keys: Optional[List[str]] = field(default_factory=list) save_images: Optional[bool] = None image_dir: Optional[str] = None @@ -68,6 +69,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "HeadlessConfig": "file-name": "file_name", "image-dir": "image_dir", "save-images": "save_images", + "selected-keys": "selected_keys", } normalized: Dict[str, Any] = {} @@ -90,6 +92,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "HeadlessConfig": "required_columns", "duplicate_columns", "target_columns", + "selected_keys", ): if key in normalized: normalized[key] = _normalize_list(normalized[key]) diff --git a/aidrin/headless/runners.py b/aidrin/headless/runners.py index 2ff9d160..a3bf4782 100644 --- a/aidrin/headless/runners.py +++ b/aidrin/headless/runners.py @@ -1,7 +1,9 @@ import os import json +from contextlib import contextmanager +from contextvars import ContextVar from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterator, List, Optional from aidrin.file_handling.file_parser import read_file from aidrin.structured_data_metrics.add_noise import return_noisy_stats @@ -48,6 +50,22 @@ _EXCEL_TYPES = {".xls", ".xlsx", ".xlsm", ".xlsb"} _EXCEL_KEY = ".xls, .xlsb, .xlsx, .xlsm" +# Optional HDF5/Zarr path selection for the current headless call (CLI/API/batch). +_selected_keys_ctx: ContextVar[Optional[List[str]]] = ContextVar( + "aidrin_selected_keys", default=None +) + + +@contextmanager +def using_selected_keys(selected_keys: Optional[List[str]] = None) -> Iterator[None]: + """Apply ``selected_keys`` to all ``_build_file_info`` calls in this context.""" + keys = [str(k).strip() for k in (selected_keys or []) if str(k).strip()] or None + token = _selected_keys_ctx.set(keys) + try: + yield + finally: + _selected_keys_ctx.reset(token) + class NullTask: def update_state(self, *args: Any, **kwargs: Any) -> None: @@ -68,10 +86,16 @@ def _normalize_file_type(file_type: Optional[str], file_path: str) -> Optional[s def _build_file_info( - file_path: str, file_type: Optional[str], file_name: Optional[str] + file_path: str, + file_type: Optional[str], + file_name: Optional[str], + selected_keys: Optional[List[str]] = None, ) -> tuple: normalized_type = _normalize_file_type(file_type, file_path) final_name = file_name or os.path.basename(file_path) + keys = selected_keys if selected_keys is not None else _selected_keys_ctx.get() + if keys: + return (file_path, final_name, normalized_type, list(keys)) return (file_path, final_name, normalized_type) diff --git a/docs/source/cli_usage.rst b/docs/source/cli_usage.rst index 040aa1e1..22fc7db7 100644 --- a/docs/source/cli_usage.rst +++ b/docs/source/cli_usage.rst @@ -20,6 +20,14 @@ Quick Start # Run a batch of metrics from a YAML config aidrin batch /path/to/my_project/batch_config.yaml + # HDF5 / Zarr: pick compatible 1D arrays (comma-separated paths) + aidrin run completeness /path/to/store.zarr --selected-keys age,income + aidrin summarize /path/to/file.h5 --selected-keys S1/X,S1/Y + +Install Zarr support with ``pip install 'aidrin[zarr]'`` (Python >= 3.11). Local web upload +does not accept ``.zarr`` directories; use the CLI or library. Multi-dimensional grids are not +auto-flattened — select 1D (or a single 2D) arrays only. + ---- Sample Dataset @@ -207,6 +215,9 @@ Results are printed as JSON to stdout. Redirect to a file to save: target-column: approved +For HDF5/Zarr multi-array stores, add ``selected-keys`` (list or comma-separated string) +with compatible 1D paths, for example ``selected-keys: [age, income]``. + **Example** — fairness analysis on the sample dataset: .. code-block:: yaml diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 28d9f9f4..7f55d2e0 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -633,5 +633,65 @@ def test_no_command_exits_nonzero(self): self.assertNotEqual(code, 0) +# =========================================================================== +# selected-keys (HDF5 / Zarr) +# =========================================================================== + + +class TestSelectedKeysCLI(unittest.TestCase): + """CLI --selected-keys for multi-array Zarr stores.""" + + @classmethod + def setUpClass(cls): + pytest = __import__("pytest") + zarr = pytest.importorskip("zarr") + cls._tmpdir = tempfile.TemporaryDirectory() + store = os.path.join(cls._tmpdir.name, "pick.zarr") + root = zarr.open_group(store, mode="w") + for name, length in (("age", 20), ("income", 20), ("meta", 1)): + arr = root.create_array(name, shape=(length,), dtype="f8") + arr[:] = np.arange(length, dtype=np.float64) + cls.store = store + + @classmethod + def tearDownClass(cls): + cls._tmpdir.cleanup() + + def test_run_completeness_with_selected_keys(self): + stdout, stderr, code = _run_cli( + "run", + "completeness", + self.store, + "--selected-keys", + "age,income", + ) + self.assertEqual(code, 0, msg=stderr) + payload = json.loads(stdout) + scores = payload.get("Completeness scores", {}) + self.assertIn("age", scores) + self.assertIn("income", scores) + self.assertNotIn("meta", scores) + + def test_summarize_with_selected_keys(self): + stdout, stderr, code = _run_cli( + "summarize", + self.store, + "--selected-keys", + "age", + ) + self.assertEqual(code, 0, msg=stderr) + payload = json.loads(stdout) + self.assertEqual(payload["shape"]["columns"], 1) + self.assertIn("age", payload.get("numerical", {})) + + def test_build_file_info_includes_selected_keys(self): + from aidrin.headless.runners import _build_file_info, using_selected_keys + + with using_selected_keys(["S1/X", "S1/Y"]): + info = _build_file_info(self.store, None, None) + self.assertEqual(info[2], ".zarr") + self.assertEqual(info[3], ["S1/X", "S1/Y"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_structured_readers.py b/tests/unit/test_structured_readers.py index f3aa0b3c..ab72e594 100644 --- a/tests/unit/test_structured_readers.py +++ b/tests/unit/test_structured_readers.py @@ -188,9 +188,8 @@ def test_zarr_empty_store(tmp_path, logger): def _write_multidim_store(path): - """3D array shaped like (time, lat, lon) for subset/reduce tests.""" + """3D array shaped like (time, lat, lon) — not tabular without aggregation.""" root = zarr.open_group(str(path), mode="w") - # time=5, lat=4, lon=3 — values = time index for easy spatial-mean checks data = np.zeros((5, 4, 3), dtype=np.float64) for t in range(5): data[t, :, :] = float(t) @@ -198,52 +197,18 @@ def _write_multidim_store(path): arr[:] = data -def test_zarr_multidim_requires_reduce(tmp_path, logger): +def test_zarr_multidim_refused(tmp_path, logger): + """ndim >= 3 must not be averaged or flattened for metrics.""" store = tmp_path / "grid.zarr" _write_multidim_store(store) df = zarrReader(str(store), logger, selected_keys=["tmax_grid"]).read() assert df is None -def test_zarr_spatial_mean_reduce(tmp_path, logger): - store = tmp_path / "grid.zarr" - _write_multidim_store(store) - df = zarrReader( - str(store), - logger, - selected_keys=["tmax_grid"], - reduce="spatial_mean", - ).read() - assert df is not None - assert list(df.columns) == ["tmax_grid"] - assert len(df) == 5 - assert list(df["tmax_grid"]) == [0.0, 1.0, 2.0, 3.0, 4.0] - - -def test_zarr_subset_then_spatial_mean(tmp_path, logger): - store = tmp_path / "grid.zarr" - _write_multidim_store(store) - # First 3 time steps, subset of lat/lon - df = zarrReader( - str(store), - logger, - selected_keys=["tmax_grid"], - subset={0: slice(0, 3), 1: slice(0, 2), 2: slice(0, 2)}, - reduce="spatial_mean", - ).read() - assert df is not None - assert len(df) == 3 - assert list(df["tmax_grid"]) == [0.0, 1.0, 2.0] - - -def test_zarr_read_file_passes_subset_reduce(tmp_path): - store = tmp_path / "grid.zarr" - _write_multidim_store(store) - df = read_file( - (str(store), "grid.zarr", ".zarr", ["tmax_grid"]), - reduce="spatial_mean", - subset={0: slice(1, 4)}, - ) +def test_zarr_read_file_selected_keys(tmp_path): + store = tmp_path / "pick.zarr" + _write_grouped_hierarchical_store(store) + df = read_file((str(store), "pick.zarr", ".zarr", ["S1/X", "S1/Y"])) assert df is not None - assert len(df) == 3 - assert list(df["tmax_grid"]) == [1.0, 2.0, 3.0] + assert list(df.columns) == ["S1/X", "S1/Y"] + assert len(df) == 10 From 66879907db1b4481d62cd8e696496b1060c735b6 Mon Sep 17 00:00:00 2001 From: Dhanushkumar Jv Date: Tue, 11 Aug 2026 01:17:04 +0530 Subject: [PATCH 5/5] Accept selected_keys on RemoteExecutor summarize and data-quality --- aidrin/compute/executor.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/aidrin/compute/executor.py b/aidrin/compute/executor.py index d892ac23..0d6c69ad 100644 --- a/aidrin/compute/executor.py +++ b/aidrin/compute/executor.py @@ -15,7 +15,7 @@ """ from dataclasses import asdict, is_dataclass -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional from aidrin.compute import client from aidrin.compute.profiles import RemoteTarget @@ -138,11 +138,16 @@ def summarize_dataset( file_path: str, file_type: Optional[str] = None, max_features: Optional[int] = None, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: - return self._call( - "summarize", - {"file_path": file_path, "file_type": file_type, "max_features": max_features}, - ) + payload: Dict[str, Any] = { + "file_path": file_path, + "file_type": file_type, + "max_features": max_features, + } + if selected_keys: + payload["selected_keys"] = selected_keys + return self._call("summarize", payload) def run_data_quality( self, @@ -151,17 +156,18 @@ def run_data_quality( file_name: Optional[str] = None, verbose: bool = False, strip_visualizations: bool = True, + selected_keys: Optional[List[str]] = None, ) -> Dict[str, Any]: - return self._call( - "data_quality", - { - "file_path": file_path, - "file_type": file_type, - "file_name": file_name, - "verbose": verbose, - "strip_visualizations": strip_visualizations, - }, - ) + payload: Dict[str, Any] = { + "file_path": file_path, + "file_type": file_type, + "file_name": file_name, + "verbose": verbose, + "strip_visualizations": strip_visualizations, + } + if selected_keys: + payload["selected_keys"] = selected_keys + return self._call("data_quality", payload) def run_batch_metrics( self,