From 285ae8ac6f2bc105488c1c5b1ee9becd10915e9f Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 11:26:33 +0200 Subject: [PATCH 01/97] Add backend-agnostic spool index prototype (#648) Six-table summary index: sources, patches (frozen structural columns plus time/distance envelopes), attrs (lazily-added typed columns, one per attr/kind), coords (tall, typed min/max/step), attr_meta, meta_data. - DuckDB, SQLite (STRICT), and Parquet-manifest backends behind one abstract interface; shared SQL generation with a small dialect layer. - Ingest from PatchSummary with pint base-SI unit normalization and sanitized dynamic column names. - Query layer implements the selector semantics spec: attrs-first name resolution, kind dispatch, envelope candidacy with no false negatives, regex as pandas residual. - Contract test suite runs identically against all three backends (36 tests x 3). - Secondary indexes (SQLite EXISTS otherwise quadratic) and dataframe bulk ingest for DuckDB (executemany binds row-at-a-time). --- dascore/io/index/__init__.py | 21 + dascore/io/index/backend.py | 433 ++++++++++++++++++ dascore/io/index/dialect.py | 64 +++ dascore/io/index/duck.py | 72 +++ dascore/io/index/ingest.py | 318 +++++++++++++ dascore/io/index/lite.py | 50 ++ dascore/io/index/parq.py | 115 +++++ dascore/io/index/query.py | 300 ++++++++++++ dascore/io/index/schema.py | 133 ++++++ .../test_io/test_index/test_index_contract.py | 427 +++++++++++++++++ 10 files changed, 1933 insertions(+) create mode 100644 dascore/io/index/__init__.py create mode 100644 dascore/io/index/backend.py create mode 100644 dascore/io/index/dialect.py create mode 100644 dascore/io/index/duck.py create mode 100644 dascore/io/index/ingest.py create mode 100644 dascore/io/index/lite.py create mode 100644 dascore/io/index/parq.py create mode 100644 dascore/io/index/query.py create mode 100644 dascore/io/index/schema.py create mode 100644 tests/test_io/test_index/test_index_contract.py diff --git a/dascore/io/index/__init__.py b/dascore/io/index/__init__.py new file mode 100644 index 000000000..4fe2c10f9 --- /dev/null +++ b/dascore/io/index/__init__.py @@ -0,0 +1,21 @@ +""" +Backend-agnostic spool index package. + +Provides a normalized, summary-only index of patch metadata (sources, +patches, attrs, coords) with interchangeable storage backends. See +`.scratch/spool_index_design.md` on the spool-index-backend branch and +GitHub discussion #648 for the design. +""" + +from __future__ import annotations + +from dascore.io.index.backend import AbstractIndexBackend, get_backend +from dascore.io.index.ingest import summaries_to_records +from dascore.io.index.query import Query + +__all__ = [ + "AbstractIndexBackend", + "Query", + "get_backend", + "summaries_to_records", +] diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py new file mode 100644 index 000000000..3c9410d4a --- /dev/null +++ b/dascore/io/index/backend.py @@ -0,0 +1,433 @@ +""" +Abstract index backend and the shared SQL implementation. + +Backends persist the six-table schema and answer flat-relation queries. +All engine differences live in `dialect.py` plus a handful of hooks; the +write/query logic here is shared so the contract test suite exercises +identical semantics on every backend. +""" + +from __future__ import annotations + +import abc +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +import dascore as dc +from dascore.io.index.dialect import BaseDialect +from dascore.io.index.ingest import SourceRecord, attr_column_name +from dascore.io.index.query import Query, apply_residuals, build_query_sql +from dascore.io.index.schema import ( + COORDS, + INDEX_VERSION, + INDEXES, + KIND_STORAGE, + PATCHES, + SOURCES, + TABLES, + WHAT_IS_THIS, +) + +# Structural columns whose ns-integer storage maps to pandas time types. +_TIME_COLS = {"time_min": "datetime", "time_max": "datetime", "time_step": "timedelta"} + + +def adapt_params(params) -> list: + """Convert numpy scalars (and NaN) to plain python for DB drivers.""" + out = [] + for p in params: + if hasattr(p, "item"): # numpy scalar + p = p.item() + if isinstance(p, float) and np.isnan(p): + p = None + out.append(p) + return out + + +class AbstractIndexBackend(abc.ABC): + """Interface every index backend must implement.""" + + @abc.abstractmethod + def write_sources(self, records: list[SourceRecord]) -> None: + """Insert or replace sources (and dependents) transactionally.""" + + @abc.abstractmethod + def delete_sources(self, source_paths: list[str]) -> None: + """Remove sources and all dependent rows.""" + + @abc.abstractmethod + def query(self, query: Query) -> pd.DataFrame: + """Return the flat patch-row relation matching a query.""" + + @abc.abstractmethod + def get_sources(self) -> pd.DataFrame: + """Return the sources table.""" + + @abc.abstractmethod + def get_metadata(self) -> dict: + """Return index-level metadata.""" + + @abc.abstractmethod + def attr_names(self) -> set[str]: + """Return original attr names known to the index.""" + + @abc.abstractmethod + def coord_names(self) -> set[str]: + """Return coord names known to the index.""" + + @abc.abstractmethod + def close(self) -> None: + """Release resources.""" + + +class SQLIndexBackend(AbstractIndexBackend): + """Shared implementation for SQL-speaking backends.""" + + dialect: BaseDialect + + def __init__(self): + self._ensure_schema() + + # --- hooks each engine provides --------------------------------- + + @abc.abstractmethod + def _execute(self, sql: str, params=()) -> None: + """Execute one statement.""" + + @abc.abstractmethod + def _executemany(self, sql: str, seq_of_params) -> None: + """Execute one statement for many parameter tuples.""" + + @abc.abstractmethod + def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: + """Execute a SELECT and return a dataframe.""" + + @abc.abstractmethod + def _begin(self) -> None: + """Start a transaction.""" + + @abc.abstractmethod + def _commit(self) -> None: + """Commit the open transaction.""" + + @abc.abstractmethod + def _rollback(self) -> None: + """Roll back the open transaction.""" + + # --- schema ------------------------------------------------------ + + def _ensure_schema(self) -> None: + for name, columns in TABLES.items(): + self._execute(self.dialect.create_table(name, columns)) + for index_name, table, column in INDEXES: + self._execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " f"ON {table} ({column})" + ) + meta = self._fetch_df("SELECT * FROM meta_data") + if meta.empty: + self._execute( + "INSERT INTO meta_data VALUES (?, ?, ?, ?)", + (WHAT_IS_THIS, INDEX_VERSION, dc.__version__, time.time_ns()), + ) + + def _attr_meta(self) -> pd.DataFrame: + return self._fetch_df("SELECT * FROM attr_meta") + + def _next_id(self, table: str, column: str) -> int: + df = self._fetch_df(f"SELECT max({column}) AS m FROM {table}") + value = df["m"].iloc[0] + return 1 if pd.isnull(value) else int(value) + 1 + + def _ensure_attr_columns(self, records: list[SourceRecord]) -> None: + """Lazily add typed attr columns and register them in attr_meta.""" + known = { + (row.attr_name, row.value_kind) for row in self._attr_meta().itertuples() + } + needed: dict[tuple[str, str], str | None] = {} + for record in records: + for patch in record.patches: + for name, typed in patch.attrs.items(): + key = (name, typed.kind) + if key not in known and key not in needed: + needed[key] = typed.units + for (name, kind), units in needed.items(): + column = attr_column_name(name, kind) + self._execute(self.dialect.add_column("attrs", column, KIND_STORAGE[kind])) + self._execute( + "INSERT INTO attr_meta VALUES (?, ?, ?, ?)", + (name, kind, column, units), + ) + + def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: + """Insert many rows; engines override for faster bulk paths.""" + if not rows: + return + quoted = ", ".join(self.dialect.quote(c) for c in columns) + marks = ", ".join("?" for _ in columns) + sql = f"INSERT INTO {self.dialect.quote(table)} ({quoted}) VALUES ({marks})" + self._executemany(sql, rows) + + # --- writes ------------------------------------------------------ + + def write_sources(self, records: list[SourceRecord]) -> None: + """ + Insert or replace sources and all dependent rows, atomically. + + Rows are batched per table (attrs grouped by column signature) so + columnar engines aren't punished by row-at-a-time inserts. + """ + self._begin() + try: + self._delete_by_paths([r.source_path for r in records]) + self._ensure_attr_columns(records) + source_id = self._next_id("sources", "source_id") + patch_id = self._next_id("patches", "patch_id") + now = time.time_ns() + source_rows, patch_rows, coord_rows = [], [], [] + attr_groups: dict[tuple[str, ...], list] = {} + for record in records: + source_rows.append( + ( + source_id, + record.base_uri, + record.source_path, + record.source_format, + record.format_version, + record.mtime_ns, + record.size_bytes, + now, + ) + ) + for patch in record.patches: + patch_rows.append( + ( + patch_id, + source_id, + patch.source_patch_id, + patch.n_dims, + patch.dims, + patch.shape, + patch.sample_count_total, + patch.time_min, + patch.time_max, + patch.time_step, + patch.distance_min, + patch.distance_max, + patch.distance_step, + ) + ) + columns = tuple( + attr_column_name(name, tv.kind) + for name, tv in patch.attrs.items() + ) + attr_groups.setdefault(columns, []).append( + [patch_id, *(tv.value for tv in patch.attrs.values())] + ) + coord_rows.extend( + ( + patch_id, + c.coord_name, + c.value_kind, + c.dtype, + c.coord_dims, + c.length, + c.units, + c.min_num, + c.max_num, + c.step_num, + c.min_ns, + c.max_ns, + c.step_ns, + c.min_str, + c.max_str, + c.is_monotonic, + c.is_relative, + c.coord_hash, + ) + for c in patch.coords + ) + patch_id += 1 + source_id += 1 + self._bulk_insert("sources", tuple(SOURCES), source_rows) + self._bulk_insert("patches", tuple(PATCHES), patch_rows) + for columns, rows in attr_groups.items(): + self._bulk_insert("attrs", ("patch_id", *columns), rows) + self._bulk_insert("coords", tuple(COORDS), list(coord_rows)) + self._execute("UPDATE meta_data SET last_indexed_ns = ?", (now,)) + except Exception: + self._rollback() + raise + self._commit() + + def _delete_by_paths(self, source_paths: list[str]) -> None: + if not source_paths: + return + marks = ", ".join("?" for _ in source_paths) + ids = self._fetch_df( + f"SELECT source_id FROM sources WHERE source_path IN ({marks})", + source_paths, + )["source_id"].tolist() + if not ids: + return + id_marks = ", ".join("?" for _ in ids) + for sql in ( + f"DELETE FROM coords WHERE patch_id IN " + f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", + f"DELETE FROM attrs WHERE patch_id IN " + f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", + f"DELETE FROM patches WHERE source_id IN ({id_marks})", + f"DELETE FROM sources WHERE source_id IN ({id_marks})", + ): + self._execute(sql, ids) + + def delete_sources(self, source_paths: list[str]) -> None: + """Remove sources and all dependent rows.""" + self._begin() + try: + self._delete_by_paths(source_paths) + except Exception: + self._rollback() + raise + self._commit() + + # --- queries ----------------------------------------------------- + + def query(self, query: Query | None = None) -> pd.DataFrame: + """Return the flat patch-row relation for a query.""" + query = query if query is not None else Query() + attr_meta = self._attr_meta() + sql, params, residuals = build_query_sql(query, self.dialect, attr_meta) + df = self._fetch_df(sql, params) + df = self._flatten(df, attr_meta) + if residuals: + df = apply_residuals(df, residuals) + return df.reset_index(drop=True) + + def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: + """Post-process raw SQL output into the flat-relation contract.""" + out = df.copy() + # structural time columns: ns ints -> numpy time types + for col, flavor in _TIME_COLS.items(): + if col in out: + as_int = out[col].astype("float64") # NaN-safe intermediate + if flavor == "datetime": + out[col] = pd.to_datetime(as_int, unit="ns") + else: + out[col] = pd.to_timedelta(as_int, unit="ns") + # typed attr columns -> original names (coalesce multi-kind attrs) + for name in attr_meta["attr_name"].unique(): + rows = attr_meta[attr_meta["attr_name"] == name] + kinds = set(rows["value_kind"]) + series = None + for row in rows.itertuples(): + if row.column_name not in out: + continue + col = out[row.column_name] + if row.value_kind == "time": + col = pd.to_datetime(col.astype("float64"), unit="ns") + elif row.value_kind == "dur": + col = pd.to_timedelta(col.astype("float64"), unit="ns") + elif row.value_kind == "bool": + col = col.astype("boolean") + series = col if series is None else series.where(series.notna(), col) + out = out.drop(columns=[row.column_name]) + if series is not None: + if kinds == {"str"}: + # flat-contract convention: missing strings are "" + series = series.fillna("") + out[name] = series + # flat-contract names for source columns + renames = { + "source_path": "path", + "source_format": "file_format", + "format_version": "file_version", + } + out = out.rename(columns=renames) + if "base_uri" in out: + has_base = out["base_uri"].notna() + out.loc[has_base, "path"] = ( + out.loc[has_base, "base_uri"].str.rstrip("/") + + "/" + + out.loc[has_base, "path"] + ) + out = out.drop(columns=["base_uri"]) + return out.drop(columns=["source_id"], errors="ignore") + + # --- introspection ----------------------------------------------- + + def get_sources(self) -> pd.DataFrame: + """Return the sources table.""" + return self._fetch_df("SELECT * FROM sources") + + def get_metadata(self) -> dict: + """Return index-level metadata.""" + return self._fetch_df("SELECT * FROM meta_data").iloc[0].to_dict() + + def attr_names(self) -> set[str]: + """Return original attr names known to the index.""" + return set(self._attr_meta()["attr_name"]) + + def coord_names(self) -> set[str]: + """Return coord names known to the index.""" + return set( + self._fetch_df("SELECT DISTINCT coord_name FROM coords")["coord_name"] + ) + + +def resolve_query( + backend: AbstractIndexBackend, _attrs=None, _coords=None, **kwargs +) -> Query: + """ + Resolve bare kwargs into a Query: attrs first, then coords. + + Implements section 1 of the selector spec; raises on unknown names or + names supplied in more than one namespace. + """ + from dascore.io.index.query import InvalidSpoolQueryError + + attrs = dict(_attrs or {}) + coords = dict(_coords or {}) + known_attrs = backend.attr_names() + known_coords = backend.coord_names() + for name, value in kwargs.items(): + if name in attrs or name in coords: + msg = f"{name!r} given as both a bare kwarg and in _attrs/_coords." + raise InvalidSpoolQueryError(msg) + if name in known_attrs: + attrs[name] = value + elif name in known_coords: + coords[name] = value + else: + msg = ( + f"{name!r} is neither an attribute nor a coordinate of any " + f"patch in this spool." + ) + raise InvalidSpoolQueryError(msg) + for name in attrs: + if name not in known_attrs: + raise InvalidSpoolQueryError(f"Unknown attribute {name!r}.") + for name in coords: + if name not in known_coords: + raise InvalidSpoolQueryError(f"Unknown coordinate {name!r}.") + return Query(attrs=attrs, coords=coords) + + +def get_backend(path: str | Path, kind: str = "duckdb") -> AbstractIndexBackend: + """Create an index backend of the given kind at path.""" + if kind == "duckdb": + from dascore.io.index.duck import DuckDBBackend + + return DuckDBBackend(path) + if kind == "sqlite": + from dascore.io.index.lite import SQLiteBackend + + return SQLiteBackend(path) + if kind == "parquet": + from dascore.io.index.parq import ParquetBackend + + return ParquetBackend(path) + msg = f"Unknown index backend {kind!r}." + raise ValueError(msg) diff --git a/dascore/io/index/dialect.py b/dascore/io/index/dialect.py new file mode 100644 index 000000000..e997160ac --- /dev/null +++ b/dascore/io/index/dialect.py @@ -0,0 +1,64 @@ +""" +SQL dialect translation for index backends. + +Everything engine-specific lives here: type names, identifier quoting, +glob matching, and table DDL generation. The rest of the package emits +logical types and parametrized SQL with `?` placeholders. +""" + +from __future__ import annotations + +from collections.abc import Mapping + + +class BaseDialect: + """Shared SQL generation for engines close to the standard.""" + + # logical type -> engine type + type_map: Mapping[str, str] = { + "int64": "BIGINT", + "float64": "DOUBLE", + "str": "VARCHAR", + "bool": "BOOLEAN", + } + strict_suffix = "" + + def quote(self, identifier: str) -> str: + """Quote an identifier.""" + return '"' + identifier.replace('"', '""') + '"' + + def create_table(self, name: str, columns: Mapping[str, str]) -> str: + """Return DDL for one table from logical column types.""" + cols = ", ".join( + f"{self.quote(col)} {self.type_map[typ]}" for col, typ in columns.items() + ) + quoted = self.quote(name) + return f"CREATE TABLE IF NOT EXISTS {quoted} ({cols}){self.strict_suffix}" + + def add_column(self, table: str, column: str, logical_type: str) -> str: + """Return DDL to add one nullable column.""" + return ( + f"ALTER TABLE {self.quote(table)} " + f"ADD COLUMN {self.quote(column)} {self.type_map[logical_type]}" + ) + + def glob(self, column_sql: str) -> str: + """Return a parametrized unix-glob match expression.""" + return f"{column_sql} GLOB ?" + + +class DuckDBDialect(BaseDialect): + """Dialect for DuckDB.""" + + +class SQLiteDialect(BaseDialect): + """Dialect for SQLite; STRICT tables enforce the type contract.""" + + # SQLite STRICT tables accept INTEGER/REAL/TEXT (and INT for bool). + type_map = { + "int64": "INTEGER", + "float64": "REAL", + "str": "TEXT", + "bool": "INTEGER", + } + strict_suffix = " STRICT" diff --git a/dascore/io/index/duck.py b/dascore/io/index/duck.py new file mode 100644 index 000000000..289e86c9e --- /dev/null +++ b/dascore/io/index/duck.py @@ -0,0 +1,72 @@ +"""DuckDB index backend.""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + +from dascore.io.index.backend import SQLIndexBackend, adapt_params +from dascore.io.index.dialect import DuckDBDialect + + +def duck_bulk_insert(con, dialect, table: str, columns: tuple, rows: list) -> None: + """ + Bulk-insert rows through a registered dataframe. + + DuckDB's executemany binds row-at-a-time in Python (about 60x slower + than SQLite for ingest); routing bulk rows through its dataframe + scanner keeps ingest columnar. + """ + if not rows: + return + df = pd.DataFrame( + [adapt_params(r) for r in rows], columns=list(columns), dtype=object + ) + con.register("_bulk_rows", df) + try: + quoted = ", ".join(dialect.quote(c) for c in columns) + con.execute( + f"INSERT INTO {dialect.quote(table)} ({quoted}) " "SELECT * FROM _bulk_rows" + ) + finally: + con.unregister("_bulk_rows") + + +class DuckDBBackend(SQLIndexBackend): + """Index backend storing tables in a single DuckDB file.""" + + dialect = DuckDBDialect() + + def __init__(self, path: str | Path, read_only: bool = False): + import duckdb + + self._con = duckdb.connect(str(path), read_only=read_only) + super().__init__() + + def _execute(self, sql: str, params=()) -> None: + self._con.execute(sql, adapt_params(params)) + + def _executemany(self, sql: str, seq_of_params) -> None: + rows = [adapt_params(p) for p in seq_of_params] + if rows: + self._con.executemany(sql, rows) + + def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: + return self._con.execute(sql, adapt_params(params)).df() + + def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: + duck_bulk_insert(self._con, self.dialect, table, columns, rows) + + def _begin(self) -> None: + self._con.execute("BEGIN TRANSACTION") + + def _commit(self) -> None: + self._con.execute("COMMIT") + + def _rollback(self) -> None: + self._con.execute("ROLLBACK") + + def close(self) -> None: + """Close the database connection.""" + self._con.close() diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py new file mode 100644 index 000000000..54d32ee9a --- /dev/null +++ b/dascore/io/index/ingest.py @@ -0,0 +1,318 @@ +""" +Convert patch summaries into normalized index records. + +This module is backend-independent: it turns `PatchSummary` objects into +plain records (dicts/dataclasses) using only the four primitive storage +types. All unit-bearing numeric values are normalized to pint base SI +units here, so cross-patch comparisons in the index are always valid. +""" + +from __future__ import annotations + +import re +import warnings +from dataclasses import dataclass, field +from functools import cache + +import numpy as np +import pandas as pd + +from dascore.core.summary import PatchSummary +from dascore.io.index.schema import KINDS, RESERVED_ATTR_COLUMNS +from dascore.units import get_quantity +from dascore.utils.time import to_datetime64, to_int, to_timedelta64 + +_SANITIZE_RE = re.compile(r"[^a-z0-9_]+") + +# Attrs handled structurally or intentionally excluded from the index. +_SKIPPED_ATTRS = frozenset({"history", "dims", "coords"}) + + +@dataclass(frozen=True) +class TypedValue: + """A value with its index kind and (canonical) units.""" + + kind: str + value: float | int | str | bool + units: str | None = None + + def __post_init__(self): + assert self.kind in KINDS + + +@dataclass(frozen=True) +class CoordRecord: + """One row of the coords table (typed columns split by kind).""" + + coord_name: str + value_kind: str + dtype: str + coord_dims: str + length: int | None + units: str | None + min_num: float | None = None + max_num: float | None = None + step_num: float | None = None + min_ns: int | None = None + max_ns: int | None = None + step_ns: int | None = None + min_str: str | None = None + max_str: str | None = None + is_monotonic: bool | None = None + is_relative: bool | None = None + coord_hash: str | None = None + + +@dataclass(frozen=True) +class PatchRecord: + """One patch: structural fields, typed attrs, coord rows.""" + + source_patch_id: str + dims: str + shape: str + n_dims: int + sample_count_total: int | None + time_min: int | None + time_max: int | None + time_step: int | None + distance_min: float | None + distance_max: float | None + distance_step: float | None + attrs: dict[str, TypedValue] = field(default_factory=dict) + coords: tuple[CoordRecord, ...] = () + + +@dataclass(frozen=True) +class SourceRecord: + """One source (scan unit) and the patches it emitted.""" + + source_path: str + source_format: str + format_version: str + base_uri: str | None = None + mtime_ns: int | None = None + size_bytes: int | None = None + patches: tuple[PatchRecord, ...] = () + + +def sanitize_attr_name(name: str) -> str: + """Return a lowercase [a-z0-9_] identifier for an attr name.""" + out = _SANITIZE_RE.sub("_", name.lower()).strip("_") + if not out or out[0].isdigit(): + out = f"a_{out}" + return out + + +def attr_column_name(name: str, kind: str) -> str: + """Return the attrs-table column for an attr name and kind.""" + return f"{sanitize_attr_name(name)}__{kind}" + + +@cache +def _base_unit_info(unit_str: str) -> tuple[float, str]: + """Return (scale factor to SI base, canonical base unit string).""" + quant = get_quantity(unit_str).to_base_units() + return float(quant.magnitude), str(quant.units) + + +def _is_missing(value) -> bool: + """Return True for values that mean 'not present'.""" + if value is None or (isinstance(value, str) and value == ""): + return True + try: + return bool(pd.isnull(value)) + except (TypeError, ValueError): + return False + + +def typed_value(value) -> TypedValue | None: + """ + Classify a python/numpy scalar into a TypedValue, or None to skip. + + Unit-bearing quantities are converted to base SI units; the canonical + unit string is recorded so queries can convert consistently. + """ + if _is_missing(value): + return None + # bool must precede int (bool is a subclass of int). + if isinstance(value, bool | np.bool_): + return TypedValue("bool", bool(value)) + if isinstance(value, np.datetime64): + return TypedValue("time", to_int(value)) + if isinstance(value, np.timedelta64): + return TypedValue("dur", to_int(value)) + # pint scalar quantity or unit. + if hasattr(value, "units"): + magnitude = getattr(value, "magnitude", 1) + if isinstance(magnitude, np.ndarray): + return None # array quantities are not scalar attrs + factor, base = _base_unit_info(str(value.units)) + return TypedValue("num", float(magnitude) * factor, units=base) + if isinstance(value, int | np.integer | float | np.floating): + return TypedValue("num", float(value)) + if isinstance(value, str): + return TypedValue("str", value) + # datetime/timedelta and anything datetime-like numpy missed. + try: + return TypedValue("time", to_int(to_datetime64(value))) + except Exception: + pass + return None # complex attrs (sequences, dicts, ...) are skipped + + +def _extract_attrs(summary: PatchSummary) -> dict[str, TypedValue]: + """Get indexable typed attrs from a patch summary.""" + raw = summary.attrs.model_dump() + out = {} + for name, value in raw.items(): + if name in _SKIPPED_ATTRS or name.startswith("_"): + continue + if sanitize_attr_name(name) in RESERVED_ATTR_COLUMNS: + warnings.warn(f"Skipping reserved attr name {name!r}.", UserWarning) + continue + typed = typed_value(value) + if typed is not None: + out[name] = typed + return out + + +def _coord_record(name: str, summary) -> CoordRecord | None: + """Convert one CoordSummary into a CoordRecord.""" + common = dict( + coord_name=name, + dtype=summary.dtype, + coord_dims=",".join(summary.dims), + length=summary.len, + units=str(summary.units) if summary.units is not None else None, + coord_hash=getattr(summary, "fingerprint", None), + ) + dtype = np.dtype(summary.dtype) if summary.dtype else None + if dtype is not None and np.issubdtype(dtype, np.datetime64): + step = summary.step + return CoordRecord( + value_kind="time", + is_relative=False, + min_ns=to_int(to_datetime64(summary.min)), + max_ns=to_int(to_datetime64(summary.max)), + step_ns=None if pd.isnull(step) else to_int(to_timedelta64(step)), + **common, + ) + if dtype is not None and np.issubdtype(dtype, np.timedelta64): + step = summary.step + return CoordRecord( + value_kind="time", + is_relative=True, + min_ns=to_int(to_timedelta64(summary.min)), + max_ns=to_int(to_timedelta64(summary.max)), + step_ns=None if pd.isnull(step) else to_int(to_timedelta64(step)), + **common, + ) + if dtype is not None and np.issubdtype(dtype, np.number): + factor = 1.0 + if summary.units is not None: + factor, base = _base_unit_info(str(summary.units)) + common["units"] = base + step = summary.step + return CoordRecord( + value_kind="num", + min_num=float(summary.min) * factor, + max_num=float(summary.max) * factor, + step_num=None if pd.isnull(step) else float(step) * factor, + **common, + ) + if dtype is not None and (dtype.kind in "US" or dtype == object): + return CoordRecord( + value_kind="str", + min_str=str(summary.min), + max_str=str(summary.max), + **common, + ) + return None # unsupported coord representation: skip, per design + + +def _envelope(coords: tuple[CoordRecord, ...], name: str, kind: str): + """Pull the (min, max, step) envelope for one coord if present.""" + for rec in coords: + if rec.coord_name != name or rec.value_kind != kind: + continue + if kind == "time" and not rec.is_relative: + return rec.min_ns, rec.max_ns, rec.step_ns + if kind == "num": + return rec.min_num, rec.max_num, rec.step_num + return None, None, None + + +def patch_record(summary: PatchSummary) -> PatchRecord: + """Convert one PatchSummary into a PatchRecord.""" + coords = tuple( + rec + for name, csum in summary.coords.items() + if (rec := _coord_record(name, csum)) is not None + ) + time_min, time_max, time_step = _envelope(coords, "time", "time") + dist_min, dist_max, dist_step = _envelope(coords, "distance", "num") + shape = tuple(int(x) for x in summary.shape) + return PatchRecord( + source_patch_id=summary.source_patch_id or "", + dims=",".join(summary.dims), + shape=",".join(str(x) for x in shape), + n_dims=len(summary.dims), + sample_count_total=int(np.prod(shape)) if shape else None, + time_min=time_min, + time_max=time_max, + time_step=time_step, + distance_min=dist_min, + distance_max=dist_max, + distance_step=dist_step, + attrs=_extract_attrs(summary), + coords=coords, + ) + + +def summaries_to_records( + summaries: list[PatchSummary], + base_uri: str | None = None, + mtimes_ns: dict[str, int] | None = None, + sizes_bytes: dict[str, int] | None = None, +) -> list[SourceRecord]: + """ + Group patch summaries by source and convert to SourceRecords. + + Parameters + ---------- + summaries + Patch summaries, e.g. from `dc.scan`. + base_uri + Optional common root; source paths are stored relative to it. + mtimes_ns, sizes_bytes + Optional maps of source_path -> stat values. When omitted the + caller is responsible for change detection. + """ + by_source: dict[str, list[PatchSummary]] = {} + for summary in summaries: + by_source.setdefault(str(summary.source_path), []).append(summary) + out = [] + for path, group in by_source.items(): + first = group[0] + patches = [] + for num, summary in enumerate(group): + record = patch_record(summary) + if record.source_patch_id == "" and len(group) > 1: + # positional identity within the source, per design doc + record = PatchRecord(**{**record.__dict__, "source_patch_id": str(num)}) + patches.append(record) + store_path = path + if base_uri and path.startswith(base_uri): + store_path = path[len(base_uri) :].lstrip("/") + out.append( + SourceRecord( + source_path=store_path, + base_uri=base_uri, + source_format=first.source_format, + format_version=first.source_version, + mtime_ns=(mtimes_ns or {}).get(path), + size_bytes=(sizes_bytes or {}).get(path), + patches=tuple(patches), + ) + ) + return out diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py new file mode 100644 index 000000000..0250c162e --- /dev/null +++ b/dascore/io/index/lite.py @@ -0,0 +1,50 @@ +"""SQLite index backend (stdlib sqlite3, STRICT tables).""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pandas as pd + +from dascore.io.index.backend import SQLIndexBackend, adapt_params +from dascore.io.index.dialect import SQLiteDialect + + +def _adapt(params): + """Convert numpy/py types sqlite3 can't bind natively.""" + return [int(p) if isinstance(p, bool) else p for p in adapt_params(params)] + + +class SQLiteBackend(SQLIndexBackend): + """Index backend storing tables in a single SQLite file.""" + + dialect = SQLiteDialect() + + def __init__(self, path: str | Path): + self._con = sqlite3.connect(str(path)) + # autocommit off; we manage transactions explicitly. + self._con.isolation_level = None + super().__init__() + + def _execute(self, sql: str, params=()) -> None: + self._con.execute(sql, _adapt(params)) + + def _executemany(self, sql: str, seq_of_params) -> None: + self._con.executemany(sql, [_adapt(p) for p in seq_of_params]) + + def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: + return pd.read_sql_query(sql, self._con, params=_adapt(params)) + + def _begin(self) -> None: + self._con.execute("BEGIN") + + def _commit(self) -> None: + self._con.execute("COMMIT") + + def _rollback(self) -> None: + self._con.execute("ROLLBACK") + + def close(self) -> None: + """Close the database connection.""" + self._con.close() diff --git a/dascore/io/index/parq.py b/dascore/io/index/parq.py new file mode 100644 index 000000000..e26e03b19 --- /dev/null +++ b/dascore/io/index/parq.py @@ -0,0 +1,115 @@ +""" +Parquet-manifest index backend. + +Tables live as immutable parquet files plus a small manifest; readers +never need locks (a half-written update is invisible until the manifest +swap). This prototype materializes the tables in an in-memory DuckDB for +querying/mutation and dumps changed tables to new parquet files on +commit, replacing the manifest atomically. +""" + +from __future__ import annotations + +import json +import os +import uuid +from pathlib import Path + +import pandas as pd + +from dascore.io.index.backend import SQLIndexBackend, adapt_params +from dascore.io.index.dialect import DuckDBDialect +from dascore.io.index.schema import TABLES + +_MANIFEST = "manifest.json" + + +class ParquetBackend(SQLIndexBackend): + """Index backend storing tables as parquet files + manifest.""" + + dialect = DuckDBDialect() + + def __init__(self, path: str | Path): + import duckdb + + self._dir = Path(path) + self._dir.mkdir(parents=True, exist_ok=True) + self._con = duckdb.connect(":memory:") + self._manifest = self._read_manifest() + for table, filename in self._manifest.get("tables", {}).items(): + file_path = str(self._dir / filename).replace("'", "''") + self._con.execute( + f"CREATE TABLE {self.dialect.quote(table)} AS " + f"SELECT * FROM read_parquet('{file_path}')" + ) + super().__init__() + + def _read_manifest(self) -> dict: + manifest_path = self._dir / _MANIFEST + if manifest_path.exists(): + with manifest_path.open() as fi: + return json.load(fi) + return {"tables": {}} + + # --- SQL hooks (all against the in-memory duckdb) ----------------- + + def _execute(self, sql: str, params=()) -> None: + self._con.execute(sql, adapt_params(params)) + + def _executemany(self, sql: str, seq_of_params) -> None: + rows = [adapt_params(p) for p in seq_of_params] + if rows: + self._con.executemany(sql, rows) + + def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: + return self._con.execute(sql, adapt_params(params)).df() + + def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: + from dascore.io.index.duck import duck_bulk_insert + + duck_bulk_insert(self._con, self.dialect, table, columns, rows) + + def _begin(self) -> None: + self._con.execute("BEGIN TRANSACTION") + + def _commit(self) -> None: + self._con.execute("COMMIT") + self._persist() + + def _rollback(self) -> None: + self._con.execute("ROLLBACK") + + # --- persistence --------------------------------------------------- + + def _persist(self) -> None: + """Write all tables to new parquet files and swap the manifest.""" + new_tables = {} + for table in TABLES: + filename = f"{table}-{uuid.uuid4().hex[:12]}.parquet" + target = str(self._dir / filename).replace("'", "''") + self._con.execute( + f"COPY {self.dialect.quote(table)} TO '{target}' (FORMAT PARQUET)" + ) + new_tables[table] = filename + old = self._manifest.get("tables", {}) + self._manifest = {"tables": new_tables} + tmp = self._dir / (_MANIFEST + ".tmp") + with tmp.open("w") as fi: + json.dump(self._manifest, fi) + os.replace(tmp, self._dir / _MANIFEST) + # best-effort cleanup of superseded files (readers using the old + # manifest may still hold them open; deletion failing is fine). + for filename in old.values(): + try: + (self._dir / filename).unlink(missing_ok=True) + except OSError: + pass + + def _ensure_schema(self) -> None: + super()._ensure_schema() + if not (self._dir / _MANIFEST).exists(): + self._persist() + + def close(self) -> None: + """Close the in-memory database.""" + self._con.close() diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py new file mode 100644 index 000000000..5fe1ab09d --- /dev/null +++ b/dascore/io/index/query.py @@ -0,0 +1,300 @@ +""" +Query model and SQL generation for the spool index. + +Implements the selector semantics spec (see +`.scratch/selector_semantics_spec.md`): the index only produces +candidates — predicates the summary cannot evaluate exactly are the +caller's responsibility at patch-load time. SQL generation is shared by +all backends; anything a dialect cannot push down is applied as a pandas +residual filter with identical semantics. +""" + +from __future__ import annotations + +import fnmatch +import re +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd + +from dascore.exceptions import ParameterError +from dascore.io.index.dialect import BaseDialect +from dascore.io.index.ingest import typed_value + +_GLOB_CHARS = frozenset("*?[") + + +class InvalidSpoolQueryError(ParameterError): + """Raised when a spool query references unknown names or bad values.""" + + +@dataclass(frozen=True) +class Query: + """ + A resolved spool query. + + Name resolution (bare kwargs -> attrs first, then coords) happens + above this layer; a Query already knows which namespace each + predicate belongs to. + """ + + attrs: dict = field(default_factory=dict) + coords: dict = field(default_factory=dict) + + +def _is_collection(value) -> bool: + """True for non-string collections (membership predicates).""" + if isinstance(value, str | bytes): + return False + if isinstance(value, np.ndarray): + return True + return isinstance(value, list | tuple | set | frozenset) + + +def _is_range(value) -> bool: + """True for a 2-tuple range (possibly with open bounds).""" + return isinstance(value, tuple) and len(value) == 2 + + +def _coerce_scalar(value, target_kinds: set[str]): + """ + Coerce a query scalar to (kind, storable value). + + Follows the coercion table in the selector spec; datetime-like + strings become time queries only when the target has a time kind. + """ + typed = typed_value(value) + if typed is None: + msg = f"Cannot use {value!r} as a spool query value." + raise InvalidSpoolQueryError(msg) + if typed.kind == "str" and "time" in target_kinds: + try: + retyped = typed_value(np.datetime64(pd.Timestamp(value), "ns")) + return retyped.kind, retyped.value + except (ValueError, TypeError): + pass + return typed.kind, typed.value + + +def _range_bounds(value, target_kinds: set[str]): + """Return (kind, lo, hi) from a range tuple, handling open bounds.""" + lo_raw, hi_raw = value + lo = hi = None + kind = None + for raw, name in ((lo_raw, "lo"), (hi_raw, "hi")): + if raw is None or raw is Ellipsis: + continue + knd, val = _coerce_scalar(raw, target_kinds) + if kind is not None and knd != kind: + msg = f"Range bounds {value!r} have mixed kinds ({kind}, {knd})." + raise InvalidSpoolQueryError(msg) + kind = knd + if name == "lo": + lo = val + else: + hi = val + if kind is None: + msg = f"Range {value!r} has no usable bounds." + raise InvalidSpoolQueryError(msg) + if lo is not None and hi is not None and lo > hi: + msg = f"Range {value!r} has lo > hi after coercion." + raise InvalidSpoolQueryError(msg) + return kind, lo, hi + + +@dataclass +class _Where: + """Accumulates WHERE clauses and parameters.""" + + clauses: list[str] = field(default_factory=list) + params: list = field(default_factory=list) + + def add(self, clause: str, *params): + self.clauses.append(clause) + self.params.extend(params) + + @property + def sql(self) -> str: + return " AND ".join(self.clauses) if self.clauses else "TRUE" + + +def build_attr_clause( + where: _Where, + dialect: BaseDialect, + attr_meta: pd.DataFrame, + name: str, + value, +) -> re.Pattern | None: + """ + Add SQL for one attr predicate; return a residual pattern if the + predicate must be re-applied in pandas (regex). + """ + rows = attr_meta[attr_meta["attr_name"] == name] + if rows.empty: + msg = f"{name!r} is not an attribute of any patch in this spool." + raise InvalidSpoolQueryError(msg) + kinds = set(rows["value_kind"]) + columns = dict(zip(rows["value_kind"], rows["column_name"])) + + def col(kind): + return f"a.{dialect.quote(columns[kind])}" + + if isinstance(value, re.Pattern): + # Regex is a residual filter; SQL only requires the attr be + # present (str kind) so candidates are a superset. + if "str" not in kinds: + where.add("FALSE") + return None + where.add(f"{col('str')} IS NOT NULL") + return value + if _is_range(value): + kind, lo, hi = _range_bounds(value, kinds) + if kind not in kinds: + where.add("FALSE") + return None + if lo is not None: + where.add(f"{col(kind)} >= ?", lo) + if hi is not None: + where.add(f"{col(kind)} <= ?", hi) + return None + if _is_collection(value): + coerced = [_coerce_scalar(v, kinds) for v in value] + by_kind: dict[str, list] = {} + for kind, val in coerced: + by_kind.setdefault(kind, []).append(val) + subclauses = [] + params = [] + for kind, vals in by_kind.items(): + if kind not in kinds: + continue + marks = ", ".join("?" for _ in vals) + subclauses.append(f"{col(kind)} IN ({marks})") + params.extend(vals) + if not subclauses: + where.add("FALSE") + else: + where.add("(" + " OR ".join(subclauses) + ")", *params) + return None + if isinstance(value, str) and _GLOB_CHARS & set(value): + if "str" not in kinds: + where.add("FALSE") + return None + where.add(dialect.glob(col("str")), value) + return None + kind, val = _coerce_scalar(value, kinds) + if kind not in kinds: + where.add("FALSE") + return None + where.add(f"{col(kind)} = ?", val) + return None + + +def build_coord_clause( + where: _Where, + dialect: BaseDialect, + name: str, + value, +) -> None: + """ + Add an EXISTS clause on the coords table for one coord predicate. + + Candidacy only: envelope overlap, never false negatives. Exact + membership/boolean masks are applied at patch load, above this layer. + """ + if _is_range(value): + kind, lo, hi = _range_bounds(value, {"time", "num", "str"}) + elif _is_collection(value): + arr = np.asarray(list(value) if isinstance(value, set) else value) + if arr.dtype == bool: + # boolean masks are patch-local; no index predicate at all, + # but the coord must exist on the patch. + kind = lo = hi = None + else: + kind, lo = _coerce_scalar(arr.min(), {"time", "num", "str"}) + _, hi = _coerce_scalar(arr.max(), {"time", "num", "str"}) + else: + kind, val = _coerce_scalar(value, {"time", "num", "str"}) + lo = hi = val + + min_col, max_col = { + "time": ("min_ns", "max_ns"), + "dur": ("min_ns", "max_ns"), + "num": ("min_num", "max_num"), + "str": ("min_str", "max_str"), + None: (None, None), + }[kind] + conditions = ["c.patch_id = p.patch_id", "c.coord_name = ?"] + params: list = [name] + if kind is not None: + if kind in ("time", "dur"): + # absolute queries match absolute coords, durations relative. + conditions.append("c.is_relative = ?") + params.append(kind == "dur") + kind_match = "time" + else: + kind_match = kind + conditions.append("c.value_kind = ?") + params.append(kind_match) + if lo is not None: + conditions.append(f"c.{max_col} >= ?") + params.append(lo) + if hi is not None: + conditions.append(f"c.{min_col} <= ?") + params.append(hi) + where.add( + "EXISTS (SELECT 1 FROM coords c WHERE " + " AND ".join(conditions) + ")", + *params, + ) + + +def build_query_sql( + query: Query, + dialect: BaseDialect, + attr_meta: pd.DataFrame, +) -> tuple[str, list, dict[str, re.Pattern]]: + """ + Build the flat-relation SELECT for a query. + + Returns (sql, params, residuals) where residuals maps attr names to + regex patterns that must be re-applied to the resulting dataframe. + """ + where = _Where() + residuals: dict[str, re.Pattern] = {} + for name, value in query.attrs.items(): + residual = build_attr_clause(where, dialect, attr_meta, name, value) + if residual is not None: + residuals[name] = residual + for name, value in query.coords.items(): + build_coord_clause(where, dialect, name, value) + # attr columns selected explicitly: `a.*` would duplicate patch_id and + # engines disagree on how to dedupe result column names. + attr_cols = "".join( + f", a.{dialect.quote(col)}" for col in attr_meta["column_name"].unique() + ) + sql = ( + "SELECT s.source_path, s.base_uri, s.source_format, s.format_version, " + f"p.*{attr_cols} " + "FROM patches p " + "JOIN sources s ON s.source_id = p.source_id " + "LEFT JOIN attrs a ON a.patch_id = p.patch_id " + f"WHERE {where.sql} " + "ORDER BY p.time_min NULLS LAST, p.patch_id" + ) + return sql, where.params, residuals + + +def apply_residuals(df: pd.DataFrame, residuals: dict[str, re.Pattern]) -> pd.DataFrame: + """Apply regex residual filters to the flat relation.""" + for name, pattern in residuals.items(): + col = df[name] + keep = col.map( + lambda x: bool(pattern.search(x)) if isinstance(x, str) else False + ) + df = df[keep] + return df + + +def glob_match(value, pattern: str) -> bool: + """Reference glob semantics (used by pandas fallbacks and tests).""" + return isinstance(value, str) and fnmatch.fnmatch(value, pattern) diff --git a/dascore/io/index/schema.py b/dascore/io/index/schema.py new file mode 100644 index 000000000..77674b673 --- /dev/null +++ b/dascore/io/index/schema.py @@ -0,0 +1,133 @@ +""" +Logical schema for the spool index. + +The schema is defined in backend-neutral terms; only four primitive +storage types are used (int64, float64, str, bool) so any SQL-ish backend +can represent it. Times and durations are always epoch/plain nanoseconds +stored as int64 — never engine-native timestamp types. +""" + +from __future__ import annotations + +from types import MappingProxyType + +# Version of the index schema, independent of dascore's version. +INDEX_VERSION = 1 +# Identity string so any tool can sanity-check what it opened. +WHAT_IS_THIS = "dascore_spool_index" + +# Value kinds for typed attr columns and coord rows. +KINDS = ("num", "str", "bool", "time", "dur") +# Storage type (logical) backing each kind. +KIND_STORAGE = MappingProxyType( + { + "num": "float64", + "str": "str", + "bool": "bool", + "time": "int64", # epoch ns + "dur": "int64", # ns + } +) + +META_DATA = MappingProxyType( + { + "what_is_this": "str", + "index_version": "int64", + "dascore_version": "str", + "last_indexed_ns": "int64", + } +) + +SOURCES = MappingProxyType( + { + "source_id": "int64", + "base_uri": "str", + "source_path": "str", + "source_format": "str", + "format_version": "str", + "mtime_ns": "int64", + "size_bytes": "int64", + "last_indexed_ns": "int64", + } +) + +# Frozen structural table; nothing dynamic is ever added here. The +# time/distance envelopes are cached summaries of the two conventional +# dims (hot path), not attr promotion. +PATCHES = MappingProxyType( + { + "patch_id": "int64", + "source_id": "int64", + "source_patch_id": "str", + "n_dims": "int64", + "dims": "str", + "shape": "str", + "sample_count_total": "int64", + "time_min": "int64", # epoch ns; NULL for relative-time patches + "time_max": "int64", + "time_step": "int64", + "distance_min": "float64", # canonical SI (m) + "distance_max": "float64", + "distance_step": "float64", + } +) + +# attrs table starts with only the key; typed columns (`__`) +# are added lazily at ingest. +ATTRS_BASE = MappingProxyType({"patch_id": "int64"}) + +ATTR_META = MappingProxyType( + { + "attr_name": "str", # original (unsanitized) attr name + "value_kind": "str", + "column_name": "str", # sanitized column in the attrs table + "units": "str", # canonical unit for num kinds, nullable + } +) + +COORDS = MappingProxyType( + { + "patch_id": "int64", + "coord_name": "str", + "value_kind": "str", # num | time | str + "dtype": "str", + "coord_dims": "str", + "length": "int64", + "units": "str", # original unit string; numeric values stored SI + "min_num": "float64", + "max_num": "float64", + "step_num": "float64", + "min_ns": "int64", + "max_ns": "int64", + "step_ns": "int64", + "min_str": "str", + "max_str": "str", + "is_monotonic": "bool", + "is_relative": "bool", + "coord_hash": "str", + } +) + +TABLES = MappingProxyType( + { + "meta_data": META_DATA, + "sources": SOURCES, + "patches": PATCHES, + "attrs": ATTRS_BASE, + "attr_meta": ATTR_META, + "coords": COORDS, + } +) + +# Names which can never be dynamic attr columns. +RESERVED_ATTR_COLUMNS = frozenset({"patch_id"}) + +# Secondary indexes: without these, engines that use nested-loop plans +# (SQLite) go quadratic on the correlated coords EXISTS subquery. +INDEXES = ( + ("idx_coords_patch", "coords", "patch_id"), + ("idx_coords_name", "coords", "coord_name"), + ("idx_attrs_patch", "attrs", "patch_id"), + ("idx_patches_source", "patches", "source_id"), + ("idx_sources_path", "sources", "source_path"), +) diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py new file mode 100644 index 000000000..976a06e6a --- /dev/null +++ b/tests/test_io/test_index/test_index_contract.py @@ -0,0 +1,427 @@ +""" +Contract tests for spool index backends. + +Every backend must pass this suite unchanged; it encodes the selector +semantics spec and the summary-only/no-false-negatives contract from the +index design doc (see discussion #648). +""" + +from __future__ import annotations + +import re + +import numpy as np +import pandas as pd +import pytest + +from dascore.core.summary import PatchSummary +from dascore.io.index import Query, get_backend, summaries_to_records +from dascore.io.index.backend import resolve_query +from dascore.io.index.query import InvalidSpoolQueryError + +BACKENDS = ("duckdb", "sqlite", "parquet") + + +def _time_coord(t0: str, seconds: float, step_s: float = 0.004): + """Make an absolute time coord summary dict.""" + start = np.datetime64(t0, "ns") + return { + "dtype": "datetime64", + "min": start, + "max": start + np.timedelta64(int(seconds * 1e9), "ns"), + "step": np.timedelta64(int(step_s * 1e9), "ns"), + "units": "s", + "dims": ("time",), + "len": int(seconds / step_s), + } + + +def _distance_coord(d0: float, d1: float, step: float, units="m"): + """Make a numeric distance coord summary dict.""" + return { + "dtype": "float64", + "min": d0, + "max": d1, + "step": step, + "units": units, + "dims": ("distance",), + "len": int((d1 - d0) / step) + 1, + } + + +def make_summaries() -> list[PatchSummary]: + """A deliberately heterogeneous set of patch summaries.""" + das1 = PatchSummary( + attrs={ + "station": "STA1", + "network": "NW", + "tag": "raw", + "data_type": "strain_rate", + "gauge_length": 10, + }, + coords={ + "time": _time_coord("2024-01-01T00:00:00", 60), + "distance": _distance_coord(0, 1000, 1), + }, + dims=("time", "distance"), + shape=(15000, 1001), + dtype="float32", + source_path="das/file_1.h5", + source_format="PRODML", + source_version="2.1", + ) + das2 = PatchSummary( + attrs={ + "station": "STA2", + "network": "NW", + "tag": "raw", + "data_type": "strain_rate", + "gauge_length": 10.0, + }, + coords={ + "time": _time_coord("2024-01-01T00:01:00", 60), + "distance": _distance_coord(0, 1000, 1), + }, + dims=("time", "distance"), + shape=(15000, 1001), + dtype="float32", + source_path="das/file_2.h5", + source_format="PRODML", + source_version="2.1", + ) + # correlogram: relative (timedelta) lag_time coord, shot_number attr + correlogram = PatchSummary( + attrs={"tag": "corr", "shot_number": 42, "data_type": ""}, + coords={ + "lag_time": { + "dtype": "timedelta64", + "min": np.timedelta64(-5_000_000_000, "ns"), + "max": np.timedelta64(5_000_000_000, "ns"), + "step": np.timedelta64(10_000_000, "ns"), + "dims": ("lag_time",), + "len": 1001, + }, + "distance": _distance_coord(0, 500, 5), + }, + dims=("lag_time", "distance"), + shape=(1001, 101), + dtype="float64", + source_path="products/corr_1.h5", + source_format="DASDAE", + source_version="1", + ) + # PSD-like product with distance in feet (tests SI normalization) + psd = PatchSummary( + attrs={"tag": "psd", "shot_number": "unknown"}, + coords={ + "frequency": { + "dtype": "float64", + "min": 0.0, + "max": 500.0, + "step": 0.5, + "units": "Hz", + "dims": ("frequency",), + "len": 1001, + }, + "distance": _distance_coord(0.0, 3280.0, 3.28, units="ft"), + }, + dims=("frequency", "distance"), + shape=(1001, 1001), + dtype="float64", + source_path="products/psd_1.h5", + source_format="DASDAE", + source_version="1", + ) + return [das1, das2, correlogram, psd] + + +@pytest.fixture(scope="function", params=BACKENDS) +def backend(request, tmp_path): + """An index backend of each kind, freshly ingested.""" + path = tmp_path / f"index_{request.param}" + back = get_backend(path, kind=request.param) + back.write_sources(summaries_to_records(make_summaries())) + yield back + back.close() + + +class TestFlatRelation: + """The flat patch-row relation contract.""" + + def test_row_per_patch(self, backend): + """Row per patch.""" + df = backend.query() + assert len(df) == 4 + + def test_structural_columns(self, backend): + """Structural columns.""" + df = backend.query() + for col in ("path", "file_format", "file_version", "dims", "shape"): + assert col in df.columns + assert pd.api.types.is_datetime64_dtype(df["time_min"]) + assert pd.api.types.is_timedelta64_dtype(df["time_step"]) + + def test_attr_columns_use_original_names(self, backend): + """Attr columns use original names.""" + df = backend.query() + assert "station" in df.columns + assert "gauge_length" in df.columns + assert set(df["station"].replace("", None).dropna()) == {"STA1", "STA2"} + + def test_missing_str_attrs_are_empty_string(self, backend): + """Missing str attrs are empty string.""" + df = backend.query() + corr = df[df["tag"] == "corr"] + assert (corr["station"] == "").all() + + def test_relative_time_patches_have_null_time_min(self, backend): + """Relative time patches have null time min.""" + df = backend.query() + corr = df[df["tag"] == "corr"] + assert corr["time_min"].isnull().all() + + def test_ordering_deterministic(self, backend): + """Ordering deterministic.""" + df1, df2 = backend.query(), backend.query() + pd.testing.assert_frame_equal(df1, df2) + # NULLS LAST: relative-time patches sort after absolute ones. + nulls = df1["time_min"].isnull().to_numpy() + assert not nulls[: (~nulls).sum()].any() + + +class TestAttrPredicates: + """Attr predicates are exact at the index.""" + + def test_equality(self, backend): + """Equality.""" + df = backend.query(Query(attrs={"station": "STA1"})) + assert len(df) == 1 + assert df["station"].iloc[0] == "STA1" + + def test_glob(self, backend): + """Glob.""" + df = backend.query(Query(attrs={"station": "STA*"})) + assert len(df) == 2 + + def test_regex(self, backend): + """Regex.""" + df = backend.query(Query(attrs={"station": re.compile(r"STA\d")})) + assert len(df) == 2 + + def test_membership(self, backend): + """Membership.""" + df = backend.query(Query(attrs={"station": ["STA1", "STA2", "NOPE"]})) + assert len(df) == 2 + + def test_int_matches_float_storage(self, backend): + """Int matches float storage.""" + # gauge_length stored from int 10 and float 10.0; int query hits both + df = backend.query(Query(attrs={"gauge_length": 10})) + assert len(df) == 2 + + def test_range(self, backend): + """Range.""" + df = backend.query(Query(attrs={"gauge_length": (5, 15)})) + assert len(df) == 2 + + def test_open_range(self, backend): + """Open range.""" + df = backend.query(Query(attrs={"gauge_length": (5, None)})) + assert len(df) == 2 + + def test_kind_mismatch_matches_nothing(self, backend): + """Kind mismatch matches nothing.""" + # station is a str attr; numeric query is valid but matches nothing + df = backend.query(Query(attrs={"station": 5})) + assert df.empty + + def test_mixed_kind_attr(self, backend): + """Mixed kind attr.""" + # shot_number exists as num (42) and str ("unknown") + num = backend.query(Query(attrs={"shot_number": 42})) + assert list(num["tag"]) == ["corr"] + txt = backend.query(Query(attrs={"shot_number": "unknown"})) + assert list(txt["tag"]) == ["psd"] + + +class TestCoordPredicates: + """Coord predicates: envelope candidacy, never false negatives.""" + + def test_time_range(self, backend): + """Time range.""" + t = (np.datetime64("2024-01-01T00:00:30"), np.datetime64("2024-01-01T00:00:40")) + df = backend.query(Query(coords={"time": t})) + assert list(df["station"]) == ["STA1"] + + def test_time_range_overlap_both(self, backend): + """Time range overlap both.""" + t = (np.datetime64("2024-01-01T00:00:30"), np.datetime64("2024-01-01T00:01:30")) + df = backend.query(Query(coords={"time": t})) + assert set(df["station"]) == {"STA1", "STA2"} + + def test_absolute_time_excludes_relative(self, backend): + """Absolute time excludes relative.""" + t = (np.datetime64("1990-01-01"), np.datetime64("2100-01-01")) + df = backend.query(Query(coords={"time": t})) + assert "corr" not in set(df["tag"]) + + def test_relative_time_coord(self, backend): + """Relative time coord.""" + lag = (np.timedelta64(0, "s"), np.timedelta64(2, "s")) + df = backend.query(Query(coords={"lag_time": lag})) + assert list(df["tag"]) == ["corr"] + + def test_numeric_coord_si_normalized(self, backend): + """Numeric coord si normalized.""" + # psd distance is 0-3280 ft = 0-999.7 m; a 900-950 m query hits it + df = backend.query(Query(coords={"distance": (900, 950)})) + assert "psd" in set(df["tag"]) + + def test_scalar_coord(self, backend): + """Scalar coord.""" + df = backend.query(Query(coords={"frequency": 100})) + assert list(df["tag"]) == ["psd"] + + def test_array_membership_envelope(self, backend): + """Array membership envelope.""" + values = np.array([10.0, 20.0, 480.0]) + df = backend.query(Query(coords={"distance": values})) + assert len(df) == 4 # candidacy: all patches overlap the envelope + + def test_coord_missing_excludes_patch(self, backend): + """Coord missing excludes patch.""" + df = backend.query(Query(coords={"frequency": (0, 1000)})) + assert list(df["tag"]) == ["psd"] + + +class TestNameResolution: + """Bare kwargs resolve attrs first, then coords; unknown raises.""" + + def test_attr_wins(self, backend): + """Attr wins.""" + query = resolve_query(backend, station="STA1") + assert "station" in query.attrs + + def test_coord_fallback(self, backend): + """Coord fallback.""" + query = resolve_query(backend, lag_time=(0, 1)) + assert "lag_time" in query.coords + + def test_unknown_raises(self, backend): + """Unknown raises.""" + with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): + resolve_query(backend, wavelength=(1, 2)) + + def test_double_specification_raises(self, backend): + """Double specification raises.""" + with pytest.raises(InvalidSpoolQueryError, match="both"): + resolve_query(backend, station="STA1", _attrs={"station": "STA2"}) + + def test_explicit_namespaces(self, backend): + """Explicit namespaces.""" + query = resolve_query( + backend, _attrs={"tag": "raw"}, _coords={"distance": (0, 10)} + ) + assert query.attrs == {"tag": "raw"} and "distance" in query.coords + + +class TestNoFalseNegatives: + """Property: reference-matching patches always appear in results.""" + + def test_random_time_ranges(self, backend): + """Random time ranges.""" + rng = np.random.default_rng(42) + summaries = make_summaries() + base = np.datetime64("2024-01-01T00:00:00").astype("datetime64[ns]") + for _ in range(25): + lo = base + np.timedelta64(int(rng.integers(-60, 180)), "s") + hi = lo + np.timedelta64(int(rng.integers(1, 120)), "s") + result_paths = set(backend.query(Query(coords={"time": (lo, hi)}))["path"]) + for summary in summaries: + tcoord = summary.coords.get("time") + if tcoord is None or "datetime" not in str(tcoord.dtype): + continue + overlaps = tcoord.min <= hi and tcoord.max >= lo + if overlaps: + assert str(summary.source_path) in result_paths + + def test_random_numeric_ranges(self, backend): + """Random numeric ranges.""" + rng = np.random.default_rng(7) + factor = {"m": 1.0, "ft": 0.3048} + summaries = make_summaries() + for _ in range(25): + lo = float(rng.uniform(-100, 1000)) + hi = lo + float(rng.uniform(1, 500)) + result_paths = set( + backend.query(Query(coords={"distance": (lo, hi)}))["path"] + ) + for summary in summaries: + dcoord = summary.coords.get("distance") + if dcoord is None: + continue + scale = factor[str(dcoord.units.units)] if dcoord.units else 1.0 + if dcoord.min * scale <= hi and dcoord.max * scale >= lo: + assert str(summary.source_path) in result_paths + + +class TestSourceLifecycle: + """Source-scoped transactional replacement and deletion.""" + + def test_replace_source_drops_stale_rows(self, backend): + """Replace source drops stale rows.""" + summaries = [s for s in make_summaries() if "file_1" in str(s.source_path)] + structured = summaries[0].dump_structured() + structured["attrs"] = {"station": "NEW1"} + modified = PatchSummary(**structured) + backend.write_sources(summaries_to_records([modified])) + df = backend.query() + assert len(df) == 4 # still one row for that source + assert "STA1" not in set(df["station"]) + assert "NEW1" in set(df["station"]) + + def test_delete_cascades(self, backend): + """Delete cascades.""" + backend.delete_sources(["das/file_1.h5"]) + df = backend.query() + assert len(df) == 3 + assert "das/file_1.h5" not in set(df["path"]) + + def test_reopen_persists(self, backend, tmp_path): + """Reopen persists.""" + kind = type(backend).__name__.replace("Backend", "").lower() + path_map = { + "duckdb": tmp_path / "index_duckdb", + "sqlite": tmp_path / "index_sqlite", + "parquet": tmp_path / "index_parquet", + } + backend.close() + reopened = get_backend(path_map[kind], kind=kind) + try: + assert len(reopened.query()) == 4 + finally: + reopened.close() + # reopen once more so fixture teardown close() has a live handle + reopened_again = get_backend(path_map[kind], kind=kind) + backend.__dict__.update(reopened_again.__dict__) + + +class TestMetadata: + """Index metadata and introspection.""" + + def test_metadata(self, backend): + """Metadata.""" + meta = backend.get_metadata() + assert meta["what_is_this"] == "dascore_spool_index" + assert meta["index_version"] == 1 + + def test_names(self, backend): + """Names.""" + assert {"station", "tag", "shot_number"} <= backend.attr_names() + assert {"time", "distance", "lag_time", "frequency"} <= backend.coord_names() + + def test_sources(self, backend): + """Sources.""" + sources = backend.get_sources() + assert len(sources) == 4 + assert set(sources["source_format"]) == {"PRODML", "DASDAE"} From 54627d5836f8385fbe2fda960b935d70bcef872f Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 11:35:33 +0200 Subject: [PATCH 02/97] Wire database index into DirectorySpool - DBDirectoryIndexer: directory walk via _iter_filesystem, per-source (mtime_ns, size_bytes) change detection instead of a global watermark, stale-source removal folded into update(), scan of changed files only. - DirectorySpool gains an index_engine parameter selecting the duckdb, sqlite, or parquet backend; default behavior unchanged (HDF5 index). - Derived spools share the indexer connection (deepcopy-safe), matching the single-writer model. - Integration tests: patch loading, time select, chunk merge, and add/modify/delete lifecycle against real files for all backends. --- dascore/clients/dirspool.py | 14 +- dascore/io/index/indexer.py | 135 +++++++++++++++++++ dascore/io/index/ingest.py | 13 +- tests/test_io/test_index/test_db_dirspool.py | 110 +++++++++++++++ 4 files changed, 268 insertions(+), 4 deletions(-) create mode 100644 dascore/io/index/indexer.py create mode 100644 tests/test_io/test_index/test_db_dirspool.py diff --git a/dascore/clients/dirspool.py b/dascore/clients/dirspool.py index 4397ddb57..5aa0db23f 100644 --- a/dascore/clients/dirspool.py +++ b/dascore/clients/dirspool.py @@ -42,6 +42,10 @@ class DirectorySpool(DataFrameSpool): will save time in indexing. select_kwargs Dict of keyword arguments to restrict output contents. + index_engine + If set, use the database index backend of this kind ("duckdb", + "sqlite", or "parquet") instead of the HDF5 index. Experimental; + see the spool index design discussion (#648). """ _drop_columns = ("file_format", "file_version", "path", "source_patch_id") @@ -54,6 +58,7 @@ def __init__( preferred_format: str | None = None, select_kwargs: dict | None = None, merge_kwargs: dict | None = None, + index_engine: str | None = None, ): super().__init__(select_kwargs=select_kwargs, merge_kwargs=merge_kwargs) # Init file spool from another file spool @@ -64,7 +69,14 @@ def __init__( elif isinstance(base_path, AbstractIndexer): self.indexer = base_path elif isinstance(base_path, Path | str | UPath): - self.indexer = DirectoryIndexer(base_path, index_path=index_path) + if index_engine is not None: + from dascore.io.index.indexer import DBDirectoryIndexer + + self.indexer = DBDirectoryIndexer( + base_path, engine=index_engine, index_path=index_path + ) + else: + self.indexer = DirectoryIndexer(base_path, index_path=index_path) assert hasattr(self, "indexer"), "indexer not set." self._preferred_format = preferred_format diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py new file mode 100644 index 000000000..c4db40668 --- /dev/null +++ b/dascore/io/index/indexer.py @@ -0,0 +1,135 @@ +""" +A directory indexer backed by the generic spool index. + +Drop-in alternative to `dascore.io.indexer.DirectoryIndexer`: it walks a +directory, detects new/changed/removed sources by per-source +(mtime, size) comparison, scans only what changed, and answers content +queries from the index backend. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +from typing_extensions import Self + +import dascore as dc +from dascore.constants import PROGRESS_LEVELS +from dascore.io.index.backend import get_backend, resolve_query +from dascore.io.index.ingest import summaries_to_records +from dascore.io.indexer import AbstractIndexer +from dascore.utils.misc import _iter_filesystem + +# Structural columns the spool machinery must not see: unique-per-patch +# values block chunk merge-compatibility grouping, which compares all +# non-private columns. +_SPOOL_HIDDEN_COLUMNS = ("n_dims", "sample_count_total", "shape") + + +class DBDirectoryIndexer(AbstractIndexer): + """ + Index a directory of fiber files with a database backend. + + Parameters + ---------- + path + The directory to index. + engine + The backend kind: "duckdb", "sqlite", or "parquet". + index_path + Where to keep the index; defaults to a hidden entry at the top of + the data directory. + """ + + ext: str | None = None + + def __init__( + self, + path: str | Path, + engine: str = "duckdb", + index_path: str | Path | None = None, + ): + self.path = Path(path).absolute() + self.engine = engine + if index_path is None: + index_path = self.path / f".dascore_index_{engine}" + self.index_path = Path(index_path) + self._backend = get_backend(self.index_path, kind=engine) + + def __str__(self) -> str: + return f"{self.__class__.__name__} ({self.engine}) managing: {self.path}" + + __repr__ = __str__ + + def __deepcopy__(self, memo) -> Self: + """ + Derived spools share the indexer (and its live DB connection). + + DataFrameSpool copies spool state on select/chunk; the index + connection is read-shared, matching the single-writer model. + """ + return self + + def _current_files(self) -> dict[str, tuple[int, int, Path]]: + """Map relative posix path -> (mtime_ns, size, absolute path).""" + out = {} + for file_path in _iter_filesystem(self.path, ext=self.ext): + stat = file_path.stat() + rel = file_path.relative_to(self.path).as_posix() + out[rel] = (stat.st_mtime_ns, stat.st_size, file_path) + return out + + def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: + """ + Update the index: scan new/changed sources, drop removed ones. + + Change detection compares each source's stored (mtime_ns, + size_bytes) against the filesystem — never a global watermark — + and stale-source removal is folded in (the walk is the dominant + cost; removal afterwards is nearly free). + """ + current = self._current_files() + stored = { + row.source_path: (row.mtime_ns, row.size_bytes) + for row in self._backend.get_sources().itertuples() + } + stale = [path for path in stored if path not in current] + changed = [ + rel + for rel, (mtime, size, _) in current.items() + if stored.get(rel) != (mtime, size) + ] + if stale: + self._backend.delete_sources(stale) + if changed: + abs_paths = [current[rel][2] for rel in changed] + summaries = dc.scan(abs_paths, progress=progress) + # scan reports absolute source paths; stat maps use them too + records = summaries_to_records( + summaries, + relative_to=str(self.path), + mtimes_ns={str(current[r][2]): current[r][0] for r in changed}, + sizes_bytes={str(current[r][2]): current[r][1] for r in changed}, + ) + if records: + self._backend.write_sources(records) + return self + + def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: + """ + Query the index, returning the spool-facing flat relation. + + Bare kwargs resolve attrs-first then coords; `_attrs`/`_coords` + disambiguate explicitly (see the selector semantics spec). + """ + query = resolve_query(self._backend, _attrs=_attrs, _coords=_coords, **kwargs) + df = self._backend.query(query) + df = df.drop(columns=list(_SPOOL_HIDDEN_COLUMNS), errors="ignore") + return df.rename(columns={"patch_id": "_patch_id"}) + + __call__ = get_contents + + def close(self) -> None: + """Close the backend.""" + self._backend.close() diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 54d32ee9a..20d3d36b5 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -272,6 +272,7 @@ def patch_record(summary: PatchSummary) -> PatchRecord: def summaries_to_records( summaries: list[PatchSummary], base_uri: str | None = None, + relative_to: str | None = None, mtimes_ns: dict[str, int] | None = None, sizes_bytes: dict[str, int] | None = None, ) -> list[SourceRecord]: @@ -283,7 +284,12 @@ def summaries_to_records( summaries Patch summaries, e.g. from `dc.scan`. base_uri - Optional common root; source paths are stored relative to it. + Optional common root persisted with each source (remote spools); + source paths are stored relative to it. + relative_to + Optional local spool root: source paths are stored relative to it + but the root itself is *not* persisted (local directory spools + resolve against their current root, per the design doc). mtimes_ns, sizes_bytes Optional maps of source_path -> stat values. When omitted the caller is responsible for change detection. @@ -302,8 +308,9 @@ def summaries_to_records( record = PatchRecord(**{**record.__dict__, "source_patch_id": str(num)}) patches.append(record) store_path = path - if base_uri and path.startswith(base_uri): - store_path = path[len(base_uri) :].lstrip("/") + root = base_uri or relative_to + if root and path.startswith(root): + store_path = path[len(root) :].lstrip("/") out.append( SourceRecord( source_path=store_path, diff --git a/tests/test_io/test_index/test_db_dirspool.py b/tests/test_io/test_index/test_db_dirspool.py new file mode 100644 index 000000000..42784ec7d --- /dev/null +++ b/tests/test_io/test_index/test_db_dirspool.py @@ -0,0 +1,110 @@ +""" +Integration tests: DirectorySpool running on the database index. + +Exercises the full path — directory walk, scan, ingest, query, patch +loading, chunk — against real files for every backend. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +import dascore as dc +from dascore.clients.dirspool import DirectorySpool +from dascore.examples import spool_to_directory + +BACKENDS = ("duckdb", "sqlite", "parquet") + + +@pytest.fixture(scope="class") +def spool_directory(tmp_path_factory): + """A directory of DASDAE files from the random example spool.""" + spool = dc.get_example_spool("random_das") + return spool_to_directory(spool, path=tmp_path_factory.mktemp("db_spool")) + + +@pytest.fixture(params=BACKENDS) +def db_spool(request, spool_directory): + """A DirectorySpool using each database index engine.""" + spool = DirectorySpool(spool_directory, index_engine=request.param) + out = spool.update(progress=None) + yield out + out.indexer.close() + + +class TestDBDirectorySpool: + """DirectorySpool wired to the database index.""" + + def test_length(self, db_spool): + """One entry per patch in the source spool.""" + assert len(db_spool) == 3 + + def test_contents_columns(self, db_spool): + """The contents df carries what the spool machinery needs.""" + df = db_spool.get_contents() + for col in ("path", "time_min", "time_max", "time_step", "dims"): + assert col in df.columns + + def test_load_patches(self, db_spool): + """Every indexed patch is loadable and matches the source data.""" + source = dc.get_example_spool("random_das") + source_starts = {patch.get_coord("time").min() for patch in source} + loaded_starts = set() + for patch in db_spool: + assert patch.data.size > 0 + loaded_starts.add(patch.get_coord("time").min()) + assert loaded_starts == source_starts + + def test_select_time(self, db_spool): + """Time select narrows the spool.""" + df = db_spool.get_contents() + t0 = df["time_min"].min().to_datetime64() + sub = db_spool.select(time=(t0, t0 + np.timedelta64(2, "s"))) + assert len(sub) >= 1 + patch = sub[0] + assert patch.get_coord("time").max() <= t0 + np.timedelta64(2, "s") + + def test_chunk_merge(self, db_spool): + """Contiguous patches merge with chunk(time=None).""" + merged = db_spool.chunk(time=None) + assert len(merged) == 1 + patch = merged[0] + assert patch.data.ndim == 2 + + def test_update_is_incremental(self, db_spool): + """A second update with no changes rescans nothing.""" + indexer = db_spool.indexer + before = indexer._backend.get_sources()["last_indexed_ns"].max() + db_spool.update(progress=None) + after = indexer._backend.get_sources()["last_indexed_ns"].max() + assert before == after + + +class TestUpdateLifecycle: + """New, modified, and deleted files are tracked per source.""" + + @pytest.fixture(params=BACKENDS) + def fresh(self, request, tmp_path): + """A modifiable spool directory + db spool of each engine.""" + spool = dc.get_example_spool("random_das") + path = spool_to_directory(spool, path=tmp_path / "data") + out = DirectorySpool(path, index_engine=request.param).update(progress=None) + yield path, out, request.param + out.indexer.close() + + def test_new_file_found(self, fresh): + """A file added after indexing appears on the next update.""" + path, spool, _engine = fresh + patch = dc.get_example_patch() + patch.io.write(path / "new_file.hdf5", "dasdae") + updated = spool.update(progress=None) + assert len(updated) == 4 + + def test_deleted_file_removed(self, fresh): + """A deleted file's rows are dropped on the next update.""" + path, spool, _engine = fresh + target = next(iter(path.glob("*.hdf5"))) + target.unlink() + updated = spool.update(progress=None) + assert len(updated) == 2 From 5f9b45a3f4ca6d63be184a3b4fc2f16f4a65815b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 11:44:22 +0200 Subject: [PATCH 03/97] Batch IN-clause parameters in source deletion SQLite caps bound variables (32766 by default), so replacing or deleting many sources in one call overflowed the parameter list at ~500k sources. Chunk both the path lookup and the id deletions. Also keep a failed rollback from masking the original write error. --- dascore/io/index/backend.py | 48 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 3c9410d4a..19450b95d 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -11,6 +11,7 @@ import abc import time +from contextlib import suppress from pathlib import Path import numpy as np @@ -258,30 +259,41 @@ def write_sources(self, records: list[SourceRecord]) -> None: self._bulk_insert("coords", tuple(COORDS), list(coord_rows)) self._execute("UPDATE meta_data SET last_indexed_ns = ?", (now,)) except Exception: - self._rollback() + # A failed rollback must not mask the original error. + with suppress(Exception): + self._rollback() raise self._commit() + # Batch size for IN (...) parameter lists; SQLite caps bound + # variables (32766 by default) so large replacements must chunk. + _in_clause_batch = 5000 + def _delete_by_paths(self, source_paths: list[str]) -> None: if not source_paths: return - marks = ", ".join("?" for _ in source_paths) - ids = self._fetch_df( - f"SELECT source_id FROM sources WHERE source_path IN ({marks})", - source_paths, - )["source_id"].tolist() - if not ids: - return - id_marks = ", ".join("?" for _ in ids) - for sql in ( - f"DELETE FROM coords WHERE patch_id IN " - f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", - f"DELETE FROM attrs WHERE patch_id IN " - f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", - f"DELETE FROM patches WHERE source_id IN ({id_marks})", - f"DELETE FROM sources WHERE source_id IN ({id_marks})", - ): - self._execute(sql, ids) + batch = self._in_clause_batch + ids: list = [] + for start in range(0, len(source_paths), batch): + chunk = source_paths[start : start + batch] + marks = ", ".join("?" for _ in chunk) + found = self._fetch_df( + f"SELECT source_id FROM sources WHERE source_path IN ({marks})", + chunk, + )["source_id"].tolist() + ids.extend(found) + for start in range(0, len(ids), batch): + chunk = ids[start : start + batch] + id_marks = ", ".join("?" for _ in chunk) + for sql in ( + f"DELETE FROM coords WHERE patch_id IN " + f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", + f"DELETE FROM attrs WHERE patch_id IN " + f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", + f"DELETE FROM patches WHERE source_id IN ({id_marks})", + f"DELETE FROM sources WHERE source_id IN ({id_marks})", + ): + self._execute(sql, chunk) def delete_sources(self, source_paths: list[str]) -> None: """Remove sources and all dependent rows.""" From 6a2ba9c68c40f57dda597fe8ad2d0e66dcbf1ec2 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 17:58:55 +0200 Subject: [PATCH 04/97] Add randomized heterogeneity stress test; fix two bugs it caught Stress test: 300 summaries with randomized dims (1-3 of 10 names incl. a hostile one), coord dtypes (datetime/timedelta/float/int/str), units, and attrs with mixed kinds per name and names that collide after sanitization. Verifies ingest completeness and the no-false-negative query contract against references computed from the raw summaries. Fixes: - attr names that sanitize to the same identifier ("Shot Number" vs "shot_number") collided on the attrs column; attr_meta is now the single source of truth for column names with deterministic numeric suffixes on collision. - multi-kind attrs coalesce in object space; pandas nullable BooleanArray refuses cross-dtype fills in Series.where. --- dascore/io/index/backend.py | 38 ++- .../test_index/test_heterogeneity_stress.py | 254 ++++++++++++++++++ 2 files changed, 283 insertions(+), 9 deletions(-) create mode 100644 tests/test_io/test_index/test_heterogeneity_stress.py diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 19450b95d..9c381ed7f 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -142,25 +142,42 @@ def _next_id(self, table: str, column: str) -> int: value = df["m"].iloc[0] return 1 if pd.isnull(value) else int(value) + 1 - def _ensure_attr_columns(self, records: list[SourceRecord]) -> None: - """Lazily add typed attr columns and register them in attr_meta.""" - known = { - (row.attr_name, row.value_kind) for row in self._attr_meta().itertuples() + def _ensure_attr_columns( + self, records: list[SourceRecord] + ) -> dict[tuple[str, str], str]: + """ + Lazily add typed attr columns; return the (name, kind) -> column map. + + attr_meta is the single source of truth for column names: distinct + attr names can sanitize to the same identifier ("Shot Number" vs + "shot_number"), so collisions get a deterministic numeric suffix. + """ + mapping = { + (row.attr_name, row.value_kind): row.column_name + for row in self._attr_meta().itertuples() } + taken = set(mapping.values()) needed: dict[tuple[str, str], str | None] = {} for record in records: for patch in record.patches: for name, typed in patch.attrs.items(): key = (name, typed.kind) - if key not in known and key not in needed: + if key not in mapping and key not in needed: needed[key] = typed.units for (name, kind), units in needed.items(): - column = attr_column_name(name, kind) + column = base = attr_column_name(name, kind) + suffix = 2 + while column in taken: + column = f"{base}_{suffix}" + suffix += 1 + taken.add(column) self._execute(self.dialect.add_column("attrs", column, KIND_STORAGE[kind])) self._execute( "INSERT INTO attr_meta VALUES (?, ?, ?, ?)", (name, kind, column, units), ) + mapping[(name, kind)] = column + return mapping def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: """Insert many rows; engines override for faster bulk paths.""" @@ -183,7 +200,7 @@ def write_sources(self, records: list[SourceRecord]) -> None: self._begin() try: self._delete_by_paths([r.source_path for r in records]) - self._ensure_attr_columns(records) + column_map = self._ensure_attr_columns(records) source_id = self._next_id("sources", "source_id") patch_id = self._next_id("patches", "patch_id") now = time.time_ns() @@ -221,8 +238,7 @@ def write_sources(self, records: list[SourceRecord]) -> None: ) ) columns = tuple( - attr_column_name(name, tv.kind) - for name, tv in patch.attrs.items() + column_map[(name, tv.kind)] for name, tv in patch.attrs.items() ) attr_groups.setdefault(columns, []).append( [patch_id, *(tv.value for tv in patch.attrs.values())] @@ -344,6 +360,10 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: col = pd.to_timedelta(col.astype("float64"), unit="ns") elif row.value_kind == "bool": col = col.astype("boolean") + if len(rows) > 1: + # multi-kind attrs coalesce in object space; typed + # extension arrays refuse cross-dtype fills + col = col.astype(object).where(col.notna(), np.nan) series = col if series is None else series.where(series.notna(), col) out = out.drop(columns=[row.column_name]) if series is not None: diff --git a/tests/test_io/test_index/test_heterogeneity_stress.py b/tests/test_io/test_index/test_heterogeneity_stress.py new file mode 100644 index 000000000..0a05b3ae3 --- /dev/null +++ b/tests/test_io/test_index/test_heterogeneity_stress.py @@ -0,0 +1,254 @@ +""" +Randomized heterogeneity stress test for the spool index. + +Generates hundreds of patch summaries with randomized dimension names, +coord dtypes/units, attr names/kinds (including hostile names that +collide after sanitization), and verifies ingest, counts, and the +no-false-negative query contract on every backend. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from dascore.core.summary import PatchSummary +from dascore.io.index import Query, get_backend, summaries_to_records +from dascore.units import get_quantity + +BACKENDS = ("duckdb", "sqlite", "parquet") +N_PATCHES = 300 + +_DIM_POOL = ( + "time", + "distance", + "lag_time", + "frequency", + "velocity", + "depth", + "channel_number", + "offset", + "azimuth", + "Gauge Length (m)", # hostile coord name +) +_NUM_UNITS = (None, "m", "ft", "Hz", "1/m", "km") +_ATTR_NAMES = ( + "station", + "tag", + "shot_number", + "Shot Number", # sanitizes identically to shot_number + "GaugeLength", + "über_attr", + "9lives", + "data quality!", + "processing_level", +) + + +def _random_coord(rng, name): + """Build one random coord summary dict.""" + kind = rng.choice(["datetime", "timedelta", "float", "int", "str"]) + length = int(rng.integers(2, 5000)) + if kind == "datetime": + start = np.datetime64("2024-01-01", "ns") + np.timedelta64( + int(rng.integers(0, 10**7)), "s" + ) + span = np.timedelta64(int(rng.integers(1, 10**5)), "s") + return { + "dtype": "datetime64", + "min": start, + "max": start + span, + "step": span / max(length - 1, 1) if rng.random() > 0.3 else None, + "dims": (name,), + "len": length, + } + if kind == "timedelta": + lo = np.timedelta64(int(rng.integers(-(10**6), 0)), "ms") + hi = np.timedelta64(int(rng.integers(1, 10**6)), "ms") + return { + "dtype": "timedelta64", + "min": lo, + "max": hi, + "step": None, + "dims": (name,), + "len": length, + } + if kind == "str": + lo, hi = sorted( + [f"A{rng.integers(0, 100):03d}", f"Z{rng.integers(0, 100):03d}"] + ) + return {"dtype": "str", "min": lo, "max": hi, "dims": (name,), "len": length} + lo = float(rng.uniform(-1000, 1000)) + hi = lo + float(rng.uniform(0.001, 5000)) + units = rng.choice(_NUM_UNITS) + out = { + "dtype": "float64" if kind == "float" else "int64", + "min": lo if kind == "float" else int(lo), + "max": hi if kind == "float" else int(hi) + 1, + "step": None, + "dims": (name,), + "len": length, + } + if units is not None: + out["units"] = units + return out + + +def _random_attrs(rng) -> dict: + """Build a random attr dict with mixed kinds and hostile names.""" + out = {} + for name in _ATTR_NAMES: + roll = rng.random() + if roll < 0.35: + continue # attr missing on this patch + if name in ("station", "tag"): + # typed str fields on PatchAttrs; only extra attrs vary kind + out[name] = f"v{rng.integers(0, 50)}" + continue + kind = rng.choice(["str", "int", "float", "bool", "time", "quantity"]) + if kind == "str": + out[name] = f"v{rng.integers(0, 50)}" + elif kind == "int": + out[name] = int(rng.integers(-1000, 1000)) + elif kind == "float": + out[name] = float(rng.uniform(-1000, 1000)) + elif kind == "bool": + out[name] = bool(rng.random() > 0.5) + elif kind == "time": + out[name] = np.datetime64("2024-01-01") + np.timedelta64( + int(rng.integers(0, 10**6)), "s" + ) + else: + out[name] = float(rng.uniform(0, 100)) * get_quantity( + str(rng.choice(["m", "ft", "s"])) + ) + return out + + +def make_random_summaries(n: int, seed: int = 0) -> list[PatchSummary]: + """Generate n randomized heterogeneous patch summaries.""" + rng = np.random.default_rng(seed) + out = [] + for i in range(n): + n_dims = int(rng.integers(1, 4)) + dims = tuple(rng.choice(_DIM_POOL, size=n_dims, replace=False)) + coords = {name: _random_coord(rng, name) for name in dims} + shape = tuple(int(coords[d]["len"]) for d in dims) + out.append( + PatchSummary( + attrs=_random_attrs(rng), + coords=coords, + dims=dims, + shape=shape, + dtype="float32", + source_path=f"stress/file_{i:05d}.h5", + source_format="DASDAE", + source_version="1", + ) + ) + return out + + +@pytest.fixture(scope="module") +def summaries(): + """The randomized summary population (deterministic seed).""" + return make_random_summaries(N_PATCHES, seed=42) + + +@pytest.fixture(params=BACKENDS) +def backend(request, tmp_path_factory, summaries): + """Each backend ingesting the random population.""" + path = tmp_path_factory.mktemp("stress") / f"idx_{request.param}" + back = get_backend(path, kind=request.param) + back.write_sources(summaries_to_records(summaries)) + yield back + back.close() + + +class TestStressIngest: + """Every random population ingests completely.""" + + def test_all_patches_indexed(self, backend): + """One flat row per summary.""" + assert len(backend.query()) == N_PATCHES + + def test_all_coords_present(self, backend, summaries): + """Every generated coord name is known to the index.""" + expected = {name for s in summaries for name in s.coords} + assert expected <= backend.coord_names() + + def test_sanitize_collision_attrs_distinct(self, backend, summaries): + """Attrs whose names sanitize identically stay distinct.""" + assert "shot_number" in backend.attr_names() + assert "Shot Number" in backend.attr_names() + df = backend.query() + assert "shot_number" in df.columns and "Shot Number" in df.columns + # every generated attr that carried a value is indexed + expected = { + name + for s in summaries + for name, value in s.attrs.model_dump().items() + if name in _ATTR_NAMES and value not in (None, "") + } + assert expected <= backend.attr_names() + + +class TestStressNoFalseNegatives: + """Random range queries never miss a matching patch.""" + + def test_random_numeric_coord_ranges(self, backend, summaries): + """Numeric coord queries: reference computed from raw summaries.""" + rng = np.random.default_rng(1) + num_coords = ["distance", "frequency", "velocity", "depth", "offset"] + for _ in range(20): + name = str(rng.choice(num_coords)) + lo = float(rng.uniform(-500, 500)) + hi = lo + float(rng.uniform(1, 2000)) + got = set(backend.query(Query(coords={name: (lo, hi)}))["path"]) + for summary in summaries: + csum = summary.coords.get(name) + if csum is None: + continue + # numeric queries must NOT match datetime/timedelta coords + # (kind dispatch) — and numpy's type hierarchy makes + # timedelta64 a signedinteger subtype, so check dtype.kind. + if np.dtype(csum.dtype).kind not in "iuf": + continue + factor = 1.0 + if csum.units is not None: + factor = float( + get_quantity(str(csum.units)).to_base_units().magnitude + ) + if float(csum.min) * factor <= hi and float(csum.max) * factor >= lo: + assert str(summary.source_path) in got, (name, lo, hi) + + def test_random_time_ranges(self, backend, summaries): + """Absolute time queries against datetime coords.""" + rng = np.random.default_rng(2) + base = np.datetime64("2024-01-01", "ns") + for _ in range(20): + lo = base + np.timedelta64(int(rng.integers(0, 10**7)), "s") + hi = lo + np.timedelta64(int(rng.integers(1, 10**6)), "s") + got = set(backend.query(Query(coords={"time": (lo, hi)}))["path"]) + for summary in summaries: + csum = summary.coords.get("time") + if csum is None or "datetime" not in str(csum.dtype): + continue + if csum.min <= hi and csum.max >= lo: + assert str(summary.source_path) in got + + def test_attr_equality_roundtrip(self, backend, summaries): + """Str attr equality returns every patch carrying that value.""" + rng = np.random.default_rng(3) + for _ in range(10): + summary = summaries[int(rng.integers(0, len(summaries)))] + attrs = { + k: v + for k, v in summary.attrs.model_dump().items() + if isinstance(v, str) and v and k in set(_ATTR_NAMES) + } + if not attrs: + continue + name, value = next(iter(attrs.items())) + got = set(backend.query(Query(attrs={name: value}))["path"]) + assert str(summary.source_path) in got From 5d180a9a0785afe178b77e047b738c2318fdfb9a Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 18:46:41 +0200 Subject: [PATCH 05/97] Remove PyTables index and the tables dependency The database index replaces the HDF5/PyTables index entirely: - Delete HDFPatchIndexManager, _HDF5Store, open_hdf5_file, the kernel query, and the PyTables reader/writer wrappers; H5Reader stands alone. io/indexer.py keeps AbstractIndexer and the index-location map helpers. - Drop tables>=3.7 from dependencies (it was only used by the index; all FiberIO readers use h5py) and remove the pytables warning filters. Add duckdb as an optional extra (needed for the duckdb/parquet engines; sqlite default has no extra dependency). - DirectorySpool now defaults to index_engine="sqlite". DBDirectoryIndexer gains: read-only-archive index relocation (config-backed, engine-keyed map), update(paths=...), one automatic update on first query of a brand-new index, and directory-format scan units - a directory FiberIO (e.g. XMLBinary) indexes as one source keyed by the directory with aggregate stats (max member mtime, summed size), mirroring dc.scan's skip protocol. Every visited path gets a sources row even when it yields no patches, so non-fiber files don't force perpetual rescans; the root-as-source path is spelled ".". - Ingest hardening: container/array values can no longer slip through the datetime fallback into the index. - Tests: port conftest and io tests from tables to h5py (pytables duck-typing tests use importorskip); rewrite indexer tests for DBDirectoryIndexer; add an edge-case suite bringing dascore/io/index to 100% line coverage. The #583 skip-warning test now asserts the fixed behavior (index-level distance selection) and the febus chunk test passes conflict="keep_first", matching in-memory spool semantics that the old index masked by dropping non-schema attrs. --- dascore/clients/dirspool.py | 24 +- dascore/io/index/indexer.py | 174 ++++++- dascore/io/index/ingest.py | 7 +- dascore/io/indexer.py | 287 +----------- dascore/utils/hdf5.py | 443 +----------------- docs/contributing/new_format.qmd | 2 +- docs/tutorial/file_io.qmd | 17 +- environment.yml | 1 - pyproject.toml | 10 +- tests/conftest.py | 12 +- tests/test_clients/test_dirspool.py | 41 +- tests/test_io/test_febus/test_febusg1.py | 11 +- tests/test_io/test_h5simple/test_h5simple.py | 8 +- .../test_index/test_index_edge_cases.py | 372 +++++++++++++++ tests/test_io/test_indexer.py | 308 ++++-------- tests/test_io/test_prodml/test_prod_ml.py | 8 +- tests/test_io/test_terra15/test_terra15.py | 6 +- tests/test_utils/test_hdf_utils.py | 188 +------- tests/test_utils/test_io_utils.py | 27 +- 19 files changed, 742 insertions(+), 1204 deletions(-) create mode 100644 tests/test_io/test_index/test_index_edge_cases.py diff --git a/dascore/clients/dirspool.py b/dascore/clients/dirspool.py index 5aa0db23f..cbae28e50 100644 --- a/dascore/clients/dirspool.py +++ b/dascore/clients/dirspool.py @@ -1,7 +1,7 @@ """ A spool for working with file systems. -The spool uses a simple hdf5 index for keeping track of files. +The spool uses a database index (sqlite by default) to track files. """ from __future__ import annotations @@ -18,7 +18,8 @@ from dascore.constants import PROGRESS_LEVELS from dascore.core.spool import BaseSpool, DataFrameSpool, MemorySpool from dascore.exceptions import MissingPatchError -from dascore.io.indexer import AbstractIndexer, DirectoryIndexer +from dascore.io.index.indexer import DBDirectoryIndexer +from dascore.io.indexer import AbstractIndexer from dascore.utils.docs import compose_docstring from dascore.utils.pd import adjust_segments @@ -43,9 +44,9 @@ class DirectorySpool(DataFrameSpool): select_kwargs Dict of keyword arguments to restrict output contents. index_engine - If set, use the database index backend of this kind ("duckdb", - "sqlite", or "parquet") instead of the HDF5 index. Experimental; - see the spool index design discussion (#648). + The database backend for the index: "sqlite" (default, no extra + dependencies), "duckdb", or "parquet" (both require duckdb). See + the spool index design discussion (#648). """ _drop_columns = ("file_format", "file_version", "path", "source_patch_id") @@ -58,7 +59,7 @@ def __init__( preferred_format: str | None = None, select_kwargs: dict | None = None, merge_kwargs: dict | None = None, - index_engine: str | None = None, + index_engine: str = "sqlite", ): super().__init__(select_kwargs=select_kwargs, merge_kwargs=merge_kwargs) # Init file spool from another file spool @@ -69,14 +70,9 @@ def __init__( elif isinstance(base_path, AbstractIndexer): self.indexer = base_path elif isinstance(base_path, Path | str | UPath): - if index_engine is not None: - from dascore.io.index.indexer import DBDirectoryIndexer - - self.indexer = DBDirectoryIndexer( - base_path, engine=index_engine, index_path=index_path - ) - else: - self.indexer = DirectoryIndexer(base_path, index_path=index_path) + self.indexer = DBDirectoryIndexer( + base_path, engine=index_engine, index_path=index_path + ) assert hasattr(self, "indexer"), "indexer not set." self._preferred_format = preferred_format diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index c4db40668..42e7fd29a 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -9,17 +9,26 @@ from __future__ import annotations +from contextlib import suppress from pathlib import Path import pandas as pd from typing_extensions import Self import dascore as dc +from dascore.compat import UPath +from dascore.config import config_attr from dascore.constants import PROGRESS_LEVELS from dascore.io.index.backend import get_backend, resolve_query -from dascore.io.index.ingest import summaries_to_records -from dascore.io.indexer import AbstractIndexer +from dascore.io.index.ingest import SourceRecord, summaries_to_records +from dascore.io.indexer import ( + AbstractIndexer, + _directory_writable, + _get_index_map, + _update_index_map, +) from dascore.utils.misc import _iter_filesystem +from dascore.utils.paths import requires_local_directory # Structural columns the spool machinery must not see: unique-per-patch # values block chunk merge-compatibility grouping, which compares all @@ -43,20 +52,59 @@ class DBDirectoryIndexer(AbstractIndexer): """ ext: str | None = None + # user-level file tracking index locations for unwritable data dirs + index_map_path: Path = config_attr("directory_index_map_path") def __init__( self, path: str | Path, - engine: str = "duckdb", + engine: str = "sqlite", index_path: str | Path | None = None, ): + path = UPath(path).absolute() if isinstance(path, UPath) else Path(path) + requires_local_directory(path, label="DBDirectoryIndexer") self.path = Path(path).absolute() self.engine = engine - if index_path is None: - index_path = self.path / f".dascore_index_{engine}" - self.index_path = Path(index_path) + self.index_path = Path(self._find_index_path(index_path)) + # A brand-new index triggers one automatic update on first query, + # matching the historic auto-index-on-first-access behavior. + self._initial_update_done = self.index_path.exists() self._backend = get_backend(self.index_path, kind=engine) + @property + def _index_name(self) -> str: + return f".dascore_index_{self.engine}" + + def _find_index_path(self, index_path=None) -> Path: + """ + Find where the index lives (or should live). + + Mirrors the historic DirectoryIndexer behavior: in-directory by + default; when the data directory is read-only the index lives in + the dascore cache and its location is recorded in the index map. + """ + map_key = f"{self.path}::{self.engine}" + if index_path: + update = {map_key: str(Path(index_path).absolute())} + _update_index_map(update, cache_path=str(self.index_map_path)) + return Path(index_path) + expected = self.path / self._index_name + with suppress(PermissionError): + if expected.exists(): + return expected + path_map = _get_index_map(cache_path=str(self.index_map_path)) + if out := path_map.get(map_key): + return Path(out) + if not _directory_writable(self.path): + name = f"_dascore_index_{abs(hash(self.path))}_{self.engine}" + index_path = self.index_map_path.parent / name + _update_index_map( + {map_key: str(index_path.absolute())}, + cache_path=str(self.index_map_path), + ) + return index_path + return expected + def __str__(self) -> str: return f"{self.__class__.__name__} ({self.engine}) managing: {self.path}" @@ -71,14 +119,57 @@ def __deepcopy__(self, memo) -> Self: """ return self - def _current_files(self) -> dict[str, tuple[int, int, Path]]: - """Map relative posix path -> (mtime_ns, size, absolute path).""" - out = {} - for file_path in _iter_filesystem(self.path, ext=self.ext): - stat = file_path.stat() - rel = file_path.relative_to(self.path).as_posix() - out[rel] = (stat.st_mtime_ns, stat.st_size, file_path) - return out + def _rel(self, path: Path) -> str: + """Relative posix path of a file under the spool root.""" + return Path(path).relative_to(self.path).as_posix() + + def _directory_format(self, path: Path) -> bool: + """Return True when a directory is itself one FiberIO scan unit.""" + try: + dc.get_format(path) + except Exception: + return False + return True + + def _walk(self) -> dict[str, tuple[int, int, Path]]: + """ + Walk the spool directory, honoring directory-format scan units. + + Maps relative path -> (mtime_ns, size, abs path) for every scan + unit. A directory-format unit (e.g. XMLBinary) appears as one + entry keyed by the directory, with aggregate stats — max member + mtime and summed member size — so member modification, addition, + and removal all register as a change. Mirrors the skip protocol + dc.scan uses so members are not offered individually. + """ + files: dict[str, tuple[int, int, Path]] = {} + gen = _iter_filesystem(self.path, ext=self.ext, include_directories=True) + signal = None + while True: + try: + # send(None) is equivalent to next() and also starts it + candidate = gen.send(signal) + except StopIteration: + break + signal = None + if candidate is None: # the reply to a "skip" send + continue + path = Path(candidate) + if path.is_dir(): + if self._directory_format(path): + signal = "skip" + max_mtime, total_size = 0, 0 + for sub in path.rglob("*"): + if not sub.is_file() or sub.name.startswith("."): + continue + stat = sub.stat() + max_mtime = max(max_mtime, stat.st_mtime_ns) + total_size += stat.st_size + files[self._rel(path)] = (max_mtime, total_size, path) + continue + stat = path.stat() + files[self._rel(path)] = (stat.st_mtime_ns, stat.st_size, path) + return files def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: """ @@ -87,31 +178,66 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: Change detection compares each source's stored (mtime_ns, size_bytes) against the filesystem — never a global watermark — and stale-source removal is folded in (the walk is the dominant - cost; removal afterwards is nearly free). + cost; removal afterwards is nearly free). Directory-format scan + units (e.g. XMLBinary) are rescanned whole when any member file + changes. """ - current = self._current_files() + self._initial_update_done = True + files = self._walk() stored = { - row.source_path: (row.mtime_ns, row.size_bytes) + row.source_path: ( + None + if pd.isnull(row.mtime_ns) + else (int(row.mtime_ns), int(row.size_bytes)) + ) for row in self._backend.get_sources().itertuples() } - stale = [path for path in stored if path not in current] + stale = [path for path in stored if path not in files] changed = [ rel - for rel, (mtime, size, _) in current.items() + for rel, (mtime, size, _) in files.items() if stored.get(rel) != (mtime, size) ] + if paths is not None: + # restrict the rescan (not stale removal) to the given paths + keep = set() + for one in paths: + one = Path(one) + rel = ( + one.relative_to(self.path).as_posix() + if one.is_absolute() + else one.as_posix() + ) + keep.add(rel) + changed = [rel for rel in changed if rel in keep] if stale: self._backend.delete_sources(stale) if changed: - abs_paths = [current[rel][2] for rel in changed] - summaries = dc.scan(abs_paths, progress=progress) + scan_paths = [files[rel][2] for rel in changed] + summaries = dc.scan(scan_paths, progress=progress) # scan reports absolute source paths; stat maps use them too records = summaries_to_records( summaries, relative_to=str(self.path), - mtimes_ns={str(current[r][2]): current[r][0] for r in changed}, - sizes_bytes={str(current[r][2]): current[r][1] for r in changed}, + mtimes_ns={str(p): m for _, (m, _, p) in files.items()}, + sizes_bytes={str(p): s for _, (_, s, p) in files.items()}, ) + # Every visited path gets a sources row, even when scanning + # produced no patches (e.g. a non-fiber file). Otherwise such + # files look "new" on every update and force perpetual + # rescans. + recorded = {rec.source_path for rec in records} + for rel in set(changed) - recorded: + mtime, size, _ = files[rel] + records.append( + SourceRecord( + source_path=rel, + source_format="", + format_version="", + mtime_ns=mtime, + size_bytes=size, + ) + ) if records: self._backend.write_sources(records) return self @@ -123,6 +249,8 @@ def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: Bare kwargs resolve attrs-first then coords; `_attrs`/`_coords` disambiguate explicitly (see the selector semantics spec). """ + if not self._initial_update_done: + self.update(progress=None) query = resolve_query(self._backend, _attrs=_attrs, _coords=_coords, **kwargs) df = self._backend.query(query) df = df.drop(columns=list(_SPOOL_HIDDEN_COLUMNS), errors="ignore") diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 20d3d36b5..0a05a7ca4 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -134,6 +134,10 @@ def typed_value(value) -> TypedValue | None: """ if _is_missing(value): return None + # containers/arrays are complex attrs: never indexable scalars (and + # they must not reach the datetime fallback, which accepts arrays). + if isinstance(value, np.ndarray | list | tuple | set | frozenset | dict | bytes): + return None # bool must precede int (bool is a subclass of int). if isinstance(value, bool | np.bool_): return TypedValue("bool", bool(value)) @@ -310,7 +314,8 @@ def summaries_to_records( store_path = path root = base_uri or relative_to if root and path.startswith(root): - store_path = path[len(root) :].lstrip("/") + # "." (not "") when the source IS the root (directory units) + store_path = path[len(root) :].lstrip("/") or "." out.append( SourceRecord( source_path=store_path, diff --git a/dascore/io/indexer.py b/dascore/io/indexer.py index 22cf73db3..93cce9b13 100644 --- a/dascore/io/indexer.py +++ b/dascore/io/indexer.py @@ -1,40 +1,23 @@ -"""An HDF5-based indexer for local file systems.""" +""" +Base indexer interface and index-location utilities. + +The concrete directory indexer lives in `dascore.io.index.indexer` +(`DBDirectoryIndexer`); this module holds the abstract interface and the +machinery for tracking index locations when the data directory itself is +not writable (e.g. read-only archives). +""" from __future__ import annotations import abc import json import os -import time -import warnings from contextlib import suppress from functools import cache from pathlib import Path -import pandas as pd from typing_extensions import Self -import dascore as dc -from dascore.compat import UPath -from dascore.config import config_attr, get_config -from dascore.constants import PROGRESS_LEVELS -from dascore.exceptions import InvalidIndexVersionError -from dascore.utils.hdf5 import HDFPatchIndexManager -from dascore.utils.misc import iterate -from dascore.utils.paths import requires_local_directory -from dascore.utils.pd import filter_df -from dascore.utils.time import ( - get_max_min_times, - saturate_add, - saturate_subtract, - to_timedelta64, -) - -# supported read_hdf5 kwargs -READ_HDF5_KWARGS = frozenset( - {"columns", "where", "mode", "errors", "start", "stop", "key", "chunksize"} -) - @cache def _get_index_map(cache_path) -> dict: @@ -100,257 +83,3 @@ def update(self) -> Self: Resets any previous selection. """ - - -class DirectoryIndexer(AbstractIndexer): - """ - A class for indexing a directory of dascore-readable files. - - This works by crawling the directory, getting a summary about the data it - contains, then creating a small HDF index file which can be queried later - on. - - Parameters - ---------- - path - The path to a directory containing DAS files. - index_path - The path to the index. By default, the index will be created on the - top level of the data directory. If another index is - """ - - ext = "" - _namespace = "" - _index_name = ".dascore_index.h5" # name of index file - - def __init__(self, path: str | Path, cache_size: int = 5, index_path=None): - self.max_size = cache_size - self.path = ( - UPath(path).absolute() if isinstance(path, UPath) else Path(path).absolute() - ) - requires_local_directory(self.path, label="DirectoryIndexer") - self.path = Path(self.path).absolute() - self.index_path = Path(self._find_index_file(self.path, index_path)) - self._current_index = 0 - self._index_table = HDFPatchIndexManager( - self.index_path, - self._namespace, - ) - self.cache = pd.DataFrame( - index=range(cache_size), columns="t1 t2 kwargs cindex".split() - ) - - index_map_path: Path = config_attr("directory_index_map_path") - - def _find_index_file(self, data_path, index_path=None): - """Find the path to the index file.""" - data_path = Path(data_path).absolute() - # user specified index path - if index_path: - update = {str(data_path): str(Path(index_path).absolute())} - _update_index_map(update, cache_path=str(self.index_map_path)) - return index_path - # see if expected path is in data path - expected_path = data_path / self._index_name - with suppress(PermissionError): - if expected_path.exists(): - return expected_path - # else load path map and see if it knows where the index is. - path_map = _get_index_map(cache_path=str(self.index_map_path)) - if out := path_map.get(str(data_path)): - return out - # if not, set the path to either the data path, if writable, - # else the dascore cache - if not _directory_writable(data_path): - new_path = "_dascore_index_" + str(abs(hash(data_path))) + ".h5" - index_path = self.index_map_path.parent / new_path - update = {str(data_path): str(index_path.absolute())} - _update_index_map(update, cache_path=str(self.index_map_path)) - else: - index_path = data_path / self._index_name - return index_path - - def get_contents(self, buffer=None, **kwargs) -> pd.DataFrame: - """ - Get contents of directory with specific query params. - - Parameters - ---------- - buffer - A buffer to ensure enough info is returned from hdf index. - kwargs - Used to query contents. - """ - # create index if it doesn't exist - if not self.index_path.exists(): - self.update() - # if the index still doesn't exist there are no readable files, return - # empty df. - if not self.index_path.exists(): - return pd.DataFrame(columns=self._index_table.index_columns) - time_min, time_max = get_max_min_times(kwargs.pop("time", None)) - hdf5_kwargs, kwargs = self._separate_hdf5_kwargs(kwargs) - buffer = get_config().index_query_buffer if buffer is None else buffer - buffer = to_timedelta64(buffer) - # find out if the query falls within one cached times - con1 = self.cache.t1 <= time_min - con2 = self.cache.t2 >= time_max - con3 = self.cache.kwargs == self._kwargs_to_str(kwargs) - cached_index = self.cache[con1 & con2 & con3] - if not len(cached_index): # query is not cached get it from hdf5 file - index = self._index_table.get_index( - time_min=time_min, - time_max=time_max, - **hdf5_kwargs, - ) - self._set_cache(index, time_min, time_max, hdf5_kwargs) - else: - index = cached_index.iloc[0]["cindex"] - # trim down index - con1 = index["time_min"] >= saturate_add(time_max, buffer) - con2 = index["time_max"] <= saturate_subtract(time_min, buffer) - pre_filter_df = index[~(con1 | con2)] - out = pre_filter_df[ - filter_df( - pre_filter_df, - time=(time_min, time_max), - ignore_bad_kwargs=True, - **kwargs, - ) - ] - return out - - def __str__(self): - """Rep. indexer as a string.""" - msg = f"{self.__class__.__name__} managing: {self.path}" - return msg - - __repr__ = __str__ - - __call__ = get_contents - - def _separate_hdf5_kwargs(self, kwargs): - """Ensure kwargs are supported.""" - kdf_kwargs = {i: v for i, v in kwargs.items() if i in READ_HDF5_KWARGS} - kwargs = {i: v for i, v in kwargs.items() if i not in READ_HDF5_KWARGS} - return kdf_kwargs, kwargs - - def _set_cache(self, index, starttime, endtime, kwargs): - """Cache the current index.""" - ser = pd.Series( - { - "t1": starttime, - "t2": endtime, - "cindex": index, - "kwargs": self._kwargs_to_str(kwargs), - } - ) - self.cache.loc[self._get_next_index()] = ser - - def clear_cache(self): - """Removes all cached dataframes.""" - self.cache = pd.DataFrame( - index=range(self.max_size), columns="t1 t2 kwargs cindex".split() - ) - - def _get_next_index(self): - """ - Get the next index value on cache. - Note we can't use itertools.cycle here because it cant be pickled. - """ - if self._current_index == len(self.cache.index) - 1: - self._current_index = 0 - else: - self._current_index += 1 - return self.cache.index[self._current_index] - - def _kwargs_to_str(self, kwargs): - """Convert kwargs to a string.""" - keys = sorted(list(kwargs.keys())) - out = str([(item, kwargs[item]) for item in keys]) - return out - - def _get_mtime(self, only_new=True): - """Return an iterator of potential un-indexed files.""" - # get mtime, subtract a bit to avoid odd bugs - mtime = None - # getting last updated might need the db so only call once. - last_updated = self._index_table.last_updated_timestamp if only_new else None - if last_updated is not None and only_new: - mtime = last_updated - 0.001 - # get paths to iterate - return mtime - - def _get_paths(self, paths): - path = self.path - if paths is None: - paths = path - else: - paths = [ - f"{path}/{x}" if str(path) not in str(x) else str(x) - for x in iterate(paths) - ] - return paths - - def _enforce_min_version(self): - """Ensure the minimum version is met, else delete index file.""" - try: - self._index_table.validate_version() - except InvalidIndexVersionError: - msg = ( - f"The index file at {self.path} is not compatible with this" - f" version of DASCore ({dc.__last_version__}). " - f"Recreating the index now." - ) - warnings.warn(msg, UserWarning) - os.remove(self.index_path) - self.update() - - def get_index_metadata(self): - """Return a dict of metadata about the index.""" - self.update() - up_time = dc.to_datetime64(self._index_table.last_updated_timestamp) - out = { - "index_version": self._index_table._index_version, - "last_update": up_time, - } - return out - - def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: - """ - Updates the contents of the Indexer. - - Also resets any previous selection. - - Parameters - ---------- - paths - A sequence of paths to limit the updates, if None, index all - the contents of directory. - progress - The type of progress bar to use. None disables progress bar and - "basic" is best for low latency scenarios. - """ - self._enforce_min_version() # delete index if schema has changed - update_time = time.time() - timestamp = self._get_mtime(only_new=True) - paths = self._get_paths(paths) - df = dc.scan_to_df( - path=paths, - timestamp=timestamp, - progress=progress, - ext=self.ext, - ) - # Put contents found into database. - if not df.empty: - # Some users were surprised the spool wasn't sorted. We still cant - # guarantee all spools will be sorted but we can make sure most are - # by sorting the contents before dumping to index. - if "time_min" in df.columns: - df = df.sort_values("time_min").reset_index(drop=True) - # ensure the base path is not in the path column - assert "path" in set(df.columns), f"{df} has no path column" - self._index_table.write_update(df, update_time, base_path=self.path) - # clear cache out when new traces are added - self.clear_cache() - return self diff --git a/dascore/utils/hdf5.py b/dascore/utils/hdf5.py index 4faf7476f..f275b9d64 100644 --- a/dascore/utils/hdf5.py +++ b/dascore/utils/hdf5.py @@ -1,9 +1,4 @@ -""" -Utilities for working with HDF5 files. - -Pytables should only be imported in this module in case we need to switch -out the hdf5 backend in the future. -""" +"""Utilities for working with HDF5 files (h5py-based).""" from __future__ import annotations @@ -11,42 +6,23 @@ import os import shutil import tempfile -import time -import warnings from collections.abc import Sequence -from contextlib import contextmanager, suppress +from contextlib import suppress from functools import partial from pathlib import Path -from typing import Literal import numpy as np import pandas as pd -import tables from h5py import File as H5pyFile -from packaging.version import parse as get_version -from pandas.io.common import stringify_path -from tables import ClosedNodeError -from tables import File as PyTablesFile -import dascore as dc from dascore.compat import UPath -from dascore.config import config_attr, get_config -from dascore.constants import max_lens, remote_hdf5_tuned_protocols -from dascore.exceptions import InvalidFileHandlerError, InvalidIndexVersionError -from dascore.io.core import PatchFileSummary -from dascore.utils.mapping import FrozenDict +from dascore.config import get_config +from dascore.constants import remote_hdf5_tuned_protocols from dascore.utils.misc import ( _maybe_make_parent_directory, _maybe_unpack, - cached_method, - suppress_warnings, unbyte, ) -from dascore.utils.pd import ( - _remove_base_path, - fill_defaults_from_pydantic, - list_ser_to_str, -) from dascore.utils.remote_io import ( _FallbackFileObj, _get_cached_local_file, @@ -54,11 +30,6 @@ get_local_handle, is_no_range_http_error, ) -from dascore.utils.time import get_max_min_times, to_datetime64, to_int, to_timedelta64 - -HDF5ExtError = tables.HDF5ExtError -NoSuchNodeError = tables.NoSuchNodeError -NodeError = tables.NodeError ns_to_datetime = partial(pd.to_datetime, unit="ns") ns_to_timedelta = partial(pd.to_timedelta, unit="ns") @@ -202,407 +173,7 @@ def open_h5_resource( raise NotImplementedError(msg) -class _HDF5Store(pd.HDFStore): - """ - A work-around for pandas HDF5 store not accepting - pytables.File objects. - """ - - def __init__( # pragma: no cover - self, - path, - mode: str = "a", - complevel: int | None = None, - complib=None, - fletcher32: bool = False, - **kwargs, - ) -> None: - if isinstance(path, str | Path): - self._path = stringify_path(path) - elif isinstance(path, tables.File): - self._path = stringify_path(path.filename) - self._mode = "a" if mode is None else mode - self._handle = None - self._complevel = complevel if complevel else 0 - self._complib = complib - self._fletcher32 = fletcher32 - self._filters = None - if isinstance(path, tables.File): - self._handle = path - else: - self.open(mode) - - -@contextmanager -def open_hdf5_file( - path_or_handler: Path | str | tables.File, - mode: Literal["r", "w", "a"] = "r", -) -> tables.File: - """ - A helper function for getting a `tables.file.File` object. - - If a file reference (str or Path) is passed this context manager will - close the file when it exists. - - Parameters - ---------- - path_or_handler - The input - mode - The mode in which to open the file. - - Raises - ------ - InvalidBuffer if a writable mode is requested from a read only handler. - """ - - def _validate_mode(current_mode, desired_mode): - """Ensure modes are compatible else raise.""" - if desired_mode == "r": - return - # if a or w is desired the current mode should be w - if not current_mode == "w": - msg = ( - f"A HDF5 file handler with mode 'r' was provided but " - f"mode: {desired_mode} was requested." - ) - raise InvalidFileHandlerError(msg) - - if isinstance(path_or_handler, str | Path): - # Note: We suppress DataTypeWarnings because pytables fails to read - # 8 bit enum indicating true or false written by h5py. See: - # https://github.com/PyTables/PyTables/issues/647 - with suppress_warnings(tables.DataTypeWarning): - with tables.open_file(path_or_handler, mode) as fi: - yield fi - elif isinstance(path_or_handler, tables.File): - _validate_mode(path_or_handler.mode, mode) - yield path_or_handler - - -def _get_kernel_query(starttime: int, endtime: int, buffer: int): - """ - Create a HDF5 kernel query based on start and end times. - - This is necessary because hdf5 doesn't accept inverted conditions. - A slight buffer is applied to the ranges to make sure no edge files - are excluded. - """ - t1 = starttime - buffer - t2 = endtime + buffer - con = ( - f"(time_min>{t1:d} & time_min<{t2:d}) | " - f"((time_max>{t1:d} & time_max<{t2:d}) | " - f"(time_min<{t1:d} & time_max>{t2:d}))" - ) - return con - - -class HDFPatchIndexManager: - """ - A class for writing/querying an index table of summary patch info to hdf5. - - It creates a table of patch summary info, a table of metadata and a time - stamp of the last time it was updated. - """ - - # string column sizes in hdf5 table - _min_itemsize = max_lens - # columns which should be indexed for fast querying - _query_columns = ("time_min", "time_max") - # functions applied to encode dataframe before saving to hdf5 - _column_encoders = FrozenDict( - { - "time_min": lambda x: to_int(to_datetime64(x)), - "time_max": lambda x: to_int(to_datetime64(x)), - "time_step": lambda x: to_int(to_timedelta64(x)), - "dims": list_ser_to_str, - "path": lambda x: x.astype(str), - } - ) - # functions to apply to decode dataframe after loading from hdf file - _column_decoders = FrozenDict( - { - "time_min": ns_to_datetime, - "time_max": ns_to_datetime, - "time_step": ns_to_timedelta, - } - ) - # base model which determines fields - _base_model = PatchFileSummary - # any fields to skip - _skip_fields = () - # The minimum version of dascore required to read this index. If an older - # version is used an error will be raised. - _min_version = "0.0.13" - - def __init__(self, path, namespace=""): - super().__init__() - self.namespace = namespace - self.path = path - - @property - def index_columns(self): - """Get the columns used for indexing.""" - out = set(self._base_model.model_fields) - set(self._skip_fields) - return tuple(out) - - buffer: np.timedelta64 = config_attr("index_query_buffer") - complib: str = config_attr("hdf_index_complib") - complevel: int = config_attr("hdf_index_complevel") - max_retries: int = config_attr("hdf_index_max_retries") - - # columns which should be indexed for fast querying - @property - def _time_node(self): - """The node/table where the update time information is stored.""" - return "/".join([self.namespace, "last_updated"]) - - @property - def _index_node(self): - """Return the node/table where the index information is stored.""" - return "/".join([self.namespace, "index"]) - - @property - def _meta_node(self): - """The node/table where the update metadata is stored.""" - return "/".join([self.namespace, "metadata"]) - - def encode_table(self, df, path=None): - """Encode the table for writing to hdf5.""" - # apply column encoders, make paths relative to reference path - # and drop any non-index columns. - cols = set(df.columns) - for col, func in self._column_encoders.items(): - if col not in cols: - continue - df[col] = func(df[col]) - out = ( - df.pipe(fill_defaults_from_pydantic, self._base_model) - .loc[:, list(self.index_columns)] - .assign(path=lambda x: _remove_base_path(x["path"], path)) - ) - # there shouldn't be any null values in index now - assert not out.isnull().any().any(), "null values found in index" - return out - - def decode_table(self, df): - """Decode the table from hdf5.""" - # ensure the base path is not in the path column - for col, func in self._column_decoders.items(): - df[col] = func(df[col]) - # populate index store and update metadata - # assert not df.isnull().any().any(), "null values found in index" - return df - - def get_index(self, time_min=None, time_max=None, **kwargs): - """ - Read part of the hdf5 index from path meeting time min/max reqs. - - Parameters - ---------- - time_min - The start time of the entries to read. - time_max - The end time of the entries to read. - """ - - def _get_index(where, fail_counts=0, **kwargs): - try: - df = pd.read_hdf(self.path, self._index_node, where=where, **kwargs) - except (ClosedNodeError, Exception) as e: - # Sometimes in concurrent updates the nodes need time to open/close - # so we implement a simply "wait and retry" strategy. - # This is a bit wonky but we have found it to work well in practice. - if fail_counts >= self.max_retries: - raise e - time.sleep(0.1) - return _get_index(where, fail_counts=fail_counts + 1, **kwargs) - else: - return df - - time_min, time_max = get_max_min_times((time_min, time_max)) - where = _get_kernel_query( - time_min.view(np.int64), - time_max.view(np.int64), - self.buffer.view(np.int64), - ) - df = _get_index(where, **kwargs) - return self.decode_table(df) - - def write_update( - self, - update_df, - update_time=None, - base_path: str | Path = "", - ): - """Convert updates to dataframe, then append to index table.""" - # read in dataframe and prepare for input into hdf5 index - update_time = update_time or time.time() - df = self.encode_table(update_df.copy(), path=base_path) - with _HDF5Store(self.path) as store: - try: - nrows = store.get_storer(self._index_node).nrows - except (AttributeError, KeyError): - store.append( - self._index_node, - df, - min_itemsize=self._min_itemsize, - **self.hdf_kwargs, - ) - else: - df.index += nrows - store.append(self._index_node, df, append=True, **self.hdf_kwargs) - self._update_metadata(store, update_time) - - def _update_metadata(self, store, update_time): - # update timestamp - update_time = time.time() if update_time is None else update_time - store.put(self._time_node, pd.Series(update_time)) - # make sure meta table also exists. - # Note this is here to avoid opening the store again. - if self._meta_node not in store: - meta = self._make_meta_table() - store.put(self._meta_node, meta, format="table") - - def _read_metadata(self): - """Read the metadata table.""" - try: - with _HDF5Store(self.path, "r") as store: - out = store.get(self._meta_node) - store.close() - return out - except (FileNotFoundError, ValueError, KeyError, OSError): - with suppress(UnboundLocalError): - store.close() - self._ensure_meta_table_exists() - return pd.read_hdf(self.path, self._meta_node) - - def _ensure_meta_table_exists(self): - """If the base path exists ensure it has a meta table, if not create it.""" - if not Path(self.path).exists(): - return - with _HDF5Store(self.path) as store: - # add metadata if not in store - if self._meta_node not in store: - meta = self._make_meta_table() - store.put(self._meta_node, meta, format="table") - - def _make_meta_table(self): - """Get a dataframe of meta info.""" - meta = dict( - dascore_version=dc.__last_version__, - ) - return pd.DataFrame(meta, index=[0]) - - @property - def hdf_kwargs(self) -> dict: - """A dict of hdf_kwargs to pass to PyTables.""" - return dict( - complib=self.complib, - complevel=self.complevel, - format="table", - data_columns=list(self._query_columns), - ) - - @cached_method - def validate_version(self): - """Handles issues with version mismatches.""" - # get the version from file, if the file doesnt exist then None - version = self._version_or_none - if version is not None: - # check if index is too old to be read by this version of the parser. - # If this is the case, users of this class should handle its - # re-creation. - min_version_tuple = get_version(self._min_version) - index_version = get_version(version) - if min_version_tuple > index_version: - msg = ( - f"The indexing schema has changed since {self._min_version} " - f"and must be regenerated." - ) - raise InvalidIndexVersionError(msg) - # check if index was created with newer version of dascore - dascore_version = get_version(dc.__last_version__) - if index_version > dascore_version: - msg = ( - f"The index was created with a newer version of dascore (" - f"{version}), you are running ({dc.__last_version__}), " - f"You may encounter problems, consider updating DASCore." - ) - warnings.warn(msg) - - @property - def _index_version(self) -> str: - """Get the version of dascore used to create the index.""" - return self._read_metadata()["dascore_version"].iloc[0] - - @property - def has_index(self) -> bool: - """Return True if an index table has been written.""" - expected_node = "/".join([self.namespace, "metadata"]) - with open_hdf5_file(self.path) as h5: - try: - h5.get_node(expected_node) - except NoSuchNodeError: - return False - else: - return True - - @property - def _version_or_none(self) -> str | None: - """Return the version string or None if it doesn't yet exist.""" - try: - version = self._index_version - except FileNotFoundError: - return - return version - - @property - def last_updated_timestamp(self) -> float | None: - """Return the last modified time stored in the index, else None.""" - try: - out = pd.read_hdf(self.path, self._time_node)[0] - except (OSError, IndexError, ValueError, KeyError, AttributeError): - out = None - return out - - -class PyTablesReader(PyTablesFile): - """A thin wrapper around pytables File object for reading.""" - - mode = "r" - constructor = PyTablesFile - - @classmethod - def get_handle(cls, resource): - """Get the File object from various sources.""" - if isinstance(resource, cls | PyTablesFile): - return resource - try: - _maybe_make_parent_directory(resource) - return cls.constructor(resource, mode=cls.mode) - except TypeError: - msg = f"Couldn't get handle from {resource} using {cls}" - raise NotImplementedError(msg) - - -class LocalPyTablesReader(PyTablesReader): - """A PyTables reader which first materializes remote resources locally.""" - - @classmethod - def get_handle(cls, resource): - """Get a local-file-backed PyTables handle.""" - return get_local_handle(resource, super().get_handle) - - -class PyTablesWriter(PyTablesReader): - """A thin wrapper around pytables File object for writing.""" - - mode = "a" - - -class H5Reader(PyTablesReader): +class H5Reader: """A thin wrapper around h5py for reading files. Remote UPath resources stay remote-first and transparently retry against @@ -631,7 +202,7 @@ def get_handle(cls, resource): """ Get the HDF5 handle from local paths, remote paths, or open handles. - Unlike PyTablesReader, h5py can consume a binary file object via the + h5py can consume a binary file object via the ``fileobj`` driver, so remote UPath inputs stay streaming-based here. """ if isinstance(resource, cls | _ManagedH5pyFile): @@ -743,8 +314,6 @@ def get_handle(cls, resource): # These are left here for backward compatibility, but should not be # used in new code. -HDF5Writer = PyTablesWriter -HDF5Reader = PyTablesReader def unpack_scalar_h5_dataset(dataset): diff --git a/docs/contributing/new_format.qmd b/docs/contributing/new_format.qmd index 7cd600c5e..f4f6dcf3a 100644 --- a/docs/contributing/new_format.qmd +++ b/docs/contributing/new_format.qmd @@ -163,7 +163,7 @@ loading the full data array. ## Support for Streams/Buffers -Rather than using paths for the IO methods as shown above, it is better practice to write a `FiberIO` which supports the [python stream interface](https://docs.python.org/3/library/io.html#io.BufferedIOBase) or an opened HDF5 file in the form of a `pytables.File` or `h5py.File` object. There are a few reasons for this: +Rather than using paths for the IO methods as shown above, it is better practice to write a `FiberIO` which supports the [python stream interface](https://docs.python.org/3/library/io.html#io.BufferedIOBase) or an opened HDF5 file in the form of an `h5py.File` object. There are a few reasons for this: * More types of inputs can be supported, including steaming file contents from the web or in-memory streams like [`BytesIO`](https://docs.python.org/3/library/io.html#io.BytesIO). * It is usually more efficient since open-file handles can be automatically reused. diff --git a/docs/tutorial/file_io.qmd b/docs/tutorial/file_io.qmd index f9975dc20..23269ae54 100644 --- a/docs/tutorial/file_io.qmd +++ b/docs/tutorial/file_io.qmd @@ -135,26 +135,27 @@ The `Patch.io` namespace also includes functionality for converting `Patch` inst ## Directory Indexer -The 'DirectoryIndexer' is used to track the contents of a directory which -contains fiber data. It creates a small, hidden HDF index file at the top -of the directory which can be efficiently queried for directory contents +The `DBDirectoryIndexer` is used to track the contents of a directory which +contains fiber data. It creates a small, hidden database index (sqlite by +default; duckdb and parquet backends are also available) at the top of the +directory which can be efficiently queried for directory contents (it is used internally by the `DirectorySpool`). ```{python} #| output: false import dascore -from dascore.io.indexer import DirectoryIndexer +from dascore.io.index.indexer import DBDirectoryIndexer from dascore import examples as ex # Get a directory with several files diverse_spool = dascore.get_example_spool('diverse_das') path = ex.spool_to_directory(diverse_spool) -# Create an indexer and update the index. This will include any new files -# with timestamps newer than the last update, or create a new HDF index file -# if one does not yet exist. -indexer = DirectoryIndexer(path).update() +# Create an indexer and update the index. This scans new or changed files +# (detected by per-file modification time and size), removes entries of +# deleted files, and creates the index if one does not yet exist. +indexer = DBDirectoryIndexer(path).update() # get the contents of the directory's files df = indexer.get_contents() diff --git a/environment.yml b/environment.yml index 52f85af2f..6c108387c 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,6 @@ dependencies: - pooch>=1.2 - xarray - pre-commit - - pytables - h5py - matplotlib>=3.5 - scipy>=1.15.0 diff --git a/pyproject.toml b/pyproject.toml index be9f14a48..ca4e3d921 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,8 +53,7 @@ dependencies = [ "pooch>=1.2", "pydantic>2.1", "rich", - "tables>=3.7", - "typing_extensions>=4.12", + "typing_extensions>=4.12", "universal-pathlib", "pint>=0.24.4", "scipy>=1.15", @@ -62,7 +61,12 @@ dependencies = [ [project.optional-dependencies] +duckdb = [ + "duckdb", +] + extras = [ + "duckdb", "xarray", "netCDF4", "h5netcdf", @@ -242,8 +246,6 @@ norecursedirs = [ "worktrees", ] filterwarnings = [ - # Ignore hdf5 warnings from pytables, See pytables #1035 - 'ignore::Warning:tables:' ] markers = [ "network: tests that require network-style filesystem access", diff --git a/tests/conftest.py b/tests/conftest.py index 66689e721..fb88b00f9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,12 +8,11 @@ from contextlib import suppress from pathlib import Path +import h5py import matplotlib import numpy as np import pandas as pd import pytest -import tables as tb -import tables.parameters import dascore as dc import dascore.examples as ex @@ -85,9 +84,6 @@ def pytest_sessionstart(session): if os.environ.get("CI", False): matplotlib.use("Agg") - # need to set nodes to 32 to avoid crash on p3.11. See pytables#977. - tables.parameters.NODE_CACHE_SLOTS = 32 - # Test-time debug defaults are applied by fixture to avoid state leakage. @@ -609,9 +605,9 @@ def generic_hdf5(tmp_path_factory): parent.mkdir() path = parent / "simple.hdf5" - with tb.open_file(str(path), "w") as fi: - group = fi.create_group("/", "bob") - fi.create_carray(group, "data", obj=random_state.rand(10)) + with h5py.File(str(path), "w") as fi: + group = fi.create_group("bob") + group.create_dataset("data", data=random_state.rand(10)) return path diff --git a/tests/test_clients/test_dirspool.py b/tests/test_clients/test_dirspool.py index 2336acbfb..823160d43 100644 --- a/tests/test_clients/test_dirspool.py +++ b/tests/test_clients/test_dirspool.py @@ -14,8 +14,6 @@ from dascore.clients.dirspool import DirectorySpool from dascore.constants import ONE_SECOND from dascore.exceptions import MissingPatchError, ParameterError -from dascore.io.core import PatchFileSummary -from dascore.utils.hdf5 import HDFPatchIndexManager from dascore.utils.misc import register_func, suppress_warnings DIRECTORY_SPOOLS = [] @@ -233,8 +231,16 @@ def test_index_len(self, basic_index_df, two_patch_directory): def test_index_columns(self, basic_index_df): """Ensure expected columns show up in the index.""" - schema_fields = list(PatchFileSummary.model_fields) - assert set(basic_index_df).issuperset(schema_fields) + expected = { + "path", + "file_format", + "file_version", + "dims", + "time_min", + "time_max", + "time_step", + } + assert set(basic_index_df).issuperset(expected) def test_patches_extracted(self, basic_file_spool): """Ensure the patches can be extracted.""" @@ -483,10 +489,10 @@ class TestGetContents: """Tests for getting the contents of the spool.""" def test_str_columns_in_dataframe(self, diverse_directory_spool): - """Ensure all the string columns are in index.""" + """Ensure the conventional string columns are in the index.""" df = diverse_directory_spool.get_contents() - expected = HDFPatchIndexManager._min_itemsize - assert set(df.columns).issuperset(set(expected)) + expected = {"path", "file_format", "file_version", "dims", "station"} + assert set(df.columns).issuperset(expected) class TestIndexing: @@ -611,13 +617,15 @@ def test_select_non_distance(self, non_distance_dir_spool): assert coord.max() <= depth_tup[1] def test_differing_distances(self, dist_differ_spool): - """Ensure iteration still works with conditions described in #583.""" + """Iteration works cleanly under the conditions described in #583. + + The generic index stores per-file distance ranges, so the select + already excluded the short-distance file; no patch needs the + historic skip-with-warning workaround. + """ assert len(dist_differ_spool) - # #583 would raise on iterating. Verify that a warning is issued when - # a patch is skipped due to coordinate mismatch. - with pytest.warns(UserWarning, match="Skipping patch at index.*#583"): - for patch in dist_differ_spool: - assert isinstance(patch, dc.Patch) + for patch in dist_differ_spool: + assert isinstance(patch, dc.Patch) def test_missing_patch_error_catchable_as_index_error(self, dist_differ_spool): """ @@ -632,12 +640,9 @@ def test_missing_patch_error_catchable_as_index_error(self, dist_differ_spool): except IndexError: pass - @pytest.mark.xfail() def test_selected_out_distance_shortens_spool(self, dist_differ_spool): - """Selecting outside of distance range should reduce spool length.""" - # Need to implement new indexing before this will pass. - with suppress_warnings(UserWarning): - assert len(dist_differ_spool) == 1 + """Selecting outside of distance range reduces spool length (#583).""" + assert len(dist_differ_spool) == 1 def test_iteration_unexpected_index_error(self, basic_file_spool): """ diff --git a/tests/test_io/test_febus/test_febusg1.py b/tests/test_io/test_febus/test_febusg1.py index 0031ecf32..410e40757 100644 --- a/tests/test_io/test_febus/test_febusg1.py +++ b/tests/test_io/test_febus/test_febusg1.py @@ -231,12 +231,19 @@ def g1_two_file_directory(self, tmp_path_factory): return out_dir def test_chunk_all_time_merges_to_single_patch(self, g1_two_file_directory): - """Ensure chunk(time=None) merges both g1 files into one patch.""" + """Ensure chunk(time=None) merges both g1 files into one patch. + + The files carry per-file attrs (temperature, freqoffset) that + differ, so conflict="keep_first" is required — matching what an + in-memory spool of the same patches requires. (The old HDF5 index + silently dropped these attrs, which masked the conflict for + directory spools.) + """ spool = dc.spool(g1_two_file_directory) # These weren't directly adjacent files so we adjust the tolerance. match = "There is a gap in the patch along dimension time" with pytest.warns(UserWarning, match=match): - merged = spool.chunk(time=None, tolerance=3) + merged = spool.chunk(time=None, tolerance=3, conflict="keep_first") assert len(merged) == 1 def test_mtx_read_raises(self, g1_mtx_buffer): diff --git a/tests/test_io/test_h5simple/test_h5simple.py b/tests/test_io/test_h5simple/test_h5simple.py index 382e1e94d..198672765 100644 --- a/tests/test_io/test_h5simple/test_h5simple.py +++ b/tests/test_io/test_h5simple/test_h5simple.py @@ -7,7 +7,6 @@ import h5py import numpy as np import pytest -import tables import dascore as dc from dascore.io.h5simple.utils import ( @@ -33,8 +32,8 @@ def h5simple_with_dim_attrs_path(self, tmp_path_factory): new_path = tmp_path_factory.mktemp("h5simple_dim_attrs") / "simple.h5" shutil.copy2(basic_path, new_path) - with tables.open_file(new_path, "a") as h5: - h5.root._v_attrs["dims"] = "distance,time" + with h5py.File(new_path, "a") as h5: + h5.attrs["dims"] = "distance,time" return new_path def test_no_snap(self, h5simple_path): @@ -54,6 +53,7 @@ class TestH5SimpleInternalHelpers: def test_get_root_attrs_supports_pytables(self, tmp_path): """PyTables handles should expose root attrs through the helper.""" path = tmp_path / "root_attrs.h5" + tables = pytest.importorskip("tables") with tables.open_file(path, "w") as h5: h5.root._v_attrs["dims"] = "distance,time" attrs = _get_root_attrs(h5) @@ -62,6 +62,7 @@ def test_get_root_attrs_supports_pytables(self, tmp_path): def test_iter_root_arrays_supports_pytables(self, tmp_path): """PyTables root arrays should still be discoverable by helper code.""" path = tmp_path / "root_arrays.h5" + tables = pytest.importorskip("tables") with tables.open_file(path, "w") as h5: h5.create_array("/", "data", obj=np.arange(3)) names = [name for name, _node in _iter_root_arrays(h5)] @@ -70,6 +71,7 @@ def test_iter_root_arrays_supports_pytables(self, tmp_path): def test_get_attr_names_supports_pytables_attrs(self, tmp_path): """PyTables attr containers should still expose their stored keys.""" path = tmp_path / "attr_names.h5" + tables = pytest.importorskip("tables") with tables.open_file(path, "w") as h5: h5.root._v_attrs["dims"] = "distance,time" out = _get_attr_names(h5.root._v_attrs) diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py new file mode 100644 index 000000000..af85e06cc --- /dev/null +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -0,0 +1,372 @@ +""" +Edge-case and error-path tests for the index package. + +Complements the contract suite: exercises failure branches, kind +mismatches, rollbacks, and the directory-format walk so the package has +full line coverage. +""" + +from __future__ import annotations + +import re + +import numpy as np +import pandas as pd +import pytest +from test_index_contract import make_summaries + +from dascore.core.summary import PatchSummary +from dascore.io.index import Query, get_backend, summaries_to_records +from dascore.io.index.backend import adapt_params, resolve_query +from dascore.io.index.indexer import DBDirectoryIndexer +from dascore.io.index.ingest import ( + _coord_record, + typed_value, +) +from dascore.io.index.ingest import ( + summaries_to_records as s2r, +) +from dascore.io.index.query import InvalidSpoolQueryError, glob_match +from dascore.units import get_quantity + +BACKENDS = ("duckdb", "sqlite", "parquet") + + +@pytest.fixture(scope="module") +def backend(tmp_path_factory): + """One duckdb backend with the contract summaries plus extras.""" + extra = PatchSummary( + attrs={ + "tag": "extra", + "trigger_time": np.datetime64("2024-06-01T00:00:00", "ns"), + "window": np.timedelta64(10, "s"), + }, + coords={ + "time": { + "dtype": "datetime64", + "min": np.datetime64("2024-06-01T00:00:00", "ns"), + "max": np.datetime64("2024-06-01T00:01:00", "ns"), + "dims": ("time",), + "len": 100, + }, + }, + dims=("time",), + shape=(100,), + dtype="float32", + source_path="extras/trigger.h5", + source_format="DASDAE", + source_version="1", + ) + path = tmp_path_factory.mktemp("edge") / "idx.duckdb" + back = get_backend(path, kind="duckdb") + back.write_sources(summaries_to_records([*make_summaries(), extra])) + yield back + back.close() + + +class TestAdaptAndBackendBasics: + """Small helpers and backend plumbing.""" + + def test_adapt_params_nan_becomes_none(self): + """NaN floats bind as NULL.""" + assert adapt_params([float("nan"), 1])[0] is None + + def test_unknown_backend_kind_raises(self, tmp_path): + """Asking for a nonexistent engine errors clearly.""" + with pytest.raises(ValueError, match="Unknown index backend"): + get_backend(tmp_path / "x", kind="mongodb") + + @pytest.mark.parametrize("kind", BACKENDS) + def test_bulk_insert_empty_rows_noop(self, tmp_path, kind): + """Empty bulk inserts are no-ops on every backend.""" + back = get_backend(tmp_path / f"i_{kind}", kind=kind) + back._bulk_insert("attr_meta", ("attr_name",), []) + back._executemany( + "INSERT INTO attr_meta VALUES (?, ?, ?, ?)", + [("a", "num", "a__num", None)], + ) + assert len(back._attr_meta()) == 1 + back.close() + + @pytest.mark.parametrize("kind", BACKENDS) + def test_write_failure_rolls_back(self, tmp_path, kind): + """A failing write leaves the index unchanged.""" + back = get_backend(tmp_path / f"r_{kind}", kind=kind) + records = summaries_to_records(make_summaries()) + back.write_sources(records[:1]) + before = len(back.query()) + + def boom(*args, **kwargs): + raise RuntimeError("simulated failure") + + back._bulk_insert = boom + with pytest.raises(RuntimeError, match="simulated"): + back.write_sources(records[1:]) + del back.__dict__["_bulk_insert"] + assert len(back.query()) == before + back.close() + + @pytest.mark.parametrize("kind", BACKENDS) + def test_delete_failure_rolls_back(self, tmp_path, kind): + """A failing delete leaves the index unchanged.""" + back = get_backend(tmp_path / f"d_{kind}", kind=kind) + back.write_sources(summaries_to_records(make_summaries())) + before = len(back.query()) + + def boom(paths): + raise RuntimeError("simulated failure") + + back._delete_by_paths = boom + with pytest.raises(RuntimeError, match="simulated"): + back.delete_sources(["das/file_1.h5"]) + del back.__dict__["_delete_by_paths"] + assert len(back.query()) == before + back.close() + + def test_delete_no_paths_noop(self, backend): + """Deleting an empty path list does nothing.""" + before = len(backend.query()) + backend.delete_sources([]) + assert len(backend.query()) == before + + def test_flatten_skips_absent_columns(self, backend): + """attr_meta rows without a matching result column are skipped.""" + df = backend._fetch_df("SELECT patch_id FROM patches LIMIT 2") + out = backend._flatten(df, backend._attr_meta()) + assert len(out) == 2 + + def test_duration_attr_roundtrip(self, backend): + """dur-kind attrs come back as timedeltas.""" + df = backend.query(Query(attrs={"window": np.timedelta64(10, "s")})) + assert len(df) == 1 + assert pd.api.types.is_timedelta64_dtype(df["window"]) + + +class TestResolveQueryErrors: + """Explicit-namespace validation.""" + + def test_unknown_attr_in_explicit_namespace(self, backend): + """Unknown key in _attrs raises.""" + with pytest.raises(InvalidSpoolQueryError, match="Unknown attribute"): + resolve_query(backend, _attrs={"nope": 1}) + + def test_unknown_coord_in_explicit_namespace(self, backend): + """Unknown key in _coords raises.""" + with pytest.raises(InvalidSpoolQueryError, match="Unknown coordinate"): + resolve_query(backend, _coords={"nope": (1, 2)}) + + +class TestQueryValueEdges: + """Coercion, kind-mismatch, and malformed-value behavior.""" + + def test_none_value_raises(self, backend): + """None is not a valid predicate value.""" + with pytest.raises(InvalidSpoolQueryError, match="Cannot use"): + backend.query(Query(attrs={"station": None})) + + def test_datetime_string_matches_time_attr(self, backend): + """A datetime-like string queries a time-kind attr.""" + df = backend.query(Query(attrs={"trigger_time": "2024-06-01T00:00:00"})) + assert list(df["tag"]) == ["extra"] + + def test_mixed_kind_range_raises(self, backend): + """Range bounds of different kinds raise.""" + with pytest.raises(InvalidSpoolQueryError, match="mixed kinds"): + backend.query(Query(attrs={"gauge_length": ("a", 5)})) + + def test_fully_open_range_raises(self, backend): + """A range with no usable bounds raises.""" + with pytest.raises(InvalidSpoolQueryError, match="no usable bounds"): + backend.query(Query(attrs={"gauge_length": (None, ...)})) + + def test_inverted_range_raises(self, backend): + """Lo > hi raises.""" + with pytest.raises(InvalidSpoolQueryError, match="lo > hi"): + backend.query(Query(attrs={"gauge_length": (5, 1)})) + + def test_unknown_attr_in_query_raises(self, backend): + """A Query naming an unknown attr raises at SQL build.""" + with pytest.raises(InvalidSpoolQueryError, match="not an attribute"): + backend.query(Query(attrs={"nope": 1})) + + def test_regex_on_non_str_attr_empty(self, backend): + """Regex against a numeric-only attr matches nothing.""" + df = backend.query(Query(attrs={"gauge_length": re.compile("x")})) + assert df.empty + + def test_range_kind_mismatch_empty(self, backend): + """A numeric range on a str-only attr matches nothing.""" + df = backend.query(Query(attrs={"station": (1, 2)})) + assert df.empty + + def test_membership_mixed_kinds(self, backend): + """Wrong-kind members are ignored; right-kind ones match.""" + df = backend.query(Query(attrs={"station": ["STA1", 5]})) + assert list(df["station"]) == ["STA1"] + + def test_membership_all_wrong_kind_empty(self, backend): + """All-wrong-kind membership matches nothing.""" + df = backend.query(Query(attrs={"station": [1, 2]})) + assert df.empty + + def test_glob_on_non_str_attr_empty(self, backend): + """Glob against a numeric-only attr matches nothing.""" + df = backend.query(Query(attrs={"gauge_length": "1*"})) + assert df.empty + + def test_boolean_array_coord_requires_presence_only(self, backend): + """Boolean masks are patch-local; index only checks coord presence.""" + mask = np.array([True, False, True]) + df = backend.query(Query(coords={"distance": mask})) + assert len(df) == 4 # every patch with a distance coord + + def test_glob_match_helper(self): + """Reference glob semantics.""" + assert glob_match("STA1", "STA*") + assert not glob_match(5, "STA*") + + +class TestIngestEdges: + """typed_value and record-building edge cases.""" + + def test_plain_array_skipped(self): + """Arrays are complex attrs; skipped.""" + assert typed_value(np.array([1, 2])) is None + + def test_array_quantity_skipped(self): + """Array-valued quantities are skipped.""" + assert typed_value(np.array([1.0, 2.0]) * get_quantity("m")) is None + + def test_reserved_attr_name_warns(self): + """An attr named patch_id is skipped with a warning.""" + summary = PatchSummary( + attrs={"patch_id": 5, "tag": "x"}, + coords={ + "distance": { + "dtype": "float64", + "min": 0.0, + "max": 1.0, + "dims": ("distance",), + "len": 2, + } + }, + dims=("distance",), + shape=(2,), + dtype="float32", + source_path="a.h5", + source_format="DASDAE", + source_version="1", + ) + with pytest.warns(UserWarning, match="reserved attr name"): + records = s2r([summary]) + assert "patch_id" not in records[0].patches[0].attrs + + def test_unsupported_coord_dtype_skipped(self): + """A coord with no usable dtype produces no record.""" + + class _Stub: + dtype = "" + dims = ("x",) + len = 2 + units = None + fingerprint = None + min = 0 + max = 1 + step = None + + assert _coord_record("x", _Stub()) is None + + def test_multipatch_source_gets_positional_ids(self): + """Multi-patch sources get positional source_patch_ids.""" + base = make_summaries()[0].dump_structured() + one = PatchSummary(**base) + two = PatchSummary(**{**base, "attrs": {"station": "STA9"}}) + records = s2r([one, two]) + assert len(records) == 1 + ids = [p.source_patch_id for p in records[0].patches] + assert ids == ["0", "1"] + + +class TestIndexerEdges: + """DBDirectoryIndexer edge behavior.""" + + def test_auto_update_on_first_query(self, tmp_path, random_patch): + """A brand-new index triggers one update on first query.""" + random_patch.io.write(tmp_path / "one.hdf5", "dasdae") + indexer = DBDirectoryIndexer(tmp_path) + assert len(indexer()) == 1 # no explicit update() call + + def test_directory_format_unit(self, tmp_path): + """Directory-format sources (xml binary) group as one scan unit.""" + import sys + + sys.path.insert(0, "tests/test_io/test_xml_binary") + from test_xml_binary import metadata + + sub = tmp_path / "unit" + sub.mkdir() + (sub / "metadata.xml").write_text(metadata) + rand = np.random.default_rng(0).random((5000, 10)).astype("float32") + for name in ( + "DAS_20240530T011500_000000Z.raw", + "DAS_20240530T011530_000000Z.raw", + ): + with (sub / name).open("wb") as fi: + rand.tofile(fi) + indexer = DBDirectoryIndexer(tmp_path).update(progress=None) + df = indexer() + assert len(df) == 2 + # unchanged: second update rescans nothing + before = indexer._backend.get_sources()["last_indexed_ns"].max() + indexer.update(progress=None) + after = indexer._backend.get_sources()["last_indexed_ns"].max() + assert before == after + indexer.close() + + +class TestDirSpoolPassthrough: + """DirectorySpool accepts a prebuilt indexer.""" + + def test_spool_from_indexer(self, tmp_path, random_patch): + """Passing an indexer instance to DirectorySpool works.""" + from dascore.clients.dirspool import DirectorySpool + + random_patch.io.write(tmp_path / "one.hdf5", "dasdae") + indexer = DBDirectoryIndexer(tmp_path, engine="duckdb") + spool = DirectorySpool(indexer).update(progress=None) + assert len(spool) == 1 + + +class TestFinalCoverage: + """Remaining edge branches.""" + + def test_datetime_object_becomes_time(self): + """A python datetime routes through the datetime fallback.""" + import datetime + + out = typed_value(datetime.datetime(2024, 1, 1)) + assert out is not None and out.kind == "time" + + def test_arbitrary_object_skipped(self): + """Unclassifiable objects are skipped.""" + + class _Odd: + """Not datetime-convertible, not a scalar.""" + + assert typed_value(_Odd()) is None + + def test_parquet_cleanup_failure_tolerated(self, tmp_path, monkeypatch): + """A failed unlink of superseded parquet files is not an error.""" + import pathlib + + back = get_backend(tmp_path / "pq", kind="parquet") + back.write_sources(summaries_to_records(make_summaries()[:1])) + + def bad_unlink(self, missing_ok=False): + raise OSError("simulated busy file") + + monkeypatch.setattr(pathlib.Path, "unlink", bad_unlink) + back.write_sources(summaries_to_records(make_summaries()[1:2])) + monkeypatch.undo() + assert len(back.query()) == 2 + back.close() diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index d0b89b03f..12de26adb 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -7,39 +7,27 @@ import shutil from contextlib import suppress from pathlib import Path -from unittest.mock import patch -import numpy as np import pandas as pd import pytest -from packaging.version import parse as get_version from upath import UPath -import dascore as dc from dascore.config import set_config -from dascore.examples import spool_to_directory from dascore.exceptions import InvalidSpoolError -from dascore.io.indexer import DirectoryIndexer -from dascore.utils.hdf5 import HDFPatchIndexManager +from dascore.io.index.indexer import DBDirectoryIndexer from dascore.utils.patch import get_patch_names @pytest.fixture(scope="class") def basic_indexer(two_patch_directory): - """Return and indexer on the basic spool directory.""" - return DirectoryIndexer(two_patch_directory) - - -@pytest.fixture(scope="class") -def adjacent_indexer(adjacent_spool_directory): - """Return and indexer on the basic spool directory.""" - return DirectoryIndexer(adjacent_spool_directory).update() + """Return an indexer on the basic spool directory.""" + return DBDirectoryIndexer(two_patch_directory).update(progress=None) @pytest.fixture(scope="class") def diverse_indexer(diverse_spool_directory): - """Return and indexer on the basic spool directory.""" - return DirectoryIndexer(diverse_spool_directory).update() + """Return an indexer on the diverse spool directory.""" + return DBDirectoryIndexer(diverse_spool_directory).update(progress=None) @pytest.fixture(scope="class") @@ -48,23 +36,11 @@ def diverse_df(diverse_indexer): return diverse_indexer() -@pytest.fixture() -def diverse_df_reset_cache(diverse_indexer): - """Return the indexer with a reset cache.""" - return DirectoryIndexer(diverse_indexer.path) - - -@pytest.fixture(params=[diverse_indexer, diverse_df_reset_cache]) -def diverse_ind(request): - """Aggregate the diverse indexers.""" - return request.getfixturevalue(request.param.__name__) - - @pytest.fixture() def empty_index(tmp_path_factory): """Create an index around an empty directory.""" path = tmp_path_factory.mktemp("index_created_test") - return DirectoryIndexer(path).update() + return DBDirectoryIndexer(path).update(progress=None) class TestFindIndex: @@ -73,66 +49,66 @@ class TestFindIndex: @pytest.fixture() def unwritable_directory(self, tmp_path_factory): """Return an un-writable directory.""" - # currently this doesn't work on windows so we need to skip any test - # that depend on this fixture if running on windows if "windows" in platform.system().lower(): pytest.skip("Cant run this test on windows") path = tmp_path_factory.mktemp("read_only_data_file") os.chmod(path, 0o444) - return path + yield path + os.chmod(path, 0o755) @pytest.fixture() def directory_indexer_bad_cache(self, tmp_path_factory): - """Create a subclass of indexer which has a bd index_map file.""" + """Create a bad index_map file.""" path = tmp_path_factory.mktemp("corrupt_cache_test") cache_path = path / "corrupt_cache.json" - with cache_path.open("wt") as fi: fi.write("{'bad': 'json'") return cache_path def test_directory_cant_write(self, unwritable_directory): """Ensure correct path is found when a read-only directory is used.""" - dir_index = DirectoryIndexer(unwritable_directory) + dir_index = DBDirectoryIndexer(unwritable_directory) index_path = dir_index.index_path index_map_path = dir_index.index_map_path assert index_map_path.parent == index_path.parent def test_specify_index_path(self, tmp_path_factory): - """Ensure specifying a Path works.""" + """Ensure specifying a Path works and is remembered.""" data_path = tmp_path_factory.mktemp("data_dir") - index_path = tmp_path_factory.mktemp("index_dir") / "index.h5" - dir_index = DirectoryIndexer(data_path, index_path=index_path) + index_path = tmp_path_factory.mktemp("index_dir") / "index.sqlite" + dir_index = DBDirectoryIndexer(data_path, index_path=index_path) assert dir_index.index_path == index_path - # loading a new data dir should now remember where this is. - dir_index2 = DirectoryIndexer(data_path) + # loading the same data dir should now remember where this is. + dir_index2 = DBDirectoryIndexer(data_path) assert dir_index2.index_path == index_path def test_writeable_dir_index_not_there(self, tmp_path_factory): - """Tests for when there is writeable directory.""" + """Tests for when there is a writeable directory.""" path = tmp_path_factory.mktemp("normal_indexer_test") - dir_indexer = DirectoryIndexer(path) + dir_indexer = DBDirectoryIndexer(path) assert dir_indexer.index_path.parent == path def test_writable_dir_index_exists(self, tmp_path_factory): """A test case where the index does exist.""" path = tmp_path_factory.mktemp("normal_indexer_test") - index_path = path / DirectoryIndexer._index_name - index_path.open("w").close() - dir_indexer = DirectoryIndexer(path) - assert dir_indexer.index_path == index_path - - def test_corrupt_cache( - self, - directory_indexer_bad_cache, - tmp_path_factory, - ): - """Ensure a corrupted cache doesnt crash indexing. See #508.""" + first = DBDirectoryIndexer(path) + second = DBDirectoryIndexer(path) + assert first.index_path == second.index_path + assert first.index_path.exists() + + def test_engines_get_separate_indices(self, tmp_path_factory): + """Different engines on one directory must not share an index.""" + path = tmp_path_factory.mktemp("multi_engine_test") + sqlite = DBDirectoryIndexer(path, engine="sqlite") + duckdb = DBDirectoryIndexer(path, engine="duckdb") + assert sqlite.index_path != duckdb.index_path + + def test_corrupt_cache(self, directory_indexer_bad_cache, tmp_path_factory): + """Ensure a corrupted cache doesn't crash indexing. See #508.""" path = tmp_path_factory.mktemp("corrupt_cache_test") - # Test passes if this doesn't raise should not raise. assert directory_indexer_bad_cache.exists() with set_config(directory_index_map_path=directory_indexer_bad_cache): - DirectoryIndexer(path) + DBDirectoryIndexer(path) assert not directory_indexer_bad_cache.exists() def test_remote_directory_not_supported(self): @@ -140,11 +116,11 @@ def test_remote_directory_not_supported(self): path = UPath("memory://dascore/indexer") (path / "file.txt").write_text("x") with pytest.raises(InvalidSpoolError, match="local filesystem"): - DirectoryIndexer(path) + DBDirectoryIndexer(path) def test_local_upath_normalized_to_path(self, tmp_path): """Local UPath inputs should normalize to pathlib.Path internally.""" - out = DirectoryIndexer(UPath(tmp_path)) + out = DBDirectoryIndexer(UPath(tmp_path)) assert isinstance(out.path, Path) assert out.path == Path(tmp_path).absolute() @@ -152,7 +128,7 @@ def test_index_map_path_comes_from_config(self, tmp_path): """Index map paths should be sourced from runtime configuration.""" index_map_path = tmp_path / "cache_paths.json" with set_config(directory_index_map_path=index_map_path): - out = DirectoryIndexer(tmp_path) + out = DBDirectoryIndexer(tmp_path) assert out.index_map_path == index_map_path @@ -164,24 +140,11 @@ def test_str_repr(self, basic_indexer): out = str(basic_indexer) assert "object at" not in out - def test_version(self, basic_indexer): - """Ensure the version written to file is correct.""" - updated = basic_indexer.update() - index_version = updated._index_table._index_version - assert index_version == dc.__last_version__ - assert get_version(index_version) > get_version("0.0.1") - - def test_update_does_not_reconstruct_patch_summary_from_flat_dicts( - self, two_patch_directory - ): - """Indexer update should not reconstruct PatchSummary from flat rows.""" - indexer = DirectoryIndexer(two_patch_directory) - with patch( - "dascore.core.summary.PatchSummary.model_validate", - side_effect=AssertionError("unexpected PatchSummary.model_validate call"), - ): - updated = indexer.update() - assert updated.index_path.exists() + def test_metadata(self, basic_indexer): + """The index records its schema version and identity.""" + meta = basic_indexer._backend.get_metadata() + assert meta["what_is_this"] == "dascore_spool_index" + assert meta["index_version"] >= 1 class TestGetContents: @@ -197,175 +160,104 @@ def test_get_contents(self, basic_indexer, two_patch_directory): names_files = {x.name for x in files} assert names_df == names_files - def test_filter_large_starttime(self, diverse_df, diverse_ind): - """Ensure the index can be filtered by end time.""" + def test_filter_time_after(self, diverse_df, diverse_indexer): + """Half-open time range keeps every file overlapping it.""" max_starttime = diverse_df["time_min"].max() - filtered = diverse_df[diverse_df["time_min"] >= max_starttime] - out = diverse_ind(time_min=max_starttime) - assert len(out) == len(filtered) + expected = diverse_df[diverse_df["time_max"] >= max_starttime] + out = diverse_indexer(time=(max_starttime, None)) + assert len(out) == len(expected) - def test_filter_small_starttime(self, diverse_df, diverse_ind): - """Ensure the index can be filtered by start time.""" + def test_filter_time_before(self, diverse_df, diverse_indexer): + """Half-open time range keeps every file overlapping it.""" min_endtime = diverse_df["time_max"].min() - filtered = diverse_df[diverse_df["time_max"] <= min_endtime] - out = diverse_ind(time_max=min_endtime) - assert len(out) == len(filtered) + expected = diverse_df[diverse_df["time_min"] <= min_endtime] + out = diverse_indexer(time=(None, min_endtime)) + assert len(out) == len(expected) - def test_filter_station_exact(self, diverse_df, diverse_ind): - """Ensure contents can be filtered on time.""" - # tests for filtering with exact station name + def test_filter_station_exact(self, diverse_df, diverse_indexer): + """Ensure contents can be filtered on an attr.""" exact_name = diverse_df["station"].unique()[0] - new_df = diverse_ind(station=exact_name) + new_df = diverse_indexer(station=exact_name) assert (new_df["station"] == exact_name).all() - def test_filter_isin(self, diverse_df, diverse_ind): - """Ensure contents can be filtered on time.""" - # tests for filtering with exact station name - exact_name = diverse_df["station"].unique()[0] - new_df = diverse_ind(station=exact_name) - assert (new_df["station"] == exact_name).all() + def test_filter_isin(self, diverse_df, diverse_indexer): + """Ensure contents can be filtered with a collection.""" + # empty strings mean "attr missing" and are not queryable (spec). + stations = [x for x in diverse_df["station"].unique() if x] + new_df = diverse_indexer(station=stations[:2]) + assert set(new_df["station"]) <= set(stations[:2]) + assert len(new_df) def test_empty_index(self, empty_index): """An empty index should return an empty dataframe.""" df = empty_index() assert df.empty - def test_default_buffer_comes_from_config(self, basic_indexer, monkeypatch): - """Configured index buffer should be used when no explicit buffer is passed.""" - seen = {} - - def _fake_to_timedelta64(value): - seen["buffer"] = value - return value - - monkeypatch.setattr("dascore.io.indexer.to_timedelta64", _fake_to_timedelta64) - with set_config(index_query_buffer=np.timedelta64(5, "s")): - basic_indexer.get_contents() - assert seen["buffer"] == np.timedelta64(5, "s") - - def test_explicit_buffer_overrides_config(self, basic_indexer): - """Explicit get_contents buffer should override config defaults.""" - seen = {} - - def _fake_to_timedelta64(value): - seen["buffer"] = value - return value - - with set_config(index_query_buffer=np.timedelta64(10, "s")): - with pytest.MonkeyPatch.context() as monkeypatch: - monkeypatch.setattr( - "dascore.io.indexer.to_timedelta64", _fake_to_timedelta64 - ) - basic_indexer.get_contents(buffer=np.timedelta64(0, "s")) - assert seen["buffer"] == np.timedelta64(0, "s") - - -class TestHDFPatchIndexManager: - """Tests for config-backed HDF index defaults.""" - - def test_buffer_comes_from_config(self, tmp_path): - """Index manager buffer property should reflect runtime config.""" - manager = HDFPatchIndexManager(tmp_path / "index.h5") - with set_config(index_query_buffer=np.timedelta64(7, "s")): - assert manager.buffer == np.timedelta64(7, "s") - class TestUpdate: - """Tests for updating index.""" - - def make_simple_index_with_version(self, path, version): - """Helper function to make a simple index with desired version.""" - patch = dc.get_example_patch() - spool_to_directory([patch], path) - # this ensure the version is set to fake version - old_version = dc.__last_version__ - # for some reason monkeypatch fixture wasnt setting version back - # so I had to manually set and revert dascore version. - setattr(dc, "__last_version__", version) - spool = dc.spool(path).update() - setattr(dc, "__last_version__", old_version) - # ensure version monkey patch worked. - meta = spool.indexer.get_index_metadata() - assert meta["index_version"] == version - return path + """Tests for updating the index.""" @pytest.fixture(scope="class") def spool_directory_with_non_das_file(self, two_patch_directory, tmp_path_factory): """Create a directory with some das files and some non-das files.""" new = tmp_path_factory.mktemp("unreadable_test") / "sub" shutil.copytree(two_patch_directory, new) - indexer = DirectoryIndexer(new) - # remove index if it exists with suppress(FileNotFoundError): - indexer.index_path.unlink() - # add a non das file + for index in Path(new).glob(".dascore_index*"): + index.unlink() with open(new / "not_das.open", "w") as fi: fi.write("cant be das, can it?") return new - @pytest.fixture() - def index_old_version(self, monkeypatch, tmp_path_factory): - """Create an index which has an old, incompatible version.""" - # cant use random_patch fixture due to scope-mismatch w/ monkeypatch. - path = tmp_path_factory.mktemp("index_old_version ") - self.make_simple_index_with_version(path, "0.0.1") - return path - - @pytest.fixture() - def index_new_version(self, monkeypatch, tmp_path_factory): - """Create an index which has an old, incompatible version.""" - # cant use random_patch fixture due to scope-mismatch w/ monkeypatch. - path = tmp_path_factory.mktemp("index_new_version ") - # a ridiculously high version - fake_version = "1000.0.1" - assert get_version(fake_version) > get_version(dc.__last_version__) - self.make_simple_index_with_version(path, fake_version) - return path - def test_add_one_patch(self, empty_index, random_patch): """Ensure a new patch added to the directory shows up.""" path = empty_index.path / get_patch_names(random_patch).iloc[0] random_patch.io.write(path, file_format="dasdae") - new_index = empty_index.update() + new_index = empty_index.update(progress=None) contents = new_index() assert len(contents) == 1 def test_index_with_bad_file(self, spool_directory_with_non_das_file): """Ensure if one file is not readable index continues.""" - indexer = DirectoryIndexer(spool_directory_with_non_das_file) - # if this doesn't fail the test passes - updated = indexer.update() - assert isinstance(updated, DirectoryIndexer) - - def test_old_index_recreated(self, index_old_version): - """Ensure the old index is recreated when update is called.""" - msg = "Recreating the index now." - with pytest.warns(UserWarning, match=msg): - dc.spool(index_old_version).update() - - def test_new_version_warnings(self, index_new_version): - """Ensure an index file with a newer version of dascore issues a warning.""" - msg = "The index was created with a newer version of dascore" - dc.spool(index_new_version) - with pytest.warns(UserWarning, match=msg): - dc.spool(index_new_version).update() + indexer = DBDirectoryIndexer(spool_directory_with_non_das_file) + updated = indexer.update(progress=None) + assert isinstance(updated, DBDirectoryIndexer) + assert len(updated()) == 2 + + def test_removed_file_dropped(self, two_patch_directory, tmp_path_factory): + """A deleted file's rows disappear on the next update.""" + new = tmp_path_factory.mktemp("removed_file_test") / "sub" + shutil.copytree(two_patch_directory, new) + for index in Path(new).glob(".dascore_index*"): + index.unlink() + indexer = DBDirectoryIndexer(new).update(progress=None) + assert len(indexer()) == 2 + next(iter(Path(new).glob("*.hdf5"))).unlink() + assert len(indexer.update(progress=None)()) == 1 + + def test_noop_update_rescans_nothing(self, basic_indexer): + """Unchanged sources are not rescanned.""" + before = basic_indexer._backend.get_sources()["last_indexed_ns"].max() + basic_indexer.update(progress=None) + after = basic_indexer._backend.get_sources()["last_indexed_ns"].max() + assert before == after def test_update_with_specific_paths(self, basic_indexer): - """Test updating with specific file paths to cover _get_paths method.""" - # Get files in the directory + """Updating with specific paths restricts the rescan.""" files = list(basic_indexer.path.rglob("*.hdf5")) - assert len(files) > 0, "Need at least one file for testing" + assert len(files) > 0 + updated = basic_indexer.update(paths=[files[0].name], progress=None) + assert len(updated()) >= 1 + updated2 = basic_indexer.update(paths=[str(files[0])], progress=None) + assert len(updated2()) >= 1 + - # Test update with specific paths (relative paths) - relative_paths = [f.name for f in files[:1]] # Use just first file - updated = basic_indexer.update(paths=relative_paths) - contents = updated() +class TestNameResolution: + """Unknown names raise per the selector spec.""" - # Should have at least the file we specified - assert len(contents) >= 1 + def test_unknown_name_raises(self, basic_indexer): + """Names in neither namespace error clearly (#435).""" + from dascore.io.index.query import InvalidSpoolQueryError - # Test update with absolute paths - absolute_paths = [str(f) for f in files[:1]] - updated2 = basic_indexer.update(paths=absolute_paths) - contents2 = updated2() - assert len(contents2) >= 1 + with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): + basic_indexer(bad_dimension=(1, 2)) diff --git a/tests/test_io/test_prodml/test_prod_ml.py b/tests/test_io/test_prodml/test_prod_ml.py index 6f3fd519d..5e8ad374d 100644 --- a/tests/test_io/test_prodml/test_prod_ml.py +++ b/tests/test_io/test_prodml/test_prod_ml.py @@ -7,7 +7,6 @@ import h5py import pandas as pd import pytest -import tables import dascore as dc from dascore.core.coords import get_coord @@ -47,12 +46,9 @@ def issue_221_patch_path(self, tmp_path_factory): tmp_path = tmp_path_factory.mktemp("issue_221") path = dc.utils.downloader.fetch("prodml_2.0.h5") new_path = shutil.copy2(path, tmp_path / "prod_2_monkey_patched.h5") - with tables.open_file(new_path, "a") as fi: + with h5py.File(new_path, "a") as fi: # monkey patch dimensions to simulate issue. - new_dims = "time, locus" - parent_node = fi.root.Acquisition["Raw[0]"] - node = parent_node["RawData"] - node._v_attrs.Dimensions = new_dims + fi["Acquisition/Raw[0]/RawData"].attrs["Dimensions"] = "time, locus" return new_path @pytest.fixture(scope="class") diff --git a/tests/test_io/test_terra15/test_terra15.py b/tests/test_io/test_terra15/test_terra15.py index 903a05738..ab04b522b 100644 --- a/tests/test_io/test_terra15/test_terra15.py +++ b/tests/test_io/test_terra15/test_terra15.py @@ -5,10 +5,10 @@ import shutil from typing import ClassVar +import h5py import numpy as np import pandas as pd import pytest -import tables import dascore as dc from dascore.io.terra15 import Terra15FormatterV4 @@ -22,8 +22,8 @@ def missing_gps_terra15_hdf5(self, terra15_v5_path, tmp_path_factory): """Creates a terra15 file with missing GPS Time.""" new = tmp_path_factory.mktemp("missing_gps") / "missing.hdf5" shutil.copy(terra15_v5_path, new) - with tables.open_file(new, "a") as fi: - fi.root.data_product.gps_time._f_remove() + with h5py.File(new, "a") as fi: + del fi["data_product/gps_time"] return new def test_missing_gps_time(self, missing_gps_terra15_hdf5): diff --git a/tests/test_utils/test_hdf_utils.py b/tests/test_utils/test_hdf_utils.py index ee87017d1..7efb1258c 100644 --- a/tests/test_utils/test_hdf_utils.py +++ b/tests/test_utils/test_hdf_utils.py @@ -3,27 +3,17 @@ from __future__ import annotations from contextlib import closing -from pathlib import Path import h5py -import pandas as pd import pytest -import tables -from tables.exceptions import ClosedNodeError -import dascore as dc -from dascore.config import set_config -from dascore.exceptions import InvalidFileHandlerError from dascore.utils.downloader import fetch from dascore.utils.hdf5 import ( H5Reader, - HDFPatchIndexManager, - LocalPyTablesReader, - PyTablesWriter, + H5Writer, extract_h5_attrs, get_h5py_file, h5_matches_structure, - open_hdf5_file, ) @@ -36,172 +26,24 @@ def h5_example_file(): fi.close() -class TestGetHDF5Handlder: - """Tests for opening an HDF5 file from various inputs.""" +class TestH5Readers: + """Tests for the h5py-based reader/writer handles.""" - @pytest.fixture() - def simple_hdf_path(self, tmp_path_factory): - """Create a hdf5 file in a temporary directory.""" - new = tmp_path_factory.mktemp("dummy_hdf5") / "test.h5" - with tables.open_file(str(new), mode="w") as fi: - bob = fi.create_group(fi.root, name="bob") - bob._v_attrs["lightening"] = 1 - return Path(new) - - @pytest.fixture() - def simple_hdf_file_handler_read(self, simple_hdf_path): - """Return a tables file handler in read mode.""" - with tables.open_file(simple_hdf_path, mode="r") as fi: - yield fi - - @pytest.fixture() - def simple_hdf_file_handler_append(self, simple_hdf_path): - """Return a tables file handler in append mode.""" - with tables.open_file(simple_hdf_path, mode="a") as fi: - yield fi - - def test_path_read(self, simple_hdf_path): - """Ensure passing a path works.""" - with open_hdf5_file(simple_hdf_path) as fi: - assert isinstance(fi, tables.File) - - def test_table_file_read(self, simple_hdf_file_handler_read): - """Ensure a tables file also works.""" - with open_hdf5_file(simple_hdf_file_handler_read) as fi: - assert isinstance(fi, tables.File) - - def test_read_only_filehandle_raises(self, simple_hdf_file_handler_read): - """If write is requested but read handler is provided an error should raise.""" - with pytest.raises(InvalidFileHandlerError, match="but mode"): - with open_hdf5_file(simple_hdf_file_handler_read, mode="w"): - pass - - def test_read_with_write_filehandler(self, simple_hdf_file_handler_append): - """ - Ensure a file handler is returned if read mode is requested but write - mode is provided. This works because write is a superset of read - functionality. - """ - with open_hdf5_file(simple_hdf_file_handler_append, mode="r") as fi: - assert isinstance(fi, tables.File) - - -class TestHDFPatchIndexManager: - """Tests for the HDF5 index manager.""" - - @pytest.fixture - def index_manager(self, tmp_path_factory): - """Create a new index.""" - path = Path(tmp_path_factory.mktemp("example")) / ".index" - return HDFPatchIndexManager(path) - - @pytest.fixture - def index_manager_with_content(self, index_manager, random_spool): - """Add content to the index manager.""" - spool_df = dc.scan_to_df(random_spool) - index_manager.write_update(spool_df) - return index_manager - - def test_extra_columns(self, index_manager, random_spool): - """ - Only the columns used for indexing should be kept, extras discarded. - - Here we include a column with types that can't be serialized. If the - write_update works the test passes. - """ - df = dc.scan_to_df(random_spool).assign( - bad_cols=[[] for _ in range(len(random_spool))] - ) - index_manager.write_update(df) - - def test_empty_tuple(self, index_manager, random_spool): - """Empty dims should convert to empty string.""" - df = dc.scan_to_df(random_spool).assign( - dims=[() for _ in range(len(random_spool))], - ) - index_manager.write_update(df) - - def test_has_content(self, index_manager_with_content, tmp_path): - """`has_index` should return True if data have been written else False.""" - assert index_manager_with_content.has_index - # create hdf5 file with no index - path = tmp_path / "empty.h5" - df = pd.DataFrame([1, 2, 3], columns=["first"]) - df.to_hdf(str(path), key="df") - # assert it doesn't have an index - assert not HDFPatchIndexManager(path).has_index - - def test_closed_node_error(self, index_manager_with_content, monkeypatch): - """ - Test for when the file fails to open. This is a bit contrived but the - closed node issues does happen sometimes in multiple thread environments. - """ - failed_count = 0 - old_func = pd.read_hdf - - def _new_read(*args, **kwargs): - nonlocal failed_count - if failed_count < 1: - failed_count += 1 - raise ClosedNodeError("Simulated failed node opening") - else: - return old_func(*args, **kwargs) - - monkeypatch.setattr(pd, "read_hdf", _new_read) - - df = index_manager_with_content.get_index() - assert len(df) - - # now insure the exception propagates - failed_count = 0 - with set_config(hdf_index_max_retries=0): - with pytest.raises(ClosedNodeError): - index_manager_with_content.get_index() - - def test_metadata_created(self, tmp_path_factory): - """Tests for getting info from a index that doesnt yet exist.""" - path = tmp_path_factory.mktemp("non_existent_index") / "index.hdf5" - with tables.open_file(path, "w"): - pass - index = HDFPatchIndexManager(path) - meta = index._read_metadata() - assert meta is not None - - def test_encode_table_skips_missing_encoded_columns(self, index_manager): - """Missing encoder columns should be ignored safely.""" - df = dc.scan_to_df(dc.get_example_spool()) - index_manager._column_encoders = dict(index_manager._column_encoders) | { - "missing_column": lambda x: x - } - out = index_manager.encode_table(df.copy(), path=None) - assert "path" in out.columns - - def test_hdf_kwargs_come_from_config(self, index_manager): - """Compression defaults should come from runtime configuration.""" - with set_config(hdf_index_complib="zlib", hdf_index_complevel=1): - out = index_manager.hdf_kwargs - assert out["complib"] == "zlib" - assert out["complevel"] == 1 - - -class TestHDFReaders: - """Tests for HDF5 readers.""" - - def test_get_handle(self, tmp_path_factory): - """Ensure we can get a handle with the class.""" + def test_writer_get_handle(self, tmp_path_factory): + """Ensure we can get a writable handle from a path or handle.""" path = tmp_path_factory.mktemp("hdf_handle_test") / "test_file.h5" - with closing(PyTablesWriter.get_handle(path)) as handle: - assert isinstance(handle, tables.File) - handle_2 = PyTablesWriter.get_handle(handle) - assert isinstance(handle_2, tables.File) + with closing(H5Writer.get_handle(path)) as handle: + handle.create_group("waveforms") + handle_2 = H5Writer.get_handle(handle) + assert handle_2 is handle - def test_local_pytables_reader_get_handle(self, tmp_path): - """The local-materializing reader should open local files directly.""" + def test_reader_get_handle(self, tmp_path): + """Ensure the reader opens existing files.""" path = tmp_path / "local_reader.h5" - with tables.open_file(path, mode="w") as h5: - h5.create_group("/", "waveforms") - with closing(LocalPyTablesReader.get_handle(path)) as handle: - assert isinstance(handle, tables.File) + with h5py.File(path, "w") as h5: + h5.create_group("waveforms") + with closing(H5Reader.get_handle(path)) as handle: + assert "waveforms" in handle class TestGetH5pyFile: diff --git a/tests/test_utils/test_io_utils.py b/tests/test_utils/test_io_utils.py index e89bfd6cb..36dcd687b 100644 --- a/tests/test_utils/test_io_utils.py +++ b/tests/test_utils/test_io_utils.py @@ -8,7 +8,6 @@ import h5py import pytest -from tables import File from upath import UPath import dascore as dc @@ -18,8 +17,6 @@ from dascore.utils.hdf5 import ( H5Reader, H5Writer, - HDF5Reader, - HDF5Writer, LocalH5Reader, open_h5_resource, ) @@ -149,14 +146,15 @@ def test_binary_stream_not_text_reader(self): def test_path_to_hdf5_reader(self, generic_hdf5): """Ensure we get a reader from tmp path reader.""" - with closing(get_handle_from_resource(generic_hdf5, HDF5Reader)) as handle: - assert isinstance(handle, File) + with closing(get_handle_from_resource(generic_hdf5, H5Reader)) as handle: + assert "bob" in handle # h5py-file-like def test_path_to_hdf5_writer(self, tmp_path): - """Ensure we get a reader from tmp path reader.""" + """Ensure we get a writer from tmp path.""" path = tmp_path / "test_hdf_writer.h5" - with closing(get_handle_from_resource(path, HDF5Writer)) as handle: - assert isinstance(handle, File) + with closing(get_handle_from_resource(path, H5Writer)) as handle: + handle.create_group("waveforms") + assert "waveforms" in handle def test_get_path(self, tmp_path): """Ensure we can get a path.""" @@ -446,9 +444,9 @@ def test_not_implemented(self): with pytest.raises(NotImplementedError): get_handle_from_resource(bad_instance, BinaryWriter) with pytest.raises(NotImplementedError): - get_handle_from_resource(bad_instance, HDF5Writer) + get_handle_from_resource(bad_instance, H5Writer) with pytest.raises(NotImplementedError): - get_handle_from_resource(bad_instance, HDF5Reader) + get_handle_from_resource(bad_instance, H5Reader) class TestIOResourceManager: @@ -471,13 +469,12 @@ def test_basic_context_manager(self, tmp_path): assert isinstance(path_from_hint, Path) path = man.get_resource(Path) assert isinstance(path, Path) - hf = man.get_resource(HDF5Writer) + hf = man.get_resource(H5Writer) fi = man.get_resource(BinaryWriter) - # Why didn't pytables implement the stream like pythons? - assert hf.isopen + assert not hf.closed assert not fi.closed - # after the context manager exists everything should be closed. - assert not hf.isopen + # after the context manager exits everything should be closed. + assert hf.closed assert fi.closed def test_get_none_resource_returns_source(self): From 2612c95aedf76b2130d03ccd7f36c74f647095ae Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 19:11:33 +0200 Subject: [PATCH 06/97] Deduplicate coordinate summaries by fingerprint Split the coords table into coord_defs (one row per unique coordinate summary) and patch_coords (patch -> name/dims -> def links). The def key is the CoordSummary fingerprint when the scan provides one (exact value identity, truncated to 128 bits) or a hash of the stored summary fields otherwise (lossless for the index, too weak for value-identity claims). Name and dims stay on the link since two patches can share values under different names. Defs are upserted with batched key lookups, reused across writes, and left orphaned on source deletion (a rebuild compacts them). Coord predicates join patch_coords x coord_defs; query timings are unchanged. This is groundwork, not a storage win: time coords are unique per file so their defs don't dedup (index grows ~40% on time-indexed archives), but shared coords collapse to single rows, chunk/merge can later recognize shared coordinates by coord_def_id equality instead of value comparison, and coord_defs is the natural home for exact coord arrays if full-coords storage lands. --- dascore/io/index/backend.py | 102 ++++++++++++------ dascore/io/index/ingest.py | 37 ++++++- dascore/io/index/query.py | 17 +-- dascore/io/index/schema.py | 33 ++++-- .../test_index/test_index_edge_cases.py | 58 ++++++++++ 5 files changed, 201 insertions(+), 46 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 9c381ed7f..f71c0b610 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -22,10 +22,11 @@ from dascore.io.index.ingest import SourceRecord, attr_column_name from dascore.io.index.query import Query, apply_residuals, build_query_sql from dascore.io.index.schema import ( - COORDS, + COORD_DEFS, INDEX_VERSION, INDEXES, KIND_STORAGE, + PATCH_COORDS, PATCHES, SOURCES, TABLES, @@ -179,6 +180,60 @@ def _ensure_attr_columns( mapping[(name, kind)] = column return mapping + def _ensure_coord_defs(self, defs_needed: dict) -> dict[str, int]: + """ + Ensure unique coord definitions exist; return def_key -> id. + + Coord summaries are deduplicated across patches: identical values + (by fingerprint, or by summary content when no fingerprint is + available) share one coord_defs row. This is what will later let + chunk/merge recognize shared coordinates by id equality. + """ + keys = list(defs_needed) + mapping: dict[str, int] = {} + batch = self._in_clause_batch + for start in range(0, len(keys), batch): + chunk = keys[start : start + batch] + marks = ", ".join("?" for _ in chunk) + found = self._fetch_df( + f"SELECT def_key, coord_def_id FROM coord_defs " + f"WHERE def_key IN ({marks})", + chunk, + ) + mapping.update( + zip(found["def_key"], (int(x) for x in found["coord_def_id"])) + ) + new_keys = [k for k in keys if k not in mapping] + next_id = self._next_id("coord_defs", "coord_def_id") + def_rows = [] + for key in new_keys: + c = defs_needed[key] + def_rows.append( + ( + next_id, + key, + c.coord_hash, + c.value_kind, + c.dtype, + c.length, + c.units, + c.min_num, + c.max_num, + c.step_num, + c.min_ns, + c.max_ns, + c.step_ns, + c.min_str, + c.max_str, + c.is_monotonic, + c.is_relative, + ) + ) + mapping[key] = next_id + next_id += 1 + self._bulk_insert("coord_defs", tuple(COORD_DEFS), def_rows) + return mapping + def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: """Insert many rows; engines override for faster bulk paths.""" if not rows: @@ -204,7 +259,8 @@ def write_sources(self, records: list[SourceRecord]) -> None: source_id = self._next_id("sources", "source_id") patch_id = self._next_id("patches", "patch_id") now = time.time_ns() - source_rows, patch_rows, coord_rows = [], [], [] + source_rows, patch_rows, link_rows = [], [], [] + defs_needed: dict[str, object] = {} attr_groups: dict[tuple[str, ...], list] = {} for record in records: source_rows.append( @@ -243,36 +299,22 @@ def write_sources(self, records: list[SourceRecord]) -> None: attr_groups.setdefault(columns, []).append( [patch_id, *(tv.value for tv in patch.attrs.values())] ) - coord_rows.extend( - ( - patch_id, - c.coord_name, - c.value_kind, - c.dtype, - c.coord_dims, - c.length, - c.units, - c.min_num, - c.max_num, - c.step_num, - c.min_ns, - c.max_ns, - c.step_ns, - c.min_str, - c.max_str, - c.is_monotonic, - c.is_relative, - c.coord_hash, - ) - for c in patch.coords - ) + for c in patch.coords: + key = c.def_key + defs_needed.setdefault(key, c) + link_rows.append((patch_id, c.coord_name, c.coord_dims, key)) patch_id += 1 source_id += 1 self._bulk_insert("sources", tuple(SOURCES), source_rows) self._bulk_insert("patches", tuple(PATCHES), patch_rows) for columns, rows in attr_groups.items(): self._bulk_insert("attrs", ("patch_id", *columns), rows) - self._bulk_insert("coords", tuple(COORDS), list(coord_rows)) + def_ids = self._ensure_coord_defs(defs_needed) + self._bulk_insert( + "patch_coords", + tuple(PATCH_COORDS), + [(pid, name, dims, def_ids[key]) for pid, name, dims, key in link_rows], + ) self._execute("UPDATE meta_data SET last_indexed_ns = ?", (now,)) except Exception: # A failed rollback must not mask the original error. @@ -301,8 +343,9 @@ def _delete_by_paths(self, source_paths: list[str]) -> None: for start in range(0, len(ids), batch): chunk = ids[start : start + batch] id_marks = ", ".join("?" for _ in chunk) + # coord_defs rows may orphan; harmless, a rebuild compacts them for sql in ( - f"DELETE FROM coords WHERE patch_id IN " + f"DELETE FROM patch_coords WHERE patch_id IN " f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", f"DELETE FROM attrs WHERE patch_id IN " f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", @@ -404,9 +447,8 @@ def attr_names(self) -> set[str]: def coord_names(self) -> set[str]: """Return coord names known to the index.""" - return set( - self._fetch_df("SELECT DISTINCT coord_name FROM coords")["coord_name"] - ) + df = self._fetch_df("SELECT DISTINCT coord_name FROM patch_coords") + return set(df["coord_name"]) def resolve_query( diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 0a05a7ca4..0f8977e83 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -9,6 +9,7 @@ from __future__ import annotations +import hashlib import re import warnings from dataclasses import dataclass, field @@ -42,7 +43,7 @@ def __post_init__(self): @dataclass(frozen=True) class CoordRecord: - """One row of the coords table (typed columns split by kind).""" + """One patch-coord entry (typed columns split by kind).""" coord_name: str value_kind: str @@ -62,6 +63,40 @@ class CoordRecord: is_relative: bool | None = None coord_hash: str | None = None + @property + def def_key(self) -> str: + """ + Deduplication key for the coord definition. + + The CoordSummary fingerprint when available ("fp:" prefix; exact + value identity), otherwise a hash of the stored summary fields + ("sum:" prefix; lossless for the index but too weak for + value-identity claims). Name and dims are patch-level and + excluded. + """ + if self.coord_hash: + # truncated: 128 bits is ample and key size shows up in the + # def_key index for archives with mostly-unique time coords + return f"fp:{self.coord_hash[:32]}" + fields = ( + self.value_kind, + self.dtype, + self.length, + self.units, + self.min_num, + self.max_num, + self.step_num, + self.min_ns, + self.max_ns, + self.step_ns, + self.min_str, + self.max_str, + self.is_monotonic, + self.is_relative, + ) + digest = hashlib.sha256(repr(fields).encode()).hexdigest()[:32] + return f"sum:{digest}" + @dataclass(frozen=True) class PatchRecord: diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 5fe1ab09d..1f07ce0a6 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -197,7 +197,8 @@ def build_coord_clause( value, ) -> None: """ - Add an EXISTS clause on the coords table for one coord predicate. + Add an EXISTS clause over patch_coords/coord_defs for one coord + predicate. Candidacy only: envelope overlap, never false negatives. Exact membership/boolean masks are applied at patch load, above this layer. @@ -224,26 +225,28 @@ def build_coord_clause( "str": ("min_str", "max_str"), None: (None, None), }[kind] - conditions = ["c.patch_id = p.patch_id", "c.coord_name = ?"] + conditions = ["pc.patch_id = p.patch_id", "pc.coord_name = ?"] params: list = [name] if kind is not None: if kind in ("time", "dur"): # absolute queries match absolute coords, durations relative. - conditions.append("c.is_relative = ?") + conditions.append("cd.is_relative = ?") params.append(kind == "dur") kind_match = "time" else: kind_match = kind - conditions.append("c.value_kind = ?") + conditions.append("cd.value_kind = ?") params.append(kind_match) if lo is not None: - conditions.append(f"c.{max_col} >= ?") + conditions.append(f"cd.{max_col} >= ?") params.append(lo) if hi is not None: - conditions.append(f"c.{min_col} <= ?") + conditions.append(f"cd.{min_col} <= ?") params.append(hi) where.add( - "EXISTS (SELECT 1 FROM coords c WHERE " + " AND ".join(conditions) + ")", + "EXISTS (SELECT 1 FROM patch_coords pc " + "JOIN coord_defs cd ON cd.coord_def_id = pc.coord_def_id " + "WHERE " + " AND ".join(conditions) + ")", *params, ) diff --git a/dascore/io/index/schema.py b/dascore/io/index/schema.py index 77674b673..741a01147 100644 --- a/dascore/io/index/schema.py +++ b/dascore/io/index/schema.py @@ -85,13 +85,17 @@ } ) -COORDS = MappingProxyType( +# Unique coordinate summaries, deduplicated across patches. The def_key +# is the CoordSummary fingerprint when the scan provides one (semantic +# value identity) or a hash of the stored summary fields otherwise +# (lossless for the index; too weak for value-identity claims). +COORD_DEFS = MappingProxyType( { - "patch_id": "int64", - "coord_name": "str", + "coord_def_id": "int64", + "def_key": "str", + "fingerprint": "str", # nullable; semantic hash from CoordSummary "value_kind": "str", # num | time | str "dtype": "str", - "coord_dims": "str", "length": "int64", "units": "str", # original unit string; numeric values stored SI "min_num": "float64", @@ -104,7 +108,17 @@ "max_str": "str", "is_monotonic": "bool", "is_relative": "bool", - "coord_hash": "str", + } +) + +# Links a patch to its coord defs; the name and dims are patch-level +# semantics (two patches can share values under different names). +PATCH_COORDS = MappingProxyType( + { + "patch_id": "int64", + "coord_name": "str", + "coord_dims": "str", + "coord_def_id": "int64", } ) @@ -115,7 +129,8 @@ "patches": PATCHES, "attrs": ATTRS_BASE, "attr_meta": ATTR_META, - "coords": COORDS, + "coord_defs": COORD_DEFS, + "patch_coords": PATCH_COORDS, } ) @@ -125,8 +140,10 @@ # Secondary indexes: without these, engines that use nested-loop plans # (SQLite) go quadratic on the correlated coords EXISTS subquery. INDEXES = ( - ("idx_coords_patch", "coords", "patch_id"), - ("idx_coords_name", "coords", "coord_name"), + ("idx_pcoords_patch", "patch_coords", "patch_id"), + ("idx_pcoords_name", "patch_coords", "coord_name"), + ("idx_pcoords_def", "patch_coords", "coord_def_id"), + ("idx_defs_key", "coord_defs", "def_key"), ("idx_attrs_patch", "attrs", "patch_id"), ("idx_patches_source", "patches", "source_id"), ("idx_sources_path", "sources", "source_path"), diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index af85e06cc..426514158 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -15,6 +15,7 @@ import pytest from test_index_contract import make_summaries +import dascore as dc from dascore.core.summary import PatchSummary from dascore.io.index import Query, get_backend, summaries_to_records from dascore.io.index.backend import adapt_params, resolve_query @@ -370,3 +371,60 @@ def bad_unlink(self, missing_ok=False): monkeypatch.undo() assert len(back.query()) == 2 back.close() + + +class TestCoordDeduplication: + """Coord summaries are stored once per unique definition.""" + + def test_shared_coord_stored_once(self, tmp_path): + """Identical distance coords across patches share one def row.""" + back = get_backend(tmp_path / "dedup", kind="duckdb") + back.write_sources(summaries_to_records(make_summaries())) + links = back._fetch_df("SELECT * FROM patch_coords") + defs = back._fetch_df("SELECT * FROM coord_defs") + assert len(defs) < len(links) + # das1 and das2 share an identical distance coord: one def, two links + dist_links = links[links["coord_name"] == "distance"] + das_defs = dist_links["coord_def_id"].value_counts() + assert (das_defs >= 2).any() + back.close() + + def test_defs_reused_across_writes(self, tmp_path): + """A second write with known coords creates no new defs.""" + back = get_backend(tmp_path / "reuse", kind="duckdb") + summaries = make_summaries() + back.write_sources(summaries_to_records(summaries[:1])) + n_defs = len(back._fetch_df("SELECT * FROM coord_defs")) + # das2 shares the distance def with das1; only time is new + back.write_sources(summaries_to_records(summaries[1:2])) + n_defs_after = len(back._fetch_df("SELECT * FROM coord_defs")) + assert n_defs_after == n_defs + 1 + back.close() + + def test_fingerprint_backed_defs(self, tmp_path): + """Summaries from real patches carry fingerprints into defs.""" + summary = PatchSummary.from_patch(dc.get_example_patch()) + structured = summary.dump_structured() + structured.update( + { + "source_path": "fp/one.h5", + "source_format": "DASDAE", + "source_version": "1", + } + ) + back = get_backend(tmp_path / "fp", kind="duckdb") + back.write_sources(summaries_to_records([PatchSummary(**structured)])) + defs = back._fetch_df("SELECT def_key, fingerprint FROM coord_defs") + assert defs["fingerprint"].notna().all() + assert defs["def_key"].str.startswith("fp:").all() + back.close() + + def test_orphan_defs_tolerated(self, tmp_path): + """Deleting sources leaves defs behind without breaking queries.""" + back = get_backend(tmp_path / "orphan", kind="duckdb") + back.write_sources(summaries_to_records(make_summaries())) + n_defs = len(back._fetch_df("SELECT * FROM coord_defs")) + back.delete_sources(["das/file_1.h5", "das/file_2.h5"]) + assert len(back._fetch_df("SELECT * FROM coord_defs")) == n_defs + assert len(back.query()) == 2 + back.close() From 5c4d39fc3a8030f20dc8a92665c29627d67defa8 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 21:43:43 +0200 Subject: [PATCH 07/97] Add per-coord envelope columns to the flat relation The flat relation now carries {name}_min/{name}_max/{name}_step for every coord in the result beyond the time/distance envelopes cached on patches, restoring parity with memory-spool dataframes (chunking on any dim now works from a directory spool). Every coord also gets a private _{name}_def_key column: the globally-stable coordinate identity that future chunk/merge grouping keys on (underscore-prefixed so it does not yet participate in merge-compatibility comparisons). --- dascore/io/index/backend.py | 69 +++++++++++++++++++ .../test_io/test_index/test_index_contract.py | 33 +++++++++ .../test_index/test_index_edge_cases.py | 23 +++++++ 3 files changed, 125 insertions(+) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index f71c0b610..128a89f80 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -373,6 +373,7 @@ def query(self, query: Query | None = None) -> pd.DataFrame: sql, params, residuals = build_query_sql(query, self.dialect, attr_meta) df = self._fetch_df(sql, params) df = self._flatten(df, attr_meta) + df = self._pivot_coords(df) if residuals: df = apply_residuals(df, residuals) return df.reset_index(drop=True) @@ -431,6 +432,74 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: out = out.drop(columns=["base_uri"]) return out.drop(columns=["source_id"], errors="ignore") + def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: + """ + Add per-coord envelope columns to the flat relation. + + Emits {name}_min/{name}_max/{name}_step for every coord in the + result beyond the time/distance envelopes already cached on + patches (memory-spool parity: chunking on any dim needs these), + plus a private _{name}_def_key column for every coord — the + globally-stable coordinate identity future chunk/merge grouping + uses (private so it does not yet participate in merge + compatibility comparisons). + """ + if out.empty or "patch_id" not in out.columns: + return out + ids = out["patch_id"].tolist() + frames = [] + batch = self._in_clause_batch + for start in range(0, len(ids), batch): + chunk = ids[start : start + batch] + marks = ", ".join("?" for _ in chunk) + frames.append( + self._fetch_df( + "SELECT pc.patch_id, pc.coord_name, cd.def_key, " + "cd.value_kind, cd.is_relative, cd.min_num, cd.max_num, " + "cd.step_num, cd.min_ns, cd.max_ns, cd.step_ns, " + "cd.min_str, cd.max_str " + "FROM patch_coords pc " + "JOIN coord_defs cd ON cd.coord_def_id = pc.coord_def_id " + f"WHERE pc.patch_id IN ({marks})", + chunk, + ) + ) + coords = pd.concat(frames, ignore_index=True) + if coords.empty: + return out + for name, group in coords.groupby("coord_name"): + mins, maxs, steps, keys = {}, {}, {}, {} + for row in group.itertuples(): + keys[row.patch_id] = row.def_key + if row.value_kind == "num": + mn, mx = row.min_num, row.max_num + st = row.step_num + elif row.value_kind == "time": + conv = ( + pd.to_timedelta + if pd.notnull(row.is_relative) and row.is_relative + else pd.to_datetime + ) + mn = conv(int(row.min_ns), unit="ns") + mx = conv(int(row.max_ns), unit="ns") + st = ( + pd.to_timedelta(int(row.step_ns), unit="ns") + if pd.notnull(row.step_ns) + else None + ) + else: + mn, mx, st = row.min_str, row.max_str, None + mins[row.patch_id], maxs[row.patch_id] = mn, mx + steps[row.patch_id] = st + out[f"_{name}_def_key"] = out["patch_id"].map(keys) + # time/distance envelopes already live on patches + if name in ("time", "distance"): + continue + out[f"{name}_min"] = out["patch_id"].map(mins) + out[f"{name}_max"] = out["patch_id"].map(maxs) + out[f"{name}_step"] = out["patch_id"].map(steps) + return out + # --- introspection ----------------------------------------------- def get_sources(self) -> pd.DataFrame: diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index 976a06e6a..d9f25bea2 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -425,3 +425,36 @@ def test_sources(self, backend): sources = backend.get_sources() assert len(sources) == 4 assert set(sources["source_format"]) == {"PRODML", "DASDAE"} + + +class TestCoordPivot: + """Per-coord envelope columns in the flat relation.""" + + def test_generic_coord_envelopes_present(self, backend): + """Non-conventional dims get {name}_min/max/step columns.""" + df = backend.query() + for col in ("lag_time_min", "lag_time_max", "frequency_min"): + assert col in df.columns + corr = df[df["tag"] == "corr"] + assert pd.api.types.is_timedelta64_dtype(corr["lag_time_min"].dtype) or ( + corr["lag_time_min"].map(lambda x: hasattr(x, "total_seconds")).all() + ) + + def test_time_distance_envelopes_not_duplicated(self, backend): + """patches-level envelopes are authoritative; pivot skips them.""" + df = backend.query() + assert pd.api.types.is_datetime64_dtype(df["time_min"]) + assert df.columns.tolist().count("distance_min") == 1 + + def test_def_key_columns_private_and_shared(self, backend): + """_{name}_def_key exists for every coord; shared coords share keys.""" + df = backend.query() + assert "_distance_def_key" in df.columns + das = df[df["station"].isin(["STA1", "STA2"])] + assert das["_distance_def_key"].nunique() == 1 + + def test_pivot_respects_query(self, backend): + """A filtered result only pivots the rows it contains.""" + df = backend.query(Query(attrs={"tag": "corr"})) + assert df["lag_time_min"].notna().all() + assert "frequency_min" not in df.columns or df["frequency_min"].isna().all() diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 426514158..8a9a97509 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -428,3 +428,26 @@ def test_orphan_defs_tolerated(self, tmp_path): assert len(back._fetch_df("SELECT * FROM coord_defs")) == n_defs assert len(back.query()) == 2 back.close() + + +class TestPivotEdge: + """Pivot with coord-less patches.""" + + def test_no_coords_patch(self, tmp_path): + """A patch with no coords pivots to nothing, without error.""" + summary = PatchSummary( + attrs={"tag": "bare"}, + coords={}, + dims=(), + shape=(), + dtype="float32", + source_path="bare.h5", + source_format="DASDAE", + source_version="1", + ) + back = get_backend(tmp_path / "bare", kind="duckdb") + back.write_sources(summaries_to_records([summary])) + df = back.query() + assert len(df) == 1 + assert not [c for c in df.columns if c.endswith("_def_key")] + back.close() From 2fa944b730a3bdebc581d24d98d4208360e733fb Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 4 Jul 2026 21:55:26 +0200 Subject: [PATCH 08/97] Make source identity (base_uri, source_path) in the write path Groundwork for merged/universal spools: base_uri is stored as "" (never NULL) so plain equality works on every engine; source replacement and deletion are scoped by base_uri; identical relative paths under different bases coexist. The flat relation prefixes base_uri onto paths only when non-empty. --- dascore/io/index/backend.py | 28 ++++++----- .../test_index/test_index_edge_cases.py | 49 ++++++++++++++++++- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 128a89f80..66880afcc 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -57,8 +57,8 @@ def write_sources(self, records: list[SourceRecord]) -> None: """Insert or replace sources (and dependents) transactionally.""" @abc.abstractmethod - def delete_sources(self, source_paths: list[str]) -> None: - """Remove sources and all dependent rows.""" + def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: + """Remove sources (identified by base_uri + path) and dependents.""" @abc.abstractmethod def query(self, query: Query) -> pd.DataFrame: @@ -254,7 +254,11 @@ def write_sources(self, records: list[SourceRecord]) -> None: """ self._begin() try: - self._delete_by_paths([r.source_path for r in records]) + by_base: dict[str, list[str]] = {} + for record in records: + by_base.setdefault(record.base_uri or "", []).append(record.source_path) + for base_uri, paths in by_base.items(): + self._delete_by_paths(paths, base_uri=base_uri) column_map = self._ensure_attr_columns(records) source_id = self._next_id("sources", "source_id") patch_id = self._next_id("patches", "patch_id") @@ -266,7 +270,7 @@ def write_sources(self, records: list[SourceRecord]) -> None: source_rows.append( ( source_id, - record.base_uri, + record.base_uri or "", record.source_path, record.source_format, record.format_version, @@ -327,7 +331,8 @@ def write_sources(self, records: list[SourceRecord]) -> None: # variables (32766 by default) so large replacements must chunk. _in_clause_batch = 5000 - def _delete_by_paths(self, source_paths: list[str]) -> None: + def _delete_by_paths(self, source_paths: list[str], base_uri: str = "") -> None: + """Delete sources by (base_uri, source_path) identity.""" if not source_paths: return batch = self._in_clause_batch @@ -336,8 +341,9 @@ def _delete_by_paths(self, source_paths: list[str]) -> None: chunk = source_paths[start : start + batch] marks = ", ".join("?" for _ in chunk) found = self._fetch_df( - f"SELECT source_id FROM sources WHERE source_path IN ({marks})", - chunk, + f"SELECT source_id FROM sources WHERE source_path IN ({marks}) " + "AND base_uri = ?", + [*chunk, base_uri], )["source_id"].tolist() ids.extend(found) for start in range(0, len(ids), batch): @@ -354,11 +360,11 @@ def _delete_by_paths(self, source_paths: list[str]) -> None: ): self._execute(sql, chunk) - def delete_sources(self, source_paths: list[str]) -> None: - """Remove sources and all dependent rows.""" + def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: + """Remove sources (identified by base_uri + path) and dependents.""" self._begin() try: - self._delete_by_paths(source_paths) + self._delete_by_paths(source_paths, base_uri=base_uri) except Exception: self._rollback() raise @@ -423,7 +429,7 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: } out = out.rename(columns=renames) if "base_uri" in out: - has_base = out["base_uri"].notna() + has_base = out["base_uri"].notna() & (out["base_uri"] != "") out.loc[has_base, "path"] = ( out.loc[has_base, "base_uri"].str.rstrip("/") + "/" diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 8a9a97509..24160518d 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -114,7 +114,7 @@ def test_delete_failure_rolls_back(self, tmp_path, kind): back.write_sources(summaries_to_records(make_summaries())) before = len(back.query()) - def boom(paths): + def boom(paths, base_uri=""): raise RuntimeError("simulated failure") back._delete_by_paths = boom @@ -307,6 +307,9 @@ def test_directory_format_unit(self, tmp_path): sub = tmp_path / "unit" sub.mkdir() (sub / "metadata.xml").write_text(metadata) + # hidden files and subdirectories inside a unit are ignored + (sub / ".hidden_state").write_text("x") + (sub / "logs").mkdir() rand = np.random.default_rng(0).random((5000, 10)).astype("float32") for name in ( "DAS_20240530T011500_000000Z.raw", @@ -451,3 +454,47 @@ def test_no_coords_patch(self, tmp_path): assert len(df) == 1 assert not [c for c in df.columns if c.endswith("_def_key")] back.close() + + +class TestCompositeSourceIdentity: + """Sources are identified by (base_uri, source_path).""" + + def test_same_path_different_base_coexist(self, tmp_path): + """Identical relative paths under different bases don't collide.""" + base = make_summaries()[0].dump_structured() + one = PatchSummary(**base) + records_a = summaries_to_records([one], base_uri="s3://bucket-a") + # base_uri strip only applies when paths share the base; set directly + records_a = [ + type(r)(**{**r.__dict__, "base_uri": "s3://bucket-a"}) for r in records_a + ] + records_b = [ + type(r)(**{**r.__dict__, "base_uri": "s3://bucket-b"}) for r in records_a + ] + back = get_backend(tmp_path / "multi", kind="duckdb") + back.write_sources(records_a) + back.write_sources(records_b) + df = back.query() + assert len(df) == 2 + prefixes = {p.split("/das/")[0] for p in df["path"]} + assert prefixes == {"s3://bucket-a", "s3://bucket-b"} + # deletion is base-scoped + back.delete_sources([records_a[0].source_path], base_uri="s3://bucket-a") + df = back.query() + assert len(df) == 1 + assert df["path"].iloc[0].startswith("s3://bucket-b") + back.close() + + def test_replacement_is_base_scoped(self, tmp_path): + """Rewriting a source under one base leaves the other base alone.""" + base = make_summaries()[0].dump_structured() + one = PatchSummary(**base) + rec = summaries_to_records([one])[0] + rec_a = type(rec)(**{**rec.__dict__, "base_uri": "s3://a"}) + rec_b = type(rec)(**{**rec.__dict__, "base_uri": "s3://b"}) + back = get_backend(tmp_path / "scoped", kind="duckdb") + back.write_sources([rec_a, rec_b]) + assert len(back.query()) == 2 + back.write_sources([rec_a]) # replace only the s3://a copy + assert len(back.query()) == 2 + back.close() From 1e0a24c059c8de7fec24dd143da4af13a50a84cf Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 5 Jul 2026 04:36:51 +0200 Subject: [PATCH 09/97] Add PatchCatalog: unified metadata engine over the index The catalog owns the index tables (via any backend) and composed selection state; resolvers turn flat-relation rows into patches (FileResolver through dc.read with trim hints -- remoteness belongs to the path layer; LiveResolver from an in-memory registry with synthetic memory:// source identities); the directory indexer plugs in as the syncer for directory-backed catalogs. Laziness: from_patches does no metadata work until the first metadata operation (backend bootstrap costs ~10s of ms; holding a patch list is free). select composes Query predicates with eager name validation and no SQL; realization runs one query per view. samples selections are patch-local; relative bounds resolve against the view envelope; coord range predicates re-apply exactly at patch load (two-stage select). Mutation is root-only; views share backend and resolver. Also: build_query_sql accepts AND-composed query sequences. --- dascore/io/index/__init__.py | 10 + dascore/io/index/backend.py | 4 +- dascore/io/index/catalog.py | 357 +++++++++++++++++++++++ dascore/io/index/query.py | 19 +- tests/test_io/test_index/test_catalog.py | 219 ++++++++++++++ 5 files changed, 599 insertions(+), 10 deletions(-) create mode 100644 dascore/io/index/catalog.py create mode 100644 tests/test_io/test_index/test_catalog.py diff --git a/dascore/io/index/__init__.py b/dascore/io/index/__init__.py index 4fe2c10f9..e0434969c 100644 --- a/dascore/io/index/__init__.py +++ b/dascore/io/index/__init__.py @@ -10,11 +10,21 @@ from __future__ import annotations from dascore.io.index.backend import AbstractIndexBackend, get_backend +from dascore.io.index.catalog import ( + FileResolver, + LiveResolver, + PatchCatalog, + PatchResolver, +) from dascore.io.index.ingest import summaries_to_records from dascore.io.index.query import Query __all__ = [ "AbstractIndexBackend", + "FileResolver", + "LiveResolver", + "PatchCatalog", + "PatchResolver", "Query", "get_backend", "summaries_to_records", diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 66880afcc..ba47fbab1 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -372,8 +372,8 @@ def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: # --- queries ----------------------------------------------------- - def query(self, query: Query | None = None) -> pd.DataFrame: - """Return the flat patch-row relation for a query.""" + def query(self, query=None) -> pd.DataFrame: + """Return the flat patch-row relation for a query (or several).""" query = query if query is not None else Query() attr_meta = self._attr_meta() sql, params, residuals = build_query_sql(query, self.dialect, attr_meta) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py new file mode 100644 index 000000000..229a2a7aa --- /dev/null +++ b/dascore/io/index/catalog.py @@ -0,0 +1,357 @@ +""" +PatchCatalog: one metadata engine for every spool type. + +The catalog owns the index tables (through a backend) and the composed +selection state; a resolver turns flat-relation rows into patches +(from files via dc.read, or from a live registry for in-memory spools); +a syncer (the directory indexer) keeps directory-backed catalogs in step +with the filesystem. See the spool index design doc and discussion #648. + +Laziness contract: creating a catalog from patches does no metadata work +until the first metadata operation (select/len/iteration), because +backend bootstrap costs ~10s of ms while holding a patch list is free. +Selection composes Query predicates without running SQL; realization +(len, to_df, iteration) runs exactly one query per view. +""" + +from __future__ import annotations + +import abc +import itertools +from collections.abc import Mapping, Sequence +from pathlib import Path + +import numpy as np +import pandas as pd + +import dascore as dc +from dascore.constants import PROGRESS_LEVELS +from dascore.io.index.backend import get_backend, resolve_query +from dascore.io.index.ingest import SourceRecord, patch_record +from dascore.io.index.query import InvalidSpoolQueryError, Query + +_MEMORY_ENGINES = frozenset({"sqlite", "duckdb"}) +_counter = itertools.count() + + +class PatchResolver(abc.ABC): + """Turn one flat-relation row into a Patch.""" + + @abc.abstractmethod + def resolve(self, row: Mapping, **trim) -> dc.Patch: + """ + Return the patch for a row. + + Trim kwargs are hints (a slice of the plan, not the query): + implementations may use them to read less, but exact trimming is + re-applied above, so ignoring them is slower, never wrong. + """ + + +class LiveResolver(PatchResolver): + """Serve patches from an in-memory registry.""" + + def __init__(self): + self._registry: dict[tuple[str, str], dc.Patch] = {} + + def register(self, path: str, source_patch_id: str, patch: dc.Patch) -> None: + """Register a live patch under its synthetic source identity.""" + self._registry[(path, source_patch_id)] = patch + + def resolve(self, row: Mapping, **trim) -> dc.Patch: + """Look the patch up; live patches ignore trim hints.""" + key = (row["path"], row.get("source_patch_id") or "") + return self._registry[key] + + +class FileResolver(PatchResolver): + """Load patches through dc.read; remoteness is the path layer's job.""" + + def __init__(self, root: Path | str | None = None): + self._root = Path(root) if root is not None else None + + def resolve(self, row: Mapping, **trim) -> dc.Patch: + """Read the patch, passing range trims down as read hints.""" + path = row["path"] + # relative paths resolve against the catalog root; URIs and + # absolute paths pass through untouched. + if self._root is not None and "://" not in str(path): + if not Path(path).is_absolute(): + path = self._root / path + kwargs = {"path": path} + if row.get("file_format"): + kwargs["file_format"] = row["file_format"] + if row.get("file_version"): + kwargs["file_version"] = row["file_version"] + return dc.read(**kwargs, **trim)[0] + + +def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): + """Build source records for live patches with synthetic identities.""" + token = next(_counter) + records = [] + for num, patch in enumerate(patches): + summary = dc.PatchSummary.from_patch(patch) + path = f"memory://catalog_{token}/{num}" + record = patch_record(summary) + records.append( + SourceRecord( + source_path=path, + source_format="memory", + format_version="", + patches=(record,), + ) + ) + resolver.register(path, record.source_patch_id, patch) + return records + + +class PatchCatalog: + """ + Query-composable metadata catalog over the spool index tables. + + Instances are lightweight views: `select` returns a new catalog + sharing the backend and resolver with composed predicates. Mutation + (`add`, `update`, `remove`) is only allowed on the root view. + """ + + def __init__( + self, + *, + backend=None, + backend_factory=None, + resolver: PatchResolver | None = None, + syncer=None, + queries: tuple[Query, ...] = (), + residuals: tuple[tuple[dict, bool, bool], ...] = (), + ): + self._backend = backend + self._backend_factory = backend_factory + self.resolver = resolver + self._syncer = syncer + self._queries = tuple(queries) + self._residuals = tuple(residuals) + self._df_cache: pd.DataFrame | None = None + + # --- construction ------------------------------------------------- + + @classmethod + def from_patches( + cls, patches: Sequence[dc.Patch] = (), engine: str = "sqlite" + ) -> PatchCatalog: + """ + Catalog over live patches. No backend work happens until the + first metadata operation. + """ + if engine not in _MEMORY_ENGINES: + msg = f"In-memory catalogs support {sorted(_MEMORY_ENGINES)}, not {engine}." + raise ValueError(msg) + resolver = LiveResolver() + pending = tuple(patches) + + def factory(): + backend = get_backend(":memory:", kind=engine) + if pending: + backend.write_sources(_live_records(pending, resolver)) + return backend + + return cls(backend_factory=factory, resolver=resolver) + + @classmethod + def from_directory( + cls, + path: str | Path, + engine: str = "sqlite", + index_path: str | Path | None = None, + ) -> PatchCatalog: + """Catalog over a directory of fiber files.""" + from dascore.io.index.indexer import DBDirectoryIndexer + + syncer = DBDirectoryIndexer(path, engine=engine, index_path=index_path) + return cls( + backend=syncer._backend, + resolver=FileResolver(root=syncer.path), + syncer=syncer, + ) + + # --- internals ------------------------------------------------------ + + @property + def backend(self): + """The index backend, bootstrapping lazily on first use.""" + if self._backend is None: + self._backend = self._backend_factory() + return self._backend + + def _view(self, queries, residuals) -> PatchCatalog: + out = PatchCatalog( + backend=self.backend, + resolver=self.resolver, + syncer=self._syncer, + queries=queries, + residuals=residuals, + ) + return out + + def _invalidate(self) -> None: + self._df_cache = None + + @property + def is_view(self) -> bool: + """True when this catalog carries selection state.""" + return bool(self._queries or self._residuals) + + def _require_root(self, operation: str) -> None: + if self.is_view: + msg = f"{operation} is only allowed on a root catalog, not a view." + raise InvalidSpoolQueryError(msg) + + # --- selection ------------------------------------------------------ + + def select( + self, + *, + _attrs: dict | None = None, + _coords: dict | None = None, + samples: bool = False, + relative: bool = False, + **kwargs, + ) -> PatchCatalog: + """ + Compose a selection; validation is eager, execution is lazy. + + samples=True selectors are patch-local (never index predicates); + relative=True bounds resolve against the current view's global + envelope, then behave as absolute ranges. + """ + if samples: + # names must be coords; validated against the index + unknown = set(kwargs) - self.coord_names() + if unknown or _attrs or _coords: + msg = ( + f"samples=True selections are coordinate-only; " + f"unknown coordinates: {sorted(unknown)}" + ) + raise InvalidSpoolQueryError(msg) + residual = (dict(kwargs), True, False) + return self._view(self._queries, (*self._residuals, residual)) + if relative: + kwargs = self._relative_to_absolute(kwargs) + query = resolve_query(self.backend, _attrs=_attrs, _coords=_coords, **kwargs) + # coord range predicates are re-applied exactly at patch load + residuals = self._residuals + if query.coords: + residuals = (*residuals, (dict(query.coords), False, False)) + return self._view((*self._queries, query), residuals) + + def _relative_to_absolute(self, kwargs: dict) -> dict: + """Resolve relative bounds against the view's global envelopes.""" + df = self.to_df() + out = {} + for name, value in kwargs.items(): + lo_col, hi_col = f"{name}_min", f"{name}_max" + if lo_col not in df.columns or df.empty: + msg = f"Cannot use relative select on unknown coord {name!r}." + raise InvalidSpoolQueryError(msg) + gmin, gmax = df[lo_col].min(), df[hi_col].max() + if not (isinstance(value, tuple) and len(value) == 2): + msg = f"relative=True requires (start, stop) ranges, got {value!r}." + raise InvalidSpoolQueryError(msg) + lo, hi = value + out[name] = ( + _offset(gmin, gmax, lo), + _offset(gmin, gmax, hi), + ) + return out + + # --- realization ------------------------------------------------------ + + def to_df(self) -> pd.DataFrame: + """The flat patch-row relation under the composed selection.""" + if self._df_cache is None: + self._df_cache = self.backend.query(list(self._queries) or None) + return self._df_cache + + def __len__(self) -> int: + return len(self.to_df()) + + def get_patch(self, index: int) -> dc.Patch: + """Materialize one patch: resolve, then exact two-stage trim.""" + row = self.to_df().iloc[index].to_dict() + trim_hint = {} + for coords, samples, _ in self._residuals: + if not samples: + trim_hint.update( + {k: v for k, v in coords.items() if isinstance(v, tuple)} + ) + patch = self.resolver.resolve(row, **trim_hint) + for coords, samples, relative in self._residuals: + usable = {k: v for k, v in coords.items() if k in patch.coords.coord_map} + if usable: + patch = patch.select(**usable, samples=samples, relative=relative) + return patch + + def __iter__(self): + for index in range(len(self)): + yield self.get_patch(index) + + # --- mutation (root only) ---------------------------------------------- + + def add(self, patches: Sequence[dc.Patch] | dc.Patch) -> PatchCatalog: + """Add live patches to the catalog.""" + self._require_root("add") + if not isinstance(self.resolver, LiveResolver): + msg = "add() currently supports in-memory catalogs only." + raise NotImplementedError(msg) + patches = [patches] if isinstance(patches, dc.Patch) else list(patches) + self.backend.write_sources(_live_records(patches, self.resolver)) + self._invalidate() + return self + + def update(self, progress: PROGRESS_LEVELS = "standard") -> PatchCatalog: + """Sync a directory-backed catalog with the filesystem.""" + self._require_root("update") + if self._syncer is not None: + self._syncer.update(progress=progress) + self._invalidate() + return self + + def remove(self, source_paths: Sequence[str], base_uri: str = "") -> PatchCatalog: + """Remove sources (and their patches) from the catalog.""" + self._require_root("remove") + self.backend.delete_sources(list(source_paths), base_uri=base_uri) + self._invalidate() + return self + + # --- introspection ------------------------------------------------------- + + def attr_names(self) -> set[str]: + """Attr names known to the index.""" + return self.backend.attr_names() + + def coord_names(self) -> set[str]: + """Coord names known to the index.""" + return self.backend.coord_names() + + def sources(self) -> pd.DataFrame: + """The sources table.""" + return self.backend.get_sources() + + def get_metadata(self) -> dict: + """Index-level metadata.""" + return self.backend.get_metadata() + + def close(self) -> None: + """Close the backend (root and all views share it).""" + if self._backend is not None: + self._backend.close() + + +def _offset(gmin, gmax, value): + """Resolve one relative bound against a global envelope.""" + if value is None or value is Ellipsis: + return None + if isinstance(gmin, pd.Timestamp) or isinstance(gmin, np.datetime64): + delta = dc.to_timedelta64(abs(value)) + return (gmin + delta) if value >= 0 else (gmax - delta) + return (gmin + value) if value >= 0 else (gmax + value) diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 1f07ce0a6..0d4896f82 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -13,6 +13,7 @@ import fnmatch import re +from collections.abc import Sequence from dataclasses import dataclass, field import numpy as np @@ -252,24 +253,26 @@ def build_coord_clause( def build_query_sql( - query: Query, + query: Query | Sequence[Query], dialect: BaseDialect, attr_meta: pd.DataFrame, ) -> tuple[str, list, dict[str, re.Pattern]]: """ - Build the flat-relation SELECT for a query. + Build the flat-relation SELECT for one or more AND-composed queries. Returns (sql, params, residuals) where residuals maps attr names to regex patterns that must be re-applied to the resulting dataframe. """ + queries = [query] if isinstance(query, Query) else list(query) where = _Where() residuals: dict[str, re.Pattern] = {} - for name, value in query.attrs.items(): - residual = build_attr_clause(where, dialect, attr_meta, name, value) - if residual is not None: - residuals[name] = residual - for name, value in query.coords.items(): - build_coord_clause(where, dialect, name, value) + for one in queries: + for name, value in one.attrs.items(): + residual = build_attr_clause(where, dialect, attr_meta, name, value) + if residual is not None: + residuals[name] = residual + for name, value in one.coords.items(): + build_coord_clause(where, dialect, name, value) # attr columns selected explicitly: `a.*` would duplicate patch_id and # engines disagree on how to dedupe result column names. attr_cols = "".join( diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py new file mode 100644 index 000000000..415f3397a --- /dev/null +++ b/tests/test_io/test_index/test_catalog.py @@ -0,0 +1,219 @@ +"""Tests for PatchCatalog: the unified spool metadata engine.""" + +from __future__ import annotations + +import numpy as np +import pytest + +import dascore as dc +from dascore.io.index import PatchCatalog +from dascore.io.index.query import InvalidSpoolQueryError + + +@pytest.fixture(scope="class") +def patches(): + """Patches from the random example spool.""" + return tuple(dc.get_example_spool("random_das")) + + +@pytest.fixture() +def live_catalog(patches): + """A catalog over live patches.""" + return PatchCatalog.from_patches(patches) + + +class TestLaziness: + """Catalog construction does no metadata work.""" + + def test_no_backend_until_needed(self, patches): + """from_patches must not bootstrap a backend.""" + catalog = PatchCatalog.from_patches(patches) + assert catalog._backend is None + + def test_first_len_bootstraps(self, live_catalog, patches): + """First metadata op creates the backend and ingests.""" + assert len(live_catalog) == len(patches) + assert live_catalog._backend is not None + + +class TestLiveRoundtrip: + """Live patches come back identical.""" + + def test_iteration_returns_same_patches(self, live_catalog, patches): + """Iterated patches are the registered objects (order: time).""" + out = list(live_catalog) + assert len(out) == len(patches) + starts = [p.get_coord("time").min() for p in out] + assert starts == sorted(starts) + assert {id(p) for p in out} == {id(p) for p in patches} + + def test_get_patch_by_index(self, live_catalog): + """Integer access works.""" + patch = live_catalog.get_patch(0) + assert isinstance(patch, dc.Patch) + + def test_add_more_patches(self, patches): + """add() ingests additional live patches.""" + catalog = PatchCatalog.from_patches(patches[:1]) + assert len(catalog) == 1 + catalog.add(patches[1]) + assert len(catalog) == 2 + + +class TestSelectComposition: + """select composes lazily with eager validation.""" + + def test_select_narrows(self, live_catalog, patches): + """A time range select excludes non-overlapping patches.""" + t0 = patches[0].get_coord("time").min() + t1 = patches[0].get_coord("time").max() + view = live_catalog.select(time=(t0, t1)) + assert len(view) == 1 + + def test_chained_selects_and(self, live_catalog, patches): + """Chained selects AND together.""" + t0 = patches[0].get_coord("time").min() + view = live_catalog.select(time=(t0, None)).select( + time=(None, t0 + np.timedelta64(1, "s")) + ) + assert len(view) == 1 + + def test_two_stage_exact_trim(self, live_catalog, patches): + """Coord range selects trim the loaded patch exactly.""" + t0 = patches[0].get_coord("time").min() + np.timedelta64(2, "s") + view = live_catalog.select(time=(t0, None)) + patch = view.get_patch(0) + assert patch.get_coord("time").min() >= t0 + + def test_unknown_name_raises_at_select(self, live_catalog): + """Validation is eager (#435).""" + with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): + live_catalog.select(bad_dim=(1, 2)) + + def test_no_sql_at_select(self, live_catalog): + """Selection composes without realizing the dataframe.""" + view = live_catalog.select(distance=(0, 10)) + assert view._df_cache is None + + def test_views_cannot_mutate(self, live_catalog, patches): + """Mutation only on the root.""" + view = live_catalog.select(distance=(0, 10)) + with pytest.raises(InvalidSpoolQueryError, match="root catalog"): + view.add(patches[0]) + + +class TestResidualSelects: + """samples/relative are patch-local (two-stage).""" + + def test_samples_never_excludes(self, live_catalog, patches): + """samples=True keeps every patch, trims on load (#447).""" + view = live_catalog.select(distance=(0, 10), samples=True) + assert len(view) == len(patches) + patch = view.get_patch(0) + # patch-level samples semantics are authoritative (0..9) + assert len(patch.get_coord("distance")) == 10 + + def test_samples_unknown_coord_raises(self, live_catalog): + """Samples selections validate coord names.""" + with pytest.raises(InvalidSpoolQueryError, match="coordinate-only"): + live_catalog.select(wavelength=(0, 10), samples=True) + + def test_relative_select(self, live_catalog): + """Relative bounds resolve against the global envelope (#362).""" + full = live_catalog.to_df() + span = (full["time_max"].max() - full["time_min"].min()).total_seconds() + view = live_catalog.select(time=(1, -1), relative=True) + patch = view.get_patch(0) + got_span = ( + patch.get_coord("time").max() - patch.get_coord("time").min() + ) / np.timedelta64(1, "s") + assert got_span <= span - 1 + + +class TestDirectoryCatalog: + """Directory-backed catalogs share machinery with live ones.""" + + @pytest.fixture(scope="class") + def spool_dir(self, tmp_path_factory): + """A directory of example files.""" + spool = dc.get_example_spool("random_das") + return dc.examples.spool_to_directory( + spool, path=tmp_path_factory.mktemp("catalog_dir") + ) + + def test_roundtrip(self, spool_dir): + """Directory catalog serves the same patches.""" + catalog = PatchCatalog.from_directory(spool_dir).update(progress=None) + patches = list(catalog) + assert len(patches) == 3 + assert all(isinstance(p, dc.Patch) for p in patches) + catalog.close() + + def test_select_and_trim(self, spool_dir): + """Two-stage select works through files too.""" + catalog = PatchCatalog.from_directory(spool_dir).update(progress=None) + df = catalog.to_df() + t0 = df["time_min"].min().to_datetime64() + np.timedelta64(2, "s") + view = catalog.select(time=(t0, None)) + patch = view.get_patch(0) + assert patch.get_coord("time").min() >= t0 + catalog.close() + + +class TestEngineGuard: + """Only in-memory-capable engines for live catalogs.""" + + def test_parquet_rejected(self): + """Parquet has no :memory: form.""" + with pytest.raises(ValueError, match="In-memory catalogs support"): + PatchCatalog.from_patches((), engine="parquet") + + +class TestCatalogEdges: + """Remaining branches: errors, passthroughs, offsets.""" + + def test_relative_on_unknown_coord_raises(self, live_catalog): + """Relative select against an absent coord errors clearly.""" + with pytest.raises(InvalidSpoolQueryError, match="unknown coord"): + live_catalog.select(wavelength=(1, -1), relative=True) + + def test_relative_requires_range(self, live_catalog): + """Relative selects take (start, stop) tuples only.""" + with pytest.raises(InvalidSpoolQueryError, match="requires"): + live_catalog.select(time=5, relative=True) + + def test_add_on_file_catalog_not_implemented(self, tmp_path, patches): + """add() is memory-only for now.""" + path = dc.examples.spool_to_directory( + dc.spool(list(patches)), path=tmp_path / "d" + ) + catalog = PatchCatalog.from_directory(path).update(progress=None) + with pytest.raises(NotImplementedError, match="in-memory"): + catalog.add(patches[0]) + catalog.close() + + def test_remove(self, patches): + """remove() drops sources by identity.""" + catalog = PatchCatalog.from_patches(patches) + target = catalog.sources()["source_path"].iloc[0] + catalog.remove([target]) + assert len(catalog) == len(patches) - 1 + + def test_introspection(self, live_catalog): + """Names, sources, and metadata pass through.""" + assert "time" in live_catalog.coord_names() + assert "tag" in live_catalog.attr_names() + assert len(live_catalog.sources()) == 3 + assert live_catalog.get_metadata()["what_is_this"] == "dascore_spool_index" + live_catalog.close() + + def test_open_relative_bound(self, live_catalog): + """Ellipsis/None bounds stay open through relative resolution.""" + view = live_catalog.select(time=(1, None), relative=True) + assert len(view) >= 1 + + def test_numeric_relative_offset(self, live_catalog): + """Relative selects work on numeric coords too.""" + view = live_catalog.select(distance=(5, -5), relative=True) + patch = view.get_patch(0) + assert patch.get_coord("distance").min() >= 5 From 374ee4c5ac5248eb725bd44eb010c06de09d0be1 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 5 Jul 2026 04:56:22 +0200 Subject: [PATCH 10/97] Run MemorySpool on the PatchCatalog Patch-list memory spools now build their managing dataframes from the catalog's flat relation and resolve patches through the shared LiveResolver, completing stage 1: one metadata engine for every spool type. Spools created from other spools/dataframes keep the legacy flat-dump path. Derived spools share the catalog (deepcopy-safe); catalogs pickle by rebuilding their backend from registered patches. Correctness fixes surfaced by the rewire: - ns-epoch integers never pass through float64 (which corrupts them by ~100 ns and breaks merge boundary arithmetic): exact masked conversion in _flatten, and duckdb/parquet fetch through arrow with integer_object_nulls (df() floats nullable BIGINTs). - numeric envelope columns coerce object-None to float NaN so sorting and chunking work on coords without steps. - all-relative time results serve timedelta envelopes so chunking on relative time keeps working (#553). - MemorySpool drops synthetic identity columns (path, file_format, file_version, source_patch_id) before chunk merge-compat comparisons, as DirectorySpool always has. - get_patch_names ignores memory:// synthetic paths and renders absent name-field columns as empty, so generated names (and DASDAE group names) are identical whichever metadata engine produced the frame. --- dascore/core/spool.py | 44 +++++++++++++++++++++++--- dascore/io/index/backend.py | 50 ++++++++++++++++++++++++------ dascore/io/index/catalog.py | 62 ++++++++++++++++++++++++++++--------- dascore/io/index/duck.py | 5 ++- dascore/io/index/parq.py | 5 ++- dascore/utils/patch.py | 19 +++++++----- 6 files changed, 146 insertions(+), 39 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 4de9cc822..7ef54e412 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -832,10 +832,15 @@ class MemorySpool(DataFrameSpool): from patches nearly free, which matters when reading many files. """ + # synthetic catalog identity columns must not join patch kwargs + # comparisons or chunk merge-compatibility checks + _drop_columns = ("patch", "path", "file_format", "file_version", "source_patch_id") + def __init__(self, data: PatchType | Sequence[PatchType] | None = None): super().__init__() self._patches: tuple[PatchType, ...] | None = None self._data = None + self._catalog = None if data is not None: if isinstance(data, dc.Patch): self._patches = (data,) @@ -851,11 +856,25 @@ def _get_df(self): data = self._patches if self._patches is not None else self._data if data is None: return None - df, source, instruction = self._get_dummy_dataframes(patches_to_df(data)) + if self._patches is not None: + # patch-list spools run on the index catalog: one metadata + # engine (and one select semantics) for every spool type. + current = self._get_catalog().to_df() + else: # spools/dataframes: legacy flat-dump path (patch column) + current = patches_to_df(data) + df, source, instruction = self._get_dummy_dataframes(current) self._source_df = source self._instruction_df = instruction return df + def _get_catalog(self): + """Get (lazily creating) the catalog for patch-list spools.""" + from dascore.io.index.catalog import PatchCatalog + + if self._catalog is None: + self._catalog = PatchCatalog.from_patches(self._patches) + return self._catalog + def _get_source_df(self): """Build the source df (happens as part of building current df).""" _ = self._df @@ -909,16 +928,26 @@ def __eq__(self, other) -> bool: def _eq_dict(self) -> dict: """Get a dict for equality checks, normalizing lazy state.""" + + def _strip_identity(df): + # synthetic per-catalog identities (memory:// paths, ids) are + # not content; equal spools must compare equal without them. + drop = ("path", "_patch_id", "source_patch_id", "file_format") + if df is None: + return df + return df.drop(columns=list(drop), errors="ignore") + out = dict(self.__dict__) # Build (if needed) and compare the dataframes; drop the inputs # they were built from, whose form can differ for equal contents. out["_cache"] = { - "_df": self._df, - "_source_df": self._source_df, - "_instruction_df": self._instruction_df, + "_df": _strip_identity(self._df), + "_source_df": _strip_identity(self._source_df), + "_instruction_df": _strip_identity(self._instruction_df), } out.pop("_patches", None) out.pop("_data", None) + out.pop("_catalog", None) return out def __rich__(self): @@ -938,7 +967,9 @@ def __rich__(self): def _load_patch(self, kwargs) -> Self: """Load the patch into memory.""" - return kwargs["patch"] + if (patch := kwargs.get("patch")) is not None: + return patch + return self._catalog.resolver.resolve(kwargs) @compose_docstring(doc=DataFrameSpool.new_from_df.__doc__) def new_from_df(self, *args, **kwargs): @@ -947,6 +978,9 @@ def new_from_df(self, *args, **kwargs): # The provided dataframes fully define the new spool; drop the # construction input so derived spools don't retain their parents. new._data = None + new._patches = None + # derived spools resolve patches through the shared catalog + new._catalog = self._catalog return new # Add specific implementation of concatenate patches. diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index ba47fbab1..f729a567a 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -37,6 +37,25 @@ _TIME_COLS = {"time_min": "datetime", "time_max": "datetime", "time_step": "timedelta"} +def _ns_to_time(series: pd.Series, flavor: str) -> pd.Series: + """ + Convert nullable ns-integer columns to datetime64/timedelta64 exactly. + + Never goes through float64: ns epochs exceed float64's 2**53 integer + range, and the resulting ~100 ns corruption breaks merge boundary + arithmetic downstream. + """ + mask = series.isna() + values = np.zeros(len(series), dtype="int64") + if (~mask).any(): + values[~mask.to_numpy()] = series[~mask].astype("int64").to_numpy() + dtype = "datetime64[ns]" if flavor == "datetime" else "timedelta64[ns]" + out = pd.Series(values.view(dtype), index=series.index) + if mask.any(): + out[mask] = pd.NaT + return out + + def adapt_params(params) -> list: """Convert numpy scalars (and NaN) to plain python for DB drivers.""" out = [] @@ -387,14 +406,15 @@ def query(self, query=None) -> pd.DataFrame: def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: """Post-process raw SQL output into the flat-relation contract.""" out = df.copy() - # structural time columns: ns ints -> numpy time types + # structural time columns: ns ints -> numpy time types (exactly) for col, flavor in _TIME_COLS.items(): if col in out: - as_int = out[col].astype("float64") # NaN-safe intermediate - if flavor == "datetime": - out[col] = pd.to_datetime(as_int, unit="ns") - else: - out[col] = pd.to_timedelta(as_int, unit="ns") + out[col] = _ns_to_time(out[col], flavor) + # numeric envelopes: engines return object columns when all-NULL; + # downstream sorting needs float64 with NaN, never object None. + for col in ("distance_min", "distance_max", "distance_step"): + if col in out: + out[col] = pd.to_numeric(out[col]) # typed attr columns -> original names (coalesce multi-kind attrs) for name in attr_meta["attr_name"].unique(): rows = attr_meta[attr_meta["attr_name"] == name] @@ -405,9 +425,9 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: continue col = out[row.column_name] if row.value_kind == "time": - col = pd.to_datetime(col.astype("float64"), unit="ns") + col = _ns_to_time(col, "datetime") elif row.value_kind == "dur": - col = pd.to_timedelta(col.astype("float64"), unit="ns") + col = _ns_to_time(col, "timedelta") elif row.value_kind == "bool": col = col.astype("boolean") if len(rows) > 1: @@ -498,12 +518,24 @@ def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: mins[row.patch_id], maxs[row.patch_id] = mn, mx steps[row.patch_id] = st out[f"_{name}_def_key"] = out["patch_id"].map(keys) - # time/distance envelopes already live on patches + kinds = set(group["value_kind"]) + # time/distance envelopes already live on patches... if name in ("time", "distance"): + col = f"{name}_min" + # ...but relative-time patches leave them NULL by design; + # when the whole result is relative, serve timedelta + # envelopes so chunking on relative time works (#553). + if col in out.columns and out[col].isnull().all() and mins: + out[f"{name}_min"] = out["patch_id"].map(mins) + out[f"{name}_max"] = out["patch_id"].map(maxs) + out[f"{name}_step"] = out["patch_id"].map(steps) continue out[f"{name}_min"] = out["patch_id"].map(mins) out[f"{name}_max"] = out["patch_id"].map(maxs) out[f"{name}_step"] = out["patch_id"].map(steps) + if kinds == {"num"}: # object-None -> float NaN for sorting + for suffix in ("_min", "_max", "_step"): + out[f"{name}{suffix}"] = pd.to_numeric(out[f"{name}{suffix}"]) return out # --- introspection ----------------------------------------------- diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 229a2a7aa..182cca831 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -119,16 +119,20 @@ def __init__( self, *, backend=None, - backend_factory=None, resolver: PatchResolver | None = None, syncer=None, + pending: tuple = (), + engine: str = "sqlite", queries: tuple[Query, ...] = (), residuals: tuple[tuple[dict, bool, bool], ...] = (), ): self._backend = backend - self._backend_factory = backend_factory self.resolver = resolver self._syncer = syncer + # live patches not yet ingested; kept (not a closure) so catalogs + # pickle and can rebuild their backend after unpickling. + self._pending = tuple(pending) + self._engine = engine self._queries = tuple(queries) self._residuals = tuple(residuals) self._df_cache: pd.DataFrame | None = None @@ -146,16 +150,7 @@ def from_patches( if engine not in _MEMORY_ENGINES: msg = f"In-memory catalogs support {sorted(_MEMORY_ENGINES)}, not {engine}." raise ValueError(msg) - resolver = LiveResolver() - pending = tuple(patches) - - def factory(): - backend = get_backend(":memory:", kind=engine) - if pending: - backend.write_sources(_live_records(pending, resolver)) - return backend - - return cls(backend_factory=factory, resolver=resolver) + return cls(resolver=LiveResolver(), pending=tuple(patches), engine=engine) @classmethod def from_directory( @@ -180,9 +175,27 @@ def from_directory( def backend(self): """The index backend, bootstrapping lazily on first use.""" if self._backend is None: - self._backend = self._backend_factory() + self._backend = get_backend(":memory:", kind=self._engine) + if self._pending: + self._backend.write_sources(_live_records(self._pending, self.resolver)) return self._backend + def __getstate__(self) -> dict: + """ + Pickle without the live DB connection. + + Live catalogs rebuild their backend from pending patches on next + use; the resolver registry (which pickled rows reference) rides + along unchanged, so already-realized views keep resolving. + """ + state = dict(self.__dict__) + state["_backend"] = None + if isinstance(self.resolver, LiveResolver) and not state["_pending"]: + # allow rebuilding the backend from the registered patches + registry = self.resolver._registry + state["_pending"] = tuple(registry.values()) + return state + def _view(self, queries, residuals) -> PatchCatalog: out = PatchCatalog( backend=self.backend, @@ -196,6 +209,15 @@ def _view(self, queries, residuals) -> PatchCatalog: def _invalidate(self) -> None: self._df_cache = None + def __deepcopy__(self, memo) -> PatchCatalog: + """ + Derived spools share the catalog (live registry + connection). + + DataFrameSpool copies spool state on select/chunk; catalog state + is read-shared, matching the single-writer model. + """ + return self + @property def is_view(self) -> bool: """True when this catalog carries selection state.""" @@ -267,9 +289,19 @@ def _relative_to_absolute(self, kwargs: dict) -> dict: # --- realization ------------------------------------------------------ def to_df(self) -> pd.DataFrame: - """The flat patch-row relation under the composed selection.""" + """ + The spool-facing flat patch-row relation under the selection. + + Unique-per-patch structural columns (patch_id and friends) are + hidden or renamed private so chunk merge-compatibility (which + compares all non-private columns) is not spuriously blocked. + """ if self._df_cache is None: - self._df_cache = self.backend.query(list(self._queries) or None) + df = self.backend.query(list(self._queries) or None) + df = df.drop( + columns=["n_dims", "sample_count_total", "shape"], errors="ignore" + ).rename(columns={"patch_id": "_patch_id"}) + self._df_cache = df return self._df_cache def __len__(self) -> int: diff --git a/dascore/io/index/duck.py b/dascore/io/index/duck.py index 289e86c9e..5bf65cc88 100644 --- a/dascore/io/index/duck.py +++ b/dascore/io/index/duck.py @@ -53,7 +53,10 @@ def _executemany(self, sql: str, seq_of_params) -> None: self._con.executemany(sql, rows) def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: - return self._con.execute(sql, adapt_params(params)).df() + # arrow keeps nullable BIGINT exact (df() would use float64, + # corrupting ns timestamps beyond float's 2**53 integer range) + reader = self._con.execute(sql, adapt_params(params)).arrow() + return reader.read_all().to_pandas(integer_object_nulls=True) def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: duck_bulk_insert(self._con, self.dialect, table, columns, rows) diff --git a/dascore/io/index/parq.py b/dascore/io/index/parq.py index e26e03b19..02f7a8936 100644 --- a/dascore/io/index/parq.py +++ b/dascore/io/index/parq.py @@ -62,7 +62,10 @@ def _executemany(self, sql: str, seq_of_params) -> None: self._con.executemany(sql, rows) def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: - return self._con.execute(sql, adapt_params(params)).df() + # arrow keeps nullable BIGINT exact (df() would use float64, + # corrupting ns timestamps beyond float's 2**53 integer range) + reader = self._con.execute(sql, adapt_params(params)).arrow() + return reader.read_all().to_pandas(integer_object_nulls=True) def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: from dascore.io.index.duck import duck_bulk_insert diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index 096a45d11..789057cd9 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -358,6 +358,7 @@ def patches_to_df( df["patch"] = None return df + @deprecate( info=( "merge_patches is deprecated. Use spool.chunk instead. " @@ -583,15 +584,17 @@ def _get_filename(path_ser, strip_extension): # Handle special cases. if "name" in col_set: return df["name"].astype(str) - if "path" in col_set and df["path"].astype(str).str.len().gt(0).any(): - return _get_filename(df["path"], strip_extension) - # Determine the requested fields and get the ones that are there. + path_ser = df["path"].astype(str) if "path" in col_set else None + if path_ser is not None: + # synthetic in-memory identities are not real file names + usable = path_ser.str.len().gt(0) & ~path_ser.str.startswith("memory://") + if usable.any(): + return _get_filename(df["path"], strip_extension) + # Determine the requested fields; absent columns render as empty so + # names don't depend on which metadata engine produced the dataframe. coord_fields = zip([f"{x}_min" for x in coords], [f"{x}_max" for x in coords]) - requested_fields = list(attrs) + list(*coord_fields) - current = set(df.columns) - fields = [x for x in requested_fields if x in current] - # Get a sub dataframe and convert any datetime things to strings. - sub = df[fields].pipe(_format_time_columns).fillna("").astype(str) + fields = list(attrs) + list(*coord_fields) + sub = df.reindex(columns=fields).pipe(_format_time_columns).fillna("").astype(str) out = f"{prefix}_{sep}" + sub[fields[0]].str.cat(sub[fields[1:]], sep=sep) return out From df628b66593c8e7248f00b74d359480059d372bf Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 5 Jul 2026 05:01:47 +0200 Subject: [PATCH 11/97] Implement selector-spec select for all spools (hard break for 0.2) DataFrameSpool.select now implements the selector semantics spec at the user-facing surface, for memory and directory spools alike: - Unknown names raise InvalidSpoolQueryError with the valid attribute and coordinate names (closes #435). Bare names resolve attrs-first, then coords. - _attrs / _coords dict kwargs disambiguate explicitly and validate against their own namespace only. - samples=True selections are coordinate-only and patch-local: they never exclude patches; the selection is recorded and applied to each patch as it loads, surviving chunk and other derived spools (closes the second failure mode of #447). - relative=True resolves range bounds against the spool's coordinate envelope, mirroring Patch.select semantics at spool scope (closes #362). InvalidSpoolQueryError moves to dascore.exceptions (avoiding a circular import); the index query module re-imports it from there. --- dascore/core/spool.py | 143 +++++++++++++++++++++- dascore/exceptions.py | 4 + dascore/io/index/query.py | 6 +- tests/test_core/test_spool_select_spec.py | 127 +++++++++++++++++++ 4 files changed, 273 insertions(+), 7 deletions(-) create mode 100644 tests/test_core/test_spool_select_spec.py diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 7ef54e412..fb0f1ef19 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -29,6 +29,7 @@ from dascore.exceptions import ( CoordMergeError, InvalidSpoolError, + InvalidSpoolQueryError, MissingPatchError, ParameterError, ) @@ -222,10 +223,24 @@ def select(self, **kwargs) -> Self: Sub-select parts of the spool. Can be used to specify dimension ranges, or unix-style matches - on string attributes. + on string attributes. Bare keyword names resolve against + attributes first, then coordinates; unknown names raise. Parameters ---------- + _attrs + A dict of attribute selections; names validate as attributes + only (disambiguates names shared with coordinates). + _coords + A dict of coordinate selections; names validate as + coordinates only. + samples + If True, selections are coordinate-only and given in sample + indices; they never exclude patches, but are applied to each + patch as it loads. + relative + If True, range bounds are relative to the spool's coordinate + envelope: positive from the start, negative from the end. **kwargs Specifies query. Can be of the form {dim_name=(start, stop)} or {attr_name=query}. @@ -403,6 +418,16 @@ def viz(self): raise AttributeError(msg) +def _relative_offset(gmin, gmax, value): + """Resolve one relative bound against a global envelope.""" + if value is None or value is Ellipsis: + return None + if isinstance(gmin, pd.Timestamp) or isinstance(gmin, np.datetime64): + delta = dc.to_timedelta64(abs(float(value))) + return (gmin + delta) if value >= 0 else (gmax - delta) + return (gmin + value) if value >= 0 else (gmax + value) + + class DataFrameSpool(BaseSpool): """An abstract class for spools whose contents are managed by a dataframe.""" @@ -419,6 +444,8 @@ class DataFrameSpool(BaseSpool): # attributes which effect merge groups for internal patches _group_columns = ("network", "station", "dims", "data_type", "tag") _drop_columns = ("patch",) + # patch-local selections (samples=True) applied as patches load + _post_selects: tuple = () def _get_df(self): """Function to get the current df.""" @@ -435,6 +462,7 @@ def __init__( self._cache = {} self._select_kwargs = {} if select_kwargs is None else select_kwargs self._merge_kwargs = {} if merge_kwargs is None else merge_kwargs + self._post_selects = () def _select_from_array(self, array) -> Self: """Create new spool with contents changed from array input.""" @@ -568,6 +596,15 @@ def _load_trimmed_patch(self, patch_kwargs, joined) -> dc.Patch: select_kwargs = self._select_kwargs if select_kwargs: patch = patch.select(**select_kwargs) + # patch-local selections (samples=True) recorded by spool.select + for post_kwargs, samples in self._post_selects: + usable = { + k: v + for k, v in post_kwargs.items() + if k in patch.dims or k in patch.coords.coord_map + } + if usable: + patch = patch.select(**usable, samples=samples) return patch def _merge_patches_streaming(self, joined, df_dict_list, merge_dim, samples): @@ -742,11 +779,113 @@ def new_from_df( new._select_kwargs.update(select_kwargs or {}) new._merge_kwargs = dict(self._merge_kwargs) new._merge_kwargs.update(merge_kwargs or {}) + new._post_selects = self._post_selects return new + def _select_namespaces(self) -> tuple[set[str], set[str]]: + """Return (attr names, coord names) selectable on this spool.""" + columns = set(self._df.columns) + coords = { + c.removesuffix("_min") + for c in columns + if c.endswith("_min") and f"{c.removesuffix('_min')}_max" in columns + } + skip = set(self._drop_columns) | {"coord_names", "dims"} + attrs = { + c + for c in columns + if not c.startswith("_") + and not c.endswith(("_min", "_max", "_step", "_units")) + and c not in skip + } + return attrs, coords + + def _resolve_select_kwargs(self, _attrs, _coords, kwargs) -> dict: + """ + Validate and merge select kwargs per the selector spec. + + Bare names resolve attrs-first, then coords; unknown names raise + (see #435). The _attrs/_coords namespaces validate against their + own side only. + """ + attrs, coords = self._select_namespaces() + out = {} + for name, value in (_attrs or {}).items(): + if name not in attrs: + msg = f"{name!r} is not an attribute of this spool." + raise InvalidSpoolQueryError(msg) + out[name] = value + for name, value in (_coords or {}).items(): + if name not in coords: + msg = f"{name!r} is not a coordinate of this spool." + raise InvalidSpoolQueryError(msg) + out[name] = value + for name, value in kwargs.items(): + if name in out: + msg = f"{name!r} given as both a bare kwarg and in _attrs/_coords." + raise InvalidSpoolQueryError(msg) + if name not in attrs and name not in coords: + msg = ( + f"{name!r} is neither an attribute nor a coordinate of " + f"this spool. Attributes: {sorted(attrs)}; " + f"coordinates: {sorted(coords)}." + ) + raise InvalidSpoolQueryError(msg) + out[name] = value + return out + + def _relative_select_kwargs(self, kwargs: dict) -> dict: + """Resolve relative bounds against the spool's global envelopes.""" + df = self._df + out = {} + for name, value in kwargs.items(): + lo_col, hi_col = f"{name}_min", f"{name}_max" + if lo_col not in df.columns: + msg = f"Cannot use relative select on {name!r}." + raise InvalidSpoolQueryError(msg) + if not (isinstance(value, tuple) and len(value) == 2): + msg = f"relative=True requires (start, stop) ranges, got {value!r}." + raise InvalidSpoolQueryError(msg) + gmin, gmax = df[lo_col].min(), df[hi_col].max() + lo, hi = value + out[name] = ( + _relative_offset(gmin, gmax, lo), + _relative_offset(gmin, gmax, hi), + ) + return out + @compose_docstring(doc=BaseSpool.select.__doc__) - def select(self, **kwargs) -> Self: + def select( + self, + *, + _attrs: dict | None = None, + _coords: dict | None = None, + samples: bool = False, + relative: bool = False, + **kwargs, + ) -> Self: """{doc}.""" + kwargs = self._resolve_select_kwargs(_attrs, _coords, kwargs) + if samples: + # sample indices are patch-local: never filter the spool, + # record the selection and apply it as patches load (#447). + _, coords = self._select_namespaces() + non_coords = set(kwargs) - coords + if non_coords: + msg = ( + f"samples=True selections are coordinate-only; got " + f"{sorted(non_coords)}." + ) + raise InvalidSpoolQueryError(msg) + new = self.new_from_df( + self._df, + source_df=self._source_df, + instruction_df=self._instruction_df, + ) + new._post_selects = (*self._post_selects, (kwargs, True)) + return new + if relative: + kwargs = self._relative_select_kwargs(kwargs) _, _, extra_kwargs = split_df_query(kwargs, self._df, ignore_bad_kwargs=True) filtered_df = adjust_segments(self._df, ignore_bad_kwargs=True, **kwargs) inst = adjust_segments( diff --git a/dascore/exceptions.py b/dascore/exceptions.py index d687136cf..a59f54ff3 100644 --- a/dascore/exceptions.py +++ b/dascore/exceptions.py @@ -31,6 +31,10 @@ class ParameterError(ValueError, DASCoreError): """Raised when something is wrong with an input parameter.""" +class InvalidSpoolQueryError(ParameterError): + """Raised when a spool query references unknown names or bad values.""" + + class PatchError(DASCoreError): """Parent class for more specific Patch Errors.""" diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 0d4896f82..ea5bec78f 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -19,17 +19,13 @@ import numpy as np import pandas as pd -from dascore.exceptions import ParameterError +from dascore.exceptions import InvalidSpoolQueryError from dascore.io.index.dialect import BaseDialect from dascore.io.index.ingest import typed_value _GLOB_CHARS = frozenset("*?[") -class InvalidSpoolQueryError(ParameterError): - """Raised when a spool query references unknown names or bad values.""" - - @dataclass(frozen=True) class Query: """ diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py new file mode 100644 index 000000000..f8ab99cac --- /dev/null +++ b/tests/test_core/test_spool_select_spec.py @@ -0,0 +1,127 @@ +""" +Selector-spec behavior of Spool.select (all spool types). + +These encode the hard-break semantics adopted for 0.2: unknown names +raise (#435), samples selections are patch-local (#447), relative ranges +work at spool level (#362), and _attrs/_coords disambiguate explicitly. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +import dascore as dc +from dascore.exceptions import InvalidSpoolQueryError + + +@pytest.fixture(scope="module", params=("memory", "directory")) +def spool(request, tmp_path_factory): + """The same patches served by each spool type.""" + base = dc.get_example_spool("random_das") + if request.param == "memory": + return dc.spool(list(base)) + path = dc.examples.spool_to_directory( + base, path=tmp_path_factory.mktemp("select_spec") + ) + return dc.spool(path).update(progress=None) + + +class TestUnknownNames: + """Unknown selector names raise eagerly (#435).""" + + def test_unknown_kwarg_raises(self, spool): + """A name that is neither attr nor coord errors.""" + with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): + spool.select(bad_dimension=(1, 2)) + + def test_unknown_attr_namespace_raises(self, spool): + """_attrs validates against attributes only.""" + with pytest.raises(InvalidSpoolQueryError, match="not an attribute"): + spool.select(_attrs={"time": (None, None)}) + + def test_unknown_coord_namespace_raises(self, spool): + """_coords validates against coordinates only.""" + with pytest.raises(InvalidSpoolQueryError, match="not a coordinate"): + spool.select(_coords={"tag": "random"}) + + def test_double_specification_raises(self, spool): + """A name can't be bare and namespaced at once.""" + with pytest.raises(InvalidSpoolQueryError, match="both"): + spool.select(tag="random", _attrs={"tag": "random"}) + + +class TestNamespaces: + """Explicit namespaces select as their bare equivalents.""" + + def test_attr_namespace(self, spool): + """_attrs behaves like the bare attr kwarg.""" + assert len(spool.select(_attrs={"tag": "random"})) == len(spool) + + def test_coord_namespace(self, spool): + """_coords behaves like the bare coord kwarg.""" + df = spool.get_contents() + t0 = df["time_min"].min() + out = spool.select(_coords={"time": (t0, t0 + np.timedelta64(2, "s"))}) + assert len(out) == 1 + + +class TestSamples: + """samples=True never excludes patches; trims on load (#447).""" + + def test_length_preserved(self, spool): + """The spool keeps every patch.""" + out = spool.select(distance=(0, 10), samples=True) + assert len(out) == len(spool) + + def test_patch_trimmed_on_load(self, spool): + """Loaded patches carry the sample trim.""" + out = spool.select(distance=(0, 10), samples=True) + patch = out[0] + assert len(patch.get_coord("distance")) == 10 + + def test_survives_chunk(self, spool): + """Post-selects propagate through derived spools.""" + out = spool.select(distance=(0, 10), samples=True).chunk(time=None) + patch = out[0] + assert len(patch.get_coord("distance")) == 10 + + def test_non_coord_raises(self, spool): + """Samples selections must name coordinates.""" + with pytest.raises(InvalidSpoolQueryError, match="coordinate-only"): + spool.select(tag="random", samples=True) + + +class TestRelative: + """relative=True resolves against the spool envelope (#362).""" + + def test_trims_both_ends(self, spool): + """One second off each end of the spool.""" + df = spool.get_contents() + gmin = df["time_min"].min() + gmax = df["time_max"].max() + out = spool.select(time=(1, -1), relative=True) + merged = out.chunk(time=None)[0] + time = merged.get_coord("time") + assert time.min() >= np.datetime64(gmin) + np.timedelta64(1, "s") + assert time.max() <= np.datetime64(gmax) - np.timedelta64(1, "s") + + def test_requires_range(self, spool): + """Scalars are rejected with a clear message.""" + with pytest.raises(InvalidSpoolQueryError, match="requires"): + spool.select(time=5, relative=True) + + +class TestExistingBehaviorKept: + """The conventional selections still work.""" + + def test_attr_glob(self, spool): + """Unix-style attr matching.""" + assert len(spool.select(tag="rand*")) == len(spool) + + def test_time_range_narrows(self, spool): + """Plain time range selection.""" + df = spool.get_contents() + t0 = df["time_min"].min() + out = spool.select(time=(t0, t0 + np.timedelta64(2, "s"))) + assert len(out) == 1 From 1eab358d6b26898457e4d0b5281d1802787f45be Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 5 Jul 2026 05:03:57 +0200 Subject: [PATCH 12/97] Simplify: shared relative_offset helper, dead code removal One relative-bound resolver (query.relative_offset) serves catalog and spool selects; ingest uses dataclasses.replace; select drops its split_df_query call, which strict name validation made dead (every validated name matches a dataframe column or coord range, so the extra kwargs were always empty). --- dascore/core/spool.py | 19 ++++--------------- dascore/io/index/catalog.py | 21 +++++++-------------- dascore/io/index/ingest.py | 4 ++-- dascore/io/index/query.py | 18 ++++++++++++++++++ 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index fb0f1ef19..6d859044d 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -63,7 +63,6 @@ filter_df, get_column_names_from_dim, get_dim_names_from_columns, - split_df_query, ) T = TypeVar("T") @@ -418,16 +417,6 @@ def viz(self): raise AttributeError(msg) -def _relative_offset(gmin, gmax, value): - """Resolve one relative bound against a global envelope.""" - if value is None or value is Ellipsis: - return None - if isinstance(gmin, pd.Timestamp) or isinstance(gmin, np.datetime64): - delta = dc.to_timedelta64(abs(float(value))) - return (gmin + delta) if value >= 0 else (gmax - delta) - return (gmin + value) if value >= 0 else (gmax + value) - - class DataFrameSpool(BaseSpool): """An abstract class for spools whose contents are managed by a dataframe.""" @@ -848,9 +837,11 @@ def _relative_select_kwargs(self, kwargs: dict) -> dict: raise InvalidSpoolQueryError(msg) gmin, gmax = df[lo_col].min(), df[hi_col].max() lo, hi = value + from dascore.io.index.query import relative_offset + out[name] = ( - _relative_offset(gmin, gmax, lo), - _relative_offset(gmin, gmax, hi), + relative_offset(gmin, gmax, lo), + relative_offset(gmin, gmax, hi), ) return out @@ -886,7 +877,6 @@ def select( return new if relative: kwargs = self._relative_select_kwargs(kwargs) - _, _, extra_kwargs = split_df_query(kwargs, self._df, ignore_bad_kwargs=True) filtered_df = adjust_segments(self._df, ignore_bad_kwargs=True, **kwargs) inst = adjust_segments( self._instruction_df, @@ -901,7 +891,6 @@ def select( # Drop rows that are no longer needed. source_df=source, instruction_df=inst, - select_kwargs=extra_kwargs, ) return out diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 182cca831..bdffbb609 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -21,14 +21,17 @@ from collections.abc import Mapping, Sequence from pathlib import Path -import numpy as np import pandas as pd import dascore as dc from dascore.constants import PROGRESS_LEVELS from dascore.io.index.backend import get_backend, resolve_query from dascore.io.index.ingest import SourceRecord, patch_record -from dascore.io.index.query import InvalidSpoolQueryError, Query +from dascore.io.index.query import ( + InvalidSpoolQueryError, + Query, + relative_offset, +) _MEMORY_ENGINES = frozenset({"sqlite", "duckdb"}) _counter = itertools.count() @@ -281,8 +284,8 @@ def _relative_to_absolute(self, kwargs: dict) -> dict: raise InvalidSpoolQueryError(msg) lo, hi = value out[name] = ( - _offset(gmin, gmax, lo), - _offset(gmin, gmax, hi), + relative_offset(gmin, gmax, lo), + relative_offset(gmin, gmax, hi), ) return out @@ -377,13 +380,3 @@ def close(self) -> None: """Close the backend (root and all views share it).""" if self._backend is not None: self._backend.close() - - -def _offset(gmin, gmax, value): - """Resolve one relative bound against a global envelope.""" - if value is None or value is Ellipsis: - return None - if isinstance(gmin, pd.Timestamp) or isinstance(gmin, np.datetime64): - delta = dc.to_timedelta64(abs(value)) - return (gmin + delta) if value >= 0 else (gmax - delta) - return (gmin + value) if value >= 0 else (gmax + value) diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 0f8977e83..5d774bc97 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -12,7 +12,7 @@ import hashlib import re import warnings -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from functools import cache import numpy as np @@ -344,7 +344,7 @@ def summaries_to_records( record = patch_record(summary) if record.source_patch_id == "" and len(group) > 1: # positional identity within the source, per design doc - record = PatchRecord(**{**record.__dict__, "source_patch_id": str(num)}) + record = replace(record, source_patch_id=str(num)) patches.append(record) store_path = path root = base_uri or relative_to diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index ea5bec78f..98acb83c9 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -297,6 +297,24 @@ def apply_residuals(df: pd.DataFrame, residuals: dict[str, re.Pattern]) -> pd.Da return df +def relative_offset(gmin, gmax, value): + """ + Resolve one relative bound against a global [gmin, gmax] envelope. + + Positive offsets measure from the start, negative from the end; + None/Ellipsis bounds stay open. Datetime envelopes take numeric + seconds offsets. + """ + import dascore as dc + + if value is None or value is Ellipsis: + return None + if isinstance(gmin, pd.Timestamp) or isinstance(gmin, np.datetime64): + delta = dc.to_timedelta64(abs(float(value))) + return (gmin + delta) if value >= 0 else (gmax - delta) + return (gmin + value) if value >= 0 else (gmax + value) + + def glob_match(value, pattern: str) -> bool: """Reference glob semantics (used by pandas fallbacks and tests).""" return isinstance(value, str) and fnmatch.fnmatch(value, pattern) From 9df68b6a1d390d4bc05425d950bedd369fef4915 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 5 Jul 2026 06:56:19 +0200 Subject: [PATCH 13/97] Reuse cached patch summaries in catalog ingest patch.summary is a cached_property; building fresh PatchSummary objects in _live_records discarded fingerprints and summaries the patch already had. Reusing them makes catalog ingest of previously-summarized patches ~3x faster (first get_contents at 300 patches: 208 -> 75 ms) and turns the remaining summary cost into once-per-patch-lifetime instead of once-per-spool. --- dascore/io/index/catalog.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index bdffbb609..d50237b35 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -94,7 +94,9 @@ def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): token = next(_counter) records = [] for num, patch in enumerate(patches): - summary = dc.PatchSummary.from_patch(patch) + # patch.summary is a cached_property: reuse fingerprints and + # summaries the patch already computed instead of rebuilding. + summary = patch.summary path = f"memory://catalog_{token}/{num}" record = patch_record(summary) records.append( From 8929f7129113d8ebe28340a1acb1d83e80026479 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 06:32:53 +0200 Subject: [PATCH 14/97] Remove DuckDB/Parquet index backends; fix selection and resolver bugs SQLite is now the only index engine: the experimental duckdb/parquet backends and their engine selection parameters are gone. Review of the catalog path also surfaced bugs and regressions, fixed here: - DirectorySpool select_kwargs restrict contents again, persist across select/update, and no longer crash on attribute-valued entries. - NULL-unit coordinate definitions stay query candidates for quantity selectors (candidacy contract: no false negatives). - An attr observed with dimensionally incompatible units across files skips the incompatible values with a warning instead of failing the whole index update. - FileResolver resolves source identity from the loaded spool: no more double full-file reads for multi-patch files and no wrong-patch binding when trims shrink positional-id reads; files trimmed to nothing raise MissingPatchError again so iteration skips them (#583). - Chunk/select instruction trims are pushed into readers again. - backend.query() fetches coordinate metadata only for the queried coords instead of a whole-relation DISTINCT scan on every query. - Cleanups: one initial-sync chokepoint (indexer.ensure_updated), slice range-form support, unshadowed error names, dialect quoting reuse, merged adjust_segments passes, and duplicate validation removal. --- dascore/clients/dirspool.py | 90 ++----- dascore/core/spool.py | 77 +++++- dascore/exceptions.py | 6 +- dascore/io/core.py | 25 +- dascore/io/index/__init__.py | 7 +- dascore/io/index/backend.py | 243 ++++++++++++++---- dascore/io/index/catalog.py | 139 +++++++--- dascore/io/index/dialect.py | 30 +-- dascore/io/index/duck.py | 75 ------ dascore/io/index/indexer.py | 28 +- dascore/io/index/ingest.py | 7 +- dascore/io/index/lite.py | 21 +- dascore/io/index/parq.py | 118 --------- dascore/io/index/query.py | 190 +++++++++++--- dascore/io/index/schema.py | 60 ++++- dascore/io/indexer.py | 10 + docs/changelog.qmd | 4 + docs/notes/notes.qmd | 2 + docs/notes/spool_index.qmd | 35 +++ docs/notes/spool_selection.qmd | 15 ++ docs/tutorial/file_io.qmd | 6 +- pyproject.toml | 5 - scripts/_templates/_quarto.yml | 6 + tests/test_clients/test_dirspool.py | 134 ++++++++-- tests/test_core/test_spool_select_spec.py | 29 +++ tests/test_io/test_index/test_catalog.py | 18 +- tests/test_io/test_index/test_db_dirspool.py | 26 +- .../test_index/test_heterogeneity_stress.py | 15 +- .../test_io/test_index/test_index_contract.py | 44 ++-- .../test_index/test_index_edge_cases.py | 174 +++++++++---- tests/test_io/test_index/test_schema.py | 72 ++++++ tests/test_io/test_indexer.py | 7 - 32 files changed, 1155 insertions(+), 563 deletions(-) delete mode 100644 dascore/io/index/duck.py delete mode 100644 dascore/io/index/parq.py create mode 100644 docs/notes/spool_index.qmd create mode 100644 docs/notes/spool_selection.qmd create mode 100644 tests/test_io/test_index/test_schema.py diff --git a/dascore/clients/dirspool.py b/dascore/clients/dirspool.py index cbae28e50..2b8bc3515 100644 --- a/dascore/clients/dirspool.py +++ b/dascore/clients/dirspool.py @@ -13,12 +13,10 @@ from rich.text import Text from typing_extensions import Self -import dascore as dc from dascore.compat import UPath from dascore.constants import PROGRESS_LEVELS -from dascore.core.spool import BaseSpool, DataFrameSpool, MemorySpool -from dascore.exceptions import MissingPatchError -from dascore.io.index.indexer import DBDirectoryIndexer +from dascore.core.spool import BaseSpool, DataFrameSpool +from dascore.io.index.catalog import FileResolver, PatchCatalog from dascore.io.indexer import AbstractIndexer from dascore.utils.docs import compose_docstring from dascore.utils.pd import adjust_segments @@ -43,10 +41,6 @@ class DirectorySpool(DataFrameSpool): will save time in indexing. select_kwargs Dict of keyword arguments to restrict output contents. - index_engine - The database backend for the index: "sqlite" (default, no extra - dependencies), "duckdb", or "parquet" (both require duckdb). See - the spool index design discussion (#648). """ _drop_columns = ("file_format", "file_version", "path", "source_patch_id") @@ -59,7 +53,6 @@ def __init__( preferred_format: str | None = None, select_kwargs: dict | None = None, merge_kwargs: dict | None = None, - index_engine: str = "sqlite", ): super().__init__(select_kwargs=select_kwargs, merge_kwargs=merge_kwargs) # Init file spool from another file spool @@ -69,11 +62,18 @@ def __init__( # Init file spool from indexer elif isinstance(base_path, AbstractIndexer): self.indexer = base_path + self._catalog = PatchCatalog( + backend=self.indexer._backend, + resolver=FileResolver(root=self.indexer.path), + syncer=self.indexer, + ) elif isinstance(base_path, Path | str | UPath): - self.indexer = DBDirectoryIndexer( - base_path, engine=index_engine, index_path=index_path + self._catalog = PatchCatalog.from_directory( + base_path, index_path=index_path ) + self.indexer = self._catalog._syncer assert hasattr(self, "indexer"), "indexer not set." + self._catalog_native = True self._preferred_format = preferred_format def __rich__(self): @@ -87,10 +87,12 @@ def __rich__(self): def _get_df(self): """Get the dataframe of current contents.""" - out = adjust_segments( + if not self._select_kwargs: + return self._source_df + # constructor select_kwargs restrict contents (docstring contract) + return adjust_segments( self._source_df, ignore_bad_kwargs=True, **self._select_kwargs ) - return out def _get_instruction_df(self): """Return instruction df on how to get from source_df to df.""" @@ -99,7 +101,7 @@ def _get_instruction_df(self): def _get_source_df(self): """Return a dataframe of sources in spool.""" - return self.indexer(**self._select_kwargs).reset_index(drop=True) + return self._catalog.to_df().reset_index(drop=True) @property def spool_path(self): @@ -114,12 +116,8 @@ def get_contents(self) -> pd.DataFrame: @compose_docstring(doc=BaseSpool.update.__doc__) def update(self, progress: PROGRESS_LEVELS = "standard") -> Self: """{doc}.""" - out = self.__class__( - base_path=self.indexer.update(progress=progress), - preferred_format=self._preferred_format, - select_kwargs=self._select_kwargs, - ) - return out + self._catalog.update(progress=progress) + return self._new_from_catalog(self._catalog) def _df_to_dict_list(self, df): """ @@ -134,51 +132,15 @@ def _df_to_dict_list(self, df): def _load_patch(self, kwargs) -> Self: """Given a row from the managed dataframe, return a patch.""" - final_kwargs = dict(kwargs) - final_kwargs.update(self._select_kwargs) - patches = self._read_patches(final_kwargs) - if patches is None: # fast path doesn't apply, use generic read. - return self._read_and_resolve_patch(final_kwargs) - if not patches: - # Iteration skips these with a warning, see #583. - msg = ( - f"No patch in {final_kwargs.get('path')} matches the " - f"requested range; it may have been trimmed to nothing." - ) - raise MissingPatchError(msg) - return patches[0] - - def _read_patches(self, kwargs) -> list[dc.Patch] | None: - """ - Read patches directly through the file's FiberIO. - - This skips the format detection of dc.read and the spool indexing - machinery applied to its output, which add up when loading many - files. Returns None when the fast path can't be safely used. - """ - fmt, version = kwargs.get("file_format"), kwargs.get("file_version") - if not fmt or not version: - # Without a concrete version get_fiberio would return the newest - # reader; let dc.read detect the file's actual version instead. - return None - fiber_io = dc.io.FiberIO.manager.get_fiberio(format=fmt, version=version) - # Only apply select kwargs when the source patch is trimmed by the - # instruction df or the spool itself; otherwise the whole file is - # wanted and selection is wasted work. + # Push trims into the reader only when the instruction row narrows + # the source (chunk/select) or constructor select_kwargs restrict + # it; otherwise the whole file is wanted and selection is wasted. + trim = {} if kwargs.get("_modified") or self._select_kwargs: - select = { + merged = {**kwargs, **self._select_kwargs} + trim = { k: v - for k, v in kwargs.items() + for k, v in merged.items() if k not in self._drop_columns and not k.startswith("_") } - else: - select = {} - spool = fiber_io.read(kwargs["path"], **select) - if not isinstance(spool, MemorySpool): - return None - patches = list(spool) - # A multi-patch file is ambiguous: the row refers to one specific - # patch. Let the generic path resolve source_patch_id. - if len(patches) > 1: - return None - return patches + return self._catalog.resolve_row(kwargs, extra_trim=trim) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 6d859044d..4dda00712 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -435,6 +435,9 @@ class DataFrameSpool(BaseSpool): _drop_columns = ("patch",) # patch-local selections (samples=True) applied as patches load _post_selects: tuple = () + # True while rows directly represent a PatchCatalog query. Operations + # which restructure/order rows switch back to the dataframe machinery. + _catalog_native = False def _get_df(self): """Function to get the current df.""" @@ -575,14 +578,14 @@ def _load_trimmed_patch(self, patch_kwargs, joined) -> dc.Patch: # If the limits of the source patch were not modified, we can just # use the select kwargs. This is important for missing coordinates # (NaN values) to not get trimmed out. - if kwargs.get("_modified"): - select_kwargs = { - i: v - for i, v in kwargs.items() - if i in patch.dims or i in patch.coords.coord_map - } - else: - select_kwargs = self._select_kwargs + source_kwargs = kwargs if kwargs.get("_modified") else self._select_kwargs + # attr-style entries (e.g. constructor select_kwargs) filter rows + # above; only coordinate entries are valid patch selections. + select_kwargs = { + i: v + for i, v in source_kwargs.items() + if i in patch.dims or i in patch.coords.coord_map + } if select_kwargs: patch = patch.select(**select_kwargs) # patch-local selections (samples=True) recorded by spool.select @@ -707,9 +710,12 @@ def _read_and_resolve_patch(self, final_kwargs) -> dc.Patch: source_patch_id = final_kwargs.get("source_patch_id", "") spool = dc.read(**final_kwargs) # Some readers consume source_patch_id internally and return the one - # matching patch without preserving that reload metadata on the patch. + # matching patch without preserving that reload metadata on the + # patch. Only trust that when it doesn't claim a different identity. if source_patch_id and len(spool) == 1: - return spool[0] + found = str(spool[0].attrs.get("_source_patch_id", "") or "") + if found in ("", str(source_patch_id)): + return spool[0] return _select_patch_from_spool(spool, source_patch_id=source_patch_id) @compose_docstring(doc=BaseSpool.chunk.__doc__) @@ -769,6 +775,9 @@ def new_from_df( new._merge_kwargs = dict(self._merge_kwargs) new._merge_kwargs.update(merge_kwargs or {}) new._post_selects = self._post_selects + # Dataframe-producing operations (chunk, sort, slice) define their + # own row/instruction plan and must not bypass it through the catalog. + new._catalog_native = False return new def _select_namespaces(self) -> tuple[set[str], set[str]]: @@ -856,6 +865,15 @@ def select( **kwargs, ) -> Self: """{doc}.""" + if self._catalog_native: + catalog = self._catalog.select( + _attrs=_attrs, + _coords=_coords, + samples=samples, + relative=relative, + **kwargs, + ) + return self._new_from_catalog(catalog) kwargs = self._resolve_select_kwargs(_attrs, _coords, kwargs) if samples: # sample indices are patch-local: never filter the spool, @@ -876,7 +894,14 @@ def select( new._post_selects = (*self._post_selects, (kwargs, True)) return new if relative: - kwargs = self._relative_select_kwargs(kwargs) + _, coords = self._select_namespaces() + coord_kwargs = { + key: value for key, value in kwargs.items() if key in coords + } + attr_kwargs = { + key: value for key, value in kwargs.items() if key not in coords + } + kwargs = {**attr_kwargs, **self._relative_select_kwargs(coord_kwargs)} filtered_df = adjust_segments(self._df, ignore_bad_kwargs=True, **kwargs) inst = adjust_segments( self._instruction_df, @@ -894,6 +919,18 @@ def select( ) return out + def _new_from_catalog(self, catalog) -> Self: + """Create a lazy catalog-native view of this spool.""" + new = self.__class__(self) + new._catalog = catalog + new._catalog_native = True + new._cache = {} + # selection composed into the catalog is dropped, but constructor + # select_kwargs (DirectorySpool contract) persist across views. + new._select_kwargs = dict(self._select_kwargs) + new._post_selects = () + return new + @compose_docstring(doc=BaseSpool.sort.__doc__) def sort(self, attribute) -> Self: """{doc}.""" @@ -981,6 +1018,12 @@ def __init__(self, data: PatchType | Sequence[PatchType] | None = None): def _get_df(self): """Build the managing dataframes from the input patches.""" + if self._catalog is not None and self._catalog_native: + current = self._catalog.to_df() + df, source, instruction = self._get_dummy_dataframes(current) + self._source_df = source + self._instruction_df = instruction + return df data = self._patches if self._patches is not None else self._data if data is None: return None @@ -1001,6 +1044,7 @@ def _get_catalog(self): if self._catalog is None: self._catalog = PatchCatalog.from_patches(self._patches) + self._catalog_native = True return self._catalog def _get_source_df(self): @@ -1076,6 +1120,7 @@ def _strip_identity(df): out.pop("_patches", None) out.pop("_data", None) out.pop("_catalog", None) + out.pop("_catalog_native", None) return out def __rich__(self): @@ -1097,7 +1142,14 @@ def _load_patch(self, kwargs) -> Self: """Load the patch into memory.""" if (patch := kwargs.get("patch")) is not None: return patch - return self._catalog.resolver.resolve(kwargs) + return self._catalog.resolve_row(kwargs) + + def _new_from_catalog(self, catalog) -> Self: + """Create a lazy memory-spool view backed by a catalog query.""" + new = self.__class__() + new._catalog = catalog + new._catalog_native = True + return new @compose_docstring(doc=DataFrameSpool.new_from_df.__doc__) def new_from_df(self, *args, **kwargs): @@ -1109,6 +1161,7 @@ def new_from_df(self, *args, **kwargs): new._patches = None # derived spools resolve patches through the shared catalog new._catalog = self._catalog + new._catalog_native = False return new # Add specific implementation of concatenate patches. diff --git a/dascore/exceptions.py b/dascore/exceptions.py index a59f54ff3..ea32e5dbf 100644 --- a/dascore/exceptions.py +++ b/dascore/exceptions.py @@ -109,7 +109,11 @@ class InvalidFileHandlerError(TypeError, DASCoreError): """Raised when a writable file handler is requested from a read handle.""" -class InvalidIndexVersionError(ValueError, DASCoreError): +class InvalidIndexError(ValueError, DASCoreError): + """Raised when a persisted index is invalid or incompatible.""" + + +class InvalidIndexVersionError(InvalidIndexError): """Raised when a version mismatch occurs in index.""" diff --git a/dascore/io/core.py b/dascore/io/core.py index 6b9c08785..777f54d77 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -38,6 +38,7 @@ InvalidFiberFileError, InvalidFiberIOError, MissingOptionalDependencyError, + MissingPatchError, ParameterError, PatchAttributeError, RemoteCacheError, @@ -257,23 +258,31 @@ def _patch_to_scan_payload(patch: dc.Patch) -> ScanPayload: def _select_patch_from_spool(spool, source_patch_id: object = "") -> dc.Patch: """Select one loaded patch from a spool using source identity.""" - - def _matches_patch_name(patch: dc.Patch, source_id: str) -> bool: - """Return True when a generated patch name matches the source id.""" - return patch.get_patch_name() == source_id - if len(spool) == 0: - msg = "index of [0] is out of bounds for spool." - raise IndexError(msg) + # Iteration skips these with a warning, see #583. + msg = ( + "No patch remained after applying load filters; the requested " + "range may have trimmed it to nothing." + ) + raise MissingPatchError(msg) if source_patch_id not in (None, ""): source_patch_id = str(source_patch_id) + # Native source ids are preserved on patch attrs by their readers. + matches = [ + patch + for patch in spool + if str(patch.attrs.get("_source_patch_id", "") or "") == source_patch_id + ] + if len(matches) == 1: + return matches[0] + # Synthesized ids are positional within the full source read. try: index = int(source_patch_id) except (TypeError, ValueError): index = None if index is not None and 0 <= index < len(spool): return spool[index] - if len(spool) == 1 and _matches_patch_name(spool[0], source_patch_id): + if len(spool) == 1 and spool[0].get_patch_name() == source_patch_id: return spool[0] msg = "Patch could not be uniquely resolved after applying load filters." raise PatchAttributeError(msg) diff --git a/dascore/io/index/__init__.py b/dascore/io/index/__init__.py index e0434969c..23f51ee03 100644 --- a/dascore/io/index/__init__.py +++ b/dascore/io/index/__init__.py @@ -1,10 +1,9 @@ """ -Backend-agnostic spool index package. +SQLite spool index package. Provides a normalized, summary-only index of patch metadata (sources, -patches, attrs, coords) with interchangeable storage backends. See -`.scratch/spool_index_design.md` on the spool-index-backend branch and -GitHub discussion #648 for the design. +patches, attrs, and coordinates. PatchCatalog is the spool-facing metadata +engine; the remaining exports support its internal index implementation. """ from __future__ import annotations diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index f729a567a..9d03e032e 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -1,16 +1,16 @@ """ -Abstract index backend and the shared SQL implementation. +Index backend interface and SQLite SQL implementation. -Backends persist the six-table schema and answer flat-relation queries. -All engine differences live in `dialect.py` plus a handful of hooks; the -write/query logic here is shared so the contract test suite exercises -identical semantics on every backend. +The backend persists the seven-table schema and answers flat-relation queries. +Storage hooks remain separate from the write/query logic so the index contract +has a clear boundary. """ from __future__ import annotations import abc import time +import warnings from contextlib import suppress from pathlib import Path @@ -18,9 +18,15 @@ import pandas as pd import dascore as dc +from dascore.exceptions import InvalidIndexError, InvalidIndexVersionError, UnitError from dascore.io.index.dialect import BaseDialect from dascore.io.index.ingest import SourceRecord, attr_column_name -from dascore.io.index.query import Query, apply_residuals, build_query_sql +from dascore.io.index.query import ( + Query, + apply_residuals, + build_query_sql, + normalize_range_forms, +) from dascore.io.index.schema import ( COORD_DEFS, INDEX_VERSION, @@ -29,9 +35,11 @@ PATCH_COORDS, PATCHES, SOURCES, + TABLE_CONSTRAINTS, TABLES, WHAT_IS_THIS, ) +from dascore.units import convert_units # Structural columns whose ns-integer storage maps to pandas time types. _TIME_COLS = {"time_min": "datetime", "time_max": "datetime", "time_step": "timedelta"} @@ -138,52 +146,182 @@ def _commit(self) -> None: def _rollback(self) -> None: """Roll back the open transaction.""" + @abc.abstractmethod + def _existing_tables(self) -> set[str]: + """Return persisted user table names.""" + + @abc.abstractmethod + def _table_columns(self, table: str) -> set[str]: + """Return persisted columns for one table.""" + # --- schema ------------------------------------------------------ def _ensure_schema(self) -> None: - for name, columns in TABLES.items(): - self._execute(self.dialect.create_table(name, columns)) - for index_name, table, column in INDEXES: - self._execute( - f"CREATE INDEX IF NOT EXISTS {index_name} " f"ON {table} ({column})" - ) - meta = self._fetch_df("SELECT * FROM meta_data") - if meta.empty: + tables = self._existing_tables() + if tables: + self._validate_schema(tables) + return + self._begin() + try: + # Another connection may have initialized the file while this + # writer waited for BEGIN IMMEDIATE. Re-check under the lock. + tables = self._existing_tables() + if tables: + self._validate_schema(tables) + self._commit() + return + for name, columns in TABLES.items(): + self._execute( + self.dialect.create_table( + name, columns, TABLE_CONSTRAINTS.get(name, ()) + ) + ) + for index_name, table, column in INDEXES: + self._execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " f"ON {table} ({column})" + ) self._execute( "INSERT INTO meta_data VALUES (?, ?, ?, ?)", (WHAT_IS_THIS, INDEX_VERSION, dc.__version__, time.time_ns()), ) + except Exception: + with suppress(Exception): + self._rollback() + raise + self._commit() + + def _validate_schema(self, tables: set[str]) -> None: + """Validate an existing index before issuing any DDL or mutation.""" + required = set(TABLES) + missing = required - tables + if missing: + msg = ( + "Existing spool index is incomplete; missing tables " + f"{sorted(missing)}. Delete it and rebuild the index." + ) + raise InvalidIndexError(msg) + meta = self._fetch_df("SELECT * FROM meta_data") + if len(meta) != 1 or meta["what_is_this"].iloc[0] != WHAT_IS_THIS: + msg = "File is not a valid DASCore spool index; delete it and rebuild." + raise InvalidIndexError(msg) + version = int(meta["index_version"].iloc[0]) + if version != INDEX_VERSION: + msg = ( + f"Spool index version {version} is incompatible with supported " + f"version {INDEX_VERSION}; delete it and rebuild." + ) + raise InvalidIndexVersionError(msg) + for table, expected in TABLES.items(): + actual = self._table_columns(table) + if not set(expected) <= actual: + absent = sorted(set(expected) - actual) + msg = ( + f"Spool index table {table!r} is missing columns {absent}; " + "delete it and rebuild." + ) + raise InvalidIndexError(msg) + attr_columns = self._table_columns("attrs") + meta_columns = set(self._attr_meta().get("column_name", ())) + if not meta_columns <= attr_columns: + absent = sorted(meta_columns - attr_columns) + msg = ( + f"Spool index attrs table is missing dynamic columns {absent}; " + "delete it and rebuild." + ) + raise InvalidIndexError(msg) def _attr_meta(self) -> pd.DataFrame: return self._fetch_df("SELECT * FROM attr_meta") + def _coord_meta(self, names=None) -> pd.DataFrame: + """ + Return distinct coordinate names, kinds, and canonical units, + optionally restricted to the given coord names. + """ + sql = ( + "SELECT DISTINCT pc.coord_name, cd.value_kind, cd.units, " + "cd.is_relative FROM patch_coords pc " + "JOIN coord_defs cd ON cd.coord_def_id = pc.coord_def_id" + ) + params: list = [] + if names is not None: + params = sorted(names) + marks = ", ".join("?" for _ in params) + sql += f" WHERE pc.coord_name IN ({marks})" + return self._fetch_df(sql, params) + def _next_id(self, table: str, column: str) -> int: df = self._fetch_df(f"SELECT max({column}) AS m FROM {table}") value = df["m"].iloc[0] return 1 if pd.isnull(value) else int(value) + 1 + @staticmethod + def _units_compatible(to_units: str, from_units: str) -> bool: + """True when one unit converts to the other (same dimensionality).""" + try: + convert_units(1.0, to_units=to_units, from_units=from_units) + except UnitError: + return False + return True + def _ensure_attr_columns( self, records: list[SourceRecord] - ) -> dict[tuple[str, str], str]: + ) -> tuple[dict[tuple[str, str], str], set[tuple[str, str, str]]]: """ - Lazily add typed attr columns; return the (name, kind) -> column map. + Lazily add typed attr columns; return the (name, kind) -> column + map and a set of (name, kind, units) values to skip. attr_meta is the single source of truth for column names: distinct attr names can sanitize to the same identifier ("Shot Number" vs "shot_number"), so collisions get a deterministic numeric suffix. + + One attr name occasionally carries dimensionally incompatible + units across files (e.g. a "resolution" in meters here, seconds + there). A single canonical unit cannot describe both, so the + incompatible values are skipped (with a warning) rather than + failing the whole index update. """ + meta = self._attr_meta() mapping = { (row.attr_name, row.value_kind): row.column_name - for row in self._attr_meta().itertuples() + for row in meta.itertuples() + } + stored_units = { + (row.attr_name, row.value_kind): ( + None if pd.isnull(row.units) else row.units + ) + for row in meta.itertuples() } taken = set(mapping.values()) - needed: dict[tuple[str, str], str | None] = {} + observed: dict[tuple[str, str], set[str | None]] = {} for record in records: for patch in record.patches: for name, typed in patch.attrs.items(): key = (name, typed.kind) - if key not in mapping and key not in needed: - needed[key] = typed.units + observed.setdefault(key, set()).add(typed.units) + needed: dict[tuple[str, str], str | None] = {} + skip_units: set[tuple[str, str, str]] = set() + for key, units_seen in observed.items(): + canonical = stored_units.get(key) if key in mapping else None + for unit in sorted(x for x in units_seen if x is not None): + if canonical is None: + canonical = unit + elif not self._units_compatible(canonical, unit): + skip_units.add((*key, unit)) + msg = ( + f"Attr {key[0]!r} has units {unit!r} incompatible " + f"with the indexed units {canonical!r}; skipping " + "these values in the index." + ) + warnings.warn(msg, UserWarning, stacklevel=2) + if key not in mapping: + needed[key] = canonical + elif stored_units.get(key) is None and canonical is not None: + self._execute( + "UPDATE attr_meta SET units = ? " + "WHERE attr_name = ? AND value_kind = ?", + (canonical, *key), + ) for (name, kind), units in needed.items(): column = base = attr_column_name(name, kind) suffix = 2 @@ -197,7 +335,7 @@ def _ensure_attr_columns( (name, kind, column, units), ) mapping[(name, kind)] = column - return mapping + return mapping, skip_units def _ensure_coord_defs(self, defs_needed: dict) -> dict[str, int]: """ @@ -205,8 +343,8 @@ def _ensure_coord_defs(self, defs_needed: dict) -> dict[str, int]: Coord summaries are deduplicated across patches: identical values (by fingerprint, or by summary content when no fingerprint is - available) share one coord_defs row. This is what will later let - chunk/merge recognize shared coordinates by id equality. + available) share one coord_defs row. Only fingerprint-backed rows + are exposed as exact coordinate identity to merge planning. """ keys = list(defs_needed) mapping: dict[str, int] = {} @@ -278,7 +416,7 @@ def write_sources(self, records: list[SourceRecord]) -> None: by_base.setdefault(record.base_uri or "", []).append(record.source_path) for base_uri, paths in by_base.items(): self._delete_by_paths(paths, base_uri=base_uri) - column_map = self._ensure_attr_columns(records) + column_map, skip_units = self._ensure_attr_columns(records) source_id = self._next_id("sources", "source_id") patch_id = self._next_id("patches", "patch_id") now = time.time_ns() @@ -316,11 +454,18 @@ def write_sources(self, records: list[SourceRecord]) -> None: patch.distance_step, ) ) + attrs = patch.attrs + if skip_units: + attrs = { + name: tv + for name, tv in attrs.items() + if (name, tv.kind, tv.units) not in skip_units + } columns = tuple( - column_map[(name, tv.kind)] for name, tv in patch.attrs.items() + column_map[(name, tv.kind)] for name, tv in attrs.items() ) attr_groups.setdefault(columns, []).append( - [patch_id, *(tv.value for tv in patch.attrs.values())] + [patch_id, *(tv.value for tv in attrs.values())] ) for c in patch.coords: key = c.def_key @@ -394,8 +539,15 @@ def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: def query(self, query=None) -> pd.DataFrame: """Return the flat patch-row relation for a query (or several).""" query = query if query is not None else Query() + queries = [query] if isinstance(query, Query) else list(query) attr_meta = self._attr_meta() - sql, params, residuals = build_query_sql(query, self.dialect, attr_meta) + # coord metadata is only consulted for coord predicates; skip the + # (whole-relation DISTINCT) scan for attr-only/empty queries. + coord_names = {name for q in queries for name in q.coords} + coord_meta = self._coord_meta(coord_names) if coord_names else pd.DataFrame() + sql, params, residuals = build_query_sql( + queries, self.dialect, attr_meta, coord_meta + ) df = self._fetch_df(sql, params) df = self._flatten(df, attr_meta) df = self._pivot_coords(df) @@ -480,7 +632,7 @@ def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: marks = ", ".join("?" for _ in chunk) frames.append( self._fetch_df( - "SELECT pc.patch_id, pc.coord_name, cd.def_key, " + "SELECT pc.patch_id, pc.coord_name, cd.def_key, cd.fingerprint, " "cd.value_kind, cd.is_relative, cd.min_num, cd.max_num, " "cd.step_num, cd.min_ns, cd.max_ns, cd.step_ns, " "cd.min_str, cd.max_str " @@ -496,7 +648,11 @@ def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: for name, group in coords.groupby("coord_name"): mins, maxs, steps, keys = {}, {}, {}, {} for row in group.itertuples(): - keys[row.patch_id] = row.def_key + # Summary-only definitions are useful for indexing/dedup but + # cannot prove coordinate value identity for merge grouping. + keys[row.patch_id] = ( + row.def_key if pd.notnull(row.fingerprint) else None + ) if row.value_kind == "num": mn, mx = row.min_num, row.max_num st = row.step_num @@ -569,14 +725,16 @@ def resolve_query( """ from dascore.io.index.query import InvalidSpoolQueryError - attrs = dict(_attrs or {}) - coords = dict(_coords or {}) + # accept the same open/slice range forms patch-level select does + attrs = {k: normalize_range_forms(v) for k, v in (_attrs or {}).items()} + coords = {k: normalize_range_forms(v) for k, v in (_coords or {}).items()} known_attrs = backend.attr_names() known_coords = backend.coord_names() for name, value in kwargs.items(): if name in attrs or name in coords: msg = f"{name!r} given as both a bare kwarg and in _attrs/_coords." raise InvalidSpoolQueryError(msg) + value = normalize_range_forms(value) if name in known_attrs: attrs[name] = value elif name in known_coords: @@ -589,26 +747,15 @@ def resolve_query( raise InvalidSpoolQueryError(msg) for name in attrs: if name not in known_attrs: - raise InvalidSpoolQueryError(f"Unknown attribute {name!r}.") + raise InvalidSpoolQueryError(f"{name!r} is not an attribute of this spool.") for name in coords: if name not in known_coords: - raise InvalidSpoolQueryError(f"Unknown coordinate {name!r}.") + raise InvalidSpoolQueryError(f"{name!r} is not a coordinate of this spool.") return Query(attrs=attrs, coords=coords) -def get_backend(path: str | Path, kind: str = "duckdb") -> AbstractIndexBackend: - """Create an index backend of the given kind at path.""" - if kind == "duckdb": - from dascore.io.index.duck import DuckDBBackend - - return DuckDBBackend(path) - if kind == "sqlite": - from dascore.io.index.lite import SQLiteBackend - - return SQLiteBackend(path) - if kind == "parquet": - from dascore.io.index.parq import ParquetBackend +def get_backend(path: str | Path) -> AbstractIndexBackend: + """Create the SQLite spool-index backend at path.""" + from dascore.io.index.lite import SQLiteBackend - return ParquetBackend(path) - msg = f"Unknown index backend {kind!r}." - raise ValueError(msg) + return SQLiteBackend(path) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index d50237b35..c5e1444ad 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -19,6 +19,7 @@ import abc import itertools from collections.abc import Mapping, Sequence +from dataclasses import dataclass from pathlib import Path import pandas as pd @@ -32,8 +33,8 @@ Query, relative_offset, ) +from dascore.utils.pd import adjust_segments -_MEMORY_ENGINES = frozenset({"sqlite", "duckdb"}) _counter = itertools.count() @@ -73,20 +74,52 @@ class FileResolver(PatchResolver): def __init__(self, root: Path | str | None = None): self._root = Path(root) if root is not None else None + def _read(self, path, row: Mapping, trim: dict, source_patch_id: str): + """Use a known FiberIO directly, falling back to format detection.""" + from dascore.core.spool import MemorySpool + + file_format = row.get("file_format") + file_version = row.get("file_version") + id_kwargs = {"source_patch_id": source_patch_id} if source_patch_id else {} + if file_format and file_version: + fiber_io = dc.io.FiberIO.manager.get_fiberio( + format=file_format, version=file_version + ) + spool = fiber_io.read(path, **id_kwargs, **trim) + if isinstance(spool, MemorySpool): + return spool + kwargs = {"path": path} + if file_format: + kwargs["file_format"] = file_format + if file_version: + kwargs["file_version"] = file_version + return dc.read(**kwargs, **id_kwargs, **trim) + def resolve(self, row: Mapping, **trim) -> dc.Patch: """Read the patch, passing range trims down as read hints.""" + from dascore.io.core import _select_patch_from_spool + path = row["path"] # relative paths resolve against the catalog root; URIs and # absolute paths pass through untouched. if self._root is not None and "://" not in str(path): if not Path(path).is_absolute(): path = self._root / path - kwargs = {"path": path} - if row.get("file_format"): - kwargs["file_format"] = row["file_format"] - if row.get("file_version"): - kwargs["file_version"] = row["file_version"] - return dc.read(**kwargs, **trim)[0] + source_patch_id = row.get("source_patch_id") or "" + if source_patch_id.isdigit(): + # Positional (synthesized) ids index the full source read; a + # trimmed read would shift or drop patches and bind the wrong + # one, so these rows read the whole source. + trim = {} + spool = self._read(path, row, trim, source_patch_id) + # Readers that consume source_patch_id return the one requested + # patch, sometimes without preserving reload metadata on it. Only + # trust that when the patch doesn't claim a different identity. + if source_patch_id and len(spool) == 1: + found = str(spool[0].attrs.get("_source_patch_id", "") or "") + if found in ("", source_patch_id): + return spool[0] + return _select_patch_from_spool(spool, source_patch_id=source_patch_id) def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): @@ -111,6 +144,13 @@ def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): return records +@dataclass +class _CatalogRevision: + """Shared mutation revision for live catalog views.""" + + value: int = 0 + + class PatchCatalog: """ Query-composable metadata catalog over the spool index tables. @@ -127,9 +167,9 @@ def __init__( resolver: PatchResolver | None = None, syncer=None, pending: tuple = (), - engine: str = "sqlite", queries: tuple[Query, ...] = (), residuals: tuple[tuple[dict, bool, bool], ...] = (), + revision: _CatalogRevision | None = None, ): self._backend = backend self.resolver = resolver @@ -137,37 +177,32 @@ def __init__( # live patches not yet ingested; kept (not a closure) so catalogs # pickle and can rebuild their backend after unpickling. self._pending = tuple(pending) - self._engine = engine self._queries = tuple(queries) self._residuals = tuple(residuals) + self._revision = revision or _CatalogRevision() self._df_cache: pd.DataFrame | None = None + self._df_cache_revision = -1 # --- construction ------------------------------------------------- @classmethod - def from_patches( - cls, patches: Sequence[dc.Patch] = (), engine: str = "sqlite" - ) -> PatchCatalog: + def from_patches(cls, patches: Sequence[dc.Patch] = ()) -> PatchCatalog: """ Catalog over live patches. No backend work happens until the first metadata operation. """ - if engine not in _MEMORY_ENGINES: - msg = f"In-memory catalogs support {sorted(_MEMORY_ENGINES)}, not {engine}." - raise ValueError(msg) - return cls(resolver=LiveResolver(), pending=tuple(patches), engine=engine) + return cls(resolver=LiveResolver(), pending=tuple(patches)) @classmethod def from_directory( cls, path: str | Path, - engine: str = "sqlite", index_path: str | Path | None = None, ) -> PatchCatalog: """Catalog over a directory of fiber files.""" from dascore.io.index.indexer import DBDirectoryIndexer - syncer = DBDirectoryIndexer(path, engine=engine, index_path=index_path) + syncer = DBDirectoryIndexer(path, index_path=index_path) return cls( backend=syncer._backend, resolver=FileResolver(root=syncer.path), @@ -178,11 +213,18 @@ def from_directory( @property def backend(self): - """The index backend, bootstrapping lazily on first use.""" + """ + The index backend, bootstrapping lazily on first use. + + Every metadata operation funnels through here, so this is also + where a brand-new directory index gets its one automatic update. + """ if self._backend is None: - self._backend = get_backend(":memory:", kind=self._engine) + self._backend = get_backend(":memory:") if self._pending: self._backend.write_sources(_live_records(self._pending, self.resolver)) + if self._syncer is not None and self._syncer.ensure_updated(): + self._invalidate() return self._backend def __getstate__(self) -> dict: @@ -208,11 +250,14 @@ def _view(self, queries, residuals) -> PatchCatalog: syncer=self._syncer, queries=queries, residuals=residuals, + revision=self._revision, ) return out def _invalidate(self) -> None: + self._revision.value += 1 self._df_cache = None + self._df_cache_revision = -1 def __deepcopy__(self, memo) -> PatchCatalog: """ @@ -251,20 +296,21 @@ def select( relative=True bounds resolve against the current view's global envelope, then behave as absolute ranges. """ + query = resolve_query(self.backend, _attrs=_attrs, _coords=_coords, **kwargs) if samples: - # names must be coords; validated against the index - unknown = set(kwargs) - self.coord_names() - if unknown or _attrs or _coords: + if query.attrs: msg = ( - f"samples=True selections are coordinate-only; " - f"unknown coordinates: {sorted(unknown)}" + "samples=True selections are coordinate-only; got attrs " + f"{sorted(query.attrs)}." ) raise InvalidSpoolQueryError(msg) - residual = (dict(kwargs), True, False) + residual = (dict(query.coords), True, False) return self._view(self._queries, (*self._residuals, residual)) - if relative: - kwargs = self._relative_to_absolute(kwargs) - query = resolve_query(self.backend, _attrs=_attrs, _coords=_coords, **kwargs) + if relative and query.coords: + query = Query( + attrs=query.attrs, + coords=self._relative_to_absolute(query.coords), + ) # coord range predicates are re-applied exactly at patch load residuals = self._residuals if query.coords: @@ -301,12 +347,33 @@ def to_df(self) -> pd.DataFrame: hidden or renamed private so chunk merge-compatibility (which compares all non-private columns) is not spuriously blocked. """ - if self._df_cache is None: + if self._df_cache is None or self._df_cache_revision != self._revision.value: df = self.backend.query(list(self._queries) or None) df = df.drop( columns=["n_dims", "sample_count_total", "shape"], errors="ignore" ).rename(columns={"patch_id": "_patch_id"}) + # SQL identifies overlapping source patches. Expose the selected + # envelopes, matching spool.get_contents() and the exact trim + # applied when each patch is materialized. Each pass copies the + # frame, so disjoint-name range sets collapse into one pass. + range_dicts = [ + ranges + for query in self._queries + if ( + ranges := { + name: value + for name, value in query.coords.items() + if isinstance(value, tuple) and len(value) == 2 + } + ) + ] + names = [name for ranges in range_dicts for name in ranges] + if range_dicts and len(set(names)) == len(names): + range_dicts = [{k: v for d in range_dicts for k, v in d.items()}] + for ranges in range_dicts: + df = adjust_segments(df, ignore_bad_kwargs=True, **ranges) self._df_cache = df + self._df_cache_revision = self._revision.value return self._df_cache def __len__(self) -> int: @@ -315,12 +382,23 @@ def __len__(self) -> int: def get_patch(self, index: int) -> dc.Patch: """Materialize one patch: resolve, then exact two-stage trim.""" row = self.to_df().iloc[index].to_dict() + return self.resolve_row(row) + + def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Patch: + """ + Resolve one flat-relation row and apply exact residual selects. + + extra_trim carries caller-side read hints (e.g. chunk instruction + ranges) merged over the view's own residual ranges; like all trim + hints they only reduce reading, exactness is re-applied above. + """ trim_hint = {} for coords, samples, _ in self._residuals: if not samples: trim_hint.update( {k: v for k, v in coords.items() if isinstance(v, tuple)} ) + trim_hint.update(extra_trim or {}) patch = self.resolver.resolve(row, **trim_hint) for coords, samples, relative in self._residuals: usable = {k: v for k, v in coords.items() if k in patch.coords.coord_map} @@ -347,7 +425,6 @@ def add(self, patches: Sequence[dc.Patch] | dc.Patch) -> PatchCatalog: def update(self, progress: PROGRESS_LEVELS = "standard") -> PatchCatalog: """Sync a directory-backed catalog with the filesystem.""" - self._require_root("update") if self._syncer is not None: self._syncer.update(progress=progress) self._invalidate() diff --git a/dascore/io/index/dialect.py b/dascore/io/index/dialect.py index e997160ac..598dbd285 100644 --- a/dascore/io/index/dialect.py +++ b/dascore/io/index/dialect.py @@ -9,29 +9,29 @@ from __future__ import annotations from collections.abc import Mapping +from typing import ClassVar class BaseDialect: """Shared SQL generation for engines close to the standard.""" - # logical type -> engine type - type_map: Mapping[str, str] = { - "int64": "BIGINT", - "float64": "DOUBLE", - "str": "VARCHAR", - "bool": "BOOLEAN", - } - strict_suffix = "" + # logical type -> engine type; concrete dialects define the values. + type_map: ClassVar[Mapping[str, str]] + strict_suffix: ClassVar[str] def quote(self, identifier: str) -> str: """Quote an identifier.""" return '"' + identifier.replace('"', '""') + '"' - def create_table(self, name: str, columns: Mapping[str, str]) -> str: + def create_table( + self, name: str, columns: Mapping[str, str], constraints: tuple[str, ...] = () + ) -> str: """Return DDL for one table from logical column types.""" - cols = ", ".join( + definitions = [ f"{self.quote(col)} {self.type_map[typ]}" for col, typ in columns.items() - ) + ] + definitions.extend(constraints) + cols = ", ".join(definitions) quoted = self.quote(name) return f"CREATE TABLE IF NOT EXISTS {quoted} ({cols}){self.strict_suffix}" @@ -47,18 +47,14 @@ def glob(self, column_sql: str) -> str: return f"{column_sql} GLOB ?" -class DuckDBDialect(BaseDialect): - """Dialect for DuckDB.""" - - class SQLiteDialect(BaseDialect): """Dialect for SQLite; STRICT tables enforce the type contract.""" # SQLite STRICT tables accept INTEGER/REAL/TEXT (and INT for bool). - type_map = { + type_map: ClassVar[Mapping[str, str]] = { "int64": "INTEGER", "float64": "REAL", "str": "TEXT", "bool": "INTEGER", } - strict_suffix = " STRICT" + strict_suffix: ClassVar[str] = " STRICT" diff --git a/dascore/io/index/duck.py b/dascore/io/index/duck.py deleted file mode 100644 index 5bf65cc88..000000000 --- a/dascore/io/index/duck.py +++ /dev/null @@ -1,75 +0,0 @@ -"""DuckDB index backend.""" - -from __future__ import annotations - -from pathlib import Path - -import pandas as pd - -from dascore.io.index.backend import SQLIndexBackend, adapt_params -from dascore.io.index.dialect import DuckDBDialect - - -def duck_bulk_insert(con, dialect, table: str, columns: tuple, rows: list) -> None: - """ - Bulk-insert rows through a registered dataframe. - - DuckDB's executemany binds row-at-a-time in Python (about 60x slower - than SQLite for ingest); routing bulk rows through its dataframe - scanner keeps ingest columnar. - """ - if not rows: - return - df = pd.DataFrame( - [adapt_params(r) for r in rows], columns=list(columns), dtype=object - ) - con.register("_bulk_rows", df) - try: - quoted = ", ".join(dialect.quote(c) for c in columns) - con.execute( - f"INSERT INTO {dialect.quote(table)} ({quoted}) " "SELECT * FROM _bulk_rows" - ) - finally: - con.unregister("_bulk_rows") - - -class DuckDBBackend(SQLIndexBackend): - """Index backend storing tables in a single DuckDB file.""" - - dialect = DuckDBDialect() - - def __init__(self, path: str | Path, read_only: bool = False): - import duckdb - - self._con = duckdb.connect(str(path), read_only=read_only) - super().__init__() - - def _execute(self, sql: str, params=()) -> None: - self._con.execute(sql, adapt_params(params)) - - def _executemany(self, sql: str, seq_of_params) -> None: - rows = [adapt_params(p) for p in seq_of_params] - if rows: - self._con.executemany(sql, rows) - - def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: - # arrow keeps nullable BIGINT exact (df() would use float64, - # corrupting ns timestamps beyond float's 2**53 integer range) - reader = self._con.execute(sql, adapt_params(params)).arrow() - return reader.read_all().to_pandas(integer_object_nulls=True) - - def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: - duck_bulk_insert(self._con, self.dialect, table, columns, rows) - - def _begin(self) -> None: - self._con.execute("BEGIN TRANSACTION") - - def _commit(self) -> None: - self._con.execute("COMMIT") - - def _rollback(self) -> None: - self._con.execute("ROLLBACK") - - def close(self) -> None: - """Close the database connection.""" - self._con.close() diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 42e7fd29a..932eeaaf7 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -44,8 +44,6 @@ class DBDirectoryIndexer(AbstractIndexer): ---------- path The directory to index. - engine - The backend kind: "duckdb", "sqlite", or "parquet". index_path Where to keep the index; defaults to a hidden entry at the top of the data directory. @@ -58,22 +56,22 @@ class DBDirectoryIndexer(AbstractIndexer): def __init__( self, path: str | Path, - engine: str = "sqlite", index_path: str | Path | None = None, ): path = UPath(path).absolute() if isinstance(path, UPath) else Path(path) requires_local_directory(path, label="DBDirectoryIndexer") self.path = Path(path).absolute() - self.engine = engine self.index_path = Path(self._find_index_path(index_path)) # A brand-new index triggers one automatic update on first query, # matching the historic auto-index-on-first-access behavior. - self._initial_update_done = self.index_path.exists() - self._backend = get_backend(self.index_path, kind=engine) + self._initial_update_done = ( + self.index_path.exists() and self.index_path.stat().st_size > 0 + ) + self._backend = get_backend(self.index_path) @property def _index_name(self) -> str: - return f".dascore_index_{self.engine}" + return ".dascore_index.sqlite3" def _find_index_path(self, index_path=None) -> Path: """ @@ -83,7 +81,7 @@ def _find_index_path(self, index_path=None) -> Path: default; when the data directory is read-only the index lives in the dascore cache and its location is recorded in the index map. """ - map_key = f"{self.path}::{self.engine}" + map_key = str(self.path) if index_path: update = {map_key: str(Path(index_path).absolute())} _update_index_map(update, cache_path=str(self.index_map_path)) @@ -96,7 +94,7 @@ def _find_index_path(self, index_path=None) -> Path: if out := path_map.get(map_key): return Path(out) if not _directory_writable(self.path): - name = f"_dascore_index_{abs(hash(self.path))}_{self.engine}" + name = f"_dascore_index_{abs(hash(self.path))}.sqlite3" index_path = self.index_map_path.parent / name _update_index_map( {map_key: str(index_path.absolute())}, @@ -105,8 +103,15 @@ def _find_index_path(self, index_path=None) -> Path: return index_path return expected + def ensure_updated(self) -> bool: + """Run the initial update if the index was never populated.""" + if self._initial_update_done: + return False + self.update(progress=None) + return True + def __str__(self) -> str: - return f"{self.__class__.__name__} ({self.engine}) managing: {self.path}" + return f"{self.__class__.__name__} managing: {self.path}" __repr__ = __str__ @@ -249,8 +254,7 @@ def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: Bare kwargs resolve attrs-first then coords; `_attrs`/`_coords` disambiguate explicitly (see the selector semantics spec). """ - if not self._initial_update_done: - self.update(progress=None) + self.ensure_updated() query = resolve_query(self._backend, _attrs=_attrs, _coords=_coords, **kwargs) df = self._backend.query(query) df = df.drop(columns=list(_SPOOL_HIDDEN_COLUMNS), errors="ignore") diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 5d774bc97..f4e2515fc 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -217,13 +217,18 @@ def _extract_attrs(summary: PatchSummary) -> dict[str, TypedValue]: def _coord_record(name: str, summary) -> CoordRecord | None: """Convert one CoordSummary into a CoordRecord.""" + fingerprint = getattr(summary, "fingerprint", None) + if fingerprint is None and getattr(summary, "is_range_like", False): + # A range summary contains its complete representation, so recover the + # same exact identity a loaded CoordRange would have produced. + fingerprint = summary.to_coord().fingerprint() common = dict( coord_name=name, dtype=summary.dtype, coord_dims=",".join(summary.dims), length=summary.len, units=str(summary.units) if summary.units is not None else None, - coord_hash=getattr(summary, "fingerprint", None), + coord_hash=fingerprint, ) dtype = np.dtype(summary.dtype) if summary.dtype else None if dtype is not None and np.issubdtype(dtype, np.datetime64): diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py index 0250c162e..a74d46b24 100644 --- a/dascore/io/index/lite.py +++ b/dascore/io/index/lite.py @@ -25,7 +25,13 @@ def __init__(self, path: str | Path): self._con = sqlite3.connect(str(path)) # autocommit off; we manage transactions explicitly. self._con.isolation_level = None - super().__init__() + self._con.execute("PRAGMA foreign_keys = ON") + self._con.execute("PRAGMA busy_timeout = 30000") + try: + super().__init__() + except Exception: + self._con.close() + raise def _execute(self, sql: str, params=()) -> None: self._con.execute(sql, _adapt(params)) @@ -37,7 +43,7 @@ def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: return pd.read_sql_query(sql, self._con, params=_adapt(params)) def _begin(self) -> None: - self._con.execute("BEGIN") + self._con.execute("BEGIN IMMEDIATE") def _commit(self) -> None: self._con.execute("COMMIT") @@ -45,6 +51,17 @@ def _commit(self) -> None: def _rollback(self) -> None: self._con.execute("ROLLBACK") + def _existing_tables(self) -> set[str]: + rows = self._con.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ).fetchall() + return {row[0] for row in rows} + + def _table_columns(self, table: str) -> set[str]: + sql = f"PRAGMA table_info({self.dialect.quote(table)})" + return {row[1] for row in self._con.execute(sql).fetchall()} + def close(self) -> None: """Close the database connection.""" self._con.close() diff --git a/dascore/io/index/parq.py b/dascore/io/index/parq.py deleted file mode 100644 index 02f7a8936..000000000 --- a/dascore/io/index/parq.py +++ /dev/null @@ -1,118 +0,0 @@ -""" -Parquet-manifest index backend. - -Tables live as immutable parquet files plus a small manifest; readers -never need locks (a half-written update is invisible until the manifest -swap). This prototype materializes the tables in an in-memory DuckDB for -querying/mutation and dumps changed tables to new parquet files on -commit, replacing the manifest atomically. -""" - -from __future__ import annotations - -import json -import os -import uuid -from pathlib import Path - -import pandas as pd - -from dascore.io.index.backend import SQLIndexBackend, adapt_params -from dascore.io.index.dialect import DuckDBDialect -from dascore.io.index.schema import TABLES - -_MANIFEST = "manifest.json" - - -class ParquetBackend(SQLIndexBackend): - """Index backend storing tables as parquet files + manifest.""" - - dialect = DuckDBDialect() - - def __init__(self, path: str | Path): - import duckdb - - self._dir = Path(path) - self._dir.mkdir(parents=True, exist_ok=True) - self._con = duckdb.connect(":memory:") - self._manifest = self._read_manifest() - for table, filename in self._manifest.get("tables", {}).items(): - file_path = str(self._dir / filename).replace("'", "''") - self._con.execute( - f"CREATE TABLE {self.dialect.quote(table)} AS " - f"SELECT * FROM read_parquet('{file_path}')" - ) - super().__init__() - - def _read_manifest(self) -> dict: - manifest_path = self._dir / _MANIFEST - if manifest_path.exists(): - with manifest_path.open() as fi: - return json.load(fi) - return {"tables": {}} - - # --- SQL hooks (all against the in-memory duckdb) ----------------- - - def _execute(self, sql: str, params=()) -> None: - self._con.execute(sql, adapt_params(params)) - - def _executemany(self, sql: str, seq_of_params) -> None: - rows = [adapt_params(p) for p in seq_of_params] - if rows: - self._con.executemany(sql, rows) - - def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: - # arrow keeps nullable BIGINT exact (df() would use float64, - # corrupting ns timestamps beyond float's 2**53 integer range) - reader = self._con.execute(sql, adapt_params(params)).arrow() - return reader.read_all().to_pandas(integer_object_nulls=True) - - def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: - from dascore.io.index.duck import duck_bulk_insert - - duck_bulk_insert(self._con, self.dialect, table, columns, rows) - - def _begin(self) -> None: - self._con.execute("BEGIN TRANSACTION") - - def _commit(self) -> None: - self._con.execute("COMMIT") - self._persist() - - def _rollback(self) -> None: - self._con.execute("ROLLBACK") - - # --- persistence --------------------------------------------------- - - def _persist(self) -> None: - """Write all tables to new parquet files and swap the manifest.""" - new_tables = {} - for table in TABLES: - filename = f"{table}-{uuid.uuid4().hex[:12]}.parquet" - target = str(self._dir / filename).replace("'", "''") - self._con.execute( - f"COPY {self.dialect.quote(table)} TO '{target}' (FORMAT PARQUET)" - ) - new_tables[table] = filename - old = self._manifest.get("tables", {}) - self._manifest = {"tables": new_tables} - tmp = self._dir / (_MANIFEST + ".tmp") - with tmp.open("w") as fi: - json.dump(self._manifest, fi) - os.replace(tmp, self._dir / _MANIFEST) - # best-effort cleanup of superseded files (readers using the old - # manifest may still hold them open; deletion failing is fine). - for filename in old.values(): - try: - (self._dir / filename).unlink(missing_ok=True) - except OSError: - pass - - def _ensure_schema(self) -> None: - super()._ensure_schema() - if not (self._dir / _MANIFEST).exists(): - self._persist() - - def close(self) -> None: - """Close the in-memory database.""" - self._con.close() diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 98acb83c9..dc968c332 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -4,9 +4,8 @@ Implements the selector semantics spec (see `.scratch/selector_semantics_spec.md`): the index only produces candidates — predicates the summary cannot evaluate exactly are the -caller's responsibility at patch-load time. SQL generation is shared by -all backends; anything a dialect cannot push down is applied as a pandas -residual filter with identical semantics. +caller's responsibility at patch-load time. Predicates SQLite cannot evaluate +exactly are applied as pandas residual filters. """ from __future__ import annotations @@ -19,11 +18,14 @@ import numpy as np import pandas as pd -from dascore.exceptions import InvalidSpoolQueryError +from dascore.exceptions import InvalidSpoolQueryError, ParameterError, UnitError from dascore.io.index.dialect import BaseDialect from dascore.io.index.ingest import typed_value +from dascore.units import convert_units +from dascore.utils.misc import sanitize_range_param _GLOB_CHARS = frozenset("*?[") +_UNSET = object() @dataclass(frozen=True) @@ -54,6 +56,19 @@ def _is_range(value) -> bool: return isinstance(value, tuple) and len(value) == 2 +def normalize_range_forms(value): + """ + Normalize the patch-level slice range form to a 2-tuple. + + Only slices are converted: bare None/Ellipsis keep their own errors, + and a fully-open range is rejected downstream as having no usable + bounds (per the selector spec). + """ + if isinstance(value, slice): + return sanitize_range_param(value) + return value + + def _coerce_scalar(value, target_kinds: set[str]): """ Coerce a query scalar to (kind, storable value). @@ -68,26 +83,59 @@ def _coerce_scalar(value, target_kinds: set[str]): if typed.kind == "str" and "time" in target_kinds: try: retyped = typed_value(np.datetime64(pd.Timestamp(value), "ns")) - return retyped.kind, retyped.value + return retyped except (ValueError, TypeError): pass - return typed.kind, typed.value + return typed + +def _normalize_unit(value) -> str | None: + """Return a nullable unit string from a dataframe value.""" + return None if value is None or pd.isnull(value) else str(value) + + +def _to_target_unit(typed, target_units: str | None, name: str): + """Validate/convert a numeric query value for one stored unit.""" + if typed.kind != "num" or typed.units is None: + return typed.value + if target_units is None: + msg = f"Cannot query unitless {name!r} with units {typed.units!r}." + raise UnitError(msg) + return convert_units(typed.value, to_units=target_units, from_units=typed.units) + + +def _range_bounds( + value, + target_kinds: set[str], + target_units: str | None | object = _UNSET, + name: str = "value", +): + """ + Return (kind, lo, hi, typed_values) from a range tuple. -def _range_bounds(value, target_kinds: set[str]): - """Return (kind, lo, hi) from a range tuple, handling open bounds.""" + Open bounds (None/Ellipsis) are skipped; typed_values carries the + coerced usable bounds so callers don't coerce twice. + """ lo_raw, hi_raw = value lo = hi = None kind = None - for raw, name in ((lo_raw, "lo"), (hi_raw, "hi")): + typed_values = [] + for raw, side in ((lo_raw, "lo"), (hi_raw, "hi")): if raw is None or raw is Ellipsis: continue - knd, val = _coerce_scalar(raw, target_kinds) + typed = _coerce_scalar(raw, target_kinds) + typed_values.append(typed) + knd = typed.kind + val = ( + typed.value + if target_units is _UNSET + else _to_target_unit(typed, target_units, name) + ) if kind is not None and knd != kind: msg = f"Range bounds {value!r} have mixed kinds ({kind}, {knd})." raise InvalidSpoolQueryError(msg) kind = knd - if name == "lo": + if side == "lo": lo = val else: hi = val @@ -97,7 +145,40 @@ def _range_bounds(value, target_kinds: set[str]): if lo is not None and hi is not None and lo > hi: msg = f"Range {value!r} has lo > hi after coercion." raise InvalidSpoolQueryError(msg) - return kind, lo, hi + return kind, lo, hi, typed_values + + +def _compatible_coord_units( + rows: pd.DataFrame, typed_values: list, name: str +) -> set[str] | None: + """ + Return stored units compatible with quantity-valued coord selectors. + + None means the query carries no units (no unit constraint at all); a + set constrains matching to those units plus NULL-unit definitions + (which can never be proven incompatible, so they stay candidates). + Raises UnitError only when every stored definition has units and none + are compatible. + """ + query_units = { + x.units for x in typed_values if x is not None and x.units is not None + } + if not query_units: + return None + first = next(iter(query_units)) + for other in query_units - {first}: + convert_units(1.0, to_units=first, from_units=other) + stored = {_normalize_unit(x) for x in rows.get("units", ())} + compatible = set() + for unit in stored - {None}: + try: + convert_units(1.0, to_units=unit, from_units=first) + except UnitError: + continue + compatible.add(unit) + if not compatible and None not in stored: + raise UnitError(f"Coordinate {name!r} has no units compatible with {first!r}.") + return compatible @dataclass @@ -133,6 +214,7 @@ def build_attr_clause( raise InvalidSpoolQueryError(msg) kinds = set(rows["value_kind"]) columns = dict(zip(rows["value_kind"], rows["column_name"])) + units = {row.value_kind: _normalize_unit(row.units) for row in rows.itertuples()} def col(kind): return f"a.{dialect.quote(columns[kind])}" @@ -146,7 +228,17 @@ def col(kind): where.add(f"{col('str')} IS NOT NULL") return value if _is_range(value): - kind, lo, hi = _range_bounds(value, kinds) + # Attr metadata has one canonical unit per typed column. + probe = next( + ( + _coerce_scalar(x, kinds) + for x in value + if x is not None and x is not Ellipsis + ), + None, + ) + target_units = units.get(probe.kind) if probe is not None else None + kind, lo, hi, _ = _range_bounds(value, kinds, target_units, name) if kind not in kinds: where.add("FALSE") return None @@ -158,8 +250,9 @@ def col(kind): if _is_collection(value): coerced = [_coerce_scalar(v, kinds) for v in value] by_kind: dict[str, list] = {} - for kind, val in coerced: - by_kind.setdefault(kind, []).append(val) + for typed in coerced: + val = _to_target_unit(typed, units.get(typed.kind), name) + by_kind.setdefault(typed.kind, []).append(val) subclauses = [] params = [] for kind, vals in by_kind.items(): @@ -179,7 +272,9 @@ def col(kind): return None where.add(dialect.glob(col("str")), value) return None - kind, val = _coerce_scalar(value, kinds) + typed = _coerce_scalar(value, kinds) + kind = typed.kind + val = _to_target_unit(typed, units.get(kind), name) if kind not in kinds: where.add("FALSE") return None @@ -190,6 +285,7 @@ def col(kind): def build_coord_clause( where: _Where, dialect: BaseDialect, + coord_meta: pd.DataFrame, name: str, value, ) -> None: @@ -200,20 +296,40 @@ def build_coord_clause( Candidacy only: envelope overlap, never false negatives. Exact membership/boolean masks are applied at patch load, above this layer. """ + rows = coord_meta[coord_meta["coord_name"] == name] + if isinstance(value, tuple) and len(value) != 2: + msg = f"Coordinate range for {name!r} must be a length 2 sequence." + raise ParameterError(msg) + kinds = set(rows["value_kind"]) or {"time", "num", "str"} + typed_values = [] if _is_range(value): - kind, lo, hi = _range_bounds(value, {"time", "num", "str"}) + kind, lo, hi, typed_values = _range_bounds(value, kinds) elif _is_collection(value): - arr = np.asarray(list(value) if isinstance(value, set) else value) + raw_values = list(value) + if not raw_values: + raise InvalidSpoolQueryError("Coordinate membership cannot be empty.") + arr = np.asarray(raw_values) if arr.dtype == bool: # boolean masks are patch-local; no index predicate at all, # but the coord must exist on the patch. kind = lo = hi = None else: - kind, lo = _coerce_scalar(arr.min(), {"time", "num", "str"}) - _, hi = _coerce_scalar(arr.max(), {"time", "num", "str"}) + typed_values = [_coerce_scalar(x, kinds) for x in raw_values] + value_kinds = {x.kind for x in typed_values} + if len(value_kinds) != 1: + raise InvalidSpoolQueryError( + f"Coordinate values for {name!r} have mixed kinds." + ) + kind = typed_values[0].kind + values = [x.value for x in typed_values] + lo, hi = min(values), max(values) else: - kind, val = _coerce_scalar(value, {"time", "num", "str"}) - lo = hi = val + typed = _coerce_scalar(value, kinds) + typed_values = [typed] + kind = typed.kind + lo = hi = typed.value + + compatible_units = _compatible_coord_units(rows, typed_values, name) min_col, max_col = { "time": ("min_ns", "max_ns"), @@ -234,6 +350,15 @@ def build_coord_clause( kind_match = kind conditions.append("cd.value_kind = ?") params.append(kind_match) + if compatible_units is not None: + # NULL-unit defs stay candidates: IN () never matches NULL and + # unitless values cannot be proven dimensionally incompatible. + if compatible_units: + marks = ", ".join("?" for _ in compatible_units) + conditions.append(f"(cd.units IN ({marks}) OR cd.units IS NULL)") + params.extend(sorted(compatible_units)) + else: + conditions.append("cd.units IS NULL") if lo is not None: conditions.append(f"cd.{max_col} >= ?") params.append(lo) @@ -252,23 +377,26 @@ def build_query_sql( query: Query | Sequence[Query], dialect: BaseDialect, attr_meta: pd.DataFrame, -) -> tuple[str, list, dict[str, re.Pattern]]: + coord_meta: pd.DataFrame, +) -> tuple[str, list, list[tuple[str, re.Pattern]]]: """ Build the flat-relation SELECT for one or more AND-composed queries. - Returns (sql, params, residuals) where residuals maps attr names to - regex patterns that must be re-applied to the resulting dataframe. + coord_meta must cover every coordinate the queries reference (it may + be empty for attr-only queries). Returns (sql, params, residuals) + where residuals maps attr names to regex patterns that must be + re-applied to the resulting dataframe. """ queries = [query] if isinstance(query, Query) else list(query) where = _Where() - residuals: dict[str, re.Pattern] = {} + residuals: list[tuple[str, re.Pattern]] = [] for one in queries: for name, value in one.attrs.items(): residual = build_attr_clause(where, dialect, attr_meta, name, value) if residual is not None: - residuals[name] = residual + residuals.append((name, residual)) for name, value in one.coords.items(): - build_coord_clause(where, dialect, name, value) + build_coord_clause(where, dialect, coord_meta, name, value) # attr columns selected explicitly: `a.*` would duplicate patch_id and # engines disagree on how to dedupe result column names. attr_cols = "".join( @@ -286,9 +414,11 @@ def build_query_sql( return sql, where.params, residuals -def apply_residuals(df: pd.DataFrame, residuals: dict[str, re.Pattern]) -> pd.DataFrame: +def apply_residuals( + df: pd.DataFrame, residuals: list[tuple[str, re.Pattern]] +) -> pd.DataFrame: """Apply regex residual filters to the flat relation.""" - for name, pattern in residuals.items(): + for name, pattern in residuals: col = df[name] keep = col.map( lambda x: bool(pattern.search(x)) if isinstance(x, str) else False diff --git a/dascore/io/index/schema.py b/dascore/io/index/schema.py index 741a01147..bc8d6a58d 100644 --- a/dascore/io/index/schema.py +++ b/dascore/io/index/schema.py @@ -1,10 +1,9 @@ """ Logical schema for the spool index. -The schema is defined in backend-neutral terms; only four primitive -storage types are used (int64, float64, str, bool) so any SQL-ish backend -can represent it. Times and durations are always epoch/plain nanoseconds -stored as int64 — never engine-native timestamp types. +The schema uses four primitive storage types (int64, float64, str, bool). +Times and durations are always epoch/plain nanoseconds stored as int64, +never engine-native timestamp types. """ from __future__ import annotations @@ -12,7 +11,7 @@ from types import MappingProxyType # Version of the index schema, independent of dascore's version. -INDEX_VERSION = 1 +INDEX_VERSION = 2 # Identity string so any tool can sanity-check what it opened. WHAT_IS_THIS = "dascore_spool_index" @@ -85,10 +84,10 @@ } ) -# Unique coordinate summaries, deduplicated across patches. The def_key -# is the CoordSummary fingerprint when the scan provides one (semantic -# value identity) or a hash of the stored summary fields otherwise -# (lossless for the index; too weak for value-identity claims). +# Unique coordinate summaries, deduplicated across patches. Range coordinates +# use a semantic fingerprint supplied by the scan or reconstructed exactly +# from the range summary. Non-range coordinates without a fingerprint use a +# summary hash for storage deduplication, but it is not exposed as value identity. COORD_DEFS = MappingProxyType( { "coord_def_id": "int64", @@ -134,6 +133,49 @@ } ) +# Keeping constraints beside the logical columns makes the stored contract +# explicit and keeps dynamic attr-column DDL separate from table identity. +TABLE_CONSTRAINTS = MappingProxyType( + { + "meta_data": ( + "PRIMARY KEY (what_is_this)", + f"CHECK (what_is_this = '{WHAT_IS_THIS}')", + ), + "sources": ( + "PRIMARY KEY (source_id)", + "UNIQUE (base_uri, source_path)", + "CHECK (base_uri IS NOT NULL)", + "CHECK (source_path IS NOT NULL)", + ), + "patches": ( + "PRIMARY KEY (patch_id)", + "UNIQUE (source_id, source_patch_id)", + "FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE", + ), + "attrs": ( + "PRIMARY KEY (patch_id)", + "FOREIGN KEY (patch_id) REFERENCES patches(patch_id) ON DELETE CASCADE", + ), + "attr_meta": ( + "PRIMARY KEY (attr_name, value_kind)", + "UNIQUE (column_name)", + "CHECK (value_kind IN ('num', 'str', 'bool', 'time', 'dur'))", + ), + "coord_defs": ( + "PRIMARY KEY (coord_def_id)", + "UNIQUE (def_key)", + "CHECK (value_kind IN ('num', 'time', 'str'))", + "CHECK (is_monotonic IS NULL OR is_monotonic IN (0, 1))", + "CHECK (is_relative IS NULL OR is_relative IN (0, 1))", + ), + "patch_coords": ( + "PRIMARY KEY (patch_id, coord_name)", + "FOREIGN KEY (patch_id) REFERENCES patches(patch_id) ON DELETE CASCADE", + "FOREIGN KEY (coord_def_id) REFERENCES coord_defs(coord_def_id)", + ), + } +) + # Names which can never be dynamic attr columns. RESERVED_ATTR_COLUMNS = frozenset({"patch_id"}) diff --git a/dascore/io/indexer.py b/dascore/io/indexer.py index 93cce9b13..9f703cfa5 100644 --- a/dascore/io/indexer.py +++ b/dascore/io/indexer.py @@ -83,3 +83,13 @@ def update(self) -> Self: Resets any previous selection. """ + + def ensure_updated(self) -> bool: + """ + Run the initial update if the index was never populated. + + Return True when an update actually ran. Indexers which track + their initial-population state override this; by default nothing + happens. + """ + return False diff --git a/docs/changelog.qmd b/docs/changelog.qmd index 6a9bcf6c2..e5e9abf26 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -4,6 +4,10 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f ## Unreleased API Changes +- Memory and directory spools now share a catalog-backed metadata selection path. Attribute and coordinate candidates are pushed into SQLite lazily, while exact coordinate trimming remains a patch-load operation. +- Directory indexes now use the constrained seven-table SQLite schema in `.dascore_index.sqlite3`. Experimental DuckDB and Parquet index backends and the `engine`/`index_engine` selection parameters were removed. Prototype indexes from the earlier schema must be deleted and rebuilt. +- Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces, and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. +- The legacy PyTables directory index has been replaced by SQLite. SQLite supports concurrent readers and one serialized writer on local filesystems; network filesystems with unreliable locking are not supported. - DASCore file I/O now accepts `UPath` resources across the main read, scan, spool, and write workflows. Remote file backends such as `memory://` can be used directly for supported formats, and remote directory-based formats such as `XMLBinary` now work when the backend supports listing and file reads. See the file I/O and spool tutorials for examples and current limitations. - `dc.scan(...)` now returns [`PatchSummary`](`dascore.PatchSummary`) objects rather than `PatchAttrs`. - Scan results carry metadata, coordinates, and source information without loading data. File-backed summaries contain enough source information for lazy reloads. diff --git a/docs/notes/notes.qmd b/docs/notes/notes.qmd index 85c73de09..3ee5b672d 100644 --- a/docs/notes/notes.qmd +++ b/docs/notes/notes.qmd @@ -9,3 +9,5 @@ This section of the documentation provides understanding-oriented explanation fo - [Documentation Strategy](doc_strategy.qmd) - [Fourier Transforms](dft_notes.qmd) - [Velocity to Strain Rate](velocity_to_strain_rate.qmd) +- [Spool Index](spool_index.qmd) +- [Spool Selection](spool_selection.qmd) diff --git a/docs/notes/spool_index.qmd b/docs/notes/spool_index.qmd new file mode 100644 index 000000000..004aa4c9a --- /dev/null +++ b/docs/notes/spool_index.qmd @@ -0,0 +1,35 @@ +--- +title: Spool Index +--- + +Directory and in-memory spools use the same metadata model. The persisted directory index is one SQLite file named `.dascore_index.sqlite3`; in-memory spools use the same schema in an in-memory SQLite database. The index stores summaries and source identities, not patch data. + +## Why seven tables? + +The schema separates records with different lifetimes and cardinalities. This avoids repeating source and coordinate metadata on every patch and gives SQLite enforceable ownership boundaries. + +| Table | One row per | Purpose | +|---|---|---| +| `meta_data` | index | Identifies the file and its schema version | +| `sources` | file or directory-format source | Tracks the source path, format, size, and modification time | +| `patches` | patch within a source | Stores patch identity and common time/distance envelopes | +| `attrs` | patch | Stores typed attribute values in dynamically added columns | +| `attr_meta` | attribute name and value kind | Maps original attribute names to typed storage columns and canonical units | +| `coord_defs` | unique coordinate value definition | Stores coordinate summaries and deduplicates identical definitions | +| `patch_coords` | coordinate attached to a patch | Links patches to coordinate definitions while retaining the coordinate name and dimensions | + +The last two tables are deliberately separate. Many patches can share a distance coordinate, so `coord_defs` stores it once and `patch_coords` supplies the many-to-many attachment. Range coordinates receive an exact semantic fingerprint, reconstructed from the coordinate values when a scan did not supply one. Non-range coordinates only receive a merge-compatibility identity when the scan provides an exact fingerprint. This distinction prevents a matching envelope from being mistaken for matching coordinate values while retaining the deduplication needed by future merge planning. + +`attrs` and `attr_meta` are also complementary. Attribute names are open-ended, so the index cannot define every typed column in advance. `attr_meta` records the stable mapping and units needed to interpret the columns added to `attrs` as data is ingested. + +## Lifecycle and validation + +The index is an incrementally updated cache. A directory update scans new or changed sources, transactionally replaces their patch rows, and removes rows for deleted sources. Foreign keys cascade source deletion through patches, attributes, and patch-coordinate links. Unreferenced coordinate definitions may remain available for reuse. + +The current schema version is validated before any mutation. An unrelated, incomplete, or older prototype database raises an error with instructions to delete and rebuild it; DASCore does not silently repair or migrate it. + +SQLite permits concurrent readers and serializes writers. Initialization and updates use an immediate write transaction and a 30-second busy timeout. This relies on correct local-filesystem locking; reliable operation on network filesystems with weak locking is not promised. + +## Scope + +The index answers metadata selection and identifies candidate patches. Exact coordinate selection is applied again when a patch is loaded because summary envelopes cannot prove arbitrary-coordinate membership. Chunk planning, cross-catalog federation, missing-dimension policy, and materialized derived data remain future work. diff --git a/docs/notes/spool_selection.qmd b/docs/notes/spool_selection.qmd new file mode 100644 index 000000000..f84c7e7fc --- /dev/null +++ b/docs/notes/spool_selection.qmd @@ -0,0 +1,15 @@ +--- +title: Spool Selection +--- + +`Spool.select` uses one selector model for memory and directory spools. Patch-list and directory spools compose selections in a `PatchCatalog`; ordinary metadata predicates are pushed into SQLite and remain lazy until contents, length, indexing, or iteration requires rows. + +Bare selector names resolve to attributes first and then coordinates. `_attrs={...}` and `_coords={...}` provide explicit namespaces when needed. Unknown names raise immediately instead of being ignored. + +Attribute equality, membership, ranges, and glob predicates are evaluated by the index. Regular expressions use a SQL candidate predicate and an exact residual filter; chained regular expressions are combined with AND. Quantities are converted to the canonical unit recorded by the index, and dimensionally incompatible queries raise rather than silently returning incorrect matches. Values stored without units can never be proven incompatible, so they remain candidates for quantity selectors rather than being silently excluded. + +Coordinate predicates first select patches whose summary envelopes can overlap the request. The loaded patch is then selected exactly. `samples=True` is always patch-local and therefore never excludes a patch at the index stage. `relative=True` resolves coordinate ranges against the current spool view's global envelope; attribute predicates in the same call remain unchanged. + +Operations that create a new row or instruction plan, including chunking, sorting, and slicing, switch that derived spool to dataframe planning. Exact selections already attached to the catalog still apply when source patches are resolved. + +Catalog views share their source state. Adding, removing, or rescanning sources invalidates realized metadata so existing views observe the updated catalog under their composed predicates. diff --git a/docs/tutorial/file_io.qmd b/docs/tutorial/file_io.qmd index 23269ae54..9b3359bda 100644 --- a/docs/tutorial/file_io.qmd +++ b/docs/tutorial/file_io.qmd @@ -135,11 +135,7 @@ The `Patch.io` namespace also includes functionality for converting `Patch` inst ## Directory Indexer -The `DBDirectoryIndexer` is used to track the contents of a directory which -contains fiber data. It creates a small, hidden database index (sqlite by -default; duckdb and parquet backends are also available) at the top of the -directory which can be efficiently queried for directory contents -(it is used internally by the `DirectorySpool`). +The `DBDirectoryIndexer` tracks the contents of a directory which contains fiber data. It creates a small, hidden SQLite index named `.dascore_index.sqlite3` at the top of the directory. `DirectorySpool` uses this index internally and pushes metadata selections into SQLite before loading patch data. See the [spool index note](../notes/spool_index.qmd) for the schema and lifecycle. ```{python} diff --git a/pyproject.toml b/pyproject.toml index ca4e3d921..968363b48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,12 +61,7 @@ dependencies = [ [project.optional-dependencies] -duckdb = [ - "duckdb", -] - extras = [ - "duckdb", "xarray", "netCDF4", "h5netcdf", diff --git a/scripts/_templates/_quarto.yml b/scripts/_templates/_quarto.yml index 1c46ff45e..8b2cfc950 100644 --- a/scripts/_templates/_quarto.yml +++ b/scripts/_templates/_quarto.yml @@ -220,6 +220,12 @@ website: - text: Velocity to Strain Rate href: notes/velocity_to_strain_rate.qmd + - text: Spool Index + href: notes/spool_index.qmd + + - text: Spool Selection + href: notes/spool_selection.qmd + - id: API title: "API" href: api/dascore.qmd diff --git a/tests/test_clients/test_dirspool.py b/tests/test_clients/test_dirspool.py index 823160d43..83b6f4e20 100644 --- a/tests/test_clients/test_dirspool.py +++ b/tests/test_clients/test_dirspool.py @@ -145,18 +145,29 @@ def test_merge(self, multi_patch_file_spool): class TestLoadPatchFastPath: - """Tests for the direct FiberIO read path used by _load_patch.""" + """Tests for the direct FiberIO read path owned by FileResolver.""" - def test_requires_concrete_format_and_version(self, one_directory_spool): + def test_requires_concrete_format_and_version( + self, one_directory_spool, monkeypatch + ): """ Without a concrete format and version the fast path must defer to dc.read, which detects them from the file; get_fiberio with a None version would return the newest reader, not the file's version. """ - spool = one_directory_spool - assert spool._read_patches({"file_format": "", "file_version": ""}) is None - assert spool._read_patches({"file_format": "DASDAE"}) is None - assert spool._read_patches({"file_version": "1"}) is None + resolver = one_directory_spool._catalog.resolver + sentinel = object() + monkeypatch.setattr( + "dascore.io.index.catalog.dc.read", lambda **kwargs: sentinel + ) + monkeypatch.setattr( + dc.io.FiberIO.manager, + "get_fiberio", + lambda **kwargs: pytest.fail("FiberIO fast path should not run"), + ) + assert resolver._read("path", {"file_format": ""}, {}, "") is sentinel + assert resolver._read("path", {"file_format": "DASDAE"}, {}, "") is sentinel + assert resolver._read("path", {"file_version": "1"}, {}, "") is sentinel def test_unusual_fiberio_spool_defers_to_generic_read( self, one_directory_spool, monkeypatch @@ -172,20 +183,55 @@ def read(self, *args, **kwargs): "get_fiberio", lambda format, version: _Reader(), ) - kwargs = { - "path": one_directory_spool.get_contents()["path"].iloc[0], + row = { "file_format": "DASDAE", "file_version": "1", } - assert one_directory_spool._read_patches(kwargs) is None + resolver = one_directory_spool._catalog.resolver + sentinel = object() + monkeypatch.setattr( + "dascore.io.index.catalog.dc.read", lambda **kwargs: sentinel + ) + assert resolver._read("path", row, {}, "") is sentinel - def test_multi_patch_selection_defers_to_generic_read( + def test_multi_patch_resolves_identity_without_second_read( self, one_directory_spool, random_patch, monkeypatch ): - """Fast path should not choose the first patch from multi-patch reads.""" + """Multi-patch reads resolve source identity from the loaded spool.""" class _Reader: def read(self, *args, **kwargs): + patch_1 = random_patch.update_attrs(_source_patch_id="first") + patch_2 = random_patch.update_attrs(_source_patch_id="second") + return dc.spool([patch_1, patch_2]) + + monkeypatch.setattr( + dc.io.FiberIO.manager, + "get_fiberio", + lambda format, version: _Reader(), + ) + monkeypatch.setattr( + "dascore.io.index.catalog.dc.read", + lambda **kwargs: pytest.fail("must not re-read the file"), + ) + row = { + "path": "path", + "file_format": "DASDAE", + "file_version": "1", + "source_patch_id": "second", + } + resolver = one_directory_spool._catalog.resolver + patch = resolver.resolve(row) + assert patch.attrs["_source_patch_id"] == "second" + + def test_positional_id_reads_whole_source( + self, one_directory_spool, random_patch, monkeypatch + ): + """Positional ids must ignore trim hints; a trimmed read would shift them.""" + + class _Reader: + def read(self, *args, **kwargs): + assert "time" not in kwargs, "positional ids must read untrimmed" patch_2 = random_patch.update_attrs(tag="second") return dc.spool([random_patch, patch_2]) @@ -194,17 +240,69 @@ def read(self, *args, **kwargs): "get_fiberio", lambda format, version: _Reader(), ) - path = one_directory_spool.get_contents()["path"].iloc[0] - monkeypatch.setattr(one_directory_spool, "_select_kwargs", {"tag": "second"}) - kwargs = { - "path": path, + row = { + "path": "path", "file_format": "DASDAE", "file_version": "1", - "_modified": True, - "tag": "second", + "source_patch_id": "1", } + resolver = one_directory_spool._catalog.resolver + patch = resolver.resolve(row, time=(None, None)) + assert patch.attrs["tag"] == "second" + - assert one_directory_spool._read_patches(kwargs) is None +class TestSelectKwargs: + """The select_kwargs constructor parameter restricts contents.""" + + @pytest.fixture(scope="class") + def spool_dir(self, random_spool, tmp_path_factory): + """A directory holding the random spool, one file per patch.""" + path = tmp_path_factory.mktemp("select_kwargs_dir") + for num, patch in enumerate(random_spool): + patch.io.write(path / f"patch_{num}.h5", "dasdae") + return path + + @pytest.fixture(scope="class") + def first_patch_range(self, random_spool): + """The time range of the chronologically first patch.""" + patch = sorted(random_spool, key=lambda x: x.get_coord("time").min())[0] + time = patch.get_coord("time") + return (time.min(), time.max()) + + def test_contents_restricted(self, spool_dir, random_spool, first_patch_range): + """Rows outside the requested range must not appear (regression).""" + spool = DirectorySpool( + spool_dir, select_kwargs={"time": first_patch_range} + ).update() + assert 1 <= len(spool) < len(random_spool) + contents = spool.get_contents() + assert (contents["time_min"] <= first_patch_range[1]).all() + assert (contents["time_max"] >= first_patch_range[0]).all() + for patch in spool: + time = patch.get_coord("time") + assert time.min() >= first_patch_range[0] + assert time.max() <= first_patch_range[1] + + def test_restriction_survives_select_and_update( + self, spool_dir, random_spool, first_patch_range + ): + """Derived spools keep the constructor restriction.""" + spool = DirectorySpool( + spool_dir, select_kwargs={"time": first_patch_range} + ).update() + expected = len(spool) + assert len(spool.update()) == expected + distance = random_spool[0].get_coord("distance") + sub = spool.select(distance=(distance.min(), distance.max())) + assert len(sub) == expected + + def test_attr_select_kwargs(self, spool_dir, random_spool): + """Attr-valued select_kwargs filter rows and load cleanly.""" + spool = DirectorySpool(spool_dir, select_kwargs={"tag": "random"}).update() + assert len(spool) == len(random_spool) + assert isinstance(spool[0], dc.Patch) + empty = DirectorySpool(spool_dir, select_kwargs={"tag": "no_such"}).update() + assert len(empty) == 0 class TestDirectoryIndex: diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index f8ab99cac..a2f75d94a 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -66,6 +66,29 @@ def test_coord_namespace(self, spool): assert len(out) == 1 +class TestCatalogPushdown: + """Public spool selection composes a lazy SQLite query.""" + + def test_coord_predicate_reaches_backend(self, spool, monkeypatch): + """Selection does not query all rows before applying its predicate.""" + catalog = spool._catalog or spool._get_catalog() + backend = catalog.backend + calls = [] + original = backend.query + + def wrapped(query=None): + calls.append(query) + return original(query) + + monkeypatch.setattr(backend, "query", wrapped) + selected = spool.select(time=("2020-01-03", "2020-01-04")) + assert calls == [] + assert len(selected) + queries = calls[0] + assert isinstance(queries, list) + assert queries[0].coords["time"] == ("2020-01-03", "2020-01-04") + + class TestSamples: """samples=True never excludes patches; trims on load (#447).""" @@ -111,6 +134,12 @@ def test_requires_range(self, spool): with pytest.raises(InvalidSpoolQueryError, match="requires"): spool.select(time=5, relative=True) + def test_namespaced_coord_with_attr(self, spool): + """Only the coordinate range is converted to relative offsets.""" + out = spool.select(_coords={"time": (1, -1)}, tag="random", relative=True) + assert len(out) + assert set(out.get_contents()["tag"]) == {"random"} + class TestExistingBehaviorKept: """The conventional selections still work.""" diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py index 415f3397a..88338da63 100644 --- a/tests/test_io/test_index/test_catalog.py +++ b/tests/test_io/test_index/test_catalog.py @@ -115,9 +115,14 @@ def test_samples_never_excludes(self, live_catalog, patches): def test_samples_unknown_coord_raises(self, live_catalog): """Samples selections validate coord names.""" - with pytest.raises(InvalidSpoolQueryError, match="coordinate-only"): + with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): live_catalog.select(wavelength=(0, 10), samples=True) + def test_samples_attr_raises(self, live_catalog): + """Samples selections reject attribute names.""" + with pytest.raises(InvalidSpoolQueryError, match="coordinate-only"): + live_catalog.select(tag="test", samples=True) + def test_relative_select(self, live_catalog): """Relative bounds resolve against the global envelope (#362).""" full = live_catalog.to_df() @@ -160,21 +165,12 @@ def test_select_and_trim(self, spool_dir): catalog.close() -class TestEngineGuard: - """Only in-memory-capable engines for live catalogs.""" - - def test_parquet_rejected(self): - """Parquet has no :memory: form.""" - with pytest.raises(ValueError, match="In-memory catalogs support"): - PatchCatalog.from_patches((), engine="parquet") - - class TestCatalogEdges: """Remaining branches: errors, passthroughs, offsets.""" def test_relative_on_unknown_coord_raises(self, live_catalog): """Relative select against an absent coord errors clearly.""" - with pytest.raises(InvalidSpoolQueryError, match="unknown coord"): + with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): live_catalog.select(wavelength=(1, -1), relative=True) def test_relative_requires_range(self, live_catalog): diff --git a/tests/test_io/test_index/test_db_dirspool.py b/tests/test_io/test_index/test_db_dirspool.py index 42784ec7d..db42715ed 100644 --- a/tests/test_io/test_index/test_db_dirspool.py +++ b/tests/test_io/test_index/test_db_dirspool.py @@ -2,7 +2,7 @@ Integration tests: DirectorySpool running on the database index. Exercises the full path — directory walk, scan, ingest, query, patch -loading, chunk — against real files for every backend. +loading, and chunk against real files. """ from __future__ import annotations @@ -14,8 +14,6 @@ from dascore.clients.dirspool import DirectorySpool from dascore.examples import spool_to_directory -BACKENDS = ("duckdb", "sqlite", "parquet") - @pytest.fixture(scope="class") def spool_directory(tmp_path_factory): @@ -24,10 +22,10 @@ def spool_directory(tmp_path_factory): return spool_to_directory(spool, path=tmp_path_factory.mktemp("db_spool")) -@pytest.fixture(params=BACKENDS) -def db_spool(request, spool_directory): - """A DirectorySpool using each database index engine.""" - spool = DirectorySpool(spool_directory, index_engine=request.param) +@pytest.fixture() +def db_spool(spool_directory): + """A DirectorySpool using its SQLite index.""" + spool = DirectorySpool(spool_directory) out = spool.update(progress=None) yield out out.indexer.close() @@ -84,18 +82,18 @@ def test_update_is_incremental(self, db_spool): class TestUpdateLifecycle: """New, modified, and deleted files are tracked per source.""" - @pytest.fixture(params=BACKENDS) - def fresh(self, request, tmp_path): - """A modifiable spool directory + db spool of each engine.""" + @pytest.fixture() + def fresh(self, tmp_path): + """A modifiable spool directory and database spool.""" spool = dc.get_example_spool("random_das") path = spool_to_directory(spool, path=tmp_path / "data") - out = DirectorySpool(path, index_engine=request.param).update(progress=None) - yield path, out, request.param + out = DirectorySpool(path).update(progress=None) + yield path, out out.indexer.close() def test_new_file_found(self, fresh): """A file added after indexing appears on the next update.""" - path, spool, _engine = fresh + path, spool = fresh patch = dc.get_example_patch() patch.io.write(path / "new_file.hdf5", "dasdae") updated = spool.update(progress=None) @@ -103,7 +101,7 @@ def test_new_file_found(self, fresh): def test_deleted_file_removed(self, fresh): """A deleted file's rows are dropped on the next update.""" - path, spool, _engine = fresh + path, spool = fresh target = next(iter(path.glob("*.hdf5"))) target.unlink() updated = spool.update(progress=None) diff --git a/tests/test_io/test_index/test_heterogeneity_stress.py b/tests/test_io/test_index/test_heterogeneity_stress.py index 0a05b3ae3..bff7903af 100644 --- a/tests/test_io/test_index/test_heterogeneity_stress.py +++ b/tests/test_io/test_index/test_heterogeneity_stress.py @@ -4,7 +4,7 @@ Generates hundreds of patch summaries with randomized dimension names, coord dtypes/units, attr names/kinds (including hostile names that collide after sanitization), and verifies ingest, counts, and the -no-false-negative query contract on every backend. +no-false-negative query contract on SQLite. """ from __future__ import annotations @@ -16,7 +16,6 @@ from dascore.io.index import Query, get_backend, summaries_to_records from dascore.units import get_quantity -BACKENDS = ("duckdb", "sqlite", "parquet") N_PATCHES = 300 _DIM_POOL = ( @@ -119,6 +118,8 @@ def _random_attrs(rng) -> dict: int(rng.integers(0, 10**6)), "s" ) else: + # "s" is dimensionally incompatible with "m"/"ft": ingest keeps + # the first-seen dimension and skips the rest with a warning. out[name] = float(rng.uniform(0, 100)) * get_quantity( str(rng.choice(["m", "ft", "s"])) ) @@ -155,11 +156,11 @@ def summaries(): return make_random_summaries(N_PATCHES, seed=42) -@pytest.fixture(params=BACKENDS) -def backend(request, tmp_path_factory, summaries): - """Each backend ingesting the random population.""" - path = tmp_path_factory.mktemp("stress") / f"idx_{request.param}" - back = get_backend(path, kind=request.param) +@pytest.fixture() +def backend(tmp_path_factory, summaries): + """A SQLite backend ingesting the random population.""" + path = tmp_path_factory.mktemp("stress") / "index.sqlite3" + back = get_backend(path) back.write_sources(summaries_to_records(summaries)) yield back back.close() diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index d9f25bea2..21878e2cd 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -1,7 +1,7 @@ """ -Contract tests for spool index backends. +Contract tests for the spool index backend. -Every backend must pass this suite unchanged; it encodes the selector +The SQLite backend must pass this suite; it encodes the selector semantics spec and the summary-only/no-false-negatives contract from the index design doc (see discussion #648). """ @@ -15,11 +15,11 @@ import pytest from dascore.core.summary import PatchSummary +from dascore.exceptions import UnitError from dascore.io.index import Query, get_backend, summaries_to_records from dascore.io.index.backend import resolve_query from dascore.io.index.query import InvalidSpoolQueryError - -BACKENDS = ("duckdb", "sqlite", "parquet") +from dascore.units import get_quantity def _time_coord(t0: str, seconds: float, step_s: float = 0.004): @@ -135,11 +135,12 @@ def make_summaries() -> list[PatchSummary]: return [das1, das2, correlogram, psd] -@pytest.fixture(scope="function", params=BACKENDS) -def backend(request, tmp_path): - """An index backend of each kind, freshly ingested.""" - path = tmp_path / f"index_{request.param}" - back = get_backend(path, kind=request.param) +@pytest.fixture(scope="function") +def backend(tmp_path): + """A freshly ingested SQLite index backend.""" + path = tmp_path / "index.sqlite3" + back = get_backend(path) + back._test_path = path back.write_sources(summaries_to_records(make_summaries())) yield back back.close() @@ -277,6 +278,18 @@ def test_numeric_coord_si_normalized(self, backend): df = backend.query(Query(coords={"distance": (900, 950)})) assert "psd" in set(df["tag"]) + def test_quantity_coord_converts_units(self, backend): + """Quantity selectors convert to the coordinate's canonical units.""" + meter = get_quantity("m") + df = backend.query(Query(coords={"distance": (900 * meter, 950 * meter)})) + assert "psd" in set(df["tag"]) + + def test_incompatible_quantity_coord_raises(self, backend): + """A time quantity cannot query a length coordinate.""" + second = get_quantity("s") + with pytest.raises(UnitError): + backend.query(Query(coords={"distance": (1 * second, 2 * second)})) + def test_scalar_coord(self, backend): """Scalar coord.""" df = backend.query(Query(coords={"frequency": 100})) @@ -389,20 +402,15 @@ def test_delete_cascades(self, backend): def test_reopen_persists(self, backend, tmp_path): """Reopen persists.""" - kind = type(backend).__name__.replace("Backend", "").lower() - path_map = { - "duckdb": tmp_path / "index_duckdb", - "sqlite": tmp_path / "index_sqlite", - "parquet": tmp_path / "index_parquet", - } + path = backend._test_path backend.close() - reopened = get_backend(path_map[kind], kind=kind) + reopened = get_backend(path) try: assert len(reopened.query()) == 4 finally: reopened.close() # reopen once more so fixture teardown close() has a live handle - reopened_again = get_backend(path_map[kind], kind=kind) + reopened_again = get_backend(path) backend.__dict__.update(reopened_again.__dict__) @@ -413,7 +421,7 @@ def test_metadata(self, backend): """Metadata.""" meta = backend.get_metadata() assert meta["what_is_this"] == "dascore_spool_index" - assert meta["index_version"] == 1 + assert meta["index_version"] == 2 def test_names(self, backend): """Names.""" diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 24160518d..37e3229b0 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -17,6 +17,7 @@ import dascore as dc from dascore.core.summary import PatchSummary +from dascore.exceptions import UnitError from dascore.io.index import Query, get_backend, summaries_to_records from dascore.io.index.backend import adapt_params, resolve_query from dascore.io.index.indexer import DBDirectoryIndexer @@ -30,12 +31,10 @@ from dascore.io.index.query import InvalidSpoolQueryError, glob_match from dascore.units import get_quantity -BACKENDS = ("duckdb", "sqlite", "parquet") - @pytest.fixture(scope="module") def backend(tmp_path_factory): - """One duckdb backend with the contract summaries plus extras.""" + """One SQLite backend with the contract summaries plus extras.""" extra = PatchSummary( attrs={ "tag": "extra", @@ -58,8 +57,8 @@ def backend(tmp_path_factory): source_format="DASDAE", source_version="1", ) - path = tmp_path_factory.mktemp("edge") / "idx.duckdb" - back = get_backend(path, kind="duckdb") + path = tmp_path_factory.mktemp("edge") / "index.sqlite3" + back = get_backend(path) back.write_sources(summaries_to_records([*make_summaries(), extra])) yield back back.close() @@ -72,15 +71,9 @@ def test_adapt_params_nan_becomes_none(self): """NaN floats bind as NULL.""" assert adapt_params([float("nan"), 1])[0] is None - def test_unknown_backend_kind_raises(self, tmp_path): - """Asking for a nonexistent engine errors clearly.""" - with pytest.raises(ValueError, match="Unknown index backend"): - get_backend(tmp_path / "x", kind="mongodb") - - @pytest.mark.parametrize("kind", BACKENDS) - def test_bulk_insert_empty_rows_noop(self, tmp_path, kind): - """Empty bulk inserts are no-ops on every backend.""" - back = get_backend(tmp_path / f"i_{kind}", kind=kind) + def test_bulk_insert_empty_rows_noop(self, tmp_path): + """Empty bulk inserts are no-ops.""" + back = get_backend(tmp_path / "insert.sqlite3") back._bulk_insert("attr_meta", ("attr_name",), []) back._executemany( "INSERT INTO attr_meta VALUES (?, ?, ?, ?)", @@ -89,10 +82,9 @@ def test_bulk_insert_empty_rows_noop(self, tmp_path, kind): assert len(back._attr_meta()) == 1 back.close() - @pytest.mark.parametrize("kind", BACKENDS) - def test_write_failure_rolls_back(self, tmp_path, kind): + def test_write_failure_rolls_back(self, tmp_path): """A failing write leaves the index unchanged.""" - back = get_backend(tmp_path / f"r_{kind}", kind=kind) + back = get_backend(tmp_path / "rollback.sqlite3") records = summaries_to_records(make_summaries()) back.write_sources(records[:1]) before = len(back.query()) @@ -107,10 +99,9 @@ def boom(*args, **kwargs): assert len(back.query()) == before back.close() - @pytest.mark.parametrize("kind", BACKENDS) - def test_delete_failure_rolls_back(self, tmp_path, kind): + def test_delete_failure_rolls_back(self, tmp_path): """A failing delete leaves the index unchanged.""" - back = get_backend(tmp_path / f"d_{kind}", kind=kind) + back = get_backend(tmp_path / "delete.sqlite3") back.write_sources(summaries_to_records(make_summaries())) before = len(back.query()) @@ -148,12 +139,12 @@ class TestResolveQueryErrors: def test_unknown_attr_in_explicit_namespace(self, backend): """Unknown key in _attrs raises.""" - with pytest.raises(InvalidSpoolQueryError, match="Unknown attribute"): + with pytest.raises(InvalidSpoolQueryError, match="not an attribute"): resolve_query(backend, _attrs={"nope": 1}) def test_unknown_coord_in_explicit_namespace(self, backend): """Unknown key in _coords raises.""" - with pytest.raises(InvalidSpoolQueryError, match="Unknown coordinate"): + with pytest.raises(InvalidSpoolQueryError, match="not a coordinate"): resolve_query(backend, _coords={"nope": (1, 2)}) @@ -226,6 +217,93 @@ def test_glob_match_helper(self): assert glob_match("STA1", "STA*") assert not glob_match(5, "STA*") + def test_slice_range_form(self, backend): + """Slices resolve to the same range tuples patch selects accept.""" + lo = np.datetime64("2024-06-01T00:00:00", "ns") + query = resolve_query(backend, time=slice(lo, None)) + assert query.coords["time"] == (lo, None) + df = backend.query(query) + assert list(df["tag"]) == ["extra"] + + +class TestUnitHeterogeneity: + """Mixed unit populations across sources.""" + + @staticmethod + def _summary(path, attrs=None, coord_units=None): + """One summary with a 100-200 distance coord.""" + coord = { + "dtype": "float64", + "min": 100.0, + "max": 200.0, + "dims": ("distance",), + "len": 10, + } + if coord_units is not None: + coord["units"] = coord_units + return PatchSummary( + attrs=attrs or {"tag": "units"}, + coords={"distance": coord}, + dims=("distance",), + shape=(10,), + dtype="float32", + source_path=path, + source_format="DASDAE", + source_version="1", + ) + + def test_null_unit_coord_defs_stay_candidates(self, tmp_path): + """Quantity queries must not drop unitless coord defs (candidacy).""" + back = get_backend(tmp_path / "units.sqlite3") + back.write_sources( + summaries_to_records( + [ + self._summary("with_units.h5", coord_units="m"), + self._summary("no_units.h5"), + ] + ) + ) + meters = get_quantity("m") + df = back.query(Query(coords={"distance": (150 * meters, 300 * meters)})) + assert set(df["path"]) == {"with_units.h5", "no_units.h5"} + back.close() + + def test_all_unitless_quantity_query_keeps_candidates(self, tmp_path): + """Unitless defs cannot be proven incompatible; they stay candidates.""" + back = get_backend(tmp_path / "unitless.sqlite3") + back.write_sources(summaries_to_records([self._summary("no_units.h5")])) + meters = get_quantity("m") + df = back.query(Query(coords={"distance": (150 * meters, 300 * meters)})) + assert list(df["path"]) == ["no_units.h5"] + back.close() + + def test_incompatible_units_only_raises(self, tmp_path): + """When every def carries units and none are compatible, raise.""" + back = get_backend(tmp_path / "incompat.sqlite3") + back.write_sources( + summaries_to_records([self._summary("s.h5", coord_units="s")]) + ) + meters = get_quantity("m") + with pytest.raises(UnitError, match="no units compatible"): + back.query(Query(coords={"distance": (150 * meters, 300 * meters)})) + back.close() + + def test_incompatible_attr_units_warn_not_fail(self, tmp_path): + """One rogue attr dimension must not abort the whole index update.""" + summaries = [ + self._summary("a.h5", attrs={"resolution": 1.0 * get_quantity("m")}), + self._summary("b.h5", attrs={"resolution": 1.0 * get_quantity("s")}), + ] + back = get_backend(tmp_path / "attr_units.sqlite3") + with pytest.warns(UserWarning, match="incompatible"): + back.write_sources(summaries_to_records(summaries)) + # both patches indexed; only the incompatible value is skipped + df = back.query() + assert set(df["path"]) == {"a.h5", "b.h5"} + got = back.query(Query(attrs={"resolution": (0.5, 2.0)})) + assert list(got["path"]) == ["a.h5"] + back.close() + class TestIngestEdges: """typed_value and record-building edge cases.""" @@ -297,6 +375,14 @@ def test_auto_update_on_first_query(self, tmp_path, random_patch): indexer = DBDirectoryIndexer(tmp_path) assert len(indexer()) == 1 # no explicit update() call + def test_empty_index_file_updates_on_first_query(self, tmp_path, random_patch): + """A pre-created empty SQLite path is still a new index.""" + random_patch.io.write(tmp_path / "one.hdf5", "dasdae") + index_path = tmp_path / "empty.sqlite3" + index_path.touch() + indexer = DBDirectoryIndexer(tmp_path, index_path=index_path) + assert len(indexer()) == 1 + def test_directory_format_unit(self, tmp_path): """Directory-format sources (xml binary) group as one scan unit.""" import sys @@ -336,7 +422,7 @@ def test_spool_from_indexer(self, tmp_path, random_patch): from dascore.clients.dirspool import DirectorySpool random_patch.io.write(tmp_path / "one.hdf5", "dasdae") - indexer = DBDirectoryIndexer(tmp_path, engine="duckdb") + indexer = DBDirectoryIndexer(tmp_path) spool = DirectorySpool(indexer).update(progress=None) assert len(spool) == 1 @@ -359,29 +445,13 @@ class _Odd: assert typed_value(_Odd()) is None - def test_parquet_cleanup_failure_tolerated(self, tmp_path, monkeypatch): - """A failed unlink of superseded parquet files is not an error.""" - import pathlib - - back = get_backend(tmp_path / "pq", kind="parquet") - back.write_sources(summaries_to_records(make_summaries()[:1])) - - def bad_unlink(self, missing_ok=False): - raise OSError("simulated busy file") - - monkeypatch.setattr(pathlib.Path, "unlink", bad_unlink) - back.write_sources(summaries_to_records(make_summaries()[1:2])) - monkeypatch.undo() - assert len(back.query()) == 2 - back.close() - class TestCoordDeduplication: """Coord summaries are stored once per unique definition.""" def test_shared_coord_stored_once(self, tmp_path): """Identical distance coords across patches share one def row.""" - back = get_backend(tmp_path / "dedup", kind="duckdb") + back = get_backend(tmp_path / "dedup.sqlite3") back.write_sources(summaries_to_records(make_summaries())) links = back._fetch_df("SELECT * FROM patch_coords") defs = back._fetch_df("SELECT * FROM coord_defs") @@ -394,7 +464,7 @@ def test_shared_coord_stored_once(self, tmp_path): def test_defs_reused_across_writes(self, tmp_path): """A second write with known coords creates no new defs.""" - back = get_backend(tmp_path / "reuse", kind="duckdb") + back = get_backend(tmp_path / "reuse.sqlite3") summaries = make_summaries() back.write_sources(summaries_to_records(summaries[:1])) n_defs = len(back._fetch_df("SELECT * FROM coord_defs")) @@ -415,16 +485,28 @@ def test_fingerprint_backed_defs(self, tmp_path): "source_version": "1", } ) - back = get_backend(tmp_path / "fp", kind="duckdb") + back = get_backend(tmp_path / "fp.sqlite3") back.write_sources(summaries_to_records([PatchSummary(**structured)])) defs = back._fetch_df("SELECT def_key, fingerprint FROM coord_defs") assert defs["fingerprint"].notna().all() assert defs["def_key"].str.startswith("fp:").all() back.close() + def test_irregular_coord_hashes_values(self): + """A non-range coordinate carries the hash of its complete array.""" + patch = dc.get_example_patch() + old = patch.get_coord("distance") + values = np.arange(len(old), dtype=float) + values[2:] += 0.5 + patch = patch.update_coords(distance=values) + summary = PatchSummary.from_patch(patch) + record = _coord_record("distance", summary.coords["distance"]) + assert record.coord_hash == patch.get_coord("distance").fingerprint() + assert record.def_key.startswith("fp:") + def test_orphan_defs_tolerated(self, tmp_path): """Deleting sources leaves defs behind without breaking queries.""" - back = get_backend(tmp_path / "orphan", kind="duckdb") + back = get_backend(tmp_path / "orphan.sqlite3") back.write_sources(summaries_to_records(make_summaries())) n_defs = len(back._fetch_df("SELECT * FROM coord_defs")) back.delete_sources(["das/file_1.h5", "das/file_2.h5"]) @@ -448,7 +530,7 @@ def test_no_coords_patch(self, tmp_path): source_format="DASDAE", source_version="1", ) - back = get_backend(tmp_path / "bare", kind="duckdb") + back = get_backend(tmp_path / "bare.sqlite3") back.write_sources(summaries_to_records([summary])) df = back.query() assert len(df) == 1 @@ -471,7 +553,7 @@ def test_same_path_different_base_coexist(self, tmp_path): records_b = [ type(r)(**{**r.__dict__, "base_uri": "s3://bucket-b"}) for r in records_a ] - back = get_backend(tmp_path / "multi", kind="duckdb") + back = get_backend(tmp_path / "multi.sqlite3") back.write_sources(records_a) back.write_sources(records_b) df = back.query() @@ -492,7 +574,7 @@ def test_replacement_is_base_scoped(self, tmp_path): rec = summaries_to_records([one])[0] rec_a = type(rec)(**{**rec.__dict__, "base_uri": "s3://a"}) rec_b = type(rec)(**{**rec.__dict__, "base_uri": "s3://b"}) - back = get_backend(tmp_path / "scoped", kind="duckdb") + back = get_backend(tmp_path / "scoped.sqlite3") back.write_sources([rec_a, rec_b]) assert len(back.query()) == 2 back.write_sources([rec_a]) # replace only the s3://a copy diff --git a/tests/test_io/test_index/test_schema.py b/tests/test_io/test_index/test_schema.py new file mode 100644 index 000000000..d442665e7 --- /dev/null +++ b/tests/test_io/test_index/test_schema.py @@ -0,0 +1,72 @@ +"""Schema and initialization tests for the SQLite spool index.""" + +from __future__ import annotations + +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +import pytest + +from dascore.exceptions import InvalidIndexError, InvalidIndexVersionError +from dascore.io.index import get_backend +from dascore.io.index.schema import INDEX_VERSION, TABLES + + +class TestSchemaValidation: + """Existing files are validated without repair or implicit migration.""" + + def test_unrelated_database_rejected(self, tmp_path): + """A SQLite database belonging to another application is rejected.""" + path = tmp_path / "other.sqlite3" + con = sqlite3.connect(path) + con.execute("CREATE TABLE other_app (value TEXT)") + con.close() + with pytest.raises(InvalidIndexError, match="missing tables"): + get_backend(path) + + def test_old_version_rejected(self, tmp_path): + """Prototype schemas require an explicit delete and rebuild.""" + path = tmp_path / "old.sqlite3" + backend = get_backend(path) + backend._execute("UPDATE meta_data SET index_version = ?", (INDEX_VERSION - 1,)) + backend.close() + with pytest.raises(InvalidIndexVersionError, match="delete it and rebuild"): + get_backend(path) + + def test_schema_has_foreign_keys_and_constraints(self, tmp_path): + """SQLite enforces source/patch ownership and cascades.""" + backend = get_backend(tmp_path / "index.sqlite3") + assert backend._con.execute("PRAGMA foreign_keys").fetchone()[0] == 1 + assert backend._con.execute("PRAGMA busy_timeout").fetchone()[0] == 30_000 + tables = backend._existing_tables() + assert set(TABLES) <= tables + with pytest.raises(sqlite3.IntegrityError): + backend._execute( + "INSERT INTO patches (patch_id, source_id, source_patch_id) " + "VALUES (1, 999, '0')" + ) + backend.close() + + +class TestConcurrentInitialization: + """Only one writer initializes a new index file.""" + + def test_concurrent_open(self, tmp_path): + """Connections racing to create one index all open successfully.""" + path = tmp_path / "shared.sqlite3" + barrier = Barrier(4) + + def open_index(_): + barrier.wait() + backend = get_backend(path) + metadata = backend.get_metadata() + backend.close() + return metadata["index_version"] + + with ThreadPoolExecutor(max_workers=4) as pool: + versions = list(pool.map(open_index, range(4))) + assert versions == [INDEX_VERSION] * 4 + backend = get_backend(path) + assert len(backend._fetch_df("SELECT * FROM meta_data")) == 1 + backend.close() diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index 12de26adb..4f59798f2 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -96,13 +96,6 @@ def test_writable_dir_index_exists(self, tmp_path_factory): assert first.index_path == second.index_path assert first.index_path.exists() - def test_engines_get_separate_indices(self, tmp_path_factory): - """Different engines on one directory must not share an index.""" - path = tmp_path_factory.mktemp("multi_engine_test") - sqlite = DBDirectoryIndexer(path, engine="sqlite") - duckdb = DBDirectoryIndexer(path, engine="duckdb") - assert sqlite.index_path != duckdb.index_path - def test_corrupt_cache(self, directory_indexer_bad_cache, tmp_path_factory): """Ensure a corrupted cache doesn't crash indexing. See #508.""" path = tmp_path_factory.mktemp("corrupt_cache_test") From 8cf0f04691ae6981bcbd07dfb1f904e637870696 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 07:02:09 +0200 Subject: [PATCH 15/97] Trim redundant schema indexes; use semi-join for coord predicates Benchmarked on a 200k-source synthetic SQLite index against the pre-fix state: - Every explicit index except idx_pcoords_name duplicated a PRIMARY KEY/UNIQUE autoindex; dropping them shrinks the index file ~22% (137 -> 107 MB) and speeds builds ~11% with no query-plan loss. - Coordinate predicates now use `p.patch_id IN (SELECT ...)` instead of a correlated EXISTS: SQLite evaluates the subquery once through idx_pcoords_name with a bloom filter rather than probing per patch row (coordinate window query 259 -> 121 ms). - Attr-only queries (len, get_contents, iteration setup) measured 7x faster from the earlier conditional coord-metadata fetch; spool-level memory/directory operations are unchanged within noise. --- dascore/io/index/query.py | 8 +++++--- dascore/io/index/schema.py | 17 ++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index dc968c332..67caa51d9 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -290,7 +290,7 @@ def build_coord_clause( value, ) -> None: """ - Add an EXISTS clause over patch_coords/coord_defs for one coord + Add a patch_coords/coord_defs semi-join clause for one coord predicate. Candidacy only: envelope overlap, never false negatives. Exact @@ -338,7 +338,7 @@ def build_coord_clause( "str": ("min_str", "max_str"), None: (None, None), }[kind] - conditions = ["pc.patch_id = p.patch_id", "pc.coord_name = ?"] + conditions = ["pc.coord_name = ?"] params: list = [name] if kind is not None: if kind in ("time", "dur"): @@ -365,8 +365,10 @@ def build_coord_clause( if hi is not None: conditions.append(f"cd.{min_col} <= ?") params.append(hi) + # A semi-join the engine can evaluate once (idx_pcoords_name) beats a + # correlated EXISTS probed per patch row (~2.5x on a 200k-source index). where.add( - "EXISTS (SELECT 1 FROM patch_coords pc " + "p.patch_id IN (SELECT pc.patch_id FROM patch_coords pc " "JOIN coord_defs cd ON cd.coord_def_id = pc.coord_def_id " "WHERE " + " AND ".join(conditions) + ")", *params, diff --git a/dascore/io/index/schema.py b/dascore/io/index/schema.py index bc8d6a58d..7bd2a404e 100644 --- a/dascore/io/index/schema.py +++ b/dascore/io/index/schema.py @@ -179,14 +179,9 @@ # Names which can never be dynamic attr columns. RESERVED_ATTR_COLUMNS = frozenset({"patch_id"}) -# Secondary indexes: without these, engines that use nested-loop plans -# (SQLite) go quadratic on the correlated coords EXISTS subquery. -INDEXES = ( - ("idx_pcoords_patch", "patch_coords", "patch_id"), - ("idx_pcoords_name", "patch_coords", "coord_name"), - ("idx_pcoords_def", "patch_coords", "coord_def_id"), - ("idx_defs_key", "coord_defs", "def_key"), - ("idx_attrs_patch", "attrs", "patch_id"), - ("idx_patches_source", "patches", "source_id"), - ("idx_sources_path", "sources", "source_path"), -) +# Explicit secondary indexes. Every other access path is covered by a +# PRIMARY KEY or UNIQUE autoindex above — patch_coords(patch_id, +# coord_name), sources(base_uri, source_path), patches(source_id, +# source_patch_id), coord_defs(def_key) — and duplicating them measured +# ~25% extra file size and slower writes for no query gain. +INDEXES = (("idx_pcoords_name", "patch_coords", "coord_name"),) From 803fada2a3989adf349293d0668c748656fc636c Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 07:21:43 +0200 Subject: [PATCH 16/97] Vectorize coord envelope pivot in flat-relation realization _pivot_coords converted 400k coord-link rows with per-row scalar pd.to_datetime calls (~40us each), which dominated full-relation realization on large indexes. Envelope values are now extracted on whole columns through the exact int-ns conversion _ns_to_time already provides, and near-full realizations fetch the link relation in one scan instead of dozens of batched IN queries. On the 200k-source benchmark, full get_contents-style realization drops from 19s to 4.3s (now bounded by the SQL row volume and pandas frame assembly); predicate-driven query times are unchanged. --- dascore/io/index/backend.py | 129 ++++++++++++++++++++++++------------ 1 file changed, 85 insertions(+), 44 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 9d03e032e..d50d1d9e8 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -610,6 +610,60 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: out = out.drop(columns=["base_uri"]) return out.drop(columns=["source_id"], errors="ignore") + @staticmethod + def _add_envelope_objects(coords: pd.DataFrame) -> pd.DataFrame: + """ + Add per-row envelope object columns (_env_min/_env_max/_env_step) + and the merge-identity _key column to the coord-link relation. + + Conversions run on whole columns: per-row scalar pd.to_datetime + calls cost ~40us each and dominated large realizations. + """ + kind = coords["value_kind"].to_numpy() + num_mask = kind == "num" + time_mask = kind == "time" + str_mask = ~(num_mask | time_mask) + # NULL means not relative; via to_numeric so object/float/int + # columns all coerce without pandas downcasting warnings. + relative = ( + pd.to_numeric(coords["is_relative"], errors="coerce") + .to_numpy(dtype="float64", na_value=0.0) + .astype(bool) + ) + + def _time_objects(ns_series: pd.Series, flavor: str) -> np.ndarray: + """Exact int-ns -> Timestamp/Timedelta objects (None for null).""" + series = _ns_to_time(ns_series, flavor) + return series.astype(object).where(series.notna(), None).to_numpy() + + fields = ( + ("_env_min", "min_num", "min_ns", "min_str"), + ("_env_max", "max_num", "max_ns", "max_str"), + ("_env_step", "step_num", "step_ns", None), + ) + for out_col, num_col, ns_col, str_col in fields: + values = np.empty(len(coords), dtype=object) + if num_mask.any(): + values[num_mask] = coords[num_col].to_numpy(dtype=object)[num_mask] + if str_col is not None and str_mask.any(): + values[str_mask] = coords[str_col].to_numpy(dtype=object)[str_mask] + # absolute times are datetimes, relative ones timedeltas; steps + # are timedeltas either way. + time_flavors = ( + ((time_mask & ~relative), "datetime"), + ((time_mask & relative), "timedelta"), + ) + if str_col is None: + time_flavors = ((time_mask, "timedelta"),) + for mask, flavor in time_flavors: + if mask.any(): + values[mask] = _time_objects(coords[ns_col][mask], flavor) + coords[out_col] = values + # Summary-only definitions are useful for indexing/dedup but cannot + # prove coordinate value identity for merge grouping. + coords["_key"] = coords["def_key"].where(coords["fingerprint"].notna(), None) + return coords + def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: """ Add per-coord envelope columns to the flat relation. @@ -625,54 +679,41 @@ def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: if out.empty or "patch_id" not in out.columns: return out ids = out["patch_id"].tolist() - frames = [] - batch = self._in_clause_batch - for start in range(0, len(ids), batch): - chunk = ids[start : start + batch] - marks = ", ".join("?" for _ in chunk) - frames.append( - self._fetch_df( - "SELECT pc.patch_id, pc.coord_name, cd.def_key, cd.fingerprint, " - "cd.value_kind, cd.is_relative, cd.min_num, cd.max_num, " - "cd.step_num, cd.min_ns, cd.max_ns, cd.step_ns, " - "cd.min_str, cd.max_str " - "FROM patch_coords pc " - "JOIN coord_defs cd ON cd.coord_def_id = pc.coord_def_id " - f"WHERE pc.patch_id IN ({marks})", - chunk, + link_sql = ( + "SELECT pc.patch_id, pc.coord_name, cd.def_key, cd.fingerprint, " + "cd.value_kind, cd.is_relative, cd.min_num, cd.max_num, " + "cd.step_num, cd.min_ns, cd.max_ns, cd.step_ns, " + "cd.min_str, cd.max_str " + "FROM patch_coords pc " + "JOIN coord_defs cd ON cd.coord_def_id = pc.coord_def_id" + ) + n_patches = self._fetch_df("SELECT count(*) AS n FROM patches")["n"].iloc[0] + if len(ids) * 4 >= n_patches: + # Most patches selected: one scan plus a pandas filter beats + # many batched IN queries and their frame concatenation. + coords = self._fetch_df(link_sql) + coords = coords[coords["patch_id"].isin(set(ids))].reset_index(drop=True) + else: + frames = [] + batch = self._in_clause_batch + for start in range(0, len(ids), batch): + chunk = ids[start : start + batch] + marks = ", ".join("?" for _ in chunk) + frames.append( + self._fetch_df(f"{link_sql} WHERE pc.patch_id IN ({marks})", chunk) ) - ) - coords = pd.concat(frames, ignore_index=True) + coords = pd.concat(frames, ignore_index=True) if coords.empty: return out + coords = self._add_envelope_objects(coords) for name, group in coords.groupby("coord_name"): - mins, maxs, steps, keys = {}, {}, {}, {} - for row in group.itertuples(): - # Summary-only definitions are useful for indexing/dedup but - # cannot prove coordinate value identity for merge grouping. - keys[row.patch_id] = ( - row.def_key if pd.notnull(row.fingerprint) else None - ) - if row.value_kind == "num": - mn, mx = row.min_num, row.max_num - st = row.step_num - elif row.value_kind == "time": - conv = ( - pd.to_timedelta - if pd.notnull(row.is_relative) and row.is_relative - else pd.to_datetime - ) - mn = conv(int(row.min_ns), unit="ns") - mx = conv(int(row.max_ns), unit="ns") - st = ( - pd.to_timedelta(int(row.step_ns), unit="ns") - if pd.notnull(row.step_ns) - else None - ) - else: - mn, mx, st = row.min_str, row.max_str, None - mins[row.patch_id], maxs[row.patch_id] = mn, mx - steps[row.patch_id] = st + pids = group["patch_id"] + # last row wins for duplicate patch ids, like the mapping loop + # this replaces. + keys = dict(zip(pids, group["_key"])) + mins = dict(zip(pids, group["_env_min"])) + maxs = dict(zip(pids, group["_env_max"])) + steps = dict(zip(pids, group["_env_step"])) out[f"_{name}_def_key"] = out["patch_id"].map(keys) kinds = set(group["value_kind"]) # time/distance envelopes already live on patches... From 2ff2faa1dc541e24f95f4734c753383b6f009219 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 07:49:53 +0200 Subject: [PATCH 17/97] Fetch nullable integer columns exactly; reject float ns input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pd.read_sql_query assembles nullable INTEGER columns through float64, so ns epochs (>2**53) lost ~100-200 ns whenever a fetched column held a NULL: coord envelopes on mixed-kind archives, patches.time_min/max when any patch is relative or timeless, and sources.mtime_ns — where one NULL-mtime row corrupted every stored mtime and turned each update() into a full-directory rescan. _fetch_df now fetches with dtype_backend="numpy_nullable" (exact assembly; a dtype= hint does not help, pandas rounds through float64 before casting) and converts back to classic dtypes, leaving only null-bearing integer columns as nullable Int64. _ns_to_time raises TypeError on float input so a future fetch path cannot silently reintroduce the corruption, and the backend _fetch_df contract now states the exactness requirement. Costs ~1.2s on a 200k-source full realization (4.4 -> 5.6s), the price of exact assembly at that row volume; predicate query times unchanged. --- dascore/io/index/backend.py | 19 ++++- dascore/io/index/lite.py | 35 +++++++- .../test_index/test_index_edge_cases.py | 84 ++++++++++++++++++- 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index d50d1d9e8..62ea750b9 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -51,8 +51,17 @@ def _ns_to_time(series: pd.Series, flavor: str) -> pd.Series: Never goes through float64: ns epochs exceed float64's 2**53 integer range, and the resulting ~100 ns corruption breaks merge boundary - arithmetic downstream. + arithmetic downstream. Float input means precision was already lost + upstream (a fetch path rounding nullable integers through float64), + so it is rejected rather than silently converted. """ + if series.dtype.kind == "f": + msg = ( + f"ns column {series.name!r} arrived as {series.dtype}; values " + "above 2**53 ns are already corrupted. Fetch nullable integer " + "columns exactly (e.g. pandas nullable Int64)." + ) + raise TypeError(msg) mask = series.isna() values = np.zeros(len(series), dtype="int64") if (~mask).any(): @@ -132,7 +141,13 @@ def _executemany(self, sql: str, seq_of_params) -> None: @abc.abstractmethod def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: - """Execute a SELECT and return a dataframe.""" + """ + Execute a SELECT and return a dataframe. + + Contract: nullable integer columns must round-trip exactly (use a + nullable integer dtype, never float64) — ns epochs exceed + float64's 2**53 integer range. + """ @abc.abstractmethod def _begin(self) -> None: diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py index a74d46b24..3bedae0df 100644 --- a/dascore/io/index/lite.py +++ b/dascore/io/index/lite.py @@ -5,6 +5,7 @@ import sqlite3 from pathlib import Path +import numpy as np import pandas as pd from dascore.io.index.backend import SQLIndexBackend, adapt_params @@ -16,6 +17,31 @@ def _adapt(params): return [int(p) if isinstance(p, bool) else p for p in adapt_params(params)] +def _classic_dtypes(df: pd.DataFrame) -> pd.DataFrame: + """ + Convert nullable extension columns back to classic numpy dtypes. + + Fetching with dtype_backend="numpy_nullable" is what keeps nullable + INTEGER columns exact (the default assembly rounds >2**53 ns values + through float64), but downstream spool code expects classic dtypes. + Only int columns that actually hold NULLs stay nullable (Int64) — + the exactness they exist for; consumers handle them via isna(). + """ + for name in df.columns: + col = df[name] + dtype = col.dtype + if not isinstance(dtype, pd.api.extensions.ExtensionDtype): + continue + if dtype.kind == "i": + if not col.isna().any(): + df[name] = col.to_numpy(dtype="int64") + elif dtype.kind == "f": + df[name] = col.to_numpy(dtype="float64", na_value=np.nan) + else: # string/boolean/... -> classic object with None for missing + df[name] = col.to_numpy(dtype=object, na_value=None) + return df + + class SQLiteBackend(SQLIndexBackend): """Index backend storing tables in a single SQLite file.""" @@ -40,7 +66,14 @@ def _executemany(self, sql: str, seq_of_params) -> None: self._con.executemany(sql, [_adapt(p) for p in seq_of_params]) def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: - return pd.read_sql_query(sql, self._con, params=_adapt(params)) + # numpy_nullable assembly keeps nullable INTEGER columns exact; + # the default path rounds them through float64, corrupting ns + # epochs (>2**53). A dtype= hint does NOT prevent that: pandas + # builds float64 first and casts after. + df = pd.read_sql_query( + sql, self._con, params=_adapt(params), dtype_backend="numpy_nullable" + ) + return _classic_dtypes(df) def _begin(self) -> None: self._con.execute("BEGIN IMMEDIATE") diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 37e3229b0..65a7a1290 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -19,9 +19,10 @@ from dascore.core.summary import PatchSummary from dascore.exceptions import UnitError from dascore.io.index import Query, get_backend, summaries_to_records -from dascore.io.index.backend import adapt_params, resolve_query +from dascore.io.index.backend import _ns_to_time, adapt_params, resolve_query from dascore.io.index.indexer import DBDirectoryIndexer from dascore.io.index.ingest import ( + SourceRecord, _coord_record, typed_value, ) @@ -305,6 +306,87 @@ def test_incompatible_attr_units_warn_not_fail(self, tmp_path): back.close() +class TestExactNsFetch: + """Nullable ns-integer columns must never round through float64.""" + + # an epoch-ns value float64 rounds to ...768: exactness is observable + NS = 1_752_244_251_123_456_789 + + def _summary(self, path, coords, dims): + return PatchSummary( + attrs={"tag": "ns"}, + coords=coords, + dims=dims, + shape=tuple(10 for _ in dims), + dtype="float32", + source_path=path, + source_format="DASDAE", + source_version="1", + ) + + def test_time_envelopes_exact_when_column_nullable(self, tmp_path): + """NULLs from other kinds/patches must not degrade ns columns.""" + t0 = np.datetime64(self.NS, "ns") + with_time = self._summary( + "abs.h5", + { + "event_time": { + "dtype": "datetime64", + "min": t0, + "max": t0 + np.timedelta64(60, "s"), + "dims": ("event_time",), + "len": 10, + } + }, + ("event_time",), + ) + # a numeric coord puts NULL min_ns rows in the same link fetch, + # and no time coord leaves patches.time_min NULL for this patch. + numeric_only = self._summary( + "num.h5", + { + "distance": { + "dtype": "float64", + "min": 0.0, + "max": 10.0, + "dims": ("distance",), + "len": 10, + } + }, + ("distance",), + ) + back = get_backend(tmp_path / "exact.sqlite3") + back.write_sources(summaries_to_records([with_time, numeric_only])) + df = back.query().set_index("path") + got = df.loc["abs.h5", "event_time_min"] + assert pd.Timestamp(got).value == self.NS + back.close() + + def test_mtime_ns_exact_with_null_row(self, tmp_path): + """A single NULL mtime row must not corrupt the others (rescans).""" + back = get_backend(tmp_path / "mtime.sqlite3") + records = [ + SourceRecord( + source_path="a.h5", + source_format="", + format_version="", + mtime_ns=self.NS, + size_bytes=1, + ), + SourceRecord(source_path="b.h5", source_format="", format_version=""), + ] + back.write_sources(records) + sources = back.get_sources().set_index("source_path") + assert int(sources.loc["a.h5", "mtime_ns"]) == self.NS + back.close() + + def test_float_ns_column_rejected(self): + """The conversion helper refuses already-corrupted float input.""" + series = pd.Series([1.5e18, np.nan], name="min_ns") + with pytest.raises(TypeError, match="already corrupted"): + _ns_to_time(series, "datetime") + + class TestIngestEdges: """typed_value and record-building edge cases.""" From a555ad9656d22534ba0c83aae6a06a194b83ac7d Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 11:21:43 +0200 Subject: [PATCH 18/97] Normalize NaN source_patch_id in resolvers Rows fetched through pandas represent missing text values as NaN, which is truthy, so the previous 'or ""' normalization let floats through to string methods. Surfaced by the exact nullable-column fetching change; caught by dirspool patch extraction after merging dev. --- dascore/io/index/catalog.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index c5e1444ad..cb153ce30 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -38,6 +38,16 @@ _counter = itertools.count() +def _row_source_patch_id(row: Mapping) -> str: + """Return the row's source_patch_id as a string ("" when missing). + + Rows fetched through pandas represent missing text values as NaN, + which is truthy, so a plain `or ""` does not normalize them. + """ + value = row.get("source_patch_id") + return "" if value is None or pd.isnull(value) else str(value) + + class PatchResolver(abc.ABC): """Turn one flat-relation row into a Patch.""" @@ -64,7 +74,7 @@ def register(self, path: str, source_patch_id: str, patch: dc.Patch) -> None: def resolve(self, row: Mapping, **trim) -> dc.Patch: """Look the patch up; live patches ignore trim hints.""" - key = (row["path"], row.get("source_patch_id") or "") + key = (row["path"], _row_source_patch_id(row)) return self._registry[key] @@ -105,7 +115,7 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: if self._root is not None and "://" not in str(path): if not Path(path).is_absolute(): path = self._root / path - source_patch_id = row.get("source_patch_id") or "" + source_patch_id = _row_source_patch_id(row) if source_patch_id.isdigit(): # Positional (synthesized) ids index the full source read; a # trimmed read would shift or drop patches and bind the wrong From 2e18b8073628c20fc102de9bd20e5aceb72c4cab Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 11:24:11 +0200 Subject: [PATCH 19/97] Add groupby_attrs config field The default set of attributes which partition patches during chunk/merge: conventional categorical identity attrs (network, station, data_type, data_category, tag, instrument_id, acquisition_id). Per-call group arguments will override this; quantitative attrs stay under the conflict parameter. First piece of the chunk planner work. --- dascore/config.py | 18 ++++++++++++++++++ tests/test_utils/test_config.py | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/dascore/config.py b/dascore/config.py index 8e0979279..fb4b3f466 100644 --- a/dascore/config.py +++ b/dascore/config.py @@ -51,6 +51,24 @@ class DascoreConfig(BaseModel): default="standard", description="Controls whether DASCore appends processing history to patches.", ) + groupby_attrs: tuple[str, ...] = Field( + default=( + "network", + "station", + "data_type", + "data_category", + "tag", + "instrument_id", + "acquisition_id", + ), + description=( + "Attributes which partition patches into separate groups for " + "chunk/merge operations. Patches whose values differ on any of " + "these are never combined (no error); the per-call `group` " + "argument overrides this default. Names missing from a spool " + "are ignored." + ), + ) # Local cache and index locations. downloader_cache_dir: Path = Field( diff --git a/tests/test_utils/test_config.py b/tests/test_utils/test_config.py index 162f6e10b..8cc6e44a1 100644 --- a/tests/test_utils/test_config.py +++ b/tests/test_utils/test_config.py @@ -75,3 +75,28 @@ class _UsesConfig: assert _UsesConfig().value == get_config().display_float_precision with set_config(display_float_precision=7): assert _UsesConfig().value == 7 + + def test_groupby_attrs_default(self): + """The default group attrs are the conventional identity set.""" + expected = ( + "network", + "station", + "data_type", + "data_category", + "tag", + "instrument_id", + "acquisition_id", + ) + assert get_config().groupby_attrs == expected + + def test_groupby_attrs_override(self): + """groupby_attrs round-trips through scoped set_config.""" + previous = get_config() + with set_config(groupby_attrs=("network", "station")): + assert get_config().groupby_attrs == ("network", "station") + assert get_config() == previous + + def test_groupby_attrs_coerced_to_tuple(self): + """List inputs coerce to the immutable tuple form.""" + with set_config(groupby_attrs=["tag"]): + assert get_config().groupby_attrs == ("tag",) From d7db65fce1604b5baf093d1e56e6440ca243cc30 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 11:29:47 +0200 Subject: [PATCH 20/97] Add sampling_group_tolerance config field Exposes the previously hardcoded 5% relative step tolerance that keeps differently-sampled patches in separate chunk/merge groups (chunking spec D9). --- dascore/config.py | 10 ++++++++++ tests/test_utils/test_config.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/dascore/config.py b/dascore/config.py index fb4b3f466..47fc2c1b2 100644 --- a/dascore/config.py +++ b/dascore/config.py @@ -51,6 +51,16 @@ class DascoreConfig(BaseModel): default="standard", description="Controls whether DASCore appends processing history to patches.", ) + sampling_group_tolerance: float = Field( + default=0.05, + gt=0, + description=( + "Relative sampling-interval difference above which patches are " + "never combined during chunk/merge operations. E.g. the default " + "0.05 keeps patches whose steps differ by more than 5% in " + "separate groups." + ), + ) groupby_attrs: tuple[str, ...] = Field( default=( "network", diff --git a/tests/test_utils/test_config.py b/tests/test_utils/test_config.py index 8cc6e44a1..ccf48e725 100644 --- a/tests/test_utils/test_config.py +++ b/tests/test_utils/test_config.py @@ -100,3 +100,19 @@ def test_groupby_attrs_coerced_to_tuple(self): """List inputs coerce to the immutable tuple form.""" with set_config(groupby_attrs=["tag"]): assert get_config().groupby_attrs == ("tag",) + + def test_sampling_group_tolerance_default(self): + """The default sampling group tolerance is 5%.""" + assert get_config().sampling_group_tolerance == 0.05 + + def test_sampling_group_tolerance_override(self): + """sampling_group_tolerance round-trips through scoped set_config.""" + previous = get_config() + with set_config(sampling_group_tolerance=0.01): + assert get_config().sampling_group_tolerance == 0.01 + assert get_config() == previous + + def test_sampling_group_tolerance_must_be_positive(self): + """Non-positive tolerances are rejected.""" + with pytest.raises(ValueError, match="sampling_group_tolerance"): + set_config(sampling_group_tolerance=0) From a44662bbce3ed9c5645177e1536f57209875b9aa Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 11:42:08 +0200 Subject: [PATCH 21/97] Add chunk planner over the flat relation First piece of the chunk-machinery replacement: build_chunk_plan consumes the catalog's flat relation and produces a ChunkPlan (outputs table + members/instruction table) without touching patch data, per the chunking formalities spec: - Partitioning: resolved group attrs (per-call group= overrides config groupby_attrs; explicit unknown names raise) + dims signature + structural def keys of non-chunked coords + sampling groups (config.sampling_group_tolerance) + continuity groups. Continuity is evaluated within each cell so unrelated patches can never bridge a gap (fixes a latent ChunkManager behavior where continuity was computed globally). - One canonical partition step (middle value) drives interval math and output metadata (spec D7). - Patches lacking the chunk dim raise by default (missing_dim='drop' restores exclusion; spec D2). - Complete-overlap members deduplicate deterministically to the first by (start, patch id) (spec D3); partition and output order never depend on input row order (spec section 8). - overlap >= length now raises a clear ParameterError in the shared interval math (spec D6); test updated. 30 new tests including oracle-parity checks against ChunkManager for compatible cases and determinism under row shuffling. Nothing consumes plans yet; the assembler and cutover follow. --- dascore/io/index/plan.py | 384 ++++++++++++++++++++++++++ dascore/utils/chunk.py | 7 +- tests/test_core/test_patch_chunk.py | 7 +- tests/test_io/test_index/test_plan.py | 329 ++++++++++++++++++++++ 4 files changed, 722 insertions(+), 5 deletions(-) create mode 100644 dascore/io/index/plan.py create mode 100644 tests/test_io/test_index/test_plan.py diff --git a/dascore/io/index/plan.py b/dascore/io/index/plan.py new file mode 100644 index 000000000..460c016f9 --- /dev/null +++ b/dascore/io/index/plan.py @@ -0,0 +1,384 @@ +""" +Chunk planning over the flat patch relation. + +Implements the "Chunking formalities" spec: the planner consumes the +catalog's flat relation (one row per patch: `{dim}_min/max/step` envelopes, +`_{dim}_def_key` structural identity, attr columns) and produces a +[`ChunkPlan`](`dascore.io.index.plan.ChunkPlan`) — an outputs table (one row +per output patch) plus a members table binding each output to trimmed +slices of source patches. No patch data is touched; assembly happens later. + +Portions of the interval/instruction math are ported from +`dascore.utils.chunk.ChunkManager` (which this planner replaces at +cutover) with the spec's adjudicated corrections applied. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +import numpy as np +import pandas as pd + +import dascore as dc +from dascore.exceptions import ( + ChunkError, + CoordMergeError, + InvalidSpoolQueryError, + ParameterError, +) +from dascore.utils.chunk import get_intervals +from dascore.utils.misc import get_middle_value +from dascore.utils.pd import _remove_overlaps, get_interval_columns +from dascore.utils.time import is_datetime64, is_timedelta64, to_float, to_timedelta64 + +# Columns which never participate in conflict policing and never carry to +# outputs: source bookkeeping (outputs are not file rows). +_SOURCE_COLUMNS = ("path", "file_format", "file_version", "source_patch_id") + + +@dataclass(frozen=True) +class ChunkPlan: + """ + A materialization-free description of a chunk operation. + + Attributes + ---------- + outputs + One row per output patch: `{dim}_min/max/step` for the chunked + dimension, an `output_id`, and all carried columns (group attrs, + dims, structural def keys, conflict-policed attrs). + members + Instruction rows binding outputs to sources: `output_id`, + `_patch_id`, the exact `{dim}_min/max` trim for that member, and + `_modified` (False when the member loads whole). + dim + The chunked dimension. + value + The requested chunk length (None for merge mode). + params + Resolved parameters (group attrs, tolerances, overlap, + keep_partial, conflict, snap_coords, missing_dim) — recorded, not + referencing config. + """ + + outputs: pd.DataFrame + members: pd.DataFrame + dim: str + value: Any + params: dict = field(default_factory=dict) + + @property + def merge_mode(self) -> bool: + """Return True when this plan merges (no segmenting length).""" + return self.value is None + + +def _resolve_group_attrs(group, columns) -> tuple[str, ...]: + """Resolve the group attrs: per-call > config; explicit names must exist.""" + if group is not None: + group = (group,) if isinstance(group, str) else tuple(group) + if missing := [x for x in group if x not in columns]: + msg = ( + f"group attribute(s) {missing} do not exist on any patch " + "in the spool." + ) + raise InvalidSpoolQueryError(msg) + return group + # Config (and default) names are best-effort. + return tuple(x for x in dc.get_config().groupby_attrs if x in columns) + + +def _sampling_group(step: pd.Series, tolerance: float) -> pd.Series: + """Label rows whose steps are within relative tolerance (spec 2.3).""" + col = to_float(step.values) + order = np.argsort(col) + sorted_col = col[order] + prev = np.roll(sorted_col, 1) + with np.errstate(invalid="ignore", divide="ignore"): + diff = (sorted_col - prev) / sorted_col + out_of_threshold = diff > tolerance + group = np.cumsum(out_of_threshold) + return pd.Series(group[np.argsort(order)], index=step.index) + + +def _continuity_group(start, stop, step, tolerance) -> pd.Series: + """Label maximal near-contiguous runs (spec 2.4).""" + args = np.argsort(start.to_numpy()) + start_sorted = start.iloc[args] + stop_sorted = stop.iloc[args] + step_sorted = step.iloc[args] + stop_cum_max = stop_sorted.cummax() + end_markers = stop_cum_max.shift() + step_sorted * tolerance + has_gap = start_sorted > end_markers + group = has_gap.astype(np.int64).cumsum() + return group[start.index] + + +def _partition(df, name, group_attrs, tolerance, sampling_tolerance) -> pd.Series: + """ + Return partition labels: rows sharing a label may combine (spec 2). + + Components: group attrs, dims signature, structural def keys of + non-chunked coords, sampling group, and continuity group. Continuity + is evaluated *within* each other-component cell so unrelated patches + can never bridge a gap. + """ + start, stop, step = get_interval_columns(df, name) + cols = [x for x in group_attrs if x in df.columns] + if "dims" in df.columns: + cols.append("dims") + cols += [ + x for x in df.columns if x.endswith("_def_key") and x != f"_{name}_def_key" + ] + base = ( + df.groupby(cols, dropna=False, sort=False).ngroup() + if cols + else pd.Series(0, index=df.index) + ) + samp = _sampling_group(step, sampling_tolerance) + cell = base.astype(str) + "_" + samp.astype(str) + cont = pd.Series(0, index=df.index, dtype=np.int64) + for _, index in df.groupby(cell, sort=False).groups.items(): + sub = df.loc[index] + s, e, st = get_interval_columns(sub, name) + cont.loc[index] = _continuity_group(s, e, st, tolerance).astype(np.int64) + return cell + "_" + cont.astype(str) + + +def _coerce_length_overlap(value, overlap, start_dtype): + """Coerce the chunk length/overlap to the dimension's span dtype.""" + time_like = is_datetime64(start_dtype) or is_timedelta64(start_dtype) + if time_like: + value = to_timedelta64(value) if value is not None else None + overlap = to_timedelta64(overlap) if overlap is not None else None + return value, overlap + + +def _police_columns(sub: pd.DataFrame, name, group_attrs, conflict) -> dict: + """ + Return the carried column values for one partition (spec 2.5/6.4). + + Group attrs, dims, and def keys are single-valued by construction. + Remaining public attrs must be single-valued, policed by `conflict`. + """ + dims = set(str(sub.iloc[0].get("dims", "")).split(",")) + carried: dict[str, Any] = {} + for col in sub.columns: + if col.startswith("_") or col in _SOURCE_COLUMNS: + continue + prefix = col.split("_")[0] + if prefix == name: # chunk-dim envelope columns are rebuilt + continue + values = sub[col].unique() + single = len(values) == 1 or (len(values) and pd.isnull(values).all()) + if single: + carried[col] = values[0] + continue + in_group = col in group_attrs or col == "dims" + if in_group: # partitioning guarantees this; guard anyway + carried[col] = values[0] + continue + if prefix in dims or conflict == "raise": + msg = ( + f"Cannot merge on dim {name} because all values for " + f"{col} are not equal. Consider using the `conflict` " + "argument to loosen this restriction." + ) + raise CoordMergeError(msg) + if conflict == "keep_first": + carried[col] = sub[col].iloc[0] + # conflict == "drop": omit the column entirely. + # Structural def keys carry (single-valued within a partition). + for col in sub.columns: + if col.endswith("_def_key") and col != f"_{name}_def_key": + carried[col] = sub[col].iloc[0] + return carried + + +def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFrame: + """ + Bind one partition's outputs to trimmed source slices. + + Sources are ordered by (start, _patch_id); overlapping coverage is + deduplicated so the earlier source owns the overlap (D3: complete + overlaps keep the first member, deterministically). + """ + min_name, max_name = f"{name}_min", f"{name}_max" + sub = sub.sort_values([min_name, "_patch_id"], kind="stable") + original = sub[[min_name, max_name]].reset_index(drop=True) + sub = _remove_overlaps(sub, name) + # Fully-covered sources become degenerate after start correction; they + # contribute nothing (deterministic keep-first dedup). + keep = sub[min_name].values <= sub[max_name].values + sub = sub[keep] + original = original[keep].reset_index(drop=True) + if sub.empty or outputs.empty: + return pd.DataFrame( + columns=["output_id", "_patch_id", min_name, max_name, "_modified"] + ) + src1 = sub[min_name].values + src2 = sub[max_name].values + chu1 = outputs[min_name].values + chu2 = outputs[max_name].values + # Map each output onto the source rows it draws from. + starts_ind = np.searchsorted(src1, chu1, side="right") - 1 + ends_ind = np.searchsorted(src2, chu2, side="left") + rows = [] + modified_src = sub["_modified"].values if "_modified" in sub else None + for out_num, (a, b) in enumerate(zip(starts_ind, ends_ind)): + a = max(int(a), 0) + for src_num in range(a, int(b) + 1): + if src_num >= len(sub): + continue + lo = max(src1[src_num], chu1[out_num]) + hi = min(src2[src_num], chu2[out_num]) + if lo > hi: + continue + row_mod = bool(modified_src[src_num]) if modified_src is not None else False + unchanged = ( + lo == original[min_name].iloc[src_num] + and hi == original[max_name].iloc[src_num] + and not row_mod + ) + rows.append( + { + "output_id": outputs["output_id"].iloc[out_num], + "_patch_id": sub["_patch_id"].iloc[src_num], + min_name: lo, + max_name: hi, + "_modified": not unchanged, + } + ) + return pd.DataFrame(rows) + + +def build_chunk_plan( + df: pd.DataFrame, + *, + overlap=None, + keep_partial: bool = False, + snap_coords: bool = True, + tolerance: float = 1.5, + conflict: Literal["drop", "raise", "keep_first"] = "raise", + group=None, + missing_dim: Literal["raise", "drop"] = "raise", + **kwargs, +) -> ChunkPlan: + """ + Build a chunk plan from a flat patch relation. + + Parameters mirror `Spool.chunk` (see the chunking formalities spec); + exactly one keyword names the dimension to chunk and its length + (`None`/`...` merges). + """ + if len(kwargs) != 1: + msg = ( + "Chunking only supported along one dimension. You passed " + f"kwargs: {kwargs}" + ) + raise ParameterError(msg) + ((name, value),) = kwargs.items() + value = None if value is Ellipsis else value + merge_mode = pd.isnull(value) + if merge_mode and (keep_partial or overlap): + msg = ( + "When chunk value is None (ie chunking is used for merging) " + "keep_partial and overlap are not supported." + ) + raise ParameterError(msg) + if not merge_mode: + zero = to_timedelta64(0) if is_timedelta64(value) else 0 + if value <= zero: + msg = "Chunk value must be greater than 0." + raise ParameterError(msg) + if missing_dim not in ("raise", "drop"): + msg = f"missing_dim must be 'raise' or 'drop', got {missing_dim!r}" + raise ParameterError(msg) + + min_name, max_name = f"{name}_min", f"{name}_max" + if min_name not in df.columns: + msg = f"No patch in the spool has a {name!r} dimension to chunk." + raise ChunkError(msg) + empty_members = pd.DataFrame( + columns=["output_id", "_patch_id", min_name, max_name, "_modified"] + ) + params = dict( + overlap=overlap, + keep_partial=keep_partial, + snap_coords=snap_coords, + tolerance=tolerance, + conflict=conflict, + missing_dim=missing_dim, + group=_resolve_group_attrs(group, set(df.columns)), + sampling_group_tolerance=dc.get_config().sampling_group_tolerance, + ) + # Missing chunk-dim envelopes (spec 7 / D2). + null_rows = pd.isnull(df[min_name]) | pd.isnull(df[max_name]) + if null_rows.any(): + if missing_dim == "raise": + bad = df.loc[null_rows, "_patch_id"].tolist() + msg = ( + f"{int(null_rows.sum())} patch(es) lack the chunk dimension " + f"{name!r} (patch ids {bad[:5]}...). Pass missing_dim='drop' " + "to exclude them." + ) + raise ChunkError(msg) + df = df[~null_rows] + if df.empty: + outputs = pd.DataFrame(columns=[min_name, max_name, "output_id"]) + return ChunkPlan(outputs, empty_members, name, value, params) + + labels = _partition( + df, name, params["group"], tolerance, params["sampling_group_tolerance"] + ) + value_c, overlap_c = _coerce_length_overlap(value, overlap, df[min_name].dtype) + out_frames, member_frames = [], [] + next_id = 0 + # Deterministic partition order (spec 8): by (partition min, smallest + # member patch id) — never by anything derived from input row order. + stats = df.groupby(labels, sort=False).agg( + _min=(min_name, "min"), _pid=("_patch_id", "min") + ) + part_order = stats.sort_values(["_min", "_pid"], kind="stable").index + groups = df.groupby(labels, sort=False).groups + for label in part_order: + sub = df.loc[groups[label]] + start, stop, step = get_interval_columns(sub, name) + part_step = get_middle_value(step.values) # D7: one step everywhere + g_start, g_stop = start.min(), stop.max() + if merge_mode: + start_stop = np.atleast_2d(np.asarray([g_start, g_stop])) + else: + try: + start_stop = get_intervals( + g_start, + g_stop, + value_c, + overlap=overlap_c, + step=part_step, + keep_partials=keep_partial, + ) + except ChunkError: # partition too short; skip (D8) + continue + sub_sorted = sub.sort_values([min_name, "_patch_id"], kind="stable") + carried = _police_columns(sub_sorted, name, params["group"], conflict) + outputs = pd.DataFrame(start_stop, columns=[min_name, max_name]) + outputs[f"{name}_step"] = part_step + for col, val in carried.items(): + outputs[col] = val + outputs["output_id"] = np.arange(next_id, next_id + len(outputs)) + next_id += len(outputs) + members = _build_members(sub, outputs, name) + out_frames.append(outputs) + member_frames.append(members) + if not out_frames: + msg = "Could not chunk. No segments with sufficient length found." + raise ChunkError(msg) + outputs = pd.concat(out_frames, ignore_index=True) + members = pd.concat( + [x for x in member_frames if not x.empty] or [empty_members], + ignore_index=True, + ) + return ChunkPlan(outputs, members, name, value, params) diff --git a/dascore/utils/chunk.py b/dascore/utils/chunk.py index dc80f0759..aed3e1d26 100644 --- a/dascore/utils/chunk.py +++ b/dascore/utils/chunk.py @@ -85,9 +85,10 @@ def get_intervals( # get variable and perform checks overlap = length * 0 if not overlap else overlap step = length * 0 if pd.isnull(step) else step - # Check for errors - if overlap > length: - msg = "Cant chunk when overlap is greater than chunk size" + # Check for errors. Overlap equal to length would produce zero-stride + # segments, so it is also rejected. + if overlap >= length: + msg = "Cant chunk when overlap is greater than or equal to chunk size" raise ParameterError(msg) # If the step is known, we need to account for it in the total duration # See 474. diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 869f3fc45..5b94cd1e2 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -170,10 +170,13 @@ def test_too_big_partial(self, diverse_spool): assert spool1 == spool2 def test_too_big_overlap_raises(self, diverse_spool): - """Overlap > chunk an error should raise.""" - msg = "overlap is greater than chunk size" + """Overlap >= chunk size should raise a clear error.""" + msg = "overlap is greater than or equal to chunk size" with pytest.raises(ParameterError, match=msg): diverse_spool.chunk(time=10, overlap=11) + # Equal overlap would mean zero-stride segments; also rejected. + with pytest.raises(ParameterError, match=msg): + diverse_spool.chunk(time=10, overlap=10) def test_issue_474(self, random_spool): """Ensure spools can be chunked with the duration reported by coord.""" diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py new file mode 100644 index 000000000..b7f51c62e --- /dev/null +++ b/tests/test_io/test_index/test_plan.py @@ -0,0 +1,329 @@ +"""Tests for the chunk planner (chunking formalities spec).""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +import dascore as dc +from dascore.exceptions import ( + ChunkError, + CoordMergeError, + InvalidSpoolQueryError, + ParameterError, +) +from dascore.io.index.catalog import PatchCatalog +from dascore.io.index.plan import ChunkPlan, build_chunk_plan +from dascore.utils.time import to_timedelta64 + +ONE_S = np.timedelta64(1, "s") + + +def _flat(patches) -> pd.DataFrame: + """Get the flat relation for a list of patches.""" + return PatchCatalog.from_patches(list(patches)).to_df() + + +@pytest.fixture(scope="module") +def random_flat() -> pd.DataFrame: + """Flat relation of the contiguous random_das example spool.""" + return _flat(dc.get_example_spool("random_das")) + + +@pytest.fixture(scope="module") +def diverse_flat() -> pd.DataFrame: + """Flat relation of the diverse example spool.""" + return _flat(dc.get_example_spool("diverse_das")) + + +class TestValidation: + """Parameter validation per the spec errors table.""" + + def test_no_kwargs_raises(self, random_flat): + """Exactly one dimension kwarg is required.""" + with pytest.raises(ParameterError, match="one dimension"): + build_chunk_plan(random_flat) + + def test_two_kwargs_raise(self, random_flat): + """Two chunk kwargs raise.""" + with pytest.raises(ParameterError, match="one dimension"): + build_chunk_plan(random_flat, time=10, distance=10) + + def test_non_positive_value_raises(self, random_flat): + """Chunk lengths must be positive.""" + with pytest.raises(ParameterError, match="greater than 0"): + build_chunk_plan(random_flat, time=0) + + def test_merge_mode_forbids_overlap(self, random_flat): + """Merge mode does not accept overlap/keep_partial.""" + with pytest.raises(ParameterError, match="merging"): + build_chunk_plan(random_flat, time=None, overlap=1) + with pytest.raises(ParameterError, match="merging"): + build_chunk_plan(random_flat, time=..., keep_partial=True) + + def test_overlap_ge_length_raises(self, random_flat): + """D6: overlap >= length raises cleanly.""" + with pytest.raises(ParameterError, match="overlap"): + build_chunk_plan(random_flat, time=2, overlap=2) + + def test_unknown_group_raises(self, random_flat): + """Explicit group names must exist somewhere in the spool.""" + with pytest.raises(InvalidSpoolQueryError, match="bob"): + build_chunk_plan(random_flat, time=None, group=("bob",)) + + def test_unknown_dim_raises(self, random_flat): + """Chunking a dimension no patch has raises.""" + with pytest.raises(ChunkError, match="quelle"): + build_chunk_plan(random_flat, quelle=10) + + def test_bad_missing_dim_raises(self, random_flat): + """missing_dim accepts only raise/drop.""" + with pytest.raises(ParameterError, match="missing_dim"): + build_chunk_plan(random_flat, time=None, missing_dim="bob") + + +class TestMergePlan: + """Merge-mode planning on contiguous data.""" + + def test_contiguous_spool_single_output(self, random_flat): + """A contiguous spool merges to one output.""" + plan = build_chunk_plan(random_flat, time=None) + assert isinstance(plan, ChunkPlan) + assert plan.merge_mode + assert len(plan.outputs) == 1 + out = plan.outputs.iloc[0] + assert out["time_min"] == random_flat["time_min"].min() + assert out["time_max"] == random_flat["time_max"].max() + # every source patch appears exactly once as a member + assert len(plan.members) == len(random_flat) + assert set(plan.members["_patch_id"]) == set(random_flat["_patch_id"]) + + def test_members_unmodified_when_contiguous(self, random_flat): + """Contiguous members load whole (no trims).""" + plan = build_chunk_plan(random_flat, time=None) + assert not plan.members["_modified"].any() + + def test_diverse_partitions(self, diverse_flat): + """The diverse spool partitions by identity attrs, never raising.""" + plan = build_chunk_plan(diverse_flat, time=None) + assert len(plan.outputs) > 1 + # every output's members share that output's group attr values + merged = plan.members.merge( + diverse_flat[["_patch_id", "network", "station", "tag"]], + on="_patch_id", + ).merge( + plan.outputs[["output_id", "network", "station", "tag"]], + on="output_id", + suffixes=("_src", "_out"), + ) + for col in ("network", "station", "tag"): + src, out = merged[f"{col}_src"], merged[f"{col}_out"] + equal = (src == out) | (src.isnull() & out.isnull()) + assert equal.all() + + def test_gap_splits_partition(self): + """A gap larger than tolerance yields separate outputs.""" + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) + time = p1.get_coord("time") + dt = time.step + span = time.max() - time.min() + dt + p2 = dc.get_example_patch(time_min=time.max() + dt) # contiguous + p3 = dc.get_example_patch(time_min=time.max() + span + 10 * dt) + plan = build_chunk_plan(_flat([p1, p2, p3]), time=None) + assert len(plan.outputs) == 2 + + def test_plan_records_params(self, random_flat): + """Plans record resolved parameters, not config references.""" + with dc.set_config(sampling_group_tolerance=0.02): + plan = build_chunk_plan(random_flat, time=None) + assert plan.params["sampling_group_tolerance"] == 0.02 + assert isinstance(plan.params["group"], tuple) + + +class TestSegmentPlan: + """Segment-mode planning.""" + + def test_intervals_cover_envelope(self, random_flat): + """Chunk segments tile the envelope with the requested length.""" + plan = build_chunk_plan(random_flat, time=3) + out = plan.outputs + lengths = (out["time_max"] - out["time_min"]) + out["time_step"] + expected = to_timedelta64(3) + assert (abs(lengths - expected) <= out["time_step"]).all() + + def test_members_reference_real_patches(self, random_flat): + """All members point at rows of the input relation.""" + plan = build_chunk_plan(random_flat, time=3) + assert set(plan.members["_patch_id"]) <= set(random_flat["_patch_id"]) + # member trims stay within their output's envelope + joined = plan.members.merge( + plan.outputs[["output_id", "time_min", "time_max"]], + on="output_id", + suffixes=("", "_out"), + ) + assert (joined["time_min"] >= joined["time_min_out"]).all() + assert (joined["time_max"] <= joined["time_max_out"]).all() + + def test_too_short_partition_skipped(self): + """D8: partitions shorter than the length are skipped silently.""" + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) # ~8s long + time = p1.get_coord("time") + gap = time.max() - time.min() + 100 * ONE_S + p2 = dc.get_example_patch(time_min=time.min() + gap) + df = _flat([p1, p2]) + plan = build_chunk_plan(df, time=5) + assert len(plan.outputs) == 2 # one 5s chunk per 8s partition + with pytest.raises(ChunkError, match="sufficient length"): + build_chunk_plan(df, time=100) + + def test_overlap(self, random_flat): + """Overlapping chunks step by length minus overlap.""" + plan = build_chunk_plan(random_flat, time=4, overlap=2) + starts = plan.outputs["time_min"].sort_values().values + strides = np.diff(starts) + assert (abs(strides - to_timedelta64(2)) <= to_timedelta64(0.01)).all() + + def test_middle_value_step(self): + """D7: the partition step is the middle value of member steps.""" + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) + time = p1.get_coord("time") + p2 = dc.get_example_patch(time_min=time.max() + time.step) + plan = build_chunk_plan(_flat([p1, p2]), time=None) + assert plan.outputs["time_step"].iloc[0] == time.step + + +class TestMissingDim: + """Spec section 7 (D2): patches lacking the chunk dim.""" + + @pytest.fixture() + def flat_with_null(self, random_flat): + """A flat relation with one null time envelope.""" + df = random_flat.copy() + df.loc[df.index[0], ["time_min", "time_max"]] = (pd.NaT, pd.NaT) + return df + + def test_raise_by_default(self, flat_with_null): + """Null chunk-dim envelopes raise by default.""" + with pytest.raises(ChunkError, match="missing_dim"): + build_chunk_plan(flat_with_null, time=None) + + def test_drop_opt_in(self, flat_with_null): + """missing_dim='drop' excludes the offending rows.""" + plan = build_chunk_plan(flat_with_null, time=None, missing_dim="drop") + dropped = flat_with_null["_patch_id"].iloc[0] + assert dropped not in set(plan.members["_patch_id"]) + + +class TestConflict: + """Spec 2.5: attr policing within a partition.""" + + @pytest.fixture(scope="class") + def conflicted_patches(self): + """Two contiguous patches with a differing non-group attr.""" + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) + time = p1.get_coord("time") + p2 = dc.get_example_patch(time_min=time.max() + time.step) + p2 = p2.update_attrs(data_units="m/s") + return [p1, p2] + + def test_raise(self, conflicted_patches): + """Differing non-group attrs raise by default.""" + with pytest.raises(CoordMergeError, match="data_units"): + build_chunk_plan(_flat(conflicted_patches), time=None) + + def test_keep_first(self, conflicted_patches): + """keep_first carries the first member's value.""" + df = _flat(conflicted_patches) + plan = build_chunk_plan(df, time=None, conflict="keep_first") + first_id = df.sort_values("time_min")["_patch_id"].iloc[0] + expected = df.loc[df["_patch_id"] == first_id, "data_units"].iloc[0] + assert plan.outputs["data_units"].iloc[0] == expected + + def test_drop(self, conflicted_patches): + """Drop omits the conflicting attr from outputs.""" + plan = build_chunk_plan(_flat(conflicted_patches), time=None, conflict="drop") + assert "data_units" not in plan.outputs.columns + + +class TestGroupParameter: + """Group attrs partition instead of raising.""" + + @pytest.fixture(scope="class") + def two_station_flat(self): + """Contiguous patches from two stations.""" + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) + time = p1.get_coord("time") + p2 = dc.get_example_patch(time_min=time.max() + time.step) + p3 = p1.update_attrs(station="XX2") + p4 = p2.update_attrs(station="XX2") + return _flat([p1, p2, p3, p4]) + + def test_station_partitions(self, two_station_flat): + """Different stations produce separate outputs, no error.""" + plan = build_chunk_plan(two_station_flat, time=None) + assert len(plan.outputs) == 2 + assert set(plan.outputs["station"]) == set(two_station_flat["station"]) + + def test_group_override(self, two_station_flat): + """An explicit empty group means station conflicts raise.""" + with pytest.raises(CoordMergeError, match="station"): + build_chunk_plan(two_station_flat, time=None, group=()) + + def test_config_group(self, two_station_flat): + """Config groupby_attrs drives the default partitioning.""" + with dc.set_config(groupby_attrs=("network",)): + with pytest.raises(CoordMergeError, match="station"): + build_chunk_plan(two_station_flat, time=None) + + +class TestDeterminism: + """Spec section 8.""" + + def test_repeat_identical(self, diverse_flat): + """Identical inputs give identical plans.""" + p1 = build_chunk_plan(diverse_flat, time=None) + p2 = build_chunk_plan(diverse_flat, time=None) + pd.testing.assert_frame_equal(p1.outputs, p2.outputs) + pd.testing.assert_frame_equal(p1.members, p2.members) + + def test_input_order_invariant(self, diverse_flat): + """Row order of the flat relation does not change the plan.""" + shuffled = diverse_flat.sample(frac=1, random_state=0) + p1 = build_chunk_plan(diverse_flat, time=None) + p2 = build_chunk_plan(shuffled, time=None) + cols = ["time_min", "time_max"] + pd.testing.assert_frame_equal( + p1.outputs[cols].reset_index(drop=True), + p2.outputs[cols].reset_index(drop=True), + ) + + +class TestOracleParity: + """Sanity against ChunkManager (the dev-time oracle) where compatible.""" + + def test_merge_envelopes_match(self, random_flat): + """Merge-mode output envelopes match spool.chunk(time=None).""" + spool = dc.get_example_spool("random_das") + merged = spool.chunk(time=None) + contents = merged.get_contents() + plan = build_chunk_plan(random_flat, time=None) + assert len(plan.outputs) == len(contents) + assert plan.outputs["time_min"].iloc[0] == contents["time_min"].iloc[0] + assert plan.outputs["time_max"].iloc[0] == contents["time_max"].iloc[0] + + def test_segment_envelopes_match(self, random_flat): + """Segment-mode envelopes match spool.chunk(time=3).""" + spool = dc.get_example_spool("random_das") + chunked = spool.chunk(time=3) + contents = chunked.get_contents().sort_values("time_min") + plan = build_chunk_plan(random_flat, time=3) + outs = plan.outputs.sort_values("time_min") + assert len(outs) == len(contents) + assert np.array_equal(outs["time_min"].values, contents["time_min"].values) + assert np.array_equal(outs["time_max"].values, contents["time_max"].values) From 28ac4a53e15b2da7ce3dfc63043cc5ac12222d32 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 12:05:32 +0200 Subject: [PATCH 22/97] Cut spool.chunk over to the plan-based machinery; remove ChunkManager Spool chunking now runs on build_chunk_plan: DataFrameSpool.chunk builds a ChunkPlan from its source rows and converts it to the current/instruction dataframes the loading machinery consumes. spool.chunk grows the group= and missing_dim= arguments (see the chunking spec); the hidden _group_columns tuple is gone. Merged coordinates now follow the segmented-coordinate semantics: member coords are concatenated exactly (contiguous members fuse to a plain range) and, when snap_coords=True, simplified with bounded error (no value moves more than tolerance * step). Merges whose gaps exceed the tolerance keep an exact CoordSegmented instead of being relabeled to a lying uniform coord; the old unconditional snap and its 'slightly different dt' TODO are gone. Non-monotonic member coords fall back to raw concatenation as before. snap_coords/tolerance now actually flow through the merge paths. Behavior changes (0.2, all pre-adjudicated in the chunking spec): - patches lacking the chunk dim raise unless missing_dim='drop' (D2) - differing non-chunked dim coords partition instead of raising (D4) - overlap >= length raises a clear ParameterError (D6) - continuity is evaluated within each partition cell, so an unrelated group's coverage can no longer hide a gap ChunkManager is deleted; utils/chunk.py keeps only get_intervals. Its dataframe-level tests are ported to build_chunk_plan (26 tests) and the whole suite passes (7,242). --- dascore/core/spool.py | 82 ++++-- dascore/io/index/plan.py | 70 ++++- dascore/utils/chunk.py | 438 +--------------------------- dascore/utils/patch.py | 67 +++-- tests/test_core/test_patch_chunk.py | 17 +- tests/test_utils/test_chunk.py | 242 ++++++--------- 6 files changed, 278 insertions(+), 638 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 4dda00712..300933429 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -34,7 +34,6 @@ ParameterError, ) from dascore.utils.attrs import combine_patch_attrs -from dascore.utils.chunk import ChunkManager from dascore.utils.display import get_dascore_text, get_nice_text from dascore.utils.docs import compose_docstring from dascore.utils.mapping import FrozenDict @@ -49,6 +48,7 @@ _force_patch_merge, _get_merge_dim, _get_merged_coord, + _split_coord_merge_kwargs, _spool_up, concatenate_patches, get_patch_names, @@ -174,6 +174,8 @@ def chunk( snap_coords: bool = True, tolerance: float = 1.5, conflict: Literal["drop", "raise", "keep_first"] = "raise", + group: str | Sequence[str] | None = None, + missing_dim: Literal["raise", "drop"] = "raise", **kwargs, ) -> Self: """ @@ -188,13 +190,24 @@ def chunk( If True, keep the segments which are smaller than chunk size. This often occurs because of data gaps or at end of chunks. snap_coords - If True, snap the coords on joined patches such that the spacing - remains constant. + If True (default), simplify the coordinates of joined patches to + an evenly sampled range when doing so moves no coordinate value + by more than `tolerance` samples. Merges whose gaps exceed that + keep an exact segmented coordinate instead. tolerance - The maximum number of samples a block of data can be spaced (gap) and - still be considered contiguous. + The maximum number of samples a block of data can be spaced (gap) + and still be considered contiguous. conflict {conflict_desc} + group + Attributes which partition patches into separate outputs (their + values differing is never an error). Defaults to the config + option `groupby_attrs`; unlike the default, explicitly passed + names must exist on at least one patch. Dimensions and + coordinate identities always partition implicitly. + missing_dim + What to do when patches lack the chunked dimension: "raise" + (default) or "drop" (exclude them from the output). kwargs kwargs are used to specify the dimension along which to chunk, eg: `time=10` chunks along the time axis in 10 second increments. @@ -430,8 +443,6 @@ class DataFrameSpool(BaseSpool): _select_kwargs: Mapping | None = FrozenDict() # kwargs for merging patches _merge_kwargs: Mapping | None = FrozenDict() - # attributes which effect merge groups for internal patches - _group_columns = ("network", "station", "dims", "data_type", "tag") _drop_columns = ("patch",) # patch-local selections (samples=True) applied as patches load _post_selects: tuple = () @@ -661,10 +672,13 @@ def _merge_patches_streaming(self, joined, df_dict_list, merge_dim, samples): f"{merge_dim} but found {found_dim}." ) raise CoordMergeError(msg) - conf = self._merge_kwargs.get("conflicts", None) + attr_kwargs, coord_kwargs = _split_coord_merge_kwargs(self._merge_kwargs) + conf = attr_kwargs.get("conflicts", None) drop_conflicting = conf in {"drop", "keep_first"} - new_coord = _get_merged_coord(summary_df, merge_dim, coords, drop_conflicting) - new_attrs = combine_patch_attrs(attrs, **self._merge_kwargs) + new_coord = _get_merged_coord( + summary_df, merge_dim, coords, drop_conflicting, **coord_kwargs + ) + new_attrs = combine_patch_attrs(attrs, **attr_kwargs) return dc.Patch(data=buffer, coords=new_coord, attrs=new_attrs, dims=list(dims)) def _get_dummy_dataframes(self, current): @@ -726,29 +740,57 @@ def chunk( snap_coords: bool = True, tolerance: float = 1.5, conflict: Literal["drop", "raise", "keep_first"] = "raise", + group: str | Sequence[str] | None = None, + missing_dim: Literal["raise", "drop"] = "raise", **kwargs, ) -> Self: """{doc}""" - df = self._source_df.drop(columns=list(self._drop_columns), errors="ignore") - chunker = ChunkManager( + from dascore.io.index.plan import build_chunk_plan + + source = self._source_df + working = source.drop(columns=list(self._drop_columns), errors="ignore") + if "_patch_id" in source.columns: + working = working.assign(_patch_id=source["_patch_id"]) + else: + working = working.assign(_patch_id=np.arange(len(source))) + plan = build_chunk_plan( + working, overlap=overlap, keep_partial=keep_partial, snap_coords=snap_coords, - group_columns=self._group_columns, tolerance=tolerance, conflict=conflict, + group=group, + missing_dim=missing_dim, **kwargs, ) - in_df, out_df = chunker.chunk(df) - if df.empty: - instructions = None - else: - instructions = chunker.get_instruction_df(in_df, out_df) + merge_kwargs = { + "conflicts": conflict, + "snap_coords": snap_coords, + "tolerance": tolerance, + } + if plan.outputs.empty: + empty = source.iloc[0:0] + return self.new_from_df(empty, merge_kwargs=merge_kwargs) + out_df = plan.outputs.drop(columns=["output_id"]).reset_index(drop=True) + # Instructions bind plan members back to source rows by patch id. + pid_to_index = pd.Series(source.index.values, index=working["_patch_id"].values) + names = [f"{plan.dim}_min", f"{plan.dim}_max", f"{plan.dim}_step"] + instructions = ( + plan.members.assign( + source_index=lambda x: x["_patch_id"].map(pid_to_index), + current_index=lambda x: x["output_id"], + ) + .drop(columns=["output_id", "_patch_id"]) + .loc[:, ["source_index", "current_index", *names, "_modified"]] + .set_index("source_index") + .sort_values("current_index") + ) return self.new_from_df( out_df, - source_df=self._source_df, + source_df=source, instruction_df=instructions, - merge_kwargs={"conflicts": conflict}, + merge_kwargs=merge_kwargs, ) def new_from_df( diff --git a/dascore/io/index/plan.py b/dascore/io/index/plan.py index 460c016f9..832d73559 100644 --- a/dascore/io/index/plan.py +++ b/dascore/io/index/plan.py @@ -8,13 +8,14 @@ per output patch) plus a members table binding each output to trimmed slices of source patches. No patch data is touched; assembly happens later. -Portions of the interval/instruction math are ported from -`dascore.utils.chunk.ChunkManager` (which this planner replaces at -cutover) with the spec's adjudicated corrections applied. +Portions of the interval/instruction math were ported from the old +`ChunkManager` (now removed) with the spec's adjudicated corrections +applied; `Spool.chunk` runs on these plans. """ from __future__ import annotations +import warnings from dataclasses import dataclass, field from typing import Any, Literal @@ -36,6 +37,9 @@ # Columns which never participate in conflict policing and never carry to # outputs: source bookkeeping (outputs are not file rows). _SOURCE_COLUMNS = ("path", "file_format", "file_version", "source_patch_id") +# The default continuity tolerance; looser values warn when they force +# merges (#662). +_DEFAULT_TOLERANCE = 1.5 @dataclass(frozen=True) @@ -90,6 +94,16 @@ def _resolve_group_attrs(group, columns) -> tuple[str, ...]: return tuple(x for x in dc.get_config().groupby_attrs if x in columns) +def _dim_def_key_columns(df: pd.DataFrame, name: str) -> list[str]: + """Return def-key column names for every non-chunked dimension.""" + dim_names: set[str] = set() + if "dims" in df.columns: + for dims_str in df["dims"].dropna().unique(): + dim_names.update(str(dims_str).split(",")) + dim_names.discard(name) + return [f"_{x}_def_key" for x in sorted(dim_names)] + + def _sampling_group(step: pd.Series, tolerance: float) -> pd.Series: """Label rows whose steps are within relative tolerance (spec 2.3).""" col = to_float(step.values) @@ -129,9 +143,10 @@ def _partition(df, name, group_attrs, tolerance, sampling_tolerance) -> pd.Serie cols = [x for x in group_attrs if x in df.columns] if "dims" in df.columns: cols.append("dims") - cols += [ - x for x in df.columns if x.endswith("_def_key") and x != f"_{name}_def_key" - ] + # Structural identity: def keys of non-chunked *dimensions* only + # (spec 2.2). Non-dimensional coordinate conflicts are policed at + # assembly per the `conflict` argument, never partitioned on. + cols += [x for x in _dim_def_key_columns(df, name) if x in df.columns] base = ( df.groupby(cols, dropna=False, sort=False).ngroup() if cols @@ -140,10 +155,25 @@ def _partition(df, name, group_attrs, tolerance, sampling_tolerance) -> pd.Serie samp = _sampling_group(step, sampling_tolerance) cell = base.astype(str) + "_" + samp.astype(str) cont = pd.Series(0, index=df.index, dtype=np.int64) + forced_merge = False for _, index in df.groupby(cell, sort=False).groups.items(): sub = df.loc[index] s, e, st = get_interval_columns(sub, name) - cont.loc[index] = _continuity_group(s, e, st, tolerance).astype(np.int64) + labels = _continuity_group(s, e, st, tolerance).astype(np.int64) + cont.loc[index] = labels + # See #662: warn when a loosened tolerance forces merges the + # default would not have produced. + if tolerance > _DEFAULT_TOLERANCE and not forced_merge: + default = _continuity_group(s, e, st, _DEFAULT_TOLERANCE) + forced_merge = default.nunique() > labels.nunique() + if forced_merge: + msg = ( + f"There is a gap in the patch along dimension {name} but a " + f"merge tolerance of {tolerance} was used to force merging " + "the patches. As a result, some patches in the chunked spool " + "may be unevenly sampled, or have their sampling rate increased." + ) + warnings.warn(msg, UserWarning, stacklevel=4) return cell + "_" + cont.astype(str) @@ -190,9 +220,9 @@ def _police_columns(sub: pd.DataFrame, name, group_attrs, conflict) -> dict: if conflict == "keep_first": carried[col] = sub[col].iloc[0] # conflict == "drop": omit the column entirely. - # Structural def keys carry (single-valued within a partition). - for col in sub.columns: - if col.endswith("_def_key") and col != f"_{name}_def_key": + # Structural (dimension) def keys carry — single-valued by partitioning. + for col in _dim_def_key_columns(sub, name): + if col in sub.columns: carried[col] = sub[col].iloc[0] return carried @@ -206,6 +236,7 @@ def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFra overlaps keep the first member, deterministically). """ min_name, max_name = f"{name}_min", f"{name}_max" + step_name = f"{name}_step" sub = sub.sort_values([min_name, "_patch_id"], kind="stable") original = sub[[min_name, max_name]].reset_index(drop=True) sub = _remove_overlaps(sub, name) @@ -216,8 +247,16 @@ def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFra original = original[keep].reset_index(drop=True) if sub.empty or outputs.empty: return pd.DataFrame( - columns=["output_id", "_patch_id", min_name, max_name, "_modified"] + columns=[ + "output_id", + "_patch_id", + min_name, + max_name, + step_name, + "_modified", + ] ) + steps = sub[step_name].values src1 = sub[min_name].values src2 = sub[max_name].values chu1 = outputs[min_name].values @@ -248,6 +287,7 @@ def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFra "_patch_id": sub["_patch_id"].iloc[src_num], min_name: lo, max_name: hi, + step_name: steps[src_num], "_modified": not unchanged, } ) @@ -298,7 +338,7 @@ def build_chunk_plan( raise ParameterError(msg) min_name, max_name = f"{name}_min", f"{name}_max" - if min_name not in df.columns: + if min_name not in df.columns and not df.empty: msg = f"No patch in the spool has a {name!r} dimension to chunk." raise ChunkError(msg) empty_members = pd.DataFrame( @@ -314,6 +354,12 @@ def build_chunk_plan( group=_resolve_group_attrs(group, set(df.columns)), sampling_group_tolerance=dc.get_config().sampling_group_tolerance, ) + if df.empty: + outputs = pd.DataFrame(columns=[min_name, max_name, "output_id"]) + return ChunkPlan(outputs, empty_members, name, value, params) + if "_patch_id" not in df.columns: + # Positional identity fallback for plain dataframes. + df = df.assign(_patch_id=np.arange(len(df))) # Missing chunk-dim envelopes (spec 7 / D2). null_rows = pd.isnull(df[min_name]) | pd.isnull(df[max_name]) if null_rows.any(): diff --git a/dascore/utils/chunk.py b/dascore/utils/chunk.py index aed3e1d26..c5ff8d1f3 100644 --- a/dascore/utils/chunk.py +++ b/dascore/utils/chunk.py @@ -1,28 +1,15 @@ -"""Utilities for chunking dataframes.""" +"""Utilities for chunking dataframes. -from __future__ import annotations +The interval math here is consumed by the chunk planner +(`dascore.io.index.plan`), which replaced the old ChunkManager. +""" -import warnings -from collections.abc import Collection -from functools import reduce -from typing import ClassVar +from __future__ import annotations -import numpy import numpy as np import pandas as pd -from dascore.constants import attr_conflict_description, numeric_types, timeable_types -from dascore.exceptions import ChunkError, CoordMergeError, ParameterError -from dascore.utils.docs import compose_docstring -from dascore.utils.misc import get_middle_value -from dascore.utils.pd import ( - _instructions_modified, - _remove_overlaps, - get_column_names_from_dim, - get_dim_names_from_columns, - get_interval_columns, - list_ser_to_str, -) +from dascore.exceptions import ChunkError, ParameterError from dascore.utils.time import ( is_datetime64, is_timedelta64, @@ -30,8 +17,6 @@ to_timedelta64, ) -_DEFAULT_TOLERANCE = 1.5 - def get_intervals( start, @@ -117,414 +102,3 @@ def get_intervals( else: ends[bad_ends] = stop return np.stack([starts, ends]).T - - -@compose_docstring(attr_conflict=attr_conflict_description) -class ChunkManager: - """ - A class for managing the chunking of data defined in a dataframe. - - The chunk manager handles both splitting and joining of contiguous, - or near-contiguous, blocks of data. - - Parameters - ---------- - overlap - The amount of overlap between each segment, starting with the end of - first row. Negative values can be used for inducing gaps. - group_columns - A sequence of column names which should be used for sorting groups. - keep_partial - If True, keep segments which are shorter than chunk size (at end of - contiguous blocks) - tolerance - The upper limit of a gap to tolerate in terms of the sampling - along the desired dimension. E.G., the default value means entities - with gaps <= 1.5 * {name}_step will be merged. - conflict - {attr_conflict} - **kawrgs - kwargs specify the column along which to chunk. The key specifies the - column along which to chunk, typically, `time` or `distance`, and the - value specifies the chunk size. A value of None means to chunk on all - available data (e.g. merge all data). - - Notes - ----- - This class is used internally by `dc.BaseSpool.chunk`. - """ - - # Coord fingerprints are stable IDs for exact coord contents. Chunk merges - # intentionally rewrite coord extents/min/max, so inherited fingerprint - # columns are no longer expected to remain equal across merge candidates. - _merge_ignored_columns: ClassVar[set[str]] = { - "dtype", - "time_fingerprint", - "distance_fingerprint", - } - - def __init__( - self, - overlap: timeable_types | numeric_types | None = None, - group_columns: Collection[str] | None = None, - keep_partial=False, - snap_coords=True, - tolerance=_DEFAULT_TOLERANCE, - conflict="raise", - **kwargs, - ): - self._overlap = overlap - self._group_columns = group_columns - self._keep_partials = keep_partial - self._snap_coords = snap_coords - self._tolerance = tolerance - self._name, self._value = self._validate_kwargs(kwargs) - self._attr_conflict = conflict - self._validate_chunker() - - def _validate_kwargs(self, kwargs): - """Ensure kwargs is len one and has a valid.""" - if not len(kwargs) == 1: - msg = ( - f"Chunking only supported along one dimension. You passed " - f"kwargs: {kwargs}" - ) - raise ParameterError(msg) - ((key, value),) = kwargs.items() - value = None if value is ... else value - return key, value - - def _validate_chunker(self): - """Ensure selected parameters are compatible.""" - # chunker is used for merging - if pd.isnull(self._value): - if self._keep_partials or self._overlap: - msg = ( - "When chunk value is None (ie Chunker is used for merging) " - "both _keep_partials and self._overlap must not be selected." - ) - raise ParameterError(msg) - return - # ensure chunk values are greater than 0 - zero = to_timedelta64(0) if is_timedelta64(self._value) else 0 - if self._value <= zero: - msg = "Chunk value must be greater than 0." - raise ParameterError(msg) - - def _get_continuity_group_number( - self, start, stop, step, tolerance=None - ) -> pd.Series: - """Return a series of ints indicating continuity group.""" - tolerance = self._tolerance if tolerance is None else tolerance - # start by sorting according to start time - # Use positional argsort to avoid pandas label/return-type changes - args = np.argsort(start.to_numpy()) - start_sorted, stop_sorted, step_sorted = ( - start.iloc[args], - stop.iloc[args], - step.iloc[args], - ) - # next get cummax of endtimes and detect gaps - stop_cum_max = stop_sorted.cummax() - end_markers = stop_cum_max.shift() + step_sorted * tolerance - has_gap = start_sorted > end_markers - group_num = has_gap.astype(np.int64).cumsum() - return group_num[start.index] - - def _get_sampling_group_num(self, step, tolerance=0.05) -> pd.Series: - """ - Because sampling can be off a little, this adds some tolerance for - how sampling affects groups. - - Tolerance affects how close samples have to be in order to count as - the same. 5% is used here. - """ - col = step.values - sort_args = np.argsort(col) - sorted_col = col[sort_args] - roll_forward = np.roll(sorted_col, shift=1) - diff = (sorted_col - roll_forward) / sorted_col - out_of_threshold = diff > tolerance - group_number = numpy.cumsum(out_of_threshold) - # undo sorting - out = pd.Series(group_number[np.argsort(sort_args)], index=step.index) - return out - - def _get_duration_overlap(self, duration, start, step, overlap=None): - """Get duration and overlap from kwargs.""" - overlap = overlap if overlap is not None else self._overlap - # cast step/overlap to timedelta if start is datetime or timedelta; - # a span of either dtype is a duration. - if is_datetime64(start) or is_timedelta64(start): - step = to_timedelta64(step) - overlap = to_timedelta64(overlap) - if pd.isnull(overlap): - overlap = np.asarray([0], dtype=step.dtype)[0] - return duration, overlap - - def _create_df(self, df, name, start_stop, gnum): - """Reconstruct the dataframe.""" - cols = f"{name}_min", f"{name}_max" - out = pd.DataFrame(start_stop, columns=list(cols)) - out[f"{name}_step"] = get_middle_value(df[f"{name}_step"].values) - merger = df.drop(columns=out.columns) - # get dims to determine which columns are still compared. Some test - # dfs don't have dims though, so it should still work without dims col. - dims = set(df.iloc[0].get("dims", "").split(",")) - # We exclude private columns for considering if merge can happen. - for col in set(x for x in merger.columns if not x.startswith("_")): - if col in self._merge_ignored_columns: - continue - prefix = col.split("_")[0] - # If we have specified to ignore or remove conflicting attrs - # we don't need to check them here, but we do still check dims. - if self._attr_conflict != "raise" and prefix not in dims: - continue - vals = merger[col].unique() - if len(vals) > 1: - msg = ( - f"Cannot merge on dim {self._name} because all values for " - f"{col} are not equal. Consider using the `conflict` " - f"argument to loosen this restriction." - ) - raise CoordMergeError(msg) - - assert len(vals) == 1, "Haven't yet implemented non-homogenous merging" - out[col] = vals[0] - if "dims" in out.columns: - # Keep dims dtype consistent with patches_to_df/list_ser_to_str. - out["dims"] = list_ser_to_str(out["dims"]) - # add the group number for getting instruction df later - out["_group"] = gnum - return out - - def _get_chunk_overlap_inds(self, src1, src2, chu1, chu2): - """Get an index mapping from source to chunk.""" - chunk_starts = np.searchsorted(src1, chu1, side="right") - 1 - chunk_ends = np.searchsorted(src2, chu2, side="left") - # Ensure no chunks run off the end of the source. - assert np.all(chunk_ends < len(src1)), "Invalid chunk range found" - # add 1 to end so it is an exclusive end range - return np.stack([chunk_starts, chunk_ends + 1], axis=1) - - def _get_source_and_chunk_inds(self, chunk2src_inds, s_index, c_index): - """Get ndarrays of chunk index, source index.""" - # get indices for sorted arrays - source_inds_ = np.concatenate( - [np.arange(x[0], x[1], dtype=np.int64) for x in chunk2src_inds] - ) - chunk_inds_ = np.concatenate( - [ - np.ones((x[1] - x[0]), dtype=np.int64) * num - for num, x in enumerate(chunk2src_inds) - ] - ) - # use pandas index to map back to actual indices - source_inds = s_index.values[source_inds_] - chunk_inds = c_index.values[chunk_inds_] - out = { - "source_sorted": source_inds_, - "source": source_inds, - "chunk_sorted": chunk_inds_, - "chunk": chunk_inds, - } - return out - - def _get_instructions(self, sub_source, sub_chunk): - """Get source mapping to chunk.""" - min_name, max_name = f"{self._name}_min", f"{self._name}_max" - # sort inputs based on start of range, as long as we don't reset index - # we should be ok. - sub_source = sub_source.sort_values(min_name) - sub_chunk = sub_chunk.sort_values(min_name) - # need to make sure we don't have overlaps in source df. This implicitly - # handles merging. - sub_source = _remove_overlaps(sub_source, self._name) - src1, src2, _src_step = get_interval_columns( - sub_source, self._name, arrays=True - ) - chu1, chu2, _chu_step = get_interval_columns(sub_chunk, self._name, arrays=True) - dims = get_dim_names_from_columns(sub_source) - cols2keep = get_column_names_from_dim(dims) - # next get index range for which chunk times belong to. - chunk2src_inds = self._get_chunk_overlap_inds(src1, src2, chu1, chu2) - # total length of source to chunk mapping - inds = self._get_source_and_chunk_inds( - chunk2src_inds, - sub_source.index, - sub_chunk.index, - ) - source_inds, chunk_inds = inds["source_sorted"], inds["chunk_sorted"] - # get potential start/stop times. - starts = np.stack([src1[source_inds], chu1[chunk_inds]], axis=1) - ends = np.stack([src2[source_inds], chu2[chunk_inds]], axis=1) - end_values = np.min(ends, axis=1) - start_values = np.max(starts, axis=1) - data_dict = { - min_name: start_values, - max_name: end_values, - "source_index": inds["source"], - "current_index": inds["chunk"], - } - out = pd.DataFrame(data_dict) - # populate the rest of the columns needed in instruction df. - for col in cols2keep: - if col in out.columns: - continue - out[col] = sub_source[col].values[source_inds] - out = out.sort_index() - out["_modified"] = _instructions_modified(out, sub_source) - return out - - def get_instruction_df(self, source_df, chunked_df): - """ - Get a dataframe connecting the chunked dataframe to its origin. - - This is used to connect source data to desired data after chunking - operation. - - Parameters - ---------- - source_df - The dataframe before chunking - chunked_df - The chunked dataframe (output of `chunk` method) - """ - # the group column should exist and the chunked groups should be subset - # of the source groups - assert "_group" in source_df.columns and "_group" in chunked_df.columns - chunked_groups = set(chunked_df["_group"]) - if not chunked_groups: - return pd.DataFrame(columns=[*list(source_df.columns), "_modified"]) - # chunk groups should be a subset of source groups - assert chunked_groups.issubset(set(source_df["_group"])) - # iterate each group and create instruction df - out = [] - for group in chunked_groups: - sub_source = source_df[source_df["_group"] == group] - sub_chunk = chunked_df[chunked_df["_group"] == group] - out.append(self._get_instructions(sub_source, sub_chunk)) - df = pd.concat(out, axis=0).reset_index(drop=True).set_index("source_index") - return df - - def _get_col_group(self, df, cont_g): - """Get group columns based on common columns.""" - cols = list(self._group_columns or []) - columns = [x for x in cols if x in df.columns] - col_g = cont_g * 0 if not columns else df.groupby(columns).ngroup() - return col_g - - def _get_final_group(self, samp_g, col_g, cont_g): - """Combine grouping components into final group labels.""" - group_series = [x.astype(str) for x in [samp_g, col_g, cont_g]] - return reduce(lambda x, y: x + "_" + y, group_series) - - def _groups_merge_default_groups(self, group, default_cont_g) -> bool: - """Return True if any final group contains multiple default groups.""" - group_codes = pd.factorize(group, sort=False)[0] - default_codes = pd.factorize(default_cont_g, sort=False)[0] - order = np.argsort(group_codes, kind="stable") - group_sorted = group_codes[order] - default_sorted = default_codes[order] - new_group = np.r_[True, group_sorted[1:] != group_sorted[:-1]] - group_starts = np.flatnonzero(new_group) - default_min = np.minimum.reduceat(default_sorted, group_starts) - default_max = np.maximum.reduceat(default_sorted, group_starts) - return bool(np.any(default_min != default_max)) - - def _get_group(self, df, start, stop, step): - """ - Get the group designation for df. This accounts for both time intervals - being consistent and group columns matching. - """ - cont_g = self._get_continuity_group_number(start, stop, step) - samp_g = self._get_sampling_group_num(step) - col_g = self._get_col_group(df, cont_g) - group = self._get_final_group(samp_g, col_g, cont_g) - - # Check for final merges that only occur because of a non-default - # tolerance. See #662. - if self._tolerance > _DEFAULT_TOLERANCE: - default_cont_g = self._get_continuity_group_number( - start, stop, step, tolerance=_DEFAULT_TOLERANCE - ) - if self._groups_merge_default_groups(group, default_cont_g): - msg = ( - f"There is a gap in the patch along dimension {self._name} " - f"but a merge tolerance of {self._tolerance} was used to force " - "merging the patches. As a result, some patches in the chunked " - "spool may be unevenly sampled, or have their sampling rate " - "increased." - ) - warnings.warn(msg, UserWarning, stacklevel=4) - return group - - def _get_group_dfs(self, group, dur, overlap, group_mins, group_maxs, df, step): - """Get the new dataframe for a given group.""" - out = [] - for gnum in group.unique(): - g_start, g_stop = group_mins[gnum], group_maxs[gnum] - current_df = df.loc[group[group == gnum].index] - # reconstruct DF - try: - new_start_stop = get_intervals( - g_start, - g_stop, - dur, - overlap=overlap, - step=step.loc[current_df.index].iloc[0], - keep_partials=self._keep_partials, - ) - except ChunkError: # this chunk is too short, skip. - continue - # create the newly chunked dataframe - sub_new_df = self._create_df(current_df, self._name, new_start_stop, gnum) - out.append(sub_new_df) - return out - - def _filter_nan_dfs(self, df, start, stop): - """Filter NaN out of dataframe if they occur in start/stop.""" - - def chunk( - self, - df: pd.DataFrame, - ) -> tuple[pd.DataFrame, pd.DataFrame]: - """ - Chunk a dataframe into new contiguous segments. - - The dataframe must have column names {key}_max, {key}_min, and {key}_step - where {key} is the key used in the kwargs. - - Parameters - ---------- - df - Input dataframe to chunk. - - Returns - ------- - A tuple of the original dataframe with added column '_group' and an - output dataframe with column '_group'. The _group column is used - to link the two dataframes together. - """ - if df.empty: # empty df, do nothing - return df.assign(_group=None), df.assign(_group=None) - # get series of start/stop along requested dimension - start, stop, step = get_interval_columns(df, self._name) - # Filter out any NaN in start or stop. - keep = ~(pd.isnull(start) | pd.isnull(stop)) - df, start, stop, step = df[keep], start[keep], stop[keep], step[keep] - if df.empty: # Need to check again since NaN can wipe out df. - return df.assign(_group=None), df.assign(_group=None) - dur, overlap = self._get_duration_overlap(self._value, start, step) - # get group numbers - group = self._get_group(df, start, stop, step) - # get max, min for each group and expand - group_mins = start.groupby(group).min() - group_maxs = stop.groupby(group).max() - # split/group dataframe into new chunks by iterating over each group. - out = self._get_group_dfs(group, dur, overlap, group_mins, group_maxs, df, step) - if not len(out): - msg = "Could not chunk. No segments with sufficient length found." - raise ChunkError(msg) - out = pd.concat(out, axis=0).reset_index(drop=True) - return df.assign(_group=group), out diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index 789057cd9..6d49a3350 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -26,6 +26,7 @@ ) from dascore.exceptions import ( CoordDataError, + CoordError, IncompatiblePatchError, ParameterError, PatchAttributeError, @@ -40,7 +41,6 @@ from dascore.utils.misc import ( _apply_union_indexers, _merge_tuples, - all_diffs_close_enough, get_middle_value, iterate, to_object_array, @@ -418,25 +418,52 @@ def _get_merge_dim(df) -> str | None: return dims_vary[dims_vary].index[0] -def _maybe_expected_step(df, dim): - """Get the expected step if all steps are close, else None.""" +def _middle_step(df, dim): + """Return the middle value of non-null member steps, or None.""" col = df[f"{dim}_step"].values - if all_diffs_close_enough(col): - return get_middle_value(col) - return None + valid = col[~pd.isnull(col)] + if not len(valid): + return None + return get_middle_value(valid) + +def _split_coord_merge_kwargs(merge_kwargs) -> tuple[dict, dict]: + """Split spool merge kwargs into (attr kwargs, coord kwargs).""" + merge_kwargs = dict(merge_kwargs or {}) + coord_kwargs = { + "snap_coords": merge_kwargs.pop("snap_coords", True), + "tolerance": merge_kwargs.pop("tolerance", 1.5), + } + return merge_kwargs, coord_kwargs -def _get_merged_coord(df, merge_dim, coords, drop_conflicting=False): - """Get merged coordinates, also validate anticipated sampling.""" - new_coord = merge_coord_managers( + +def _get_merged_coord( + df, merge_dim, coords, drop_conflicting=False, snap_coords=True, tolerance=1.5 +): + """ + Get merged coordinates for patches combined along merge_dim. + + The merged dimension coordinate is built by truth-preserving + concatenation of the member coords (exactly contiguous members fuse to + a plain range; recorded seams otherwise), then — when `snap_coords` — + simplified with bounded error: no value moves more than + `tolerance * step`. Merges whose gaps exceed that stay segmented + (honestly non-uniform) rather than being relabeled. + """ + from dascore.core.coords import concat_coords + + new_cm = merge_coord_managers( coords, dim=merge_dim, drop_conflicting=drop_conflicting ) - expected_step = _maybe_expected_step(df, merge_dim) - if not pd.isnull(expected_step): - new_coord = new_coord.snap(merge_dim)[0] - # TODO slightly different dt can be produced, let pass for now - # need to think more about how the merging should work. - return new_coord + try: + merged = concat_coords(*[cm.coord_map[merge_dim] for cm in coords]) + except CoordError: + # Non-monotonic (or otherwise unsegmentable) member coordinates: + # keep the raw value concatenation. + return new_cm + if snap_coords and (step := _middle_step(df, merge_dim)) is not None: + merged = merged.simplify(tolerance * np.abs(step)) + return new_cm.update(**{merge_dim: merged}) def _force_patch_merge(patch_dict_list, merge_kwargs, **kwargs): @@ -448,7 +475,7 @@ def _force_patch_merge(patch_dict_list, merge_kwargs, **kwargs): """ df = pd.DataFrame(patch_dict_list) merge_dim = _get_merge_dim(df) - merge_kwargs = merge_kwargs if merge_kwargs is not None else {} + attr_kwargs, coord_kwargs = _split_coord_merge_kwargs(merge_kwargs) if merge_dim is None: # nothing to merge, complete overlap return [patch_dict_list[0]] dims = df["dims"].iloc[0].split(",") @@ -462,10 +489,12 @@ def _force_patch_merge(patch_dict_list, merge_kwargs, **kwargs): attrs = [x.attrs for x in patches] new_data = np.concatenate(data, axis=axis) # Determine if conflicting non-dimensional coords should be dropped. - conf = merge_kwargs.get("conflicts", None) + conf = attr_kwargs.get("conflicts", None) drop_conf_coords = True if conf in {"drop", "keep_first"} else False - new_coord = _get_merged_coord(df, merge_dim, coords, drop_conf_coords) - new_attrs = combine_patch_attrs(attrs, **merge_kwargs) + new_coord = _get_merged_coord( + df, merge_dim, coords, drop_conf_coords, **coord_kwargs + ) + new_attrs = combine_patch_attrs(attrs, **attr_kwargs) patch = dc.Patch(data=new_data, coords=new_coord, attrs=new_attrs, dims=dims) new_dict = {"patch": patch} return [new_dict] diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 5b94cd1e2..418e11ad8 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -307,9 +307,13 @@ def patches_conflicting_private_coord(self, random_patch): return p1, p2 def test_merge_unequal_other(self, distance_adjacent): - """When distance values are not equal time shouldn't be merge-able.""" - with pytest.raises(CoordMergeError): - distance_adjacent.chunk(time=...) + """Unequal distance coords partition rather than raise (0.2 change). + + Patches whose non-chunked dimension coordinates differ are never + combined; they simply land in separate output patches. + """ + out = distance_adjacent.chunk(time=...) + assert len(out) == len(distance_adjacent) def test_merge_adjacent(self, adjacent_spool_no_overlap): """Test simple merge of patches.""" @@ -502,8 +506,11 @@ def test_chunk_patches_with_non_coord(self, random_patch): """Tests for chunking when some patches have non coordinate dimensions.""" patches = [random_patch.mean("time") for _ in range(3)] spool = dc.spool(patches) - chunked = spool.chunk(time=None) - # Since the time dims are NaN, this can't work. + # Losing patches silently would be data loss; this raises by default + # (0.2 change) with missing_dim="drop" restoring the old behavior. + with pytest.raises(ChunkError, match="missing_dim"): + spool.chunk(time=None) + chunked = spool.chunk(time=None, missing_dim="drop") assert not len(chunked) def test_merge_with_conflicting_private_coords( diff --git a/tests/test_utils/test_chunk.py b/tests/test_utils/test_chunk.py index acf9b689c..1b984f3fb 100644 --- a/tests/test_utils/test_chunk.py +++ b/tests/test_utils/test_chunk.py @@ -9,8 +9,9 @@ import pytest import dascore as dc -from dascore.exceptions import ParameterError -from dascore.utils.chunk import ChunkManager, get_intervals +from dascore.exceptions import ChunkError +from dascore.io.index.plan import build_chunk_plan +from dascore.utils.chunk import get_intervals from dascore.utils.time import to_timedelta64 STARTTIME = np.datetime64("2020-01-03") @@ -105,27 +106,25 @@ def test_timedelta_start_numeric_length(self): assert out[-1, 1] == stop -class TestBasicChunkDF: - """Test basic DF chunking.""" +class TestChunkPlanDF: + """Dataframe-level chunk planning (ported from the old ChunkManager tests).""" @pytest.fixture() def df_different_sample_rates(self, contiguous_df): - """Tests for a df which does have overlaps but different sampling rates.""" + """Adjacent blocks with different sampling rates.""" df1 = contiguous_df.copy() df2 = contiguous_df.copy() time_span = df1["time_max"].max() - df1["time_min"].min() df2["time_min"] += time_span df2["time_max"] += time_span df2["time_step"] = df1["time_step"] * 2 - out = pd.concat([df1, df2], axis=0).reset_index(drop=True) - return out + return pd.concat([df1, df2], axis=0).reset_index(drop=True) def test_rechunk_contiguous(self, contiguous_df): """Test rechunking with no gaps.""" time_interval = (contiguous_df["time_max"] - contiguous_df["time_min"]).max() new_time_interval = time_interval / 2 - chunker = ChunkManager(time=new_time_interval) - _, out = chunker.chunk(contiguous_df) + out = build_chunk_plan(contiguous_df, time=new_time_interval).outputs assert len(out) == 2 * len(contiguous_df) time_step = out["time_step"].iloc[0] new_interval = (out["time_max"] - out["time_min"] + time_step).max() @@ -137,17 +136,15 @@ def test_rechunk_contiguous_with_sr_separation(self, contiguous_sr_spaced_df): sr = df["time_step"] time_interval = (sr + df["time_max"] - df["time_min"]).max() new_time_interval = time_interval / 2 - chunker = ChunkManager(time=new_time_interval) - _, out = chunker.chunk(df) + out = build_chunk_plan(df, time=new_time_interval).outputs assert len(out) == 2 * len(df) new_interval = (out["time_max"] - out["time_min"]).max() assert new_interval == (new_time_interval - sr.iloc[0]) def test_rechunk_different_sr(self, df_different_sample_rates): - """Ensure segments with different sample rates don't get combined.""" + """Segments with different sample rates don't get combined.""" df = df_different_sample_rates - chunker = ChunkManager(overlap=None, time=23) - _, out = chunker.chunk(df) + out = build_chunk_plan(df, time=23).outputs dt = np.sort(np.unique(out["time_step"])) assert len(dt) == 2, "both dt should remain" # the second part of the df should start at the one minute mark @@ -170,105 +167,73 @@ def test_chunk_uses_step_from_each_group(self): "time_step": [np.timedelta64(1, "s"), np.timedelta64(10, "s")], } ) - - _, chunked = ChunkManager(time=50).chunk(df) - + chunked = build_chunk_plan(df, time=50).outputs ten_second_group = chunked[chunked["time_step"] == np.timedelta64(10, "s")] first = ten_second_group.iloc[0] assert first["time_max"] - first["time_min"] == np.timedelta64(40, "s") def test_keep_leftovers(self, contiguous_df): - """Ensure leftovers show up in df.""" - chunker = ChunkManager(overlap=None, keep_partial=True, time=28) - _, out = chunker.chunk(contiguous_df) + """Ensure leftovers show up in outputs.""" + out = build_chunk_plan(contiguous_df, keep_partial=True, time=28).outputs assert len(out) == 3 assert out["time_max"].max() == contiguous_df["time_max"].max() def test_overlap(self, contiguous_df): - """Ensure overlapping segments work.""" + """Ensure overlapping segments work, with timedelta or float overlap.""" over = to_timedelta64(10) - chunker1 = ChunkManager(overlap=over, time=20) - _, out = chunker1.chunk(contiguous_df) + out = build_chunk_plan(contiguous_df, overlap=over, time=20).outputs expected = over - contiguous_df["time_step"].iloc[0] olap = out.shift()["time_max"] - out["time_min"] assert np.all(pd.isnull(olap) | (olap == expected)) - # now ensure floats work for overlap param - chunker2 = ChunkManager(overlap=10, time=20) - _, out2 = chunker2.chunk(contiguous_df) + out2 = build_chunk_plan(contiguous_df, overlap=10, time=20).outputs assert out.equals(out2) def test_chunk_on_split(self, terra15_file_spool): """Ensure chunking which creates a slice at the end time works.""" - # this spool was selected because I first observed the issue in it. df = terra15_file_spool.get_contents() dur = (df["time_max"] - df["time_min"]).iloc[0] seg_len = dur / 3 dt = df["time_step"].iloc[0] - chunker = ChunkManager(keep_partial=True, time=seg_len) - _, chunk_df = chunker.chunk(df) + chunk_df = build_chunk_plan(df, keep_partial=True, time=seg_len).outputs duration = chunk_df["time_max"] - chunk_df["time_min"] assert duration.sum() == ((seg_len - dt) * 3) assert len(duration) == 3 assert (duration > np.timedelta64(0, "s")).all() def test_nan_in_df(self, contiguous_df): - """Ensure contiguous df with nan inside still works.""" + """A null envelope row breaks continuity when dropped.""" df = contiguous_df.copy() - # Adding null values on row 3 df.loc[3, "time_min"] = dc.to_datetime64("NaT") - # Which means new time should start in row 4 because of the gap. expected_start = df.loc[4, "time_min"] - chunker = ChunkManager(keep_partial=True, time=dc.to_timedelta64(15)) - _, chunk_df = chunker.chunk(df) - assert expected_start in set(chunk_df["time_min"]) + plan = build_chunk_plan( + df, keep_partial=True, missing_dim="drop", time=dc.to_timedelta64(15) + ) + assert expected_start in set(plan.outputs["time_min"]) def test_all_nan(self, contiguous_df): - """Ensure when all NaNs are encountered the chunked df is empty.""" + """When all rows lack the dim (and are dropped) the plan is empty.""" nat = dc.to_datetime64("NaT") df = contiguous_df.assign(time_min=nat, time_max=nat) - chunker = ChunkManager(time=dc.to_timedelta64(1.2)) - _, chunk_df = chunker.chunk(df) - assert chunk_df.empty + plan = build_chunk_plan(df, missing_dim="drop", time=dc.to_timedelta64(1.2)) + assert plan.outputs.empty def test_nan_in_sample_ok(self, contiguous_df): """Ensure a NaN in the sampling rate is ok.""" df = contiguous_df.assign(time_step=dc.to_timedelta64("NaT")) dur = (df["time_max"] - df["time_min"]).iloc[0] - chunker = ChunkManager(time=dc.to_timedelta64(dur / 2)) - _, chunk_df = chunker.chunk(df) + chunk_df = build_chunk_plan(df, time=dc.to_timedelta64(dur / 2)).outputs assert isinstance(chunk_df, pd.DataFrame) assert len(chunk_df) == 2 * len(contiguous_df) assert np.all(pd.isnull(chunk_df["time_step"])) + def test_unknown_dim_raises(self, contiguous_df): + """An unknown chunk dimension raises a clear error.""" + with pytest.raises(ChunkError, match="Time"): + build_chunk_plan(contiguous_df, Time=10) -class TestChunkExceptions: - """Tests for various exceptions from the chunk manager.""" - - def test_raises_overlap_no_chunksize(self): - """Specifying an overlap and no chunk size should raise.""" - with pytest.raises(ParameterError, match="used for merging"): - ChunkManager(time=None, overlap=10) - - def test_raises_zero_length_multiple_kwargs(self): - """Ensure multiple kwargs raises nice error.""" - with pytest.raises(ParameterError, match="along one dimension"): - ChunkManager(time=10, distance=1) - def test_raises_zero_length_chunk(self): - """Ensure zero length chunk raises.""" - with pytest.raises(ParameterError, match="must be greater than 0"): - ChunkManager(time=0) - - def test_raises_invalid_key_in_kwargs(self, contiguous_df): - """Ensure an invalid key in kwargs raises an error.""" - chunk_manager = ChunkManager(Time=10) - chunk_manager.patch = type("Patch", (object,), {"dims": ["time", "distance"]})() - with pytest.raises(ParameterError, match="Cannot chunk spool or"): - chunk_manager.chunk(contiguous_df) - - -class TestChunkToMerge: - """Tests for using chunking to merge contiguous, or overlapping, data.""" +class TestChunkPlanToMerge: + """Merge-mode planning on raw dataframes.""" @pytest.fixture() def gapy_df(self, contiguous_df): @@ -285,29 +250,26 @@ def gapy_df_unordered(self, gapy_df): def test_chunk_can_merge(self, contiguous_df): """Ensure chunk can be used to merge unspecified segment lengths.""" - cm = ChunkManager(time=None) - _, out = cm.chunk(contiguous_df) + out = build_chunk_plan(contiguous_df, time=None).outputs assert len(out) == 1 assert out["time_min"].min() == contiguous_df["time_min"].min() def test_doesnt_merge_gappy_df(self, gapy_df): """Ensure the gappy dataframe doesn't get merged.""" - cm = ChunkManager(time=None) - _, out = cm.chunk(gapy_df) + out = build_chunk_plan(gapy_df, time=None).outputs assert len(gapy_df) == len(out) - expected_durations = gapy_df["time_max"] - gapy_df["time_min"] - durations = out["time_max"] - out["time_min"] - assert expected_durations.equals(durations) + expected = (gapy_df["time_max"] - gapy_df["time_min"]).sort_values() + durations = (out["time_max"] - out["time_min"]).sort_values() + assert np.array_equal(expected.values, durations.values) def test_doesnt_merge_unordered_gappy_df(self, gapy_df_unordered): - """Ensure the gappy dataframe doesn't get merged.""" + """Row order must not affect merge results.""" df = gapy_df_unordered - cm = ChunkManager(time=None) - _, out = cm.chunk(df) + out = build_chunk_plan(df, time=None).outputs assert len(df) == len(out) - expected_durations = df["time_max"] - df["time_min"] - durations = out["time_max"] - out["time_min"] - assert expected_durations.equals(durations) + expected = (df["time_max"] - df["time_min"]).sort_values() + durations = (out["time_max"] - out["time_min"]).sort_values() + assert np.array_equal(expected.values, durations.values) def test_no_warning_when_final_groups_stay_separate(self, contiguous_df): """No warning if other group components prevent final forced merge.""" @@ -317,89 +279,69 @@ def test_no_warning_when_final_groups_stay_separate(self, contiguous_df): df.loc[1, "time_min"] = df.loc[0, "time_max"] + 5 * step.iloc[0] df.loc[1, "time_max"] = df.loc[1, "time_min"] + 10 * step.iloc[1] df["station"] = ["sta1", "sta2"] - cm = ChunkManager(time=None, tolerance=10, group_columns=("station",)) - with warnings.catch_warnings(): warnings.filterwarnings("error") - _, out = cm.chunk(df) + plan = build_chunk_plan(df, time=None, tolerance=10, group=("station",)) + assert len(plan.outputs) == 2 - assert len(out) == 2 + def test_forced_merge_warns(self, contiguous_df): + """A tolerance forcing a merge across a real gap warns (#662).""" + df = contiguous_df.iloc[:2].copy() + step = df["time_step"].iloc[0] + df.loc[0, "time_max"] = df.loc[0, "time_min"] + 10 * step + df.loc[1, "time_min"] = df.loc[0, "time_max"] + 5 * step + df.loc[1, "time_max"] = df.loc[1, "time_min"] + 10 * step + with pytest.warns(UserWarning, match="force merging"): + plan = build_chunk_plan(df, time=None, tolerance=10) + assert len(plan.outputs) == 1 def test_modified_flag_after_merge(self, contiguous_df): - """Test that the modified flag shows False for simple merge.""" - cm = ChunkManager(time=None) - # Need to remove overlapping sample so these really are contiguous - # with no overlaps. - contiguous_df = contiguous_df.assign( - time_max=lambda x: x["time_max"] - x["time_step"] - ) - source, current = cm.chunk(contiguous_df) - inst_df = cm.get_instruction_df(source, current) - assert len(current) == 1 - assert current["time_min"].min() == contiguous_df["time_min"].min() - assert not inst_df["_modified"].any() - - -class TestInstructionDF: - """Sanity checks on intermediary df.""" - - def test_indices(self, contiguous_df): - """Ensure the input/output index belong to input/output df.""" - chunker = ChunkManager(overlap=0, time=10) - in_df, out_df = chunker.chunk(contiguous_df) - instruction = chunker.get_instruction_df(in_df, out_df) - # ensure the source index is set as the index of the instruction_df - assert instruction.index.name == "source_index" - assert set(instruction.index).issubset(set(contiguous_df.index)) - assert set(instruction["current_index"]).issubset(set(out_df.index)) + """The modified flag shows False for a simple contiguous merge.""" + df = contiguous_df.assign(time_max=lambda x: x["time_max"] - x["time_step"]) + plan = build_chunk_plan(df, time=None) + assert len(plan.outputs) == 1 + assert plan.outputs["time_min"].min() == df["time_min"].min() + assert not plan.members["_modified"].any() + + +class TestPlanMembers: + """Sanity checks on the members (instruction) table.""" + + def test_ids(self, contiguous_df): + """Members reference real sources and outputs.""" + plan = build_chunk_plan(contiguous_df, overlap=0, time=10) + members = plan.members + assert set(members["_patch_id"]).issubset(set(range(len(contiguous_df)))) + assert set(members["output_id"]).issubset(set(plan.outputs["output_id"])) def test_different_group_columns(self, contiguous_df_two_stations): - """Ensure instruction df honors differences in group columns.""" + """Ensure members honor differences in group columns.""" df = contiguous_df_two_stations - chunker = ChunkManager( - overlap=0, - time=10, - group_columns=("station",), - keep_partial=True, + plan = build_chunk_plan( + df, overlap=0, time=10, group=("station",), keep_partial=True + ) + joined = plan.members.merge( + df.assign(_patch_id=np.arange(len(df)))[["_patch_id", "station"]], + on="_patch_id", + ).merge( + plan.outputs[["output_id", "station"]], + on="output_id", + suffixes=("_src", "_out"), ) - in_df, out_df = chunker.chunk(df) - instruction = chunker.get_instruction_df(in_df, out_df) - # ensure each output has exactly one station. - for _current_index, sub in instruction.groupby("current_index"): - source = df.loc[sub.index] - # there should only be on station in the source for this group - unique_stations = source["station"].unique() - assert len(unique_stations) == 1 - # ensure all stations are present. - used = in_df.loc[instruction.index] - assert set(used["station"]) == set(in_df["station"]) - assert set(used["_group"]) == set(in_df["_group"]) + assert (joined["station_src"] == joined["station_out"]).all() + assert set(plan.outputs["station"]) == set(df["station"]) def test_modified_flag_if_chunked(self, contiguous_df): """Ensure the modified flag shows up for modified rows.""" - df = contiguous_df - chunker = ChunkManager( - overlap=0, - time=5, - group_columns=("station",), - keep_partial=True, - ) - in_df, out_df = chunker.chunk(df) - instruction = chunker.get_instruction_df(in_df, out_df) - assert instruction["_modified"].all() + plan = build_chunk_plan(contiguous_df, overlap=0, time=5, keep_partial=True) + assert plan.members["_modified"].all() def test_modified_flag_no_chunk(self, contiguous_df): - """Ensure the rows that don't change limits aren't modified.""" + """Rows whose limits don't change aren't modified.""" time_diff = contiguous_df["time_max"] - contiguous_df["time_min"] df = contiguous_df.assign(time_max=lambda x: (x["time_max"] - x["time_step"])) - chunker = ChunkManager( - overlap=0, - time=time_diff.iloc[0], - group_columns=("station",), - keep_partial=True, + plan = build_chunk_plan( + df, overlap=0, time=time_diff.iloc[0], keep_partial=True ) - in_df, out_df = chunker.chunk(df) - - assert (out_df[sorted(out_df.columns)]).equals(in_df[sorted(in_df.columns)]) - instruction = chunker.get_instruction_df(in_df, out_df) - assert not instruction["_modified"].any() + assert len(plan.outputs) == len(df) + assert not plan.members["_modified"].any() From 37218bf50993a64ab270b3370af268ec9d0782cb Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 12:17:02 +0200 Subject: [PATCH 23/97] Add spool union: spool + spool combines catalogs BaseSpool.__add__ produces a lazy MemorySpool over the union of both spools' metadata: - PatchCatalog.union merges catalogs table-to-table by reconstructing source records from the member backends (records_from_backend) and re-ingesting them: coord definitions deduplicate by def key, ids are reassigned, and the same source appearing in several members keeps a single entry via (base_uri, source_path) replace semantics. - File-backed rows are absolutized against their member root so directories with different roots coexist; a CompositeResolver routes memory:// rows to the merged live registry and everything else through the file reader. In-memory patches are shared, not copied. - Selections on inputs carry over by row membership (catalog views and dataframe-layer selections both); restructured views (e.g. chunked spools) contribute their materialized patches. Chunk works across union seams: a file-backed patch and an in-memory patch that are contiguous merge into one patch with an exactly fused range coordinate. 12 new tests; full suite passes (7,254). --- dascore/core/spool.py | 47 +++++++++ dascore/io/index/catalog.py | 68 ++++++++++++ dascore/io/index/ingest.py | 123 ++++++++++++++++++++++ tests/test_io/test_index/test_union.py | 139 +++++++++++++++++++++++++ 4 files changed, 377 insertions(+) create mode 100644 tests/test_io/test_index/test_union.py diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 300933429..7632d25af 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -165,6 +165,53 @@ def __eq__(self, other) -> bool: other_dict = getattr(other, "__dict__", {}) return deep_equality_check(my_dict, other_dict) + def __add__(self, other) -> BaseSpool: + """ + Combine two spools into one containing the patches of both. + + The result is a lazy spool over the union of both spools' + metadata: file-backed patches stay unloaded, in-memory patches + are shared (not copied), and the same source appearing in both + spools keeps a single entry. Selections on the inputs carry over + by row membership. + + Examples + -------- + >>> import dascore as dc + >>> sp1 = dc.get_example_spool("random_das") + >>> sp2 = dc.get_example_spool("diverse_das") + >>> combined = sp1 + sp2 + >>> assert len(combined) == len(sp1) + len(sp2) + """ + if not isinstance(other, BaseSpool): + return NotImplemented + from dascore.io.index.catalog import PatchCatalog + + members = [] + for spool_ in (self, other): + catalog = getattr(spool_, "_catalog", None) + patch_ids = None + if catalog is not None and not getattr(spool_, "_catalog_native", False): + # Dataframe-layer selections narrow rows without touching + # the catalog; carry that membership over. Restructured + # rows (e.g. chunked views) no longer map to sources and + # contribute their materialized patches instead. + df = spool_._df + if "_patch_id" in df.columns: + patch_ids = df["_patch_id"].tolist() + else: + catalog = None + if catalog is None: + # Spools without a usable catalog contribute their + # materialized patches. + catalog = PatchCatalog.from_patches(list(spool_)) + members.append((catalog, patch_ids)) + union = PatchCatalog.union(members) + new = MemorySpool() + new._catalog = union + new._catalog_native = True + return new + @abc.abstractmethod @compose_docstring(conflict_desc=attr_conflict_description) def chunk( diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index cb153ce30..572a061c9 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -132,6 +132,32 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: return _select_patch_from_spool(spool, source_patch_id=source_patch_id) +class CompositeResolver(PatchResolver): + """ + Route rows to a live registry or the filesystem by path scheme. + + Union catalogs mix file-backed rows (absolute paths) with in-memory + rows (memory:// paths); this resolver dispatches accordingly. + """ + + def __init__(self): + self.live = LiveResolver() + self.file = FileResolver(root=None) + + def absorb(self, resolver: PatchResolver) -> None: + """Take over the live registry entries of another resolver.""" + if isinstance(resolver, LiveResolver): + self.live._registry.update(resolver._registry) + elif isinstance(resolver, CompositeResolver): + self.live._registry.update(resolver.live._registry) + + def resolve(self, row: Mapping, **trim) -> dc.Patch: + """Dispatch to the live registry or the file reader.""" + if str(row.get("path", "")).startswith("memory"): + return self.live.resolve(row, **trim) + return self.file.resolve(row, **trim) + + def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): """Build source records for live patches with synthetic identities.""" token = next(_counter) @@ -154,6 +180,17 @@ def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): return records +def _absolutize_record(record, root): + """Return a source record whose relative path is resolved against root.""" + from dataclasses import replace + + path = record.source_path + if "://" in path or Path(path).is_absolute(): + return record + resolved = str(Path(root) / (path if path != "." else "")) + return replace(record, source_path=resolved, base_uri=None) + + @dataclass class _CatalogRevision: """Shared mutation revision for live catalog views.""" @@ -203,6 +240,37 @@ def from_patches(cls, patches: Sequence[dc.Patch] = ()) -> PatchCatalog: """ return cls(resolver=LiveResolver(), pending=tuple(patches)) + @classmethod + def union(cls, catalogs: Sequence[PatchCatalog]) -> PatchCatalog: + """ + Materialize several catalogs into one in-memory catalog. + + Metadata rows are merged table-to-table (coord definitions + deduplicate by def key); file-backed rows get absolute paths so + members with different roots coexist, and the same source + appearing in several members keeps a single entry (last one + wins). For catalog views, only the selected patches transfer — + note this respects row membership, not range trims; re-select + on the result for exact envelopes. + """ + from dascore.io.index.ingest import records_from_backend + + resolver = CompositeResolver() + out = cls(resolver=resolver) + backend = out.backend + for member in catalogs: + catalog, patch_ids = member if isinstance(member, tuple) else (member, None) + if patch_ids is None and catalog.is_view: + patch_ids = catalog.to_df()["_patch_id"].tolist() + records = records_from_backend(catalog.backend, patch_ids=patch_ids) + root = getattr(catalog.resolver, "_root", None) + if root is not None: + records = [_absolutize_record(x, root) for x in records] + backend.write_sources(records) + resolver.absorb(catalog.resolver) + out._invalidate() + return out + @classmethod def from_directory( cls, diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index f4e2515fc..497c6b8f7 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -368,3 +368,126 @@ def summaries_to_records( ) ) return out + + +def _py_scalar(value): + """Convert a fetched cell to the plain python scalar records use.""" + if value is None or pd.isnull(value): + return None + if isinstance(value, np.bool_ | bool): + return bool(value) + if isinstance(value, np.integer | int): + return int(value) + if isinstance(value, np.floating | float): + return float(value) + return value + + +def records_from_backend(backend, patch_ids=None) -> list[SourceRecord]: + """ + Reconstruct source records from a backend's tables. + + This is the transfer format for merging catalogs: feeding the result + to another backend's `write_sources` re-ingests the metadata with + fresh ids, coord-def deduplication (def keys are preserved), and + replace-semantics on (base_uri, source_path) identity. + + Parameters + ---------- + backend + The index backend to read. + patch_ids + If not None, only include these patches (and only sources which + still have at least one included patch). + """ + sources = backend._fetch_df("SELECT * FROM sources") + if sources.empty: + return [] + patches = backend._fetch_df("SELECT * FROM patches") + if patch_ids is not None: + patches = patches[patches["patch_id"].isin(set(patch_ids))] + attrs = backend._fetch_df("SELECT * FROM attrs") + links = backend._fetch_df("SELECT * FROM patch_coords") + defs = backend._fetch_df("SELECT * FROM coord_defs") + meta = backend._attr_meta() + col_info = { + row.column_name: (row.attr_name, row.value_kind, _py_scalar(row.units)) + for row in meta.itertuples() + } + def_map = {int(row.coord_def_id): row for row in defs.itertuples()} + attr_rows = ( + {int(k): v for k, v in attrs.set_index("patch_id").iterrows()} + if not attrs.empty + else {} + ) + link_groups = ( + {int(k): v for k, v in links.groupby("patch_id")} if not links.empty else {} + ) + out = [] + for src in sources.itertuples(): + sub = patches[patches["source_id"] == src.source_id] + if sub.empty: + continue + patch_records = [] + for patch in sub.itertuples(): + pid = int(patch.patch_id) + typed = {} + for col, value in attr_rows.get(pid, {}).items(): + if col in col_info and not pd.isnull(value): + name, kind, units = col_info[col] + typed[name] = TypedValue( + kind=kind, value=_py_scalar(value), units=units + ) + coords = [] + for link in link_groups.get(pid, pd.DataFrame()).itertuples(): + cdef = def_map[int(link.coord_def_id)] + coords.append( + CoordRecord( + coord_name=link.coord_name, + coord_dims=link.coord_dims, + value_kind=cdef.value_kind, + dtype=_py_scalar(cdef.dtype), + length=_py_scalar(cdef.length), + units=_py_scalar(cdef.units), + min_num=_py_scalar(cdef.min_num), + max_num=_py_scalar(cdef.max_num), + step_num=_py_scalar(cdef.step_num), + min_ns=_py_scalar(cdef.min_ns), + max_ns=_py_scalar(cdef.max_ns), + step_ns=_py_scalar(cdef.step_ns), + min_str=_py_scalar(cdef.min_str), + max_str=_py_scalar(cdef.max_str), + is_monotonic=_py_scalar(cdef.is_monotonic), + is_relative=_py_scalar(cdef.is_relative), + coord_hash=_py_scalar(cdef.fingerprint), + ) + ) + patch_records.append( + PatchRecord( + source_patch_id=_py_scalar(patch.source_patch_id) or "", + dims=_py_scalar(patch.dims) or "", + shape=_py_scalar(patch.shape) or "", + n_dims=_py_scalar(patch.n_dims), + sample_count_total=_py_scalar(patch.sample_count_total), + time_min=_py_scalar(patch.time_min), + time_max=_py_scalar(patch.time_max), + time_step=_py_scalar(patch.time_step), + distance_min=_py_scalar(patch.distance_min), + distance_max=_py_scalar(patch.distance_max), + distance_step=_py_scalar(patch.distance_step), + attrs=typed, + coords=tuple(coords), + ) + ) + out.append( + SourceRecord( + source_path=_py_scalar(src.source_path) or "", + base_uri=_py_scalar(src.base_uri) or None, + source_format=_py_scalar(src.source_format) or "", + format_version=_py_scalar(src.format_version) or "", + mtime_ns=_py_scalar(src.mtime_ns), + size_bytes=_py_scalar(src.size_bytes), + patches=tuple(patch_records), + ) + ) + return out diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py new file mode 100644 index 000000000..4ec3015c8 --- /dev/null +++ b/tests/test_io/test_index/test_union.py @@ -0,0 +1,139 @@ +"""Tests for spool union (spool + spool) and catalog merging.""" + +from __future__ import annotations + +import numpy as np +import pytest + +import dascore as dc +from dascore.core.coords import CoordRange +from dascore.io.index.catalog import CompositeResolver, PatchCatalog + + +@pytest.fixture(scope="module") +def contiguous_patches(): + """Two contiguous example patches.""" + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) + time = p1.get_coord("time") + p2 = dc.get_example_patch(time_min=time.max() + time.step) + return p1, p2 + + +@pytest.fixture() +def dir_spool(contiguous_patches, tmp_path): + """A directory spool holding the first patch.""" + p1, _ = contiguous_patches + dc.write(p1, tmp_path / "a.h5", "dasdae") + return dc.spool(tmp_path).update() + + +class TestMemoryUnion: + """Unions of in-memory spools.""" + + def test_lengths_add(self): + """The union contains the patches of both.""" + sp1 = dc.get_example_spool("random_das") + sp2 = dc.get_example_spool("diverse_das") + combined = sp1 + sp2 + assert len(combined) == len(sp1) + len(sp2) + + def test_patches_shared_not_copied(self, contiguous_patches): + """In-memory patches resolve to the same objects.""" + p1, p2 = contiguous_patches + combined = dc.spool([p1]) + dc.spool([p2]) + loaded = list(combined) + assert any(x is p1 for x in loaded) + assert any(x is p2 for x in loaded) + + def test_select_on_union(self): + """Selection works over the merged metadata.""" + sp1 = dc.get_example_spool("random_das") + sp2 = dc.get_example_spool("diverse_das") + combined = sp1 + sp2 + selected = combined.select(network="das2") + assert len(selected) + for patch in selected: + assert patch.attrs.network == "das2" + + def test_non_spool_add(self): + """Adding a non-spool returns NotImplemented semantics.""" + sp1 = dc.get_example_spool("random_das") + with pytest.raises(TypeError): + _ = sp1 + 42 + + def test_chunk_across_members(self, contiguous_patches): + """Contiguous patches from different spools merge into one.""" + p1, p2 = contiguous_patches + combined = dc.spool([p1]) + dc.spool([p2]) + merged = combined.chunk(time=None) + assert len(merged) == 1 + patch = merged[0] + time = patch.get_coord("time") + assert isinstance(time, CoordRange) + assert time.min() == p1.get_coord("time").min() + assert time.max() == p2.get_coord("time").max() + + def test_selection_carries_by_membership(self): + """A selected input contributes only its selected rows.""" + sp2 = dc.get_example_spool("diverse_das") + sub = sp2.select(network="das2") + combined = dc.get_example_spool("random_das") + sub + assert len(combined) == len(dc.get_example_spool("random_das")) + len(sub) + + +class TestFileUnion: + """Unions involving file-backed spools.""" + + def test_dir_plus_memory(self, dir_spool, contiguous_patches): + """A directory spool and memory spool combine lazily.""" + _, p2 = contiguous_patches + combined = dir_spool + dc.spool([p2]) + assert len(combined) == 2 + loaded = list(combined) + assert all(x.shape for x in loaded) + + def test_chunk_across_file_and_memory(self, dir_spool, contiguous_patches): + """The union seam merges: file patch + memory patch -> one patch.""" + p1, p2 = contiguous_patches + combined = dir_spool + dc.spool([p2]) + merged = combined.chunk(time=None) + assert len(merged) == 1 + patch = merged[0] + assert patch.shape[patch.get_axis("time")] == ( + p1.shape[p1.get_axis("time")] + p2.shape[p2.get_axis("time")] + ) + + def test_same_source_dedups(self, dir_spool): + """The same source in both members keeps a single entry.""" + combined = dir_spool + dir_spool + assert len(combined) == len(dir_spool) + + def test_union_preserves_def_keys(self, dir_spool, contiguous_patches): + """Coord definitions deduplicate by def key across members.""" + _, p2 = contiguous_patches + combined = dir_spool + dc.spool([p2]) + df = combined._catalog.to_df() + # both patches share the same distance coord identity + assert df["_distance_def_key"].nunique() == 1 + + +class TestCompositeResolver: + """Resolver dispatch for union catalogs.""" + + def test_routes_memory_rows(self, contiguous_patches): + """memory:// rows go to the live registry.""" + p1, _ = contiguous_patches + cat = PatchCatalog.union([PatchCatalog.from_patches([p1])]) + assert isinstance(cat.resolver, CompositeResolver) + row = cat.to_df().iloc[0].to_dict() + assert cat.resolve_row(row) is p1 + + def test_union_of_union(self, contiguous_patches): + """Unions compose (a union catalog can be a member).""" + p1, p2 = contiguous_patches + first = PatchCatalog.union([PatchCatalog.from_patches([p1])]) + second = PatchCatalog.union([first, PatchCatalog.from_patches([p2])]) + assert len(second.to_df()) == 2 + patches = [second.resolve_row(x) for _, x in second.to_df().iterrows()] + assert {id(x) for x in patches} == {id(p1), id(p2)} From de587a2ac98ce50e363f527f8232ddec36bb8d29 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 12:45:07 +0200 Subject: [PATCH 24/97] Give patches a lineage identity; make the live registry the store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch gains _instance_id: a lazily minted uuid on the instance (never in attrs, so it cannot affect equality or leak into saved files). Copies and pickles share the identity — patches are immutable, so identical copies sharing an identity is correct — while every patch operation mints a new instance with its own. The in-memory catalog store is now the LiveResolver registry itself: a dict from memorypatch:// paths to patches, created at construction. The pending-tuple bookkeeping is gone; pickled catalogs rebuild from the registry (live) or captured source records (unions) without touching the DB connection. Behavior changes (0.2, resolved 2026-07-10): - Spools have set semantics by patch identity: dc.spool([p, p]) and dc.spool([p, deepcopy(p)]) contain one entry; use patch.new() (or any operation) to create a distinct instance. - Unions deduplicate the same in-memory patch across spools. - Unresolvable in-memory rows (e.g. from a reopened index) raise MissingPatchError with guidance instead of KeyError. Also: memorypatch:// paths excluded from filename-based patch naming (same as memory://); concatenate docstring/tests updated to use distinct instances. --- dascore/core/patch.py | 14 ++++ dascore/core/spool.py | 83 +++++++++++++++++--- dascore/io/index/catalog.py | 100 +++++++++++++++++-------- dascore/utils/patch.py | 8 +- tests/test_io/test_index/test_union.py | 51 +++++++++++++ tests/test_utils/test_patch_utils.py | 14 +++- 6 files changed, 219 insertions(+), 51 deletions(-) diff --git a/dascore/core/patch.py b/dascore/core/patch.py index 6d128e599..26953c58f 100644 --- a/dascore/core/patch.py +++ b/dascore/core/patch.py @@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence from functools import cached_property from typing import Final +from uuid import uuid4 import numpy as np from rich.text import Text @@ -262,6 +263,19 @@ def summary(self): """ return PatchSummary.from_patch(self) + @cached_property + def _instance_id(self) -> str: + """ + A stable identity for this patch instance. + + Minted lazily; once computed it rides along with copies and + pickles (patches are immutable, so identical copies sharing an + identity is correct). Patch operations produce new instances + with new identities. Never stored in attrs, so it cannot affect + equality or survive into saved files. + """ + return uuid4().hex + @property def coords(self) -> CoordManager: """ diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 7632d25af..17b68aa7e 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -274,6 +274,13 @@ def chunk( ----- [`Spool.concatenate`](`dascore.BaseSpool.concatenate`) performs a similar operation but disregards the coordinate values. + + To inspect what a chunk call will do before running it — which + output patches it produces and which slice of which source patch + feeds each one — use + [`Spool.chunk_plan`](`dascore.core.spool.DataFrameSpool.chunk_plan`), + which takes the same arguments and returns the plan without + touching any data. """ @abc.abstractmethod @@ -779,6 +786,63 @@ def _read_and_resolve_patch(self, final_kwargs) -> dc.Patch: return spool[0] return _select_patch_from_spool(spool, source_patch_id=source_patch_id) + def _chunk_working_df(self) -> pd.DataFrame: + """Return the source rows the chunk planner consumes.""" + source = self._source_df + working = source.drop(columns=list(self._drop_columns), errors="ignore") + if "_patch_id" in source.columns: + working = working.assign(_patch_id=source["_patch_id"]) + else: + working = working.assign(_patch_id=np.arange(len(source))) + return working + + def chunk_plan( + self, + overlap: numeric_types | timeable_types | None = None, + keep_partial: bool = False, + snap_coords: bool = True, + tolerance: float = 1.5, + conflict: Literal["drop", "raise", "keep_first"] = "raise", + group: str | Sequence[str] | None = None, + missing_dim: Literal["raise", "drop"] = "raise", + **kwargs, + ): + """ + Return the plan `chunk` would execute, without touching any data. + + The returned [`ChunkPlan`](`dascore.io.index.plan.ChunkPlan`) is a + read-only diagnostic: its `outputs` table describes each patch the + chunked spool would contain (envelopes, step, carried attributes), + its `members` table shows exactly which slice of which source patch + feeds each output, and `params` records every resolved parameter + (including the group attributes and sampling tolerance in effect). + Accepts the same arguments as + [`chunk`](`dascore.BaseSpool.chunk`). + + Examples + -------- + >>> import dascore as dc + >>> spool = dc.get_example_spool("random_das") + >>> plan = spool.chunk_plan(time=3) + >>> assert len(plan.outputs) == len(spool.chunk(time=3)) + >>> # See which sources contribute to the first output patch. + >>> members = plan.members + >>> first = members[members["output_id"] == 0] + """ + from dascore.io.index.plan import build_chunk_plan + + return build_chunk_plan( + self._chunk_working_df(), + overlap=overlap, + keep_partial=keep_partial, + snap_coords=snap_coords, + tolerance=tolerance, + conflict=conflict, + group=group, + missing_dim=missing_dim, + **kwargs, + ) + @compose_docstring(doc=BaseSpool.chunk.__doc__) def chunk( self, @@ -792,16 +856,8 @@ def chunk( **kwargs, ) -> Self: """{doc}""" - from dascore.io.index.plan import build_chunk_plan - source = self._source_df - working = source.drop(columns=list(self._drop_columns), errors="ignore") - if "_patch_id" in source.columns: - working = working.assign(_patch_id=source["_patch_id"]) - else: - working = working.assign(_patch_id=np.arange(len(source))) - plan = build_chunk_plan( - working, + plan = self.chunk_plan( overlap=overlap, keep_partial=keep_partial, snap_coords=snap_coords, @@ -821,7 +877,8 @@ def chunk( return self.new_from_df(empty, merge_kwargs=merge_kwargs) out_df = plan.outputs.drop(columns=["output_id"]).reset_index(drop=True) # Instructions bind plan members back to source rows by patch id. - pid_to_index = pd.Series(source.index.values, index=working["_patch_id"].values) + working_ids = self._chunk_working_df()["_patch_id"] + pid_to_index = pd.Series(source.index.values, index=working_ids.values) names = [f"{plan.dim}_min", f"{plan.dim}_max", f"{plan.dim}_step"] instructions = ( plan.members.assign( @@ -1101,7 +1158,11 @@ def __init__(self, data: PatchType | Sequence[PatchType] | None = None): elif isinstance(data, Sequence) and all( isinstance(x, dc.Patch) for x in data ): - self._patches = tuple(data) + # The same patch instance (by lineage: copies share an + # identity) appears once; spools have set semantics for + # identical in-memory patches. + unique = {x._instance_id: x for x in data} + self._patches = tuple(unique.values()) else: # eg a spool or dataframe; needs the dataframe machinery. self._data = data diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 572a061c9..6f8f85499 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -17,7 +17,6 @@ from __future__ import annotations import abc -import itertools from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path @@ -26,6 +25,7 @@ import dascore as dc from dascore.constants import PROGRESS_LEVELS +from dascore.exceptions import MissingPatchError from dascore.io.index.backend import get_backend, resolve_query from dascore.io.index.ingest import SourceRecord, patch_record from dascore.io.index.query import ( @@ -35,8 +35,6 @@ ) from dascore.utils.pd import adjust_segments -_counter = itertools.count() - def _row_source_patch_id(row: Mapping) -> str: """Return the row's source_patch_id as a string ("" when missing). @@ -63,19 +61,41 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: class LiveResolver(PatchResolver): - """Serve patches from an in-memory registry.""" + """ + Serve patches from an in-memory registry. - def __init__(self): - self._registry: dict[tuple[str, str], dc.Patch] = {} + The registry *is* the store for live catalogs: a dict from each + patch's synthetic path (`memorypatch://`) to the patch + itself. Dict construction deduplicates identical patch instances + (set semantics by lineage), and merging catalogs unions the dicts. + """ + + def __init__(self, patches: Sequence[dc.Patch] = ()): + self._registry: dict[str, dc.Patch] = { + _patch_path(patch): patch for patch in patches + } - def register(self, path: str, source_patch_id: str, patch: dc.Patch) -> None: + def register(self, path: str, patch: dc.Patch) -> None: """Register a live patch under its synthetic source identity.""" - self._registry[(path, source_patch_id)] = patch + self._registry[path] = patch def resolve(self, row: Mapping, **trim) -> dc.Patch: """Look the patch up; live patches ignore trim hints.""" - key = (row["path"], _row_source_patch_id(row)) - return self._registry[key] + path = str(row["path"]) + try: + return self._registry[path] + except KeyError: + msg = ( + f"The in-memory patch for {path} is not available in this " + "session (e.g. the row came from a reopened index). " + "In-memory patches only persist by writing them to files." + ) + raise MissingPatchError(msg) from None + + +def _patch_path(patch: dc.Patch) -> str: + """Return the synthetic source path identifying a live patch.""" + return f"memorypatch://{patch._instance_id}" class FileResolver(PatchResolver): @@ -158,16 +178,13 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: return self.file.resolve(row, **trim) -def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): - """Build source records for live patches with synthetic identities.""" - token = next(_counter) +def _live_records(registry: Mapping[str, dc.Patch]): + """Build source records for live patches keyed by their identity.""" records = [] - for num, patch in enumerate(patches): + for path, patch in registry.items(): # patch.summary is a cached_property: reuse fingerprints and # summaries the patch already computed instead of rebuilding. - summary = patch.summary - path = f"memory://catalog_{token}/{num}" - record = patch_record(summary) + record = patch_record(patch.summary) records.append( SourceRecord( source_path=path, @@ -176,7 +193,6 @@ def _live_records(patches: Sequence[dc.Patch], resolver: LiveResolver): patches=(record,), ) ) - resolver.register(path, record.source_patch_id, patch) return records @@ -213,7 +229,6 @@ def __init__( backend=None, resolver: PatchResolver | None = None, syncer=None, - pending: tuple = (), queries: tuple[Query, ...] = (), residuals: tuple[tuple[dict, bool, bool], ...] = (), revision: _CatalogRevision | None = None, @@ -221,14 +236,14 @@ def __init__( self._backend = backend self.resolver = resolver self._syncer = syncer - # live patches not yet ingested; kept (not a closure) so catalogs - # pickle and can rebuild their backend after unpickling. - self._pending = tuple(pending) self._queries = tuple(queries) self._residuals = tuple(residuals) self._revision = revision or _CatalogRevision() self._df_cache: pd.DataFrame | None = None self._df_cache_revision = -1 + # Source records for rebuilding an in-memory backend (set by + # __getstate__ so pickled catalogs survive losing the connection). + self._rebuild_records: tuple = () # --- construction ------------------------------------------------- @@ -237,8 +252,11 @@ def from_patches(cls, patches: Sequence[dc.Patch] = ()) -> PatchCatalog: """ Catalog over live patches. No backend work happens until the first metadata operation. + + The resolver's registry is the store; identical patch instances + collapse to a single entry (set semantics by lineage). """ - return cls(resolver=LiveResolver(), pending=tuple(patches)) + return cls(resolver=LiveResolver(patches)) @classmethod def union(cls, catalogs: Sequence[PatchCatalog]) -> PatchCatalog: @@ -299,8 +317,11 @@ def backend(self): """ if self._backend is None: self._backend = get_backend(":memory:") - if self._pending: - self._backend.write_sources(_live_records(self._pending, self.resolver)) + if self._rebuild_records: + self._backend.write_sources(list(self._rebuild_records)) + self._rebuild_records = () + elif registry := getattr(self.resolver, "_registry", None): + self._backend.write_sources(_live_records(registry)) if self._syncer is not None and self._syncer.ensure_updated(): self._invalidate() return self._backend @@ -309,16 +330,25 @@ def __getstate__(self) -> dict: """ Pickle without the live DB connection. - Live catalogs rebuild their backend from pending patches on next - use; the resolver registry (which pickled rows reference) rides - along unchanged, so already-realized views keep resolving. + In-memory backends (live and union catalogs) ride along as + source records and are re-ingested on next use; the resolver + registry (the store for live patches) pickles with its patches. + Directory catalogs rebuild from their index file instead. """ + from dascore.io.index.ingest import records_from_backend + state = dict(self.__dict__) state["_backend"] = None - if isinstance(self.resolver, LiveResolver) and not state["_pending"]: - # allow rebuilding the backend from the registered patches - registry = self.resolver._registry - state["_pending"] = tuple(registry.values()) + # Live catalogs rebuild from their registry without touching the + # connection (which may belong to another thread during pickling); + # other in-memory catalogs (e.g. unions) capture their rows. + needs_records = ( + self._backend is not None + and self._syncer is None + and not isinstance(self.resolver, LiveResolver) + ) + if needs_records: + state["_rebuild_records"] = tuple(records_from_backend(self._backend)) return state def _view(self, queries, residuals) -> PatchCatalog: @@ -497,7 +527,11 @@ def add(self, patches: Sequence[dc.Patch] | dc.Patch) -> PatchCatalog: msg = "add() currently supports in-memory catalogs only." raise NotImplementedError(msg) patches = [patches] if isinstance(patches, dc.Patch) else list(patches) - self.backend.write_sources(_live_records(patches, self.resolver)) + additions = {_patch_path(x): x for x in patches} + self.resolver._registry.update(additions) + # Re-adding a patch replaces its row (same identity), so this + # stays idempotent. + self.backend.write_sources(_live_records(additions)) self._invalidate() return self diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index 6d49a3350..27d3d27d4 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -616,7 +616,8 @@ def _get_filename(path_ser, strip_extension): path_ser = df["path"].astype(str) if "path" in col_set else None if path_ser is not None: # synthetic in-memory identities are not real file names - usable = path_ser.str.len().gt(0) & ~path_ser.str.startswith("memory://") + # (memory:// and memorypatch:// schemes) + usable = path_ser.str.len().gt(0) & ~path_ser.str.startswith("memory") if usable.any(): return _get_filename(df["path"], strip_extension) # Determine the requested fields; absent columns render as empty so @@ -1288,8 +1289,9 @@ def concatenate_patches( >>> spool_concat = spool.concatenate(wave_rank=None) >>> assert "wave_rank" in spool_concat[0].dims >>> - >>> # Concatenate patches in groups of 3. - >>> big_spool = dc.spool([patch] * 12) + >>> # Concatenate patches in groups of 3. Note: spools keep one + >>> # entry per patch instance, so distinct copies are needed. + >>> big_spool = dc.spool([patch.new() for _ in range(12)]) >>> spool_concat = big_spool.concatenate(time=3) >>> assert len(spool_concat) == 4 diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 4ec3015c8..c6b664088 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -137,3 +137,54 @@ def test_union_of_union(self, contiguous_patches): assert len(second.to_df()) == 2 patches = [second.resolve_row(x) for _, x in second.to_df().iterrows()] assert {id(x) for x in patches} == {id(p1), id(p2)} + + +class TestPatchIdentity: + """Set semantics by patch instance identity (lineage).""" + + def test_duplicate_instances_collapse(self): + """The same patch instance twice is one spool entry.""" + patch = dc.get_example_patch() + assert len(dc.spool([patch, patch])) == 1 + + def test_deepcopy_shares_identity(self): + """Copies of an immutable patch share its identity.""" + import copy + + patch = dc.get_example_patch() + _ = patch._instance_id # mint before copying + clone = copy.deepcopy(patch) + assert clone._instance_id == patch._instance_id + assert len(dc.spool([patch, clone])) == 1 + + def test_new_instance_distinct(self): + """patch.new() (and any patch op) mints a distinct identity.""" + patch = dc.get_example_patch() + assert len(dc.spool([patch, patch.new()])) == 2 + + def test_ops_mint_new_identity(self): + """Operations produce instances with their own identity.""" + patch = dc.get_example_patch() + other = patch.update_attrs(tag="x") + assert patch._instance_id != other._instance_id + + def test_pickle_round_trip_preserves_content(self): + """Spools of live patches pickle and rebuild their backend.""" + import pickle + + patch = dc.get_example_patch() + spool = dc.spool([patch]) + _ = len(spool) # realize the catalog + loaded = pickle.loads(pickle.dumps(spool)) + assert len(loaded) == 1 + assert loaded[0] == patch + + def test_union_pickles(self): + """Union spools survive pickling (rows ride along as records).""" + import pickle + + p1 = dc.get_example_patch() + p2 = p1.new() + combined = dc.spool([p1]) + dc.spool([p2]) + loaded = pickle.loads(pickle.dumps(combined)) + assert len(loaded) == 2 diff --git a/tests/test_utils/test_patch_utils.py b/tests/test_utils/test_patch_utils.py index b3f09eb54..160f285bf 100644 --- a/tests/test_utils/test_patch_utils.py +++ b/tests/test_utils/test_patch_utils.py @@ -30,8 +30,8 @@ get_patch_names, get_patch_window_size, get_window_axis_step, - merge_patches, merge_compatible_coords_attrs, + merge_patches, patches_to_df, stack_patches, swap_kwargs_dim_to_axis, @@ -568,8 +568,13 @@ def test_different_dims_raises(self, random_patch): concatenate_patches([p1, p2], time=None) def test_duplicate_patches_existing_dim(self, random_patch): - """Ensure duplicate patches are concatenated together.""" - spool = dc.spool([random_patch, random_patch]) + """Ensure equal (but distinct) patches are concatenated together. + + Note: the same patch *instance* twice would collapse to one entry + (spools have set semantics by patch identity); patch.new() mints + a distinct instance with equal contents. + """ + spool = dc.spool([random_patch, random_patch.new()]) out = concatenate_patches(spool, time=None) assert len(out) == 1 patch = out[0] @@ -614,7 +619,8 @@ def test_concat_chunk_to_new_dimension(self, random_patch): """Ensure the new dimension can be chunked by an int value.""" # When new_dim = 1 it should only add a new dimension to each patch # and not change the original shape. - spool = dc.spool([random_patch] * 6) + # Distinct instances: identical instances would dedup to one. + spool = dc.spool([random_patch] + [random_patch.new() for _ in range(5)]) # Test for single values along new dimension new = spool.concatenate(new_dim=1) assert len(new) == len(spool) From d3bfec1f77772db8c859306a8e025a47fd03965e Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 12:45:17 +0200 Subject: [PATCH 25/97] Add spool.chunk_plan for inspecting chunk operations A read-only diagnostic taking the same arguments as chunk and returning the ChunkPlan it would execute: the outputs table describes each patch the chunked spool would contain, the members table shows exactly which slice of which source patch feeds each output, and params records every resolved parameter. chunk() now runs on the same plan, and its docstring points users at chunk_plan. The #662 forced-merge warning now attributes to the caller's frame at any entry depth (chunk, chunk_plan, or build_chunk_plan directly). --- dascore/io/index/plan.py | 22 +++++++++++++++- tests/test_io/test_index/test_plan.py | 37 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/dascore/io/index/plan.py b/dascore/io/index/plan.py index 832d73559..0a44853e2 100644 --- a/dascore/io/index/plan.py +++ b/dascore/io/index/plan.py @@ -17,6 +17,7 @@ import warnings from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Literal import numpy as np @@ -173,10 +174,29 @@ def _partition(df, name, group_attrs, tolerance, sampling_tolerance) -> pd.Serie "the patches. As a result, some patches in the chunked spool " "may be unevenly sampled, or have their sampling rate increased." ) - warnings.warn(msg, UserWarning, stacklevel=4) + warnings.warn(msg, UserWarning, stacklevel=_user_stacklevel()) return cell + "_" + cont.astype(str) +def _user_stacklevel() -> int: + """Return the warn stacklevel pointing at the first non-dascore frame. + + Plans are built at several call depths (spool.chunk, spool.chunk_plan, + build_chunk_plan directly), so a fixed stacklevel would blame library + frames for some entries. + """ + import inspect + + package_dir = str(Path(__file__).resolve().parents[2]) + # Frames after this helper's own align exactly with warn's numbering: + # level 1 is the frame calling warn. + for level, frame_info in enumerate(inspect.stack()[1:], start=1): + filename = str(Path(frame_info.filename).resolve()) + if not filename.startswith(package_dir): + return level + return 1 + + def _coerce_length_overlap(value, overlap, start_dtype): """Coerce the chunk length/overlap to the dimension's span dtype.""" time_like = is_datetime64(start_dtype) or is_timedelta64(start_dtype) diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index b7f51c62e..74a25bc9b 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -327,3 +327,40 @@ def test_segment_envelopes_match(self, random_flat): assert len(outs) == len(contents) assert np.array_equal(outs["time_min"].values, contents["time_min"].values) assert np.array_equal(outs["time_max"].values, contents["time_max"].values) + + +class TestChunkPlanAccessor: + """Tests for the public spool.chunk_plan diagnostic.""" + + def test_matches_chunk(self): + """The plan describes exactly what chunk produces.""" + spool = dc.get_example_spool("random_das") + plan = spool.chunk_plan(time=3) + chunked = spool.chunk(time=3) + assert len(plan.outputs) == len(chunked) + contents = chunked.get_contents().sort_values("time_min") + outs = plan.outputs.sort_values("time_min") + assert np.array_equal(outs["time_min"].values, contents["time_min"].values) + + def test_records_params(self): + """Plans record the resolved parameters.""" + spool = dc.get_example_spool("random_das") + plan = spool.chunk_plan(time=None, tolerance=2.0) + assert plan.params["tolerance"] == 2.0 + assert isinstance(plan.params["group"], tuple) + assert plan.merge_mode + + def test_members_reference_sources(self): + """Members bind outputs to source patches without loading data.""" + spool = dc.get_example_spool("diverse_das") + plan = spool.chunk_plan(time=None) + assert set(plan.members["output_id"]) == set(plan.outputs["output_id"]) + assert len(plan.members) == len(spool) + + def test_directory_spool(self, tmp_path): + """chunk_plan works on file-backed spools.""" + patch = dc.get_example_patch() + dc.write(patch, tmp_path / "a.h5", "dasdae") + spool = dc.spool(tmp_path).update() + plan = spool.chunk_plan(time=None) + assert len(plan.outputs) == 1 From 85041f106ee1dcae9a2dd18df5b491fbb6781c1b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 14:07:17 +0200 Subject: [PATCH 26/97] Document spool internals with executable notes; refresh stale scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add docs/notes/spool_chunking.qmd covering the plan/assembly split, partitioning rules, missing_dim, bounded-error merged coordinates, and union spools. Refresh spool_index.qmd (flat relation, patch identity, federation — its scope section still called chunk planning and federation future work) and add the core selection contract to spool_selection.qmd. All three notes now execute assertions against the real internal objects (PatchCatalog, chunk plans, resolvers), so the doc build fails if the described behavior drifts from the implementation. Also avoid building the chunk working relation twice per chunk call. --- dascore/core/spool.py | 9 ++- docs/notes/notes.qmd | 1 + docs/notes/spool_chunking.qmd | 110 +++++++++++++++++++++++++++++++++ docs/notes/spool_index.qmd | 42 ++++++++++++- docs/notes/spool_selection.qmd | 23 +++++++ 5 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 docs/notes/spool_chunking.qmd diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 17b68aa7e..302cc2097 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -856,8 +856,12 @@ def chunk( **kwargs, ) -> Self: """{doc}""" + from dascore.io.index.plan import build_chunk_plan + source = self._source_df - plan = self.chunk_plan( + working = self._chunk_working_df() + plan = build_chunk_plan( + working, overlap=overlap, keep_partial=keep_partial, snap_coords=snap_coords, @@ -877,8 +881,7 @@ def chunk( return self.new_from_df(empty, merge_kwargs=merge_kwargs) out_df = plan.outputs.drop(columns=["output_id"]).reset_index(drop=True) # Instructions bind plan members back to source rows by patch id. - working_ids = self._chunk_working_df()["_patch_id"] - pid_to_index = pd.Series(source.index.values, index=working_ids.values) + pid_to_index = pd.Series(source.index.values, index=working["_patch_id"].values) names = [f"{plan.dim}_min", f"{plan.dim}_max", f"{plan.dim}_step"] instructions = ( plan.members.assign( diff --git a/docs/notes/notes.qmd b/docs/notes/notes.qmd index 3ee5b672d..82413e40f 100644 --- a/docs/notes/notes.qmd +++ b/docs/notes/notes.qmd @@ -11,3 +11,4 @@ This section of the documentation provides understanding-oriented explanation fo - [Velocity to Strain Rate](velocity_to_strain_rate.qmd) - [Spool Index](spool_index.qmd) - [Spool Selection](spool_selection.qmd) +- [Spool Chunking](spool_chunking.qmd) diff --git a/docs/notes/spool_chunking.qmd b/docs/notes/spool_chunking.qmd new file mode 100644 index 000000000..deb38edfb --- /dev/null +++ b/docs/notes/spool_chunking.qmd @@ -0,0 +1,110 @@ +--- +title: Spool Chunking +--- + +`Spool.chunk` runs in two stages: a **planner** decides everything from metadata alone, and **assembly** loads, trims, and combines patch data only when a patch is requested. The code cells below execute against the real machinery, so this note fails the doc build if it drifts from the implementation. + +## Plans + +The planner consumes the spool's flat metadata relation and produces a `ChunkPlan`: an `outputs` table (one row per patch the chunked spool will contain) and a `members` table (which slice of which source patch feeds each output). `Spool.chunk_plan` exposes the plan `chunk` would execute, with the same arguments. + +```{python} +import dascore as dc + +spool = dc.get_example_spool("random_das") +plan = spool.chunk_plan(time=3) +assert len(plan.outputs) == len(spool.chunk(time=3)) +assert {"output_id", "_patch_id", "_modified"}.issubset(plan.members.columns) +# Resolved parameters are recorded on the plan, never left in config. +assert isinstance(plan.params["group"], tuple) +assert plan.params["sampling_group_tolerance"] == dc.get_config().sampling_group_tolerance +``` + +Plans are deterministic: the same spool produces the same plan regardless of metadata row order, and members with `_modified=False` load whole (no per-patch selection cost). + +## Partitioning + +Patches may only combine when they agree on all of: + +1. **Group attributes** — the config option `groupby_attrs` by default (conventional categorical identity: network, station, data type/category, tag, instrument and acquisition ids), overridden per call with `group=`. Differing group values are never an error; the patches simply land in separate outputs. Explicitly passed names must exist somewhere in the spool; config names are best-effort. +2. **Structure** — the dimensions tuple and the coordinate identity of every non-chunked dimension. +3. **Sampling** — steps within the relative tolerance `config.sampling_group_tolerance` (default 5%). +4. **Continuity** — patches within `tolerance` samples of each other, evaluated within each group so unrelated patches can never bridge a gap. + +```{python} +import numpy as np + +t0 = np.datetime64("2020-01-01", "ns") +p1 = dc.get_example_patch(time_min=t0) +time = p1.get_coord("time") +p2 = dc.get_example_patch(time_min=time.max() + time.step) +p3, p4 = p1.update_attrs(station="XX2"), p2.update_attrs(station="XX2") + +# Two stations, each with two contiguous patches: two outputs, no error. +merged = dc.spool([p1, p2, p3, p4]).chunk(time=None) +assert len(merged) == 2 +``` + +Remaining (non-group, non-dimensional) attributes must be single-valued within a partition, policed by `conflict`: `"raise"` (default), `"drop"`, or `"keep_first"`. + +Patches lacking the chunked dimension raise by default; `missing_dim="drop"` excludes them instead. Losing patches silently would be data loss, so it requires the explicit opt-in. + +```{python} +import pytest +from dascore.exceptions import ChunkError + +no_time = [dc.get_example_patch().mean("time") for _ in range(2)] +with pytest.raises(ChunkError, match="missing_dim"): + dc.spool(no_time).chunk(time=None) +assert len(dc.spool(no_time).chunk(time=None, missing_dim="drop")) == 0 +``` + +## Merged coordinates + +Assembly builds the chunked dimension's coordinate by exact concatenation of the member coordinates: contiguous members fuse to a plain evenly sampled range, and every real seam is recorded. When `snap_coords=True` (default) the result is then simplified with **bounded error** — no coordinate value moves more than `tolerance * step`. A within-tolerance gap therefore comes back as an evenly sampled range whose worst label error is about half the gap (never more than the tolerance); this is the honest replacement for the old unconditional snap, whose error was unbounded. With `snap_coords=False`, or when accumulated gaps exceed the bound, the coordinate stays segmented — exactly non-uniform, with every gap queryable. + +```{python} +from dascore.core.coords import CoordRange, CoordSegmented + +# Contiguous members: exact fuse to a range. +patch = dc.spool([p1, p2]).chunk(time=None)[0] +assert isinstance(patch.get_coord("time"), CoordRange) + +# A gap forced together by a loose tolerance warns, and with +# snap_coords=False the output keeps the exact segmented coordinate. +gap_start = time.max() + 3 * time.step +p_gap = dc.get_example_patch(time_min=gap_start) +with pytest.warns(UserWarning, match="force merging"): + forced = dc.spool([p1, p_gap]).chunk( + time=None, tolerance=5, snap_coords=False + )[0] +coord = forced.get_coord("time") +assert isinstance(coord, CoordSegmented) +assert len(coord.get_discontinuities("gaps")) == 1 + +# The default simplifies the same merge to a range with bounded error: +# every value within tolerance * step of its exact position. +with pytest.warns(UserWarning, match="force merging"): + snapped = dc.spool([p1, p_gap]).chunk(time=None, tolerance=5)[0] +snapped_coord = snapped.get_coord("time") +assert isinstance(snapped_coord, CoordRange) +deviation = abs(snapped_coord.values - coord.values).max() +assert deviation <= 5 * time.step +``` + +Segmented coordinates are an in-memory representation only: a written patch must be contiguous, so saving raises unless `split=True` (or `patch.split_gaps()` is used first) to write each contiguous section as its own patch. + +## Union spools + +`spool + spool` produces a lazy spool over the union of both spools' metadata; chunking works across the seam, so contiguous patches from different spools (even file-backed and in-memory mixed) merge into one. + +```{python} +combined = dc.spool([p1]) + dc.spool([p2]) +assert len(combined.chunk(time=None)) == 1 + +# Spools have set semantics by patch instance: the same patch (or a copy +# of it) appears once; operations mint distinct instances. +assert len(dc.spool([p1, p1])) == 1 +assert len(dc.spool([p1, p1.new()])) == 2 +assert len(dc.spool([p1]) + dc.spool([p1])) == 1 +``` diff --git a/docs/notes/spool_index.qmd b/docs/notes/spool_index.qmd index 004aa4c9a..7b77c5df6 100644 --- a/docs/notes/spool_index.qmd +++ b/docs/notes/spool_index.qmd @@ -30,6 +30,46 @@ The current schema version is validated before any mutation. An unrelated, incom SQLite permits concurrent readers and serializes writers. Initialization and updates use an immediate write transaction and a 30-second busy timeout. This relies on correct local-filesystem locking; reliable operation on network filesystems with weak locking is not promised. +## The flat relation + +Spool-facing operations consume the tables through one flat relation: a dataframe with one row per patch carrying `{dim}_min/max/step` envelopes, private structural columns (`_patch_id`, `_{dim}_def_key` coordinate identities), the `dims` signature, and one column per attribute. The chunk planner and selection both operate on this relation (see the [Spool Chunking](spool_chunking.qmd) and [Spool Selection](spool_selection.qmd) notes). The cell below runs against the real catalog so this description cannot silently drift. + +```{python} +import dascore as dc +from dascore.io.index.catalog import PatchCatalog + +catalog = PatchCatalog.from_patches(list(dc.get_example_spool("random_das"))) +df = catalog.to_df() +required = {"_patch_id", "_time_def_key", "dims", "time_min", "time_max", "time_step"} +assert required.issubset(df.columns) +``` + +## Patch identity + +Every in-memory patch has a lazily minted instance identity (shared by copies — patches are immutable — while every patch operation creates a new instance with its own). The identity is the patch's synthetic source path in the index, so identity, deduplication, and resolution all agree: + +```{python} +patch = dc.get_example_patch() +cat = PatchCatalog.from_patches([patch, patch]) # one entry, not two +row = cat.to_df().iloc[0] +assert len(cat.to_df()) == 1 +assert row["path"].startswith("memorypatch://") +assert cat.resolve_row(row.to_dict()) is patch +``` + +File-backed patches are identified by `(base_uri, source_path, source_patch_id)` instead; their rows resolve through readers rather than the registry. + +## Federation + +`spool + spool` merges catalogs table-to-table: source records are reconstructed from the member backends and re-ingested, so coordinate definitions deduplicate by definition key, the same source appearing in several members keeps a single entry, and file paths are absolutized so members with different roots coexist. A composite resolver routes in-memory rows to the shared registry and everything else through file readers. + +```{python} +other = dc.get_example_patch().new() +union = dc.spool([patch]) + dc.spool([other]) +assert len(union) == 2 +assert len(dc.spool([patch]) + dc.spool([patch])) == 1 # same patch dedups +``` + ## Scope -The index answers metadata selection and identifies candidate patches. Exact coordinate selection is applied again when a patch is loaded because summary envelopes cannot prove arbitrary-coordinate membership. Chunk planning, cross-catalog federation, missing-dimension policy, and materialized derived data remain future work. +The index answers metadata selection, identifies candidate patches, and plans chunk operations — all without touching patch data. Exact coordinate selection is applied again when a patch is loaded because summary envelopes cannot prove arbitrary-coordinate membership. Materialized derived data (persisting chunked results) remains future work. diff --git a/docs/notes/spool_selection.qmd b/docs/notes/spool_selection.qmd index f84c7e7fc..62d4c89ab 100644 --- a/docs/notes/spool_selection.qmd +++ b/docs/notes/spool_selection.qmd @@ -13,3 +13,26 @@ Coordinate predicates first select patches whose summary envelopes can overlap t Operations that create a new row or instruction plan, including chunking, sorting, and slicing, switch that derived spool to dataframe planning. Exact selections already attached to the catalog still apply when source patches are resolved. Catalog views share their source state. Adding, removing, or rescanning sources invalidates realized metadata so existing views observe the updated catalog under their composed predicates. + +The core contract, executed here so drift fails the doc build: names resolve attributes-first then coordinates, unknown names raise, and coordinate ranges are exact on the loaded patches (candidacy by envelope at the index, exactness at load): + +```{python} +import pytest + +import dascore as dc +from dascore.exceptions import InvalidSpoolQueryError + +spool = dc.get_example_spool("diverse_das") +with pytest.raises(InvalidSpoolQueryError): + spool.select(not_a_name="anything") + +selected = spool.select(network="das2") +assert all(p.attrs.network == "das2" for p in selected) + +df = spool.get_contents() +t0 = df["time_min"].min() +window = (t0, t0 + dc.to_timedelta64(1)) +for patch in spool.select(time=window): + coord = patch.get_coord("time") + assert coord.min() >= window[0] and coord.max() <= window[1] +``` From 2149965e80bb3bb6f19f3ad95228fb900c555d30 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 14:20:17 +0200 Subject: [PATCH 27/97] Apply cleanup review: dedupe helpers, polymorphic union dispatch, trim waste From a four-angle review (reuse / simplification / efficiency / altitude) of the branch changes: - One implementation of relative-range resolution (query.relative_ranges_to_absolute) shared by the dataframe and catalog select paths, which had drifted copies. - One implementation of the read-result identity reconciliation (io.core._resolve_read_spool) shared by FileResolver and the filespool path. - BaseSpool.__add__ dispatches through a polymorphic _as_catalog_member() (overridden by DataFrameSpool/MemorySpool) instead of sniffing subclass internals; MemorySpool contributes its own lazy catalog, reusing cached summaries. - Resolvers expose live_entries(); CompositeResolver.absorb loses its isinstance chain. Dead LiveResolver.register removed. - is_memory_uri() is the single predicate for synthetic in-memory paths (was independent startswith checks in two modules). - The positional patch-id fallback lives once (plan._ensure_patch_id). - The #662 forced-merge warning is emitted at the build_chunk_plan boundary instead of deep inside partitioning. - Wasted work removed: merged-coordinate assembly no longer materializes the members' concatenated dim values just to discard them (merge_coord_managers grew a dim_coord pass-through); the planner groups once instead of twice; member building indexes numpy arrays instead of pandas scalars; catalog-union record rebuilding groups patches by source instead of rescanning, drops iterrows, and narrows dependent tables when membership is limited. --- dascore/core/spool.py | 100 +++++++++++++++++----------------- dascore/io/core.py | 16 ++++++ dascore/io/index/catalog.py | 53 ++++++------------ dascore/io/index/ingest.py | 15 ++++- dascore/io/index/plan.py | 72 ++++++++++++++---------- dascore/io/index/query.py | 25 +++++++++ dascore/utils/coordmanager.py | 9 +++ dascore/utils/misc.py | 11 ++++ dascore/utils/patch.py | 19 ++++--- 9 files changed, 196 insertions(+), 124 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 302cc2097..c8e8b38fc 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -187,31 +187,25 @@ def __add__(self, other) -> BaseSpool: return NotImplemented from dascore.io.index.catalog import PatchCatalog - members = [] - for spool_ in (self, other): - catalog = getattr(spool_, "_catalog", None) - patch_ids = None - if catalog is not None and not getattr(spool_, "_catalog_native", False): - # Dataframe-layer selections narrow rows without touching - # the catalog; carry that membership over. Restructured - # rows (e.g. chunked views) no longer map to sources and - # contribute their materialized patches instead. - df = spool_._df - if "_patch_id" in df.columns: - patch_ids = df["_patch_id"].tolist() - else: - catalog = None - if catalog is None: - # Spools without a usable catalog contribute their - # materialized patches. - catalog = PatchCatalog.from_patches(list(spool_)) - members.append((catalog, patch_ids)) + members = [self._as_catalog_member(), other._as_catalog_member()] union = PatchCatalog.union(members) new = MemorySpool() new._catalog = union new._catalog_native = True return new + def _as_catalog_member(self): + """ + Return (catalog, patch_ids) describing this spool for a union. + + `patch_ids` limits membership to the spool's current rows; None + means the whole catalog (or the catalog view itself carries the + selection). The base implementation materializes the patches. + """ + from dascore.io.index.catalog import PatchCatalog + + return PatchCatalog.from_patches(list(self)), None + @abc.abstractmethod @compose_docstring(conflict_desc=attr_conflict_description) def chunk( @@ -773,28 +767,41 @@ def _load_patch(self, kwargs) -> dc.Patch: def _read_and_resolve_patch(self, final_kwargs) -> dc.Patch: """Read patches for one instruction row and resolve to one patch.""" - from dascore.io.core import _select_patch_from_spool + from dascore.io.core import _resolve_read_spool source_patch_id = final_kwargs.get("source_patch_id", "") spool = dc.read(**final_kwargs) - # Some readers consume source_patch_id internally and return the one - # matching patch without preserving that reload metadata on the - # patch. Only trust that when it doesn't claim a different identity. - if source_patch_id and len(spool) == 1: - found = str(spool[0].attrs.get("_source_patch_id", "") or "") - if found in ("", str(source_patch_id)): - return spool[0] - return _select_patch_from_spool(spool, source_patch_id=source_patch_id) + return _resolve_read_spool(spool, source_patch_id) + + def _as_catalog_member(self): + """ + Return (catalog, patch_ids) describing this spool for a union. + + Catalog-native spools contribute their catalog view directly. + Dataframe-layer selections narrow rows without touching the + catalog, so their membership carries over as patch ids. + Restructured rows (e.g. chunked views) no longer map to sources + and contribute their materialized patches instead. + """ + catalog = getattr(self, "_catalog", None) + if catalog is None: + return super()._as_catalog_member() + if self._catalog_native: + return catalog, None + df = self._df + if "_patch_id" in df.columns: + return catalog, df["_patch_id"].tolist() + return super()._as_catalog_member() def _chunk_working_df(self) -> pd.DataFrame: """Return the source rows the chunk planner consumes.""" + from dascore.io.index.plan import _ensure_patch_id + source = self._source_df working = source.drop(columns=list(self._drop_columns), errors="ignore") if "_patch_id" in source.columns: working = working.assign(_patch_id=source["_patch_id"]) - else: - working = working.assign(_patch_id=np.arange(len(source))) - return working + return _ensure_patch_id(working) def chunk_plan( self, @@ -983,25 +990,9 @@ def _resolve_select_kwargs(self, _attrs, _coords, kwargs) -> dict: def _relative_select_kwargs(self, kwargs: dict) -> dict: """Resolve relative bounds against the spool's global envelopes.""" - df = self._df - out = {} - for name, value in kwargs.items(): - lo_col, hi_col = f"{name}_min", f"{name}_max" - if lo_col not in df.columns: - msg = f"Cannot use relative select on {name!r}." - raise InvalidSpoolQueryError(msg) - if not (isinstance(value, tuple) and len(value) == 2): - msg = f"relative=True requires (start, stop) ranges, got {value!r}." - raise InvalidSpoolQueryError(msg) - gmin, gmax = df[lo_col].min(), df[hi_col].max() - lo, hi = value - from dascore.io.index.query import relative_offset + from dascore.io.index.query import relative_ranges_to_absolute - out[name] = ( - relative_offset(gmin, gmax, lo), - relative_offset(gmin, gmax, hi), - ) - return out + return relative_ranges_to_absolute(self._df, kwargs) @compose_docstring(doc=BaseSpool.select.__doc__) def select( @@ -1200,6 +1191,17 @@ def _get_catalog(self): self._catalog_native = True return self._catalog + def _as_catalog_member(self): + """ + Return (catalog, patch_ids) describing this spool for a union. + + Patch-list spools contribute their own (lazily created) catalog, + reusing any summaries the patches already computed. + """ + if self._catalog is None and self._patches is not None: + self._get_catalog() + return super()._as_catalog_member() + def _get_source_df(self): """Build the source df (happens as part of building current df).""" _ = self._df diff --git a/dascore/io/core.py b/dascore/io/core.py index 777f54d77..a1f23c155 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -256,6 +256,22 @@ def _patch_to_scan_payload(patch: dc.Patch) -> ScanPayload: ) +def _resolve_read_spool(spool, source_patch_id: object = "") -> dc.Patch: + """ + Resolve one patch from a read result by source identity. + + Readers that consume source_patch_id may return the single matching + patch without preserving that reload metadata on it; only trust that + when the patch doesn't claim a different identity. + """ + source_patch_id = str(source_patch_id or "") + if source_patch_id and len(spool) == 1: + found = str(spool[0].attrs.get("_source_patch_id", "") or "") + if found in ("", source_patch_id): + return spool[0] + return _select_patch_from_spool(spool, source_patch_id=source_patch_id) + + def _select_patch_from_spool(spool, source_patch_id: object = "") -> dc.Patch: """Select one loaded patch from a spool using source identity.""" if len(spool) == 0: diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 6f8f85499..b037484e6 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -31,8 +31,9 @@ from dascore.io.index.query import ( InvalidSpoolQueryError, Query, - relative_offset, + relative_ranges_to_absolute, ) +from dascore.utils.misc import is_memory_uri from dascore.utils.pd import adjust_segments @@ -59,6 +60,10 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: re-applied above, so ignoring them is slower, never wrong. """ + def live_entries(self) -> Mapping[str, dc.Patch]: + """Return the live patches this resolver serves (path -> patch).""" + return {} + class LiveResolver(PatchResolver): """ @@ -75,9 +80,9 @@ def __init__(self, patches: Sequence[dc.Patch] = ()): _patch_path(patch): patch for patch in patches } - def register(self, path: str, patch: dc.Patch) -> None: - """Register a live patch under its synthetic source identity.""" - self._registry[path] = patch + def live_entries(self) -> Mapping[str, dc.Patch]: + """Return the live patch registry.""" + return self._registry def resolve(self, row: Mapping, **trim) -> dc.Patch: """Look the patch up; live patches ignore trim hints.""" @@ -127,7 +132,7 @@ def _read(self, path, row: Mapping, trim: dict, source_patch_id: str): def resolve(self, row: Mapping, **trim) -> dc.Patch: """Read the patch, passing range trims down as read hints.""" - from dascore.io.core import _select_patch_from_spool + from dascore.io.core import _resolve_read_spool path = row["path"] # relative paths resolve against the catalog root; URIs and @@ -142,14 +147,7 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: # one, so these rows read the whole source. trim = {} spool = self._read(path, row, trim, source_patch_id) - # Readers that consume source_patch_id return the one requested - # patch, sometimes without preserving reload metadata on it. Only - # trust that when the patch doesn't claim a different identity. - if source_patch_id and len(spool) == 1: - found = str(spool[0].attrs.get("_source_patch_id", "") or "") - if found in ("", source_patch_id): - return spool[0] - return _select_patch_from_spool(spool, source_patch_id=source_patch_id) + return _resolve_read_spool(spool, source_patch_id) class CompositeResolver(PatchResolver): @@ -164,16 +162,17 @@ def __init__(self): self.live = LiveResolver() self.file = FileResolver(root=None) + def live_entries(self) -> Mapping[str, dc.Patch]: + """Return the merged live patch registry.""" + return self.live._registry + def absorb(self, resolver: PatchResolver) -> None: """Take over the live registry entries of another resolver.""" - if isinstance(resolver, LiveResolver): - self.live._registry.update(resolver._registry) - elif isinstance(resolver, CompositeResolver): - self.live._registry.update(resolver.live._registry) + self.live._registry.update(resolver.live_entries()) def resolve(self, row: Mapping, **trim) -> dc.Patch: """Dispatch to the live registry or the file reader.""" - if str(row.get("path", "")).startswith("memory"): + if is_memory_uri(row.get("path", "")): return self.live.resolve(row, **trim) return self.file.resolve(row, **trim) @@ -427,23 +426,7 @@ def select( def _relative_to_absolute(self, kwargs: dict) -> dict: """Resolve relative bounds against the view's global envelopes.""" - df = self.to_df() - out = {} - for name, value in kwargs.items(): - lo_col, hi_col = f"{name}_min", f"{name}_max" - if lo_col not in df.columns or df.empty: - msg = f"Cannot use relative select on unknown coord {name!r}." - raise InvalidSpoolQueryError(msg) - gmin, gmax = df[lo_col].min(), df[hi_col].max() - if not (isinstance(value, tuple) and len(value) == 2): - msg = f"relative=True requires (start, stop) ranges, got {value!r}." - raise InvalidSpoolQueryError(msg) - lo, hi = value - out[name] = ( - relative_offset(gmin, gmax, lo), - relative_offset(gmin, gmax, hi), - ) - return out + return relative_ranges_to_absolute(self.to_df(), kwargs) # --- realization ------------------------------------------------------ diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 497c6b8f7..b4136cbd5 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -406,8 +406,12 @@ def records_from_backend(backend, patch_ids=None) -> list[SourceRecord]: patches = backend._fetch_df("SELECT * FROM patches") if patch_ids is not None: patches = patches[patches["patch_id"].isin(set(patch_ids))] + kept_ids = set(int(x) for x in patches["patch_id"]) attrs = backend._fetch_df("SELECT * FROM attrs") links = backend._fetch_df("SELECT * FROM patch_coords") + if patch_ids is not None: # narrow dependent tables to kept patches. + attrs = attrs[attrs["patch_id"].isin(kept_ids)] + links = links[links["patch_id"].isin(kept_ids)] defs = backend._fetch_df("SELECT * FROM coord_defs") meta = backend._attr_meta() col_info = { @@ -416,17 +420,22 @@ def records_from_backend(backend, patch_ids=None) -> list[SourceRecord]: } def_map = {int(row.coord_def_id): row for row in defs.itertuples()} attr_rows = ( - {int(k): v for k, v in attrs.set_index("patch_id").iterrows()} + {int(k): v for k, v in attrs.set_index("patch_id").to_dict("index").items()} if not attrs.empty else {} ) link_groups = ( {int(k): v for k, v in links.groupby("patch_id")} if not links.empty else {} ) + patches_by_source = ( + {int(k): v for k, v in patches.groupby("source_id")} + if not patches.empty + else {} + ) out = [] for src in sources.itertuples(): - sub = patches[patches["source_id"] == src.source_id] - if sub.empty: + sub = patches_by_source.get(int(src.source_id)) + if sub is None: continue patch_records = [] for patch in sub.itertuples(): diff --git a/dascore/io/index/plan.py b/dascore/io/index/plan.py index 0a44853e2..65d972ebb 100644 --- a/dascore/io/index/plan.py +++ b/dascore/io/index/plan.py @@ -95,6 +95,13 @@ def _resolve_group_attrs(group, columns) -> tuple[str, ...]: return tuple(x for x in dc.get_config().groupby_attrs if x in columns) +def _ensure_patch_id(df: pd.DataFrame) -> pd.DataFrame: + """Attach the positional identity fallback for plain dataframes.""" + if "_patch_id" in df.columns: + return df + return df.assign(_patch_id=np.arange(len(df))) + + def _dim_def_key_columns(df: pd.DataFrame, name: str) -> list[str]: """Return def-key column names for every non-chunked dimension.""" dim_names: set[str] = set() @@ -131,14 +138,19 @@ def _continuity_group(start, stop, step, tolerance) -> pd.Series: return group[start.index] -def _partition(df, name, group_attrs, tolerance, sampling_tolerance) -> pd.Series: +def _partition( + df, name, group_attrs, tolerance, sampling_tolerance +) -> tuple[pd.Series, bool]: """ - Return partition labels: rows sharing a label may combine (spec 2). + Return (partition labels, forced_merge): rows sharing a label may + combine (spec 2). Components: group attrs, dims signature, structural def keys of non-chunked coords, sampling group, and continuity group. Continuity is evaluated *within* each other-component cell so unrelated patches - can never bridge a gap. + can never bridge a gap. `forced_merge` is True when a loosened + tolerance merged patches the default would have kept apart (#662); + the caller owns warning about it. """ start, stop, step = get_interval_columns(df, name) cols = [x for x in group_attrs if x in df.columns] @@ -162,20 +174,10 @@ def _partition(df, name, group_attrs, tolerance, sampling_tolerance) -> pd.Serie s, e, st = get_interval_columns(sub, name) labels = _continuity_group(s, e, st, tolerance).astype(np.int64) cont.loc[index] = labels - # See #662: warn when a loosened tolerance forces merges the - # default would not have produced. if tolerance > _DEFAULT_TOLERANCE and not forced_merge: default = _continuity_group(s, e, st, _DEFAULT_TOLERANCE) forced_merge = default.nunique() > labels.nunique() - if forced_merge: - msg = ( - f"There is a gap in the patch along dimension {name} but a " - f"merge tolerance of {tolerance} was used to force merging " - "the patches. As a result, some patches in the chunked spool " - "may be unevenly sampled, or have their sampling rate increased." - ) - warnings.warn(msg, UserWarning, stacklevel=_user_stacklevel()) - return cell + "_" + cont.astype(str) + return cell + "_" + cont.astype(str), forced_merge def _user_stacklevel() -> int: @@ -281,11 +283,19 @@ def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFra src2 = sub[max_name].values chu1 = outputs[min_name].values chu2 = outputs[max_name].values + out_ids = outputs["output_id"].to_numpy() + patch_ids = sub["_patch_id"].to_numpy() + orig_min = original[min_name].to_numpy() + orig_max = original[max_name].to_numpy() + modified_src = ( + sub["_modified"].to_numpy() + if "_modified" in sub + else np.zeros(len(sub), dtype=bool) + ) # Map each output onto the source rows it draws from. starts_ind = np.searchsorted(src1, chu1, side="right") - 1 ends_ind = np.searchsorted(src2, chu2, side="left") rows = [] - modified_src = sub["_modified"].values if "_modified" in sub else None for out_num, (a, b) in enumerate(zip(starts_ind, ends_ind)): a = max(int(a), 0) for src_num in range(a, int(b) + 1): @@ -295,16 +305,15 @@ def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFra hi = min(src2[src_num], chu2[out_num]) if lo > hi: continue - row_mod = bool(modified_src[src_num]) if modified_src is not None else False unchanged = ( - lo == original[min_name].iloc[src_num] - and hi == original[max_name].iloc[src_num] - and not row_mod + lo == orig_min[src_num] + and hi == orig_max[src_num] + and not modified_src[src_num] ) rows.append( { - "output_id": outputs["output_id"].iloc[out_num], - "_patch_id": sub["_patch_id"].iloc[src_num], + "output_id": out_ids[out_num], + "_patch_id": patch_ids[src_num], min_name: lo, max_name: hi, step_name: steps[src_num], @@ -377,9 +386,7 @@ def build_chunk_plan( if df.empty: outputs = pd.DataFrame(columns=[min_name, max_name, "output_id"]) return ChunkPlan(outputs, empty_members, name, value, params) - if "_patch_id" not in df.columns: - # Positional identity fallback for plain dataframes. - df = df.assign(_patch_id=np.arange(len(df))) + df = _ensure_patch_id(df) # Missing chunk-dim envelopes (spec 7 / D2). null_rows = pd.isnull(df[min_name]) | pd.isnull(df[max_name]) if null_rows.any(): @@ -396,19 +403,26 @@ def build_chunk_plan( outputs = pd.DataFrame(columns=[min_name, max_name, "output_id"]) return ChunkPlan(outputs, empty_members, name, value, params) - labels = _partition( + labels, forced_merge = _partition( df, name, params["group"], tolerance, params["sampling_group_tolerance"] ) + if forced_merge: + msg = ( + f"There is a gap in the patch along dimension {name} but a " + f"merge tolerance of {tolerance} was used to force merging " + "the patches. As a result, some patches in the chunked spool " + "may be unevenly sampled, or have their sampling rate increased." + ) + warnings.warn(msg, UserWarning, stacklevel=_user_stacklevel()) value_c, overlap_c = _coerce_length_overlap(value, overlap, df[min_name].dtype) out_frames, member_frames = [], [] next_id = 0 # Deterministic partition order (spec 8): by (partition min, smallest # member patch id) — never by anything derived from input row order. - stats = df.groupby(labels, sort=False).agg( - _min=(min_name, "min"), _pid=("_patch_id", "min") - ) + grouped = df.groupby(labels, sort=False) + stats = grouped.agg(_min=(min_name, "min"), _pid=("_patch_id", "min")) part_order = stats.sort_values(["_min", "_pid"], kind="stable").index - groups = df.groupby(labels, sort=False).groups + groups = grouped.groups for label in part_order: sub = df.loc[groups[label]] start, stop, step = get_interval_columns(sub, name) diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 67caa51d9..74044d257 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -450,3 +450,28 @@ def relative_offset(gmin, gmax, value): def glob_match(value, pattern: str) -> bool: """Reference glob semantics (used by pandas fallbacks and tests).""" return isinstance(value, str) and fnmatch.fnmatch(value, pattern) + + +def relative_ranges_to_absolute(df, kwargs: dict) -> dict: + """ + Resolve relative (start, stop) ranges against a frame's global envelopes. + + Shared by the dataframe and catalog select paths so the relative-select + contract has exactly one implementation. + """ + out = {} + for name, value in kwargs.items(): + lo_col, hi_col = f"{name}_min", f"{name}_max" + if lo_col not in df.columns or df.empty: + msg = f"Cannot use relative select on {name!r}." + raise InvalidSpoolQueryError(msg) + if not (isinstance(value, tuple) and len(value) == 2): + msg = f"relative=True requires (start, stop) ranges, got {value!r}." + raise InvalidSpoolQueryError(msg) + gmin, gmax = df[lo_col].min(), df[hi_col].max() + lo, hi = value + out[name] = ( + relative_offset(gmin, gmax, lo), + relative_offset(gmin, gmax, hi), + ) + return out diff --git a/dascore/utils/coordmanager.py b/dascore/utils/coordmanager.py index 4e6ef04e5..dc3a1d419 100644 --- a/dascore/utils/coordmanager.py +++ b/dascore/utils/coordmanager.py @@ -19,6 +19,7 @@ def merge_coord_managers( dim: str, snap_tolerance: float | None = None, drop_conflicting: bool = False, + dim_coord=None, ) -> dc.CoordManager: """ Merge coordinate managers along a specified dimension. @@ -38,6 +39,11 @@ def merge_coord_managers( drop_conflicting If True, drop conflicting (non-dimensional) coordinates, otherwise raise an exception if they occur. + dim_coord + If provided, use this coordinate for `dim` instead of + concatenating the members' values (which materializes them); + callers which already built the merged dimension coordinate + (e.g. via `concat_coords`) pass it here to avoid that cost. """ def _get_dims(managers): @@ -112,6 +118,9 @@ def _get_merged_coords(managers, coords_to_merge): """Get the merged coordinates.""" out = {} for coord_name in coords_to_merge: + if dim_coord is not None and coord_name == dim: + out[coord_name] = (managers[0].dim_map[dim], dim_coord) + continue merge_coords = [x.coord_map[dim] for x in managers] axis = managers[0].dim_map[coord_name].index(dim) if len(units := {x.units for x in merge_coords}) != 1: diff --git a/dascore/utils/misc.py b/dascore/utils/misc.py index dd161159c..bac28c4c9 100644 --- a/dascore/utils/misc.py +++ b/dascore/utils/misc.py @@ -1123,3 +1123,14 @@ def tukey_fence(data, fence_multiplier=1.5) -> np.ndarray: q_upper = np.nanmin([q3 + diff * fence_multiplier, dmax]) lower_and_top = np.asarray([q_lower, q_upper]) return lower_and_top + + +def is_memory_uri(path) -> bool: + """ + Return True if a path is a synthetic in-memory patch identity. + + Live patches are identified by memory:// or memorypatch:// paths + (see `dascore.io.index.catalog`); such paths dispatch to in-memory + registries and are never treated as file names. + """ + return str(path).startswith("memory") diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index 27d3d27d4..6b830b4ab 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -42,6 +42,7 @@ _apply_union_indexers, _merge_tuples, get_middle_value, + is_memory_uri, iterate, to_object_array, warn_or_raise, @@ -452,18 +453,21 @@ def _get_merged_coord( """ from dascore.core.coords import concat_coords - new_cm = merge_coord_managers( - coords, dim=merge_dim, drop_conflicting=drop_conflicting - ) try: merged = concat_coords(*[cm.coord_map[merge_dim] for cm in coords]) except CoordError: # Non-monotonic (or otherwise unsegmentable) member coordinates: - # keep the raw value concatenation. - return new_cm + # fall back to raw value concatenation of the dim coord. + return merge_coord_managers( + coords, dim=merge_dim, drop_conflicting=drop_conflicting + ) if snap_coords and (step := _middle_step(df, merge_dim)) is not None: merged = merged.simplify(tolerance * np.abs(step)) - return new_cm.update(**{merge_dim: merged}) + # Passing the pre-built dim coord avoids materializing the members' + # concatenated values only to discard them. + return merge_coord_managers( + coords, dim=merge_dim, drop_conflicting=drop_conflicting, dim_coord=merged + ) def _force_patch_merge(patch_dict_list, merge_kwargs, **kwargs): @@ -616,8 +620,7 @@ def _get_filename(path_ser, strip_extension): path_ser = df["path"].astype(str) if "path" in col_set else None if path_ser is not None: # synthetic in-memory identities are not real file names - # (memory:// and memorypatch:// schemes) - usable = path_ser.str.len().gt(0) & ~path_ser.str.startswith("memory") + usable = path_ser.str.len().gt(0) & ~path_ser.map(is_memory_uri) if usable.any(): return _get_filename(df["path"], strip_extension) # Determine the requested fields; absent columns render as empty so From 456500e7c40485d0c0b841a0f1419d5bd738dd4a Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 14:36:05 +0200 Subject: [PATCH 28/97] Remove dead PyTables-era config fields and stale references The hdf_index_* config fields configured the removed PyTables index writer and had no remaining consumers. Also update a comment still justifying int indexes by pytables support and a test docstring still naming ChunkManager as the oracle. --- dascore/config.py | 14 -------------- dascore/core/coords.py | 4 ++-- tests/test_io/test_index/test_plan.py | 2 +- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/dascore/config.py b/dascore/config.py index 47fc2c1b2..84e7f1f86 100644 --- a/dascore/config.py +++ b/dascore/config.py @@ -94,20 +94,6 @@ class DascoreConfig(BaseModel): description="Time buffer applied when querying cached directory indexes.", ) - # HDF index writing. - hdf_index_complib: str = Field( - default="blosc:lz4", - description="Compression library used when writing DASCore HDF index files.", - ) - hdf_index_complevel: int = Field( - default=5, - description="Compression level used when writing DASCore HDF index files.", - ) - hdf_index_max_retries: int = Field( - default=3, - description="Maximum number of retries for concurrent HDF index access.", - ) - # Progress display. progress_basic_refresh_per_second: float = Field( default=0.25, diff --git a/dascore/core/coords.py b/dascore/core/coords.py index 5c015e1cd..a4bdcc705 100644 --- a/dascore/core/coords.py +++ b/dascore/core/coords.py @@ -1600,8 +1600,8 @@ def select( return self.empty(), out if np.all(out): return self, slice(None, None) - # Convert boolean to int indexes because these are supported for - # indexing pytables arrays but booleans are not. + # Convert boolean to int indexes; some consumers (eg lazy file + # readers) index with these where booleans are not supported. if len(self.shape) == 1: out = np.arange(len(out))[out] return self.new(values=values[out]), out diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index 74a25bc9b..27ffe447e 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -305,7 +305,7 @@ def test_input_order_invariant(self, diverse_flat): class TestOracleParity: - """Sanity against ChunkManager (the dev-time oracle) where compatible.""" + """Sanity: plans describe exactly what spool.chunk produces.""" def test_merge_envelopes_match(self, random_flat): """Merge-mode output envelopes match spool.chunk(time=None).""" From f2f4f575868f9a3cea66ab755751f2af1f79f167 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 16:12:17 +0200 Subject: [PATCH 29/97] Make pre-commit --all-files clean Removes an unused MissingPatchError import, a stale tables import in the dasdae tests (PyTables is no longer a dependency, so collection in a clean environment would fail), and two formatter fixes. --- dascore/clients/filespool.py | 1 - dascore/utils/hdf5.py | 1 + tests/test_core/test_spool.py | 4 +--- tests/test_io/test_dasdae/test_dasdae.py | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dascore/clients/filespool.py b/dascore/clients/filespool.py index 6bfbe9322..f388f895f 100644 --- a/dascore/clients/filespool.py +++ b/dascore/clients/filespool.py @@ -12,7 +12,6 @@ from dascore.compat import UPath from dascore.constants import PROGRESS_LEVELS, SpoolType from dascore.core.spool import BaseSpool, DataFrameSpool -from dascore.exceptions import MissingPatchError from dascore.io.core import FiberIO from dascore.utils.docs import compose_docstring diff --git a/dascore/utils/hdf5.py b/dascore/utils/hdf5.py index f275b9d64..bf049131e 100644 --- a/dascore/utils/hdf5.py +++ b/dascore/utils/hdf5.py @@ -34,6 +34,7 @@ ns_to_datetime = partial(pd.to_datetime, unit="ns") ns_to_timedelta = partial(pd.to_timedelta, unit="ns") + class _ManagedH5pyFile: """ DASCore's internal h5py handle wrapper with deterministic close behavior. diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index e75a0c8fc..260de7004 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -150,9 +150,7 @@ def test_derived_spools_use_df_machinery(self, patch_list): merged = spool.chunk(time=None) assert len(merged) == 1 time_coord = merged[0].get_coord("time") - expected_min = min( - x.summary.get_coord_summary("time").min for x in patch_list - ) + expected_min = min(x.summary.get_coord_summary("time").min for x in patch_list) assert time_coord.min() == expected_min def test_derived_spool_does_not_retain_parent(self, patch_list): diff --git a/tests/test_io/test_dasdae/test_dasdae.py b/tests/test_io/test_dasdae/test_dasdae.py index 1d3494b38..847501536 100644 --- a/tests/test_io/test_dasdae/test_dasdae.py +++ b/tests/test_io/test_dasdae/test_dasdae.py @@ -11,7 +11,6 @@ import numpy as np import pandas as pd import pytest -import tables import dascore as dc from dascore.compat import random_state From fa95782979f3c8cbfea76c747afca5cb683f83c6 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 16:13:53 +0200 Subject: [PATCH 30/97] Resolve envelope-column ownership by full name, not first underscore Chunking a dimension like event_time treated event_time_min/max as conflicting attrs because column ownership split on the first underscore. Ownership now matches the full coordinate name plus an interval suffix against the partition's dims. --- dascore/io/index/plan.py | 23 ++++++++++++++++++++--- tests/test_io/test_index/test_plan.py | 26 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/dascore/io/index/plan.py b/dascore/io/index/plan.py index 65d972ebb..a6a911934 100644 --- a/dascore/io/index/plan.py +++ b/dascore/io/index/plan.py @@ -208,6 +208,22 @@ def _coerce_length_overlap(value, overlap, start_dtype): return value, overlap +def _coord_owner(col: str, coord_names: set[str]) -> str | None: + """ + Return the coordinate owning an envelope column, if any. + + Ownership is decided by matching the full name against known + coordinates with an interval suffix; splitting on the first + underscore would mis-assign columns of dims like `event_time`. + """ + for suffix in ("_min", "_max", "_step", "_units"): + if col.endswith(suffix): + base = col[: -len(suffix)] + if base in coord_names: + return base + return None + + def _police_columns(sub: pd.DataFrame, name, group_attrs, conflict) -> dict: """ Return the carried column values for one partition (spec 2.5/6.4). @@ -216,12 +232,13 @@ def _police_columns(sub: pd.DataFrame, name, group_attrs, conflict) -> dict: Remaining public attrs must be single-valued, policed by `conflict`. """ dims = set(str(sub.iloc[0].get("dims", "")).split(",")) + coord_names = dims | {name} carried: dict[str, Any] = {} for col in sub.columns: if col.startswith("_") or col in _SOURCE_COLUMNS: continue - prefix = col.split("_")[0] - if prefix == name: # chunk-dim envelope columns are rebuilt + owner = _coord_owner(col, coord_names) + if owner == name: # chunk-dim envelope columns are rebuilt continue values = sub[col].unique() single = len(values) == 1 or (len(values) and pd.isnull(values).all()) @@ -232,7 +249,7 @@ def _police_columns(sub: pd.DataFrame, name, group_attrs, conflict) -> dict: if in_group: # partitioning guarantees this; guard anyway carried[col] = values[0] continue - if prefix in dims or conflict == "raise": + if owner is not None or conflict == "raise": msg = ( f"Cannot merge on dim {name} because all values for " f"{col} are not equal. Consider using the `conflict` " diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index 27ffe447e..721129fb8 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -364,3 +364,29 @@ def test_directory_spool(self, tmp_path): spool = dc.spool(tmp_path).update() plan = spool.chunk_plan(time=None) assert len(plan.outputs) == 1 + + +class TestUnderscoreDimNames: + """Dims with underscores must not confuse column ownership.""" + + def test_chunk_event_time(self): + """Adjacent patches along an underscore dim merge cleanly.""" + patch = dc.get_example_patch().rename_coords(time="event_time") + coord = patch.get_coord("event_time") + middle = coord.values[len(coord) // 2] + p1 = patch.select(event_time=(None, middle)) + p2 = patch.select(event_time=(middle + coord.step, None)) + merged = dc.spool([p1, p2]).chunk(event_time=None) + assert len(merged) == 1 + out = merged[0].get_coord("event_time") + assert out.min() == coord.min() + assert out.max() == coord.max() + + def test_segment_event_time(self): + """Segmenting along an underscore dim works too.""" + patch = dc.get_example_patch().rename_coords(time="event_time") + spool = dc.spool([patch]) + chunked = spool.chunk(event_time=2) + assert len(chunked) > 1 + for sub in chunked: + assert "event_time" in sub.dims From 347fe1a880b4c5540ab16f3dd225fea3341886b6 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 16:15:31 +0200 Subject: [PATCH 31/97] Keep patch identity deterministic and the live store in step with removal Mint _instance_id eagerly at Patch construction: lazy minting made lineage depend on whether the id happened to be accessed before a copy, so a deepcopy taken first would not share identity (review finding; the regression test previously pre-minted, hiding this). PatchCatalog.remove now also drops the removed sources from the live registry; since pickling rebuilds the backend from the registry, a stale entry would resurrect removed patches. --- dascore/core/patch.py | 15 +++------------ dascore/io/index/catalog.py | 8 +++++++- tests/test_io/test_index/test_union.py | 23 +++++++++++++++++++++-- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/dascore/core/patch.py b/dascore/core/patch.py index 26953c58f..1a02dff4f 100644 --- a/dascore/core/patch.py +++ b/dascore/core/patch.py @@ -100,6 +100,9 @@ def __init__( self._coords = coords self._attrs = attrs self._data = array(self.coords.validate_data(data)) + # Lineage identity: minted eagerly so copies made at any point + # (deepcopy/pickle carry __dict__) share it deterministically. + self._instance_id = uuid4().hex def __eq__(self, other): """Compare one Patch.""" @@ -263,18 +266,6 @@ def summary(self): """ return PatchSummary.from_patch(self) - @cached_property - def _instance_id(self) -> str: - """ - A stable identity for this patch instance. - - Minted lazily; once computed it rides along with copies and - pickles (patches are immutable, so identical copies sharing an - identity is correct). Patch operations produce new instances - with new identities. Never stored in attrs, so it cannot affect - equality or survive into saved files. - """ - return uuid4().hex @property def coords(self) -> CoordManager: diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index b037484e6..9a9cf9578 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -528,7 +528,13 @@ def update(self, progress: PROGRESS_LEVELS = "standard") -> PatchCatalog: def remove(self, source_paths: Sequence[str], base_uri: str = "") -> PatchCatalog: """Remove sources (and their patches) from the catalog.""" self._require_root("remove") - self.backend.delete_sources(list(source_paths), base_uri=base_uri) + source_paths = list(source_paths) + self.backend.delete_sources(source_paths, base_uri=base_uri) + # The live registry is the store for in-memory patches; it must + # stay in step with the backend rows (pickling rebuilds from it). + registry = self.resolver.live_entries() if self.resolver else {} + for path in source_paths: + registry.pop(path, None) self._invalidate() return self diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index c6b664088..9f011240b 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -148,11 +148,14 @@ def test_duplicate_instances_collapse(self): assert len(dc.spool([patch, patch])) == 1 def test_deepcopy_shares_identity(self): - """Copies of an immutable patch share its identity.""" + """Copies of an immutable patch share its identity. + + Identity is minted eagerly at construction, so copies share it + regardless of when they are made (no access-order dependence). + """ import copy patch = dc.get_example_patch() - _ = patch._instance_id # mint before copying clone = copy.deepcopy(patch) assert clone._instance_id == patch._instance_id assert len(dc.spool([patch, clone])) == 1 @@ -188,3 +191,19 @@ def test_union_pickles(self): combined = dc.spool([p1]) + dc.spool([p2]) loaded = pickle.loads(pickle.dumps(combined)) assert len(loaded) == 2 + + def test_remove_updates_live_registry(self): + """Removing a live source removes it from the store as well.""" + import pickle + + patch = dc.get_example_patch() + catalog = PatchCatalog.from_patches([patch]) + path = catalog.to_df().iloc[0]["path"] + catalog.remove([path]) + assert len(catalog.to_df()) == 0 + # A pickled catalog rebuilds from the registry; the removed patch + # must not resurrect. + loaded = pickle.loads(pickle.dumps(catalog)) + loaded.attr_names() # bootstrap the backend + loaded._invalidate() + assert len(loaded.to_df()) == 0 From 84e43535ffeb9ff471de24192fe637570677b2a2 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 16:17:31 +0200 Subject: [PATCH 32/97] Close SQLite connections on GC; bypass legacy index-map entries SQLiteBackend ties connection cleanup to its own collection via weakref.finalize (catalog views share the backend object, so this is the correct shared-owner lifetime); explicit close() detaches the finalizer. Realized memory spools no longer emit ResourceWarning on collection. Index-map entries recorded by older DASCore versions can point at the retired PyTables .h5 index; those are now detected (suffix or missing SQLite header) and bypassed so a fresh SQLite index is built instead of failing with 'file is not a database'. --- dascore/io/index/indexer.py | 24 +++++++- dascore/io/index/lite.py | 7 +++ .../test_index/test_index_edge_cases.py | 57 +++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 932eeaaf7..fb9dba1cd 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -73,6 +73,23 @@ def __init__( def _index_name(self) -> str: return ".dascore_index.sqlite3" + @staticmethod + def _is_legacy_or_foreign_index(path: Path) -> bool: + """Return True if an existing file is not a SQLite database. + + Older DASCore versions recorded PyTables (.h5) index locations in + the index map; passing those to sqlite3 fails with an opaque + error instead of building the replacement index. + """ + if not path.exists(): + return False + if path.suffix.lower() in (".h5", ".hdf5"): + return True + with suppress(OSError), open(path, "rb") as fh: + header = fh.read(16) + return len(header) >= 16 and not header.startswith(b"SQLite format 3") + return False + def _find_index_path(self, index_path=None) -> Path: """ Find where the index lives (or should live). @@ -92,7 +109,12 @@ def _find_index_path(self, index_path=None) -> Path: return expected path_map = _get_index_map(cache_path=str(self.index_map_path)) if out := path_map.get(map_key): - return Path(out) + mapped = Path(out) + # Index-map entries from older DASCore versions can point at + # the retired PyTables (.h5) index; those are not usable and + # a fresh SQLite index is built in their place. + if not self._is_legacy_or_foreign_index(mapped): + return mapped if not _directory_writable(self.path): name = f"_dascore_index_{abs(hash(self.path))}.sqlite3" index_path = self.index_map_path.parent / name diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py index 3bedae0df..df5630364 100644 --- a/dascore/io/index/lite.py +++ b/dascore/io/index/lite.py @@ -3,6 +3,7 @@ from __future__ import annotations import sqlite3 +import weakref from pathlib import Path import numpy as np @@ -53,6 +54,10 @@ def __init__(self, path: str | Path): self._con.isolation_level = None self._con.execute("PRAGMA foreign_keys = ON") self._con.execute("PRAGMA busy_timeout = 30000") + # Catalog views share this backend object, so tying connection + # cleanup to *its* collection is safe (close() stays idempotent + # for explicit use). + self._finalizer = weakref.finalize(self, self._con.close) try: super().__init__() except Exception: @@ -97,4 +102,6 @@ def _table_columns(self, table: str) -> set[str]: def close(self) -> None: """Close the database connection.""" + # detach the GC finalizer; closing twice is harmless but tidy. + self._finalizer.detach() self._con.close() diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 65a7a1290..910de1e60 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -662,3 +662,60 @@ def test_replacement_is_base_scoped(self, tmp_path): back.write_sources([rec_a]) # replace only the s3://a copy assert len(back.query()) == 2 back.close() + + +class TestResourceCleanup: + """Backends must not leak SQLite connections (review P2).""" + + def test_gc_closes_connection(self): + """Garbage collection closes the backend connection silently.""" + import gc + import warnings as warnings_mod + + import dascore as dc + + spool = dc.spool([dc.get_example_patch()]) + spool.get_contents() # realize the backend + with warnings_mod.catch_warnings(record=True) as caught: + warnings_mod.simplefilter("always", ResourceWarning) + del spool + gc.collect() + resource = [w for w in caught if issubclass(w.category, ResourceWarning)] + assert not resource + + def test_explicit_close_idempotent(self): + """Explicit close works and GC afterwards stays quiet.""" + from dascore.io.index.catalog import PatchCatalog + + import dascore as dc + + catalog = PatchCatalog.from_patches([dc.get_example_patch()]) + catalog.to_df() + catalog.close() + + +class TestLegacyIndexMap: + """Index-map entries pointing at retired .h5 indexes are bypassed.""" + + def test_legacy_entry_ignored(self, tmp_path): + """A mapped legacy HDF5 index does not break index creation.""" + import h5py + + import dascore as dc + from dascore.io.index.indexer import ( + DBDirectoryIndexer, + _update_index_map, + ) + + # data directory with one file, plus a fake legacy index mapping. + data_dir = tmp_path / "data" + data_dir.mkdir() + dc.write(dc.get_example_patch(), data_dir / "a.h5", "dasdae") + legacy = tmp_path / "legacy_index.h5" + with h5py.File(legacy, "w") as fh: + fh.create_dataset("x", data=[1, 2, 3]) + map_path = tmp_path / "cache_paths.json" + with dc.set_config(directory_index_map_path=map_path): + _update_index_map({str(data_dir): str(legacy)}, cache_path=str(map_path)) + spool = dc.spool(data_dir).update() + assert len(spool) == 1 From b5a0174a44793927ee3e70c5a9ada9c8dc069630 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 16:20:06 +0200 Subject: [PATCH 33/97] Define the reserved-attr contract for the flat relation Attrs whose names collide with structural storage or flat-relation columns (path, file_format, source_id, ...) previously created duplicate columns, overwrote structural values, or vanished silently. The reserved set now covers every structural column plus the spool instruction names; such attrs warn at ingest and stay on the patch without being indexed. Attrs shaped like the patch's own coordinate envelope columns are treated the same, and flattening defensively drops (rather than restores) any dynamic attr whose name already exists in the frame. --- dascore/io/index/backend.py | 12 +++++++ dascore/io/index/ingest.py | 16 +++++++-- dascore/io/index/schema.py | 34 +++++++++++++++++-- .../test_index/test_index_edge_cases.py | 28 +++++++++++++++ 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 62ea750b9..bce4e580e 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -584,6 +584,18 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: out[col] = pd.to_numeric(out[col]) # typed attr columns -> original names (coalesce multi-kind attrs) for name in attr_meta["attr_name"].unique(): + if name in out.columns: + # A dynamic attr restored onto an existing column (e.g. a + # coordinate envelope like time_min) would corrupt the + # frame; structural columns win. Reserved fixed names are + # already refused at ingest. + sanitized = attr_meta.loc[ + attr_meta["attr_name"] == name, "column_name" + ] + out = out.drop( + columns=[x for x in sanitized if x in out.columns] + ) + continue rows = attr_meta[attr_meta["attr_name"] == name] kinds = set(rows["value_kind"]) series = None diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index b4136cbd5..9de94f1ab 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -202,12 +202,24 @@ def typed_value(value) -> TypedValue | None: def _extract_attrs(summary: PatchSummary) -> dict[str, TypedValue]: """Get indexable typed attrs from a patch summary.""" raw = summary.attrs.model_dump() + # Attrs shaped like this patch's own coordinate envelope columns + # (e.g. "time_min") would collide in the flat relation. + envelope_names = { + f"{coord}_{suffix}" + for coord in getattr(summary, "coords", {}) + for suffix in ("min", "max", "step", "units") + } out = {} for name, value in raw.items(): if name in _SKIPPED_ATTRS or name.startswith("_"): continue - if sanitize_attr_name(name) in RESERVED_ATTR_COLUMNS: - warnings.warn(f"Skipping reserved attr name {name!r}.", UserWarning) + if sanitize_attr_name(name) in RESERVED_ATTR_COLUMNS or name in envelope_names: + msg = ( + f"Skipping reserved attr name {name!r}; it collides with a " + "structural index column. The attr stays on the patch but " + "is not queryable through the spool." + ) + warnings.warn(msg, UserWarning) continue typed = typed_value(value) if typed is not None: diff --git a/dascore/io/index/schema.py b/dascore/io/index/schema.py index 7bd2a404e..f3f943b4a 100644 --- a/dascore/io/index/schema.py +++ b/dascore/io/index/schema.py @@ -176,8 +176,38 @@ } ) -# Names which can never be dynamic attr columns. -RESERVED_ATTR_COLUMNS = frozenset({"patch_id"}) +# Attr names which would collide with structural storage or flat-relation +# columns. Attrs with these (sanitized) names stay on the patch but are +# not indexed; ingest warns about them. +RESERVED_ATTR_COLUMNS = frozenset( + { + # storage tables + "patch_id", + "source_id", + "source_patch_id", + "source_path", + "source_format", + "format_version", + "base_uri", + "mtime_ns", + "size_bytes", + "n_dims", + "dims", + "shape", + "sample_count_total", + "coord_def_id", + "def_key", + # flat-relation (spool-facing) names + "path", + "file_format", + "file_version", + # spool instruction machinery + "current_index", + "source_index", + "output_id", + "patch", + } +) # Explicit secondary indexes. Every other access path is covered by a # PRIMARY KEY or UNIQUE autoindex above — patch_coords(patch_id, diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 910de1e60..e61f0e824 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -719,3 +719,31 @@ def test_legacy_entry_ignored(self, tmp_path): _update_index_map({str(data_dir): str(legacy)}, cache_path=str(map_path)) spool = dc.spool(data_dir).update() assert len(spool) == 1 + + +class TestReservedAttrNames: + """Attrs colliding with structural columns are skipped with a warning.""" + + @pytest.mark.parametrize( + "name,value", + [("path", "user-path"), ("file_format", "attr-format"), ("source_id", 42)], + ) + def test_reserved_attr_warns_and_skips(self, name, value): + """Reserved names warn at ingest and never corrupt the relation.""" + import dascore as dc + + patch = dc.get_example_patch().update_attrs(**{name: value}) + with pytest.warns(UserWarning, match="reserved attr"): + df = dc.spool([patch]).get_contents() + assert not df.columns.duplicated().any() + # structural values win; the attr stays on the patch itself. + assert patch.attrs[name] == value + + def test_non_reserved_attr_round_trips(self): + """Ordinary arbitrary attrs still index and select normally.""" + import dascore as dc + + patch = dc.get_example_patch().update_attrs(experiment="exp42") + spool = dc.spool([patch]) + assert spool.get_contents()["experiment"].iloc[0] == "exp42" + assert len(spool.select(experiment="exp42")) == 1 From 0448b75d44d5689b5a300bbf36b0d78ba0c3aa37 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 16:22:36 +0200 Subject: [PATCH 34/97] Make directory spools picklable; process-backed map works again SQLiteBackend pickles by database path and reopens its own connection on unpickle (in-memory backends refuse with guidance; their owners serialize contents separately). Directory catalogs re-adopt their syncer's backend after unpickling so both keep sharing one connection. Regression tests cover pickle round trips, selected views, and Spool.map with a ProcessPoolExecutor. --- dascore/io/index/catalog.py | 17 +++++++++----- dascore/io/index/indexer.py | 6 ++--- dascore/io/index/lite.py | 24 ++++++++++++++++++- tests/test_clients/test_dirspool.py | 36 +++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 9a9cf9578..a39588ddf 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -315,12 +315,17 @@ def backend(self): where a brand-new directory index gets its one automatic update. """ if self._backend is None: - self._backend = get_backend(":memory:") - if self._rebuild_records: - self._backend.write_sources(list(self._rebuild_records)) - self._rebuild_records = () - elif registry := getattr(self.resolver, "_registry", None): - self._backend.write_sources(_live_records(registry)) + if self._syncer is not None: + # Directory catalogs re-adopt the (unpickled) syncer's + # backend; both must keep sharing one connection. + self._backend = self._syncer._backend + else: + self._backend = get_backend(":memory:") + if self._rebuild_records: + self._backend.write_sources(list(self._rebuild_records)) + self._rebuild_records = () + elif registry := getattr(self.resolver, "_registry", None): + self._backend.write_sources(_live_records(registry)) if self._syncer is not None and self._syncer.ensure_updated(): self._invalidate() return self._backend diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index fb9dba1cd..d986d92ea 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -79,12 +79,12 @@ def _is_legacy_or_foreign_index(path: Path) -> bool: Older DASCore versions recorded PyTables (.h5) index locations in the index map; passing those to sqlite3 fails with an opaque - error instead of building the replacement index. + error instead of building the replacement index. Only the file + header decides — users may legitimately choose any suffix for a + custom index path. """ if not path.exists(): return False - if path.suffix.lower() in (".h5", ".hdf5"): - return True with suppress(OSError), open(path, "rb") as fh: header = fh.read(16) return len(header) >= 16 and not header.startswith(b"SQLite format 3") diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py index df5630364..cba9f2dd0 100644 --- a/dascore/io/index/lite.py +++ b/dascore/io/index/lite.py @@ -49,7 +49,8 @@ class SQLiteBackend(SQLIndexBackend): dialect = SQLiteDialect() def __init__(self, path: str | Path): - self._con = sqlite3.connect(str(path)) + self._path = str(path) + self._con = sqlite3.connect(self._path) # autocommit off; we manage transactions explicitly. self._con.isolation_level = None self._con.execute("PRAGMA foreign_keys = ON") @@ -64,6 +65,27 @@ def __init__(self, path: str | Path): self._con.close() raise + def __getstate__(self) -> dict: + """ + Pickle by database path; the file is the durable state. + + This makes file-backed spools usable with process pools: the + receiving process reopens its own connection. In-memory backends + have no file to reopen; their owners (catalogs) serialize their + contents separately and never pickle the backend itself. + """ + if self._path == ":memory:": + msg = ( + "In-memory index backends cannot be pickled; pickle their " + "owning catalog/spool instead." + ) + raise TypeError(msg) + return {"_path": self._path} + + def __setstate__(self, state: dict) -> None: + """Reconnect to the database file.""" + self.__init__(state["_path"]) + def _execute(self, sql: str, params=()) -> None: self._con.execute(sql, _adapt(params)) diff --git a/tests/test_clients/test_dirspool.py b/tests/test_clients/test_dirspool.py index 83b6f4e20..6536721b2 100644 --- a/tests/test_clients/test_dirspool.py +++ b/tests/test_clients/test_dirspool.py @@ -758,3 +758,39 @@ def test_iteration_unexpected_index_error(self, basic_file_spool): with pytest.raises(IndexError, match="unexpected error from pandas"): for _ in basic_file_spool: pass + + +def _patch_shape(patch): + """Module-level helper (process pools need picklable functions).""" + return patch.shape + + +class TestDirectorySpoolSerialization: + """Directory spools must pickle (process-backed map depends on it).""" + + def test_pickle_round_trip(self, basic_file_spool): + """A directory spool pickles and reopens its own connection.""" + import pickle + + loaded = pickle.loads(pickle.dumps(basic_file_spool)) + assert len(loaded) == len(basic_file_spool) + assert loaded[0].shape == basic_file_spool[0].shape + + def test_pickle_selected_view(self, basic_file_spool): + """Selected views keep their selection through pickling.""" + import pickle + + df = basic_file_spool.get_contents() + sub = basic_file_spool.select(time=(df["time_min"].min(), None)) + loaded = pickle.loads(pickle.dumps(sub)) + assert len(loaded) == len(sub) + + def test_process_pool_map(self, basic_file_spool): + """Spool.map works with a process pool executor.""" + from concurrent.futures import ProcessPoolExecutor + + with ProcessPoolExecutor(max_workers=1) as client: + out = list( + basic_file_spool.map(_patch_shape, client=client, progress=False) + ) + assert len(out) == len(basic_file_spool) From aaf45fd882567ec62ae18311314f8687ac8829ef Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 16:36:45 +0200 Subject: [PATCH 35/97] Canonicalize unit-bearing coordinate selection end to end Numeric coordinate summaries are stored in canonical SI, but the raw selector leaked to both adjust_segments (quantity vs unitless frame: DimensionalityError) and the exact per-patch residual (SI-meaning bounds applied as native units, trimming the wrong physical interval on non-SI patches). Coordinate range selectors now resolve once: SI magnitudes drive the index and dataframe side, while the residual carries the same bounds as quantities in the canonical unit so Patch.select converts them to each patch's native units. Bare numbers mean canonical SI (the index contract); quantities in any compatible unit work; unitless and time-like coordinates are unchanged. Quantity bounds stay out of reader trim hints, where native-unit numbers are expected. Also: DataFrameSpool.select realizes contents first so fresh patch-list spools take the catalog path (the dataframe path bypassed canonicalization), and bare None/... selectors are no-ops on the catalog path, matching Patch.select (readers pass unset trims as None; names are still validated). Regression tests cover bare-number, quantity (SI and native unit), unitless, and directory-spool selection on feet-unit patches. --- dascore/core/patch.py | 1 - dascore/core/spool.py | 4 ++ dascore/io/index/backend.py | 14 ++-- dascore/io/index/catalog.py | 70 ++++++++++++++++++- tests/test_core/test_spool_select_spec.py | 60 ++++++++++++++++ .../test_index/test_index_edge_cases.py | 4 +- 6 files changed, 139 insertions(+), 14 deletions(-) diff --git a/dascore/core/patch.py b/dascore/core/patch.py index 1a02dff4f..27ae737bc 100644 --- a/dascore/core/patch.py +++ b/dascore/core/patch.py @@ -266,7 +266,6 @@ def summary(self): """ return PatchSummary.from_patch(self) - @property def coords(self) -> CoordManager: """ diff --git a/dascore/core/spool.py b/dascore/core/spool.py index c8e8b38fc..5b99ef697 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -1005,6 +1005,10 @@ def select( **kwargs, ) -> Self: """{doc}.""" + # Realize contents first: fresh patch-list spools only become + # catalog-native on realization, and the catalog path owns the + # full selector semantics (e.g. unit canonicalization). + _ = self._df if self._catalog_native: catalog = self._catalog.select( _attrs=_attrs, diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index bce4e580e..2cd5aef36 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -589,12 +589,8 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: # coordinate envelope like time_min) would corrupt the # frame; structural columns win. Reserved fixed names are # already refused at ingest. - sanitized = attr_meta.loc[ - attr_meta["attr_name"] == name, "column_name" - ] - out = out.drop( - columns=[x for x in sanitized if x in out.columns] - ) + sanitized = attr_meta.loc[attr_meta["attr_name"] == name, "column_name"] + out = out.drop(columns=[x for x in sanitized if x in out.columns]) continue rows = attr_meta[attr_meta["attr_name"] == name] kinds = set(rows["value_kind"]) @@ -793,6 +789,10 @@ def resolve_query( """ from dascore.io.index.query import InvalidSpoolQueryError + def _drop_noops(mapping: dict) -> dict: + """Bare None/... selectors are no-ops, matching Patch.select.""" + return {k: v for k, v in mapping.items() if v is not None and v is not Ellipsis} + # accept the same open/slice range forms patch-level select does attrs = {k: normalize_range_forms(v) for k, v in (_attrs or {}).items()} coords = {k: normalize_range_forms(v) for k, v in (_coords or {}).items()} @@ -819,7 +819,7 @@ def resolve_query( for name in coords: if name not in known_coords: raise InvalidSpoolQueryError(f"{name!r} is not a coordinate of this spool.") - return Query(attrs=attrs, coords=coords) + return Query(attrs=_drop_noops(attrs), coords=_drop_noops(coords)) def get_backend(path: str | Path) -> AbstractIndexBackend: diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index a39588ddf..dac8f77dc 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -37,6 +37,56 @@ from dascore.utils.pd import adjust_segments +def _canonical_coord_selectors(backend, coords: dict) -> tuple[dict, dict]: + """ + Split coordinate range selectors into canonical forms. + + Numeric coordinate summaries are stored in canonical SI units, so + range bounds resolve to SI magnitudes for the index/dataframe side. + The exact per-patch residual gets the same bounds as *quantities* + (in the canonical unit) so `Patch.select` converts them to each + patch's native coordinate units; re-applying raw numbers would trim + a different physical interval on non-SI patches. Bare numbers are + interpreted as canonical SI, matching the index contract. + + Coordinates without a single recorded unit (unitless, time-like, or + heterogeneous) pass through unchanged. + """ + from dascore.units import convert_units, get_quantity + + meta = backend._coord_meta(set(coords)) + units_by_name: dict[str, set] = {} + for row in meta.itertuples(): + units = getattr(row, "units", None) + if row.value_kind == "num" and units is not None and not pd.isnull(units): + units_by_name.setdefault(row.coord_name, set()).add(str(units)) + si_coords, residual_coords = {}, {} + for name, value in coords.items(): + units = units_by_name.get(name) + unit = next(iter(units)) if units and len(units) == 1 else None + if unit is None or not isinstance(value, tuple): + si_coords[name] = residual_coords[name] = value + continue + quant = get_quantity(unit) + si_bounds, residual_bounds = [], [] + for bound in value: + if bound is None or bound is Ellipsis: + si_bounds.append(None) + residual_bounds.append(None) + continue + if hasattr(bound, "units"): # pint quantity -> SI magnitude + magnitude = convert_units( + bound.magnitude, to_units=unit, from_units=bound.units + ) + else: # bare numbers are canonical SI + magnitude = float(bound) + si_bounds.append(magnitude) + residual_bounds.append(magnitude * quant) + si_coords[name] = tuple(si_bounds) + residual_coords[name] = tuple(residual_bounds) + return si_coords, residual_coords + + def _row_source_patch_id(row: Mapping) -> str: """Return the row's source_patch_id as a string ("" when missing). @@ -423,10 +473,16 @@ def select( attrs=query.attrs, coords=self._relative_to_absolute(query.coords), ) - # coord range predicates are re-applied exactly at patch load + # coord range predicates are re-applied exactly at patch load; + # the residual carries canonical quantities so per-patch native + # units are respected while the query side stays SI. residuals = self._residuals if query.coords: - residuals = (*residuals, (dict(query.coords), False, False)) + si_coords, residual_coords = _canonical_coord_selectors( + self.backend, query.coords + ) + query = Query(attrs=query.attrs, coords=si_coords) + residuals = (*residuals, (residual_coords, False, False)) return self._view((*self._queries, query), residuals) def _relative_to_absolute(self, kwargs: dict) -> dict: @@ -491,8 +547,16 @@ def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Pat trim_hint = {} for coords, samples, _ in self._residuals: if not samples: + # Quantity bounds stay out of reader hints: readers take + # numbers in their native units, so a converted-narrower + # hint could drop data exactness cannot restore. trim_hint.update( - {k: v for k, v in coords.items() if isinstance(v, tuple)} + { + k: v + for k, v in coords.items() + if isinstance(v, tuple) + and not any(hasattr(b, "units") for b in v) + } ) trim_hint.update(extra_trim or {}) patch = self.resolver.resolve(row, **trim_hint) diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index a2f75d94a..912e71852 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -154,3 +154,63 @@ def test_time_range_narrows(self, spool): t0 = df["time_min"].min() out = spool.select(time=(t0, t0 + np.timedelta64(2, "s"))) assert len(out) == 1 + + +class TestUnitCanonicalSelection: + """Coordinate selection on non-SI patches (review P1). + + The index stores numeric coordinate summaries in canonical SI, so + range bounds are interpreted as SI end to end: bare numbers mean + canonical SI, quantities convert, and the exact per-patch residual + converts back to each patch's native units. + """ + + @pytest.fixture(scope="class") + def ft_patch(self): + """An example patch with distance in feet (0..~984 ft).""" + return dc.get_example_patch().convert_units(distance="ft") + + def test_bare_numbers_are_canonical_si(self, ft_patch): + """(20, 60) means 20-60 m even on a feet-coordinate patch.""" + coord = dc.spool([ft_patch]).select(distance=(20, 60))[0].get_coord("distance") + assert float(coord.min()) >= 65 # 20 m == 65.6 ft + assert float(coord.max()) <= 197 # 60 m == 196.9 ft + + def test_quantity_selector(self, ft_patch): + """Quantity bounds select the same physical interval.""" + from dascore.units import m + + selected = dc.spool([ft_patch]).select(distance=(20 * m, 60 * m)) + assert len(selected.get_contents()) == 1 # no DimensionalityError + coord = selected[0].get_coord("distance") + assert float(coord.min()) >= 65 + assert float(coord.max()) <= 197 + + def test_quantity_in_native_units(self, ft_patch): + """Quantities in the coordinate's own units also work.""" + from dascore.units import get_quantity + + ft = get_quantity("ft") + coord = ( + dc.spool([ft_patch]) + .select(distance=(100 * ft, 200 * ft))[0] + .get_coord("distance") + ) + assert float(coord.min()) >= 99 + assert float(coord.max()) <= 201 + + def test_unitless_coords_unchanged(self): + """Coordinates without units keep plain numeric semantics.""" + patch = dc.get_example_patch() + coord = dc.spool([patch]).select(distance=(20, 60))[0].get_coord("distance") + assert 20 <= float(coord.min()) and float(coord.max()) <= 60 + + def test_directory_spool(self, ft_patch, tmp_path): + """The same semantics hold for file-backed spools.""" + from dascore.units import m + + dc.write(ft_patch, tmp_path / "ft.h5", "dasdae") + spool = dc.spool(tmp_path).update() + coord = spool.select(distance=(20 * m, 60 * m))[0].get_coord("distance") + assert float(coord.min()) >= 65 + assert float(coord.max()) <= 197 diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index e61f0e824..70b149296 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -685,9 +685,8 @@ def test_gc_closes_connection(self): def test_explicit_close_idempotent(self): """Explicit close works and GC afterwards stays quiet.""" - from dascore.io.index.catalog import PatchCatalog - import dascore as dc + from dascore.io.index.catalog import PatchCatalog catalog = PatchCatalog.from_patches([dc.get_example_patch()]) catalog.to_df() @@ -703,7 +702,6 @@ def test_legacy_entry_ignored(self, tmp_path): import dascore as dc from dascore.io.index.indexer import ( - DBDirectoryIndexer, _update_index_map, ) From e62d29d3bae851509b5adb8872f9782e97d63338 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 17:30:56 +0200 Subject: [PATCH 36/97] Keep selection construction lazy; adopt catalog without realization The unit-canonicalization fix realized the full flat relation at the top of DataFrameSpool.select just to flip fresh patch-list spools into catalog-native mode, turning a ~1 ms cold directory selection into a full unfiltered query, pivot, and cache of every row. Selection now calls a narrow _ensure_catalog hook instead: directory spools are already catalog-native (no-op), patch-list memory spools ingest into their catalog without building the flat relation, and materialized dataframe spools (post chunk/sort/slice) stay on the dataframe path. Cold-spool regression tests forbid any to_df call during selection construction for both spool types. --- dascore/core/spool.py | 23 +++++++++--- tests/test_core/test_spool_select_spec.py | 44 +++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 5b99ef697..1248b1aac 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -994,6 +994,16 @@ def _relative_select_kwargs(self, kwargs: dict) -> dict: return relative_ranges_to_absolute(self._df, kwargs) + def _ensure_catalog(self) -> None: + """ + Switch to catalog-native mode if this spool supports it. + + Must not realize the flat relation: it exists so selection can + route through the catalog on cold spools while staying lazy. + Dataframe-backed spools (post chunk/sort/slice) stay put. + """ + return + @compose_docstring(doc=BaseSpool.select.__doc__) def select( self, @@ -1005,10 +1015,10 @@ def select( **kwargs, ) -> Self: """{doc}.""" - # Realize contents first: fresh patch-list spools only become - # catalog-native on realization, and the catalog path owns the - # full selector semantics (e.g. unit canonicalization). - _ = self._df + # The catalog path owns the full selector semantics (e.g. unit + # canonicalization); adopt it where possible without realizing + # the flat relation (selection must stay lazy on cold spools). + self._ensure_catalog() if self._catalog_native: catalog = self._catalog.select( _attrs=_attrs, @@ -1195,6 +1205,11 @@ def _get_catalog(self): self._catalog_native = True return self._catalog + def _ensure_catalog(self) -> None: + """Patch-list spools ingest into a catalog; no flat realization.""" + if self._patches is not None and not self._catalog_native: + self._get_catalog() + def _as_catalog_member(self): """ Return (catalog, patch_ids) describing this spool for a union. diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 912e71852..9cf386547 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -89,6 +89,50 @@ def wrapped(query=None): assert queries[0].coords["time"] == ("2020-01-03", "2020-01-04") +class TestLazySelection: + """Selection construction never realizes the flat relation (review P1). + + Cold spools must compose the selected view without an unfiltered + backend query; realization happens on first content access. The + module-scoped ``spool`` fixture is warm by then, so these tests + build their own fresh spools. + """ + + @pytest.fixture() + def forbid_realization(self, monkeypatch): + """Return a callable that makes flat realization fail loudly.""" + from dascore.io.index.catalog import PatchCatalog + + def _boom(self): + msg = "flat relation realized during selection construction" + raise AssertionError(msg) + + def _arm(): + monkeypatch.setattr(PatchCatalog, "to_df", _boom) + + return _arm + + def test_cold_directory_select(self, tmp_path_factory, forbid_realization): + """A cold directory spool selects without touching the relation.""" + path = dc.examples.spool_to_directory( + dc.get_example_spool("random_das"), + path=tmp_path_factory.mktemp("lazy_select_dir"), + ) + dc.spool(path).update(progress=None) # build the index + fresh = dc.spool(path) + forbid_realization() + selected = fresh.select(time=("2020-01-03", "2020-01-04")) + assert selected._catalog_native + + def test_cold_memory_select(self, forbid_realization): + """A fresh patch-list spool selects via the catalog, lazily.""" + patches = list(dc.get_example_spool("random_das")) + forbid_realization() + fresh = dc.spool(patches) + selected = fresh.select(tag="random") + assert selected._catalog_native + + class TestSamples: """samples=True never excludes patches; trims on load (#447).""" From 9d7dee6d573705ffbf5c673f3901ce969fed97bf Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:11:28 +0200 Subject: [PATCH 37/97] Canonicalize all coordinate selector shapes; reject scalar/membership MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unit fix only handled range tuples. Scalar and value-membership coordinate selectors reached Patch.select uncanonicalized: a scalar raised ParameterError at load, a bare 2-list range was re-applied in native units, and a metre quantity range broke on a unitless patch in a mixed archive (the single-recorded-unit check also miscounted NULL units as a second unit). Coordinate selection now has one contract: a (start, stop) range (tuple or slice, open ends allowed) or a patch-local boolean mask. Scalars and value membership have no exact patch meaning and are rejected eagerly with a clear message; a wrong-arity range keeps the length-2 message. Numeric range bounds resolve to canonical SI for the index and dataframe side, and the residual carries a _CanonicalRange that defers its representation until each patch is known — quantities in the canonical unit for unit-bearing coordinates, bare magnitudes for unitless ones — so mixed unitful/unitless populations trim correctly in a single selection. Tests cover scalar and membership rejection, bare and quantity ranges on feet patches, mixed unitless+feet archives (bare and quantity), chained views, and the doc-note contract cells. Backend contract tests updated: hand-built scalar/membership coord Queries now raise. --- dascore/io/index/backend.py | 43 +++++- dascore/io/index/catalog.py | 129 ++++++++++++------ dascore/io/index/query.py | 35 ++--- docs/notes/spool_selection.qmd | 13 ++ tests/test_core/test_spool_select_spec.py | 55 +++++++- tests/test_io/test_index/test_catalog.py | 2 +- .../test_io/test_index/test_index_contract.py | 16 +-- 7 files changed, 217 insertions(+), 76 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 2cd5aef36..f83c68884 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -18,7 +18,12 @@ import pandas as pd import dascore as dc -from dascore.exceptions import InvalidIndexError, InvalidIndexVersionError, UnitError +from dascore.exceptions import ( + InvalidIndexError, + InvalidIndexVersionError, + ParameterError, + UnitError, +) from dascore.io.index.dialect import BaseDialect from dascore.io.index.ingest import SourceRecord, attr_column_name from dascore.io.index.query import ( @@ -793,6 +798,41 @@ def _drop_noops(mapping: dict) -> dict: """Bare None/... selectors are no-ops, matching Patch.select.""" return {k: v for k, v in mapping.items() if v is not None and v is not Ellipsis} + def _shape_coord_selector(name: str, value): + """ + Normalize one coordinate selector to the spec's accepted shapes. + + Coordinates select by range — a (start, stop) tuple or slice + with None/... open ends (a 2-element list is the legacy range + form) — or by a patch-local boolean mask. Scalars and value + membership have no exact patch-level meaning and raise here, + eagerly, rather than failing when a patch is materialized. + """ + if value is None or value is Ellipsis: + return value + if isinstance(value, np.ndarray): + if value.dtype == np.bool_: + return value + elif ( + isinstance(value, list) + and value + and all(isinstance(x, bool | np.bool_) for x in value) + ): + return np.asarray(value, dtype=bool) + elif isinstance(value, tuple | list): + # range-like: a (start, stop) pair (2-element list is the + # legacy range form). Wrong arity is a malformed range. + if len(value) != 2: + msg = f"Coordinate range for {name!r} must be a length 2 sequence." + raise ParameterError(msg) + return tuple(value) + msg = ( + f"Coordinate {name!r} accepts range selectors (a (start, stop) " + "tuple or slice, None/... for open ends) or boolean masks; " + f"scalar and membership values are not supported. Got {value!r}." + ) + raise InvalidSpoolQueryError(msg) + # accept the same open/slice range forms patch-level select does attrs = {k: normalize_range_forms(v) for k, v in (_attrs or {}).items()} coords = {k: normalize_range_forms(v) for k, v in (_coords or {}).items()} @@ -819,6 +859,7 @@ def _drop_noops(mapping: dict) -> dict: for name in coords: if name not in known_coords: raise InvalidSpoolQueryError(f"{name!r} is not a coordinate of this spool.") + coords = {k: _shape_coord_selector(k, v) for k, v in coords.items()} return Query(attrs=_drop_noops(attrs), coords=_drop_noops(coords)) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index dac8f77dc..6b83b9fc4 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -21,6 +21,7 @@ from dataclasses import dataclass from pathlib import Path +import numpy as np import pandas as pd import dascore as dc @@ -37,53 +38,87 @@ from dascore.utils.pd import adjust_segments +class _CanonicalRange: + """ + A numeric coordinate range resolved to canonical SI magnitudes. + + The exact per-patch re-select defers its representation until the + target patch is known: unit-bearing coordinates get quantities in + the canonical unit (`Patch.select` converts them to native units), + unitless coordinates get the bare magnitudes. A single eager form + cannot serve both — raw numbers trim the wrong physical interval + on non-SI patches, quantities break unitless coordinates. + """ + + __slots__ = ("magnitudes",) + + def __init__(self, magnitudes: tuple): + self.magnitudes = magnitudes + + def __repr__(self) -> str: + return f"_CanonicalRange({self.magnitudes!r})" + + def __eq__(self, other) -> bool: + return ( + isinstance(other, _CanonicalRange) and other.magnitudes == self.magnitudes + ) + + def for_patch_coord(self, coord) -> tuple: + """Return the range in the representation this coord needs.""" + from dascore.units import get_quantity + + units = getattr(coord, "units", None) + if units is None: + return self.magnitudes + base = get_quantity(str(units)).to_base_units().units + return tuple(None if mag is None else mag * base for mag in self.magnitudes) + + +def _canonical_range(value) -> _CanonicalRange | None: + """Return the canonical SI form of a numeric range, or None.""" + if not (isinstance(value, tuple) and len(value) == 2): + return None + magnitudes = [] + for bound in value: + if bound is None or bound is Ellipsis: + magnitudes.append(None) + elif hasattr(bound, "units"): # pint quantity -> SI magnitude + magnitudes.append(float(bound.to_base_units().magnitude)) + elif isinstance(bound, bool | np.bool_): + return None + elif isinstance(bound, int | float | np.integer | np.floating): + magnitudes.append(float(bound)) + else: # datetimes, strings: not a numeric range + return None + if all(mag is None for mag in magnitudes): + return None + return _CanonicalRange(tuple(magnitudes)) + + def _canonical_coord_selectors(backend, coords: dict) -> tuple[dict, dict]: """ - Split coordinate range selectors into canonical forms. + Split coordinate selectors into query-side and residual-side forms. Numeric coordinate summaries are stored in canonical SI units, so - range bounds resolve to SI magnitudes for the index/dataframe side. - The exact per-patch residual gets the same bounds as *quantities* - (in the canonical unit) so `Patch.select` converts them to each - patch's native coordinate units; re-applying raw numbers would trim - a different physical interval on non-SI patches. Bare numbers are - interpreted as canonical SI, matching the index contract. - - Coordinates without a single recorded unit (unitless, time-like, or - heterogeneous) pass through unchanged. + numeric range bounds resolve to SI magnitudes for the index and + dataframe side: bare numbers are already canonical SI (the index + contract), quantities convert. The residual keeps the range as a + `_CanonicalRange` so each patch decides its own representation at + load time, which keeps mixed unitful/unitless populations correct. + + Selectors on non-numeric coordinates (time ranges, string ranges) + and boolean masks pass through unchanged. """ - from dascore.units import convert_units, get_quantity - meta = backend._coord_meta(set(coords)) - units_by_name: dict[str, set] = {} - for row in meta.itertuples(): - units = getattr(row, "units", None) - if row.value_kind == "num" and units is not None and not pd.isnull(units): - units_by_name.setdefault(row.coord_name, set()).add(str(units)) + numeric = set(meta.loc[meta["value_kind"] == "num", "coord_name"]) si_coords, residual_coords = {}, {} for name, value in coords.items(): - units = units_by_name.get(name) - unit = next(iter(units)) if units and len(units) == 1 else None - if unit is None or not isinstance(value, tuple): + canonical = _canonical_range(value) if name in numeric else None + if canonical is None: si_coords[name] = residual_coords[name] = value - continue - quant = get_quantity(unit) - si_bounds, residual_bounds = [], [] - for bound in value: - if bound is None or bound is Ellipsis: - si_bounds.append(None) - residual_bounds.append(None) - continue - if hasattr(bound, "units"): # pint quantity -> SI magnitude - magnitude = convert_units( - bound.magnitude, to_units=unit, from_units=bound.units - ) - else: # bare numbers are canonical SI - magnitude = float(bound) - si_bounds.append(magnitude) - residual_bounds.append(magnitude * quant) - si_coords[name] = tuple(si_bounds) - residual_coords[name] = tuple(residual_bounds) + else: + si_coords[name] = canonical.magnitudes + residual_coords[name] = canonical return si_coords, residual_coords @@ -547,9 +582,10 @@ def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Pat trim_hint = {} for coords, samples, _ in self._residuals: if not samples: - # Quantity bounds stay out of reader hints: readers take - # numbers in their native units, so a converted-narrower - # hint could drop data exactness cannot restore. + # Canonical-SI and quantity bounds stay out of reader + # hints: readers take numbers in their native units, so + # a converted-narrower hint could drop data exactness + # cannot restore. trim_hint.update( { k: v @@ -561,7 +597,16 @@ def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Pat trim_hint.update(extra_trim or {}) patch = self.resolver.resolve(row, **trim_hint) for coords, samples, relative in self._residuals: - usable = {k: v for k, v in coords.items() if k in patch.coords.coord_map} + coord_map = patch.coords.coord_map + usable = { + k: ( + v.for_patch_coord(coord_map[k]) + if isinstance(v, _CanonicalRange) + else v + ) + for k, v in coords.items() + if k in coord_map + } if usable: patch = patch.select(**usable, samples=samples, relative=relative) return patch diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 74044d257..4927bd29a 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -304,30 +304,19 @@ def build_coord_clause( typed_values = [] if _is_range(value): kind, lo, hi, typed_values = _range_bounds(value, kinds) - elif _is_collection(value): - raw_values = list(value) - if not raw_values: - raise InvalidSpoolQueryError("Coordinate membership cannot be empty.") - arr = np.asarray(raw_values) - if arr.dtype == bool: - # boolean masks are patch-local; no index predicate at all, - # but the coord must exist on the patch. - kind = lo = hi = None - else: - typed_values = [_coerce_scalar(x, kinds) for x in raw_values] - value_kinds = {x.kind for x in typed_values} - if len(value_kinds) != 1: - raise InvalidSpoolQueryError( - f"Coordinate values for {name!r} have mixed kinds." - ) - kind = typed_values[0].kind - values = [x.value for x in typed_values] - lo, hi = min(values), max(values) + elif _is_collection(value) and np.asarray(value).dtype == bool: + # boolean masks are patch-local; no index predicate at all, + # but the coord must exist on the patch. + kind = lo = hi = None else: - typed = _coerce_scalar(value, kinds) - typed_values = [typed] - kind = typed.kind - lo = hi = typed.value + # Scalars and value membership have no exact patch-level + # meaning; resolve_query rejects them before SQL composition, + # so only a hand-built Query can reach this. + msg = ( + f"Coordinate {name!r} accepts range or boolean-mask " + f"selectors; got {value!r}." + ) + raise InvalidSpoolQueryError(msg) compatible_units = _compatible_coord_units(rows, typed_values, name) diff --git a/docs/notes/spool_selection.qmd b/docs/notes/spool_selection.qmd index 62d4c89ab..8754961e6 100644 --- a/docs/notes/spool_selection.qmd +++ b/docs/notes/spool_selection.qmd @@ -8,6 +8,8 @@ Bare selector names resolve to attributes first and then coordinates. `_attrs={. Attribute equality, membership, ranges, and glob predicates are evaluated by the index. Regular expressions use a SQL candidate predicate and an exact residual filter; chained regular expressions are combined with AND. Quantities are converted to the canonical unit recorded by the index, and dimensionally incompatible queries raise rather than silently returning incorrect matches. Values stored without units can never be proven incompatible, so they remain candidates for quantity selectors rather than being silently excluded. +Coordinate predicates select by range — a `(start, stop)` tuple or slice, with `None`/`...` for an open end — or by a patch-local boolean mask; scalar and value-membership coordinate selectors have no exact patch-level meaning and are rejected. Numeric coordinate summaries are stored in canonical SI units, so bare numeric range bounds are interpreted as canonical SI regardless of a patch's native coordinate units, and quantities convert. The exact per-patch trim defers its representation until each patch is known, so a mixed archive of unit-bearing and unitless patches is handled correctly in one selection. + Coordinate predicates first select patches whose summary envelopes can overlap the request. The loaded patch is then selected exactly. `samples=True` is always patch-local and therefore never excludes a patch at the index stage. `relative=True` resolves coordinate ranges against the current spool view's global envelope; attribute predicates in the same call remain unchanged. Operations that create a new row or instruction plan, including chunking, sorting, and slicing, switch that derived spool to dataframe planning. Exact selections already attached to the catalog still apply when source patches are resolved. @@ -36,3 +38,14 @@ for patch in spool.select(time=window): coord = patch.get_coord("time") assert coord.min() >= window[0] and coord.max() <= window[1] ``` + +Bare numeric coordinate bounds mean canonical SI even on a patch whose coordinate is in another unit, and scalar/membership coordinate selectors raise: + +```{python} +ft_patch = dc.get_example_patch().convert_units(distance="ft") +coord = dc.spool([ft_patch]).select(distance=(20, 60))[0].get_coord("distance") +assert float(coord.min()) >= 65 and float(coord.max()) <= 197 # 20-60 m in ft + +with pytest.raises(InvalidSpoolQueryError): + dc.spool([ft_patch]).select(distance=100) # scalar has no range meaning +``` diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 9cf386547..8359fafb7 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -175,7 +175,7 @@ def test_trims_both_ends(self, spool): def test_requires_range(self, spool): """Scalars are rejected with a clear message.""" - with pytest.raises(InvalidSpoolQueryError, match="requires"): + with pytest.raises(InvalidSpoolQueryError, match="range selectors"): spool.select(time=5, relative=True) def test_namespaced_coord_with_attr(self, spool): @@ -258,3 +258,56 @@ def test_directory_spool(self, ft_patch, tmp_path): coord = spool.select(distance=(20 * m, 60 * m))[0].get_coord("distance") assert float(coord.min()) >= 65 assert float(coord.max()) <= 197 + + def test_mixed_unitless_and_feet_bare(self, ft_patch): + """A bare range trims each patch in its own units (SI meaning).""" + plain = dc.get_example_patch() # unitless distance 0..~300 + plain = plain.update_coords( + distance=plain.get_coord("distance").set_units(None) + ) + got = dc.spool([plain, ft_patch]).select(distance=(20, 60)) + materialized = [p.get_coord("distance") for p in got] + assert len(materialized) == 2 # both overlap 20..60 m + by_units = {str(c.units): c for c in materialized} + # unitless patch: bare magnitudes applied directly + assert float(by_units["None"].min()) >= 20 + assert float(by_units["None"].max()) <= 60 + # feet patch: 20..60 m == 65.6..196.9 ft + assert float(by_units["1 ft"].min()) >= 65 + assert float(by_units["1 ft"].max()) <= 197 + + def test_mixed_unitless_and_feet_quantity(self, ft_patch): + """A metre quantity range works across a mixed population.""" + from dascore.units import m + + plain = dc.get_example_patch() + plain = plain.update_coords( + distance=plain.get_coord("distance").set_units(None) + ) + got = dc.spool([plain, ft_patch]).select(distance=(20 * m, 60 * m)) + assert len(got.get_contents()) == 2 # no UnitError on the unitless row + + def test_scalar_coord_rejected(self, ft_patch): + """A scalar coordinate selector is rejected eagerly, clearly.""" + with pytest.raises(InvalidSpoolQueryError, match="range selectors"): + dc.spool([ft_patch]).select(distance=100) + + def test_value_membership_rejected(self, ft_patch): + """A wrong-arity list is reported as a malformed range.""" + from dascore.exceptions import ParameterError + + with pytest.raises(ParameterError, match="length 2 sequence"): + dc.spool([ft_patch]).select(distance=[10, 20, 50]) + + def test_chained_views(self, ft_patch): + """Canonicalization holds across chained selections.""" + from dascore.units import m + + coord = ( + dc.spool([ft_patch]) + .select(distance=(0 * m, 90 * m)) + .select(distance=(20, 60))[0] + .get_coord("distance") + ) + assert float(coord.min()) >= 65 + assert float(coord.max()) <= 197 diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py index 88338da63..c8e41c3eb 100644 --- a/tests/test_io/test_index/test_catalog.py +++ b/tests/test_io/test_index/test_catalog.py @@ -175,7 +175,7 @@ def test_relative_on_unknown_coord_raises(self, live_catalog): def test_relative_requires_range(self, live_catalog): """Relative selects take (start, stop) tuples only.""" - with pytest.raises(InvalidSpoolQueryError, match="requires"): + with pytest.raises(InvalidSpoolQueryError, match="range selectors"): live_catalog.select(time=5, relative=True) def test_add_on_file_catalog_not_implemented(self, tmp_path, patches): diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index 21878e2cd..c85a30564 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -290,16 +290,16 @@ def test_incompatible_quantity_coord_raises(self, backend): with pytest.raises(UnitError): backend.query(Query(coords={"distance": (1 * second, 2 * second)})) - def test_scalar_coord(self, backend): - """Scalar coord.""" - df = backend.query(Query(coords={"frequency": 100})) - assert list(df["tag"]) == ["psd"] + def test_scalar_coord_rejected(self, backend): + """Scalar coord predicates have no exact patch meaning; rejected.""" + with pytest.raises(InvalidSpoolQueryError, match="range or boolean"): + backend.query(Query(coords={"frequency": 100})) - def test_array_membership_envelope(self, backend): - """Array membership envelope.""" + def test_array_membership_rejected(self, backend): + """Numeric value membership on a coord is rejected, not candidacy.""" values = np.array([10.0, 20.0, 480.0]) - df = backend.query(Query(coords={"distance": values})) - assert len(df) == 4 # candidacy: all patches overlap the envelope + with pytest.raises(InvalidSpoolQueryError, match="range or boolean"): + backend.query(Query(coords={"distance": values})) def test_coord_missing_excludes_patch(self, backend): """Coord missing excludes patch.""" From e05caf9c56c31a82f80e7116b8ba391f3de5fd80 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:13:30 +0200 Subject: [PATCH 38/97] Make SQLite backend cleanup safe under cross-thread collection The weakref finalizer closed the connection on whichever thread dropped the last backend reference. With sqlite3's default check_same_thread=True that raised ProgrammingError as an unraisable exception when a backend was created on one thread and finalized on another (e.g. a thread-pool worker), and it blocked sharing one catalog across a threaded Spool.map. On a serialized SQLite build (threadsafety == 3, the common case) the connection now opens with check_same_thread=False, so it can be used and closed from any thread. The finalizer and close() route through a helper that suppresses ProgrammingError, so the rarer thread-bound builds degrade to a freed-at-teardown handle instead of a crash. Adds a worker-thread finalization test alongside the same-thread ResourceWarning test. --- dascore/io/index/lite.py | 33 ++++++++++++++++--- .../test_index/test_index_edge_cases.py | 28 ++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py index cba9f2dd0..e3a265187 100644 --- a/dascore/io/index/lite.py +++ b/dascore/io/index/lite.py @@ -4,6 +4,7 @@ import sqlite3 import weakref +from contextlib import suppress from pathlib import Path import numpy as np @@ -12,6 +13,25 @@ from dascore.io.index.backend import SQLIndexBackend, adapt_params from dascore.io.index.dialect import SQLiteDialect +# A serialized SQLite build (threadsafety == 3) lets one connection be +# used and closed from any thread, so cross-thread garbage collection of +# a backend is safe. On rarer non-serialized builds the connection is +# thread-bound and must stay check_same_thread. +_SQLITE_SERIALIZED = sqlite3.threadsafety == 3 + + +def _safe_close(con: sqlite3.Connection) -> None: + """ + Close a connection, tolerating cross-thread finalization. + + On a serialized build closing works from any thread. On a + thread-bound build a finalizer firing on another thread would raise + ProgrammingError; suppress it (the underlying handle is freed at + interpreter teardown) rather than emit an unraisable exception. + """ + with suppress(sqlite3.ProgrammingError): + con.close() + def _adapt(params): """Convert numpy/py types sqlite3 can't bind natively.""" @@ -50,15 +70,20 @@ class SQLiteBackend(SQLIndexBackend): def __init__(self, path: str | Path): self._path = str(path) - self._con = sqlite3.connect(self._path) + # On a serialized build, drop the thread affinity so the shared + # backend can be used (and finalized) from worker threads, e.g. + # a thread-pool Spool.map over one catalog. + self._con = sqlite3.connect( + self._path, check_same_thread=not _SQLITE_SERIALIZED + ) # autocommit off; we manage transactions explicitly. self._con.isolation_level = None self._con.execute("PRAGMA foreign_keys = ON") self._con.execute("PRAGMA busy_timeout = 30000") # Catalog views share this backend object, so tying connection # cleanup to *its* collection is safe (close() stays idempotent - # for explicit use). - self._finalizer = weakref.finalize(self, self._con.close) + # for explicit use). The finalizer tolerates cross-thread firing. + self._finalizer = weakref.finalize(self, _safe_close, self._con) try: super().__init__() except Exception: @@ -126,4 +151,4 @@ def close(self) -> None: """Close the database connection.""" # detach the GC finalizer; closing twice is harmless but tidy. self._finalizer.detach() - self._con.close() + _safe_close(self._con) diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 70b149296..e8f0cf331 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -9,6 +9,7 @@ from __future__ import annotations import re +import sqlite3 import numpy as np import pandas as pd @@ -692,6 +693,33 @@ def test_explicit_close_idempotent(self): catalog.to_df() catalog.close() + def test_finalization_from_worker_thread(self): + """Dropping the last backend reference off-thread does not raise.""" + import gc + import sys + import threading + + from dascore.io.index.lite import SQLiteBackend + + holder = [SQLiteBackend(":memory:")] + unraisable = [] + original = sys.unraisablehook + + def _drop(): + holder.clear() + gc.collect() + + sys.unraisablehook = unraisable.append + try: + worker = threading.Thread(target=_drop) + worker.start() + worker.join() + gc.collect() + finally: + sys.unraisablehook = original + errors = [u for u in unraisable if u.exc_type is sqlite3.ProgrammingError] + assert not errors + class TestLegacyIndexMap: """Index-map entries pointing at retired .h5 indexes are bypassed.""" From 5f7ceb63bdd52d4f8b723fbf710eec4eb3e448b0 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:16:44 +0200 Subject: [PATCH 39/97] Reserve coordinate-envelope attr names catalog-wide Ingest reserved envelope-shaped attr names ({coord}_min/_max/_step) only against the ingesting patch's own coordinates. A valid extra attr named like another patch's coordinate envelope (e.g. event_time_min, reachable via construction-time attrs since update_attrs already rejects coord-metadata names) was indexed on one patch and then collided with a second patch's renamed coordinate envelope. Flattening silently dropped the typed attr column, so one get_contents() column's meaning depended on which patches were selected. Envelope-shaped names are now reserved catalog-wide, independent of any patch's own coordinates. Only the real per-coord envelope suffixes (min/max/step) are reserved, so ordinary attrs like data_units stay queryable, matching Patch.update_attrs. Tests cover the cross-patch collision (attr reserved, column keeps the coordinate envelope) and confirm a *_units attr still indexes. --- dascore/io/index/ingest.py | 32 ++++++++++++++----- .../test_index/test_index_edge_cases.py | 30 +++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 9de94f1ab..f4e3052d8 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -28,6 +28,23 @@ # Attrs handled structurally or intentionally excluded from the index. _SKIPPED_ATTRS = frozenset({"history", "dims", "coords"}) +# Suffixes of the per-coordinate envelope columns the flat relation +# emits ({name}_min/{name}_max/{name}_step). An attr shaped like one of +# these is reserved catalog-wide — not just against the ingesting +# patch's own coords — so a flat-relation envelope column (e.g. +# "event_time_min") can never collide with a same-named attr contributed +# by a different patch, which would make one get_contents() column's +# meaning depend on which other patches share the catalog. (units/dtype +# do not become per-coord columns, so a real attr like "data_units" +# stays queryable, matching Patch.update_attrs.) +_ENVELOPE_SUFFIXES = ("min", "max", "step") + + +def _is_envelope_shaped(name: str) -> bool: + """True if name looks like a ``{coord}_{min,max,step}`` envelope column.""" + prefix, _, suffix = name.rpartition("_") + return bool(prefix) and suffix in _ENVELOPE_SUFFIXES + @dataclass(frozen=True) class TypedValue: @@ -202,18 +219,17 @@ def typed_value(value) -> TypedValue | None: def _extract_attrs(summary: PatchSummary) -> dict[str, TypedValue]: """Get indexable typed attrs from a patch summary.""" raw = summary.attrs.model_dump() - # Attrs shaped like this patch's own coordinate envelope columns - # (e.g. "time_min") would collide in the flat relation. - envelope_names = { - f"{coord}_{suffix}" - for coord in getattr(summary, "coords", {}) - for suffix in ("min", "max", "step", "units") - } out = {} for name, value in raw.items(): if name in _SKIPPED_ATTRS or name.startswith("_"): continue - if sanitize_attr_name(name) in RESERVED_ATTR_COLUMNS or name in envelope_names: + # Reserve structural columns and any coordinate-envelope-shaped + # name (catalog-wide, not just this patch's own coords) so the + # meaning of a flat-relation column never depends on which other + # patches share the catalog. + if sanitize_attr_name(name) in RESERVED_ATTR_COLUMNS or ( + _is_envelope_shaped(name) + ): msg = ( f"Skipping reserved attr name {name!r}; it collides with a " "structural index column. The attr stays on the patch but " diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index e8f0cf331..4a602463e 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -773,3 +773,33 @@ def test_non_reserved_attr_round_trips(self): spool = dc.spool([patch]) assert spool.get_contents()["experiment"].iloc[0] == "exp42" assert len(spool.select(experiment="exp42")) == 1 + + def test_cross_patch_envelope_attr_reserved(self): + """An envelope-shaped attr is reserved even against another patch's coord.""" + import dascore as dc + + base = dc.get_example_patch() + # construction-time attrs bypass the update_attrs coord-name guard. + p1 = dc.Patch( + data=base.data, + coords=base.coords, + dims=base.dims, + attrs=dict(base.attrs) | {"event_time_min": 123.0}, + ) + p2 = dc.get_example_patch().rename_coords(time="event_time") + with pytest.warns(UserWarning, match="reserved attr name 'event_time_min'"): + spool = dc.spool([p1, p2]) + df = spool.get_contents() + backend = spool._get_catalog().backend + assert "event_time_min" not in backend.attr_names() + # the column is the coordinate envelope, not the stray attr value. + assert "event_time_min" in df.columns + assert 123.0 not in set(df["event_time_min"].dropna()) + + def test_data_units_attr_still_indexed(self): + """A real ``*_units`` attr is not an envelope column; stays queryable.""" + import dascore as dc + + patch = dc.get_example_patch().update_attrs(data_units="strain") + spool = dc.spool([patch]) + assert "data_units" in spool._get_catalog().backend.attr_names() From 177ad0922d7950a6d3dc06fc1f72926837d32945 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:25:30 +0200 Subject: [PATCH 40/97] Count patches in SQL instead of realizing the flat relation PatchCatalog.__len__ delegated to len(to_df()), so a bare count paid the full get_contents() cost: an unfiltered backend query, every attr column, a coordinate pivot, and a cached dataframe. On large archives len(spool), rich display, and split sizing became large allocations. Adds AbstractIndexBackend.count(query) with an SQL COUNT sharing the same WHERE as the flat query (extracted into a _build_where helper) but no projection, ordering, or coordinate pivot. A regex residual is not SQL-resolvable, so those fall back to counting the realized relation. PatchCatalog.__len__ counts in SQL unless the relation is already cached; DataFrameSpool.__len__ takes the catalog count for cold catalog-native views (but not when constructor select_kwargs post- filter rows outside the catalog's queries). Tests assert count equals len(to_df()) across attr, coord-range, regex-residual, and chained views, and that a cold len() performs no flat realization. --- dascore/core/spool.py | 12 +++++ dascore/io/index/backend.py | 21 +++++++++ dascore/io/index/catalog.py | 11 ++++- dascore/io/index/query.py | 57 ++++++++++++++++++++---- tests/test_io/test_index/test_catalog.py | 44 ++++++++++++++++++ 5 files changed, 135 insertions(+), 10 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 1248b1aac..d776960f2 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -554,6 +554,18 @@ def __getitem__(self, item) -> PatchType | BaseSpool: return out def __len__(self): + # A catalog-native view can count in SQL, skipping the full flat + # realization (query + attr expansion + coordinate pivot) a plain + # len(self._df) would force. Fall back to the realized frame once + # it is cached, on the dataframe path, or when constructor + # select_kwargs post-filter rows outside the catalog's queries. + if ( + self._catalog_native + and not self._select_kwargs + and getattr(self, "_catalog", None) is not None + and "_df" not in self._cache + ): + return len(self._catalog) return len(self._df) def __iter__(self): diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index f83c68884..aa948677f 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -29,6 +29,7 @@ from dascore.io.index.query import ( Query, apply_residuals, + build_count_sql, build_query_sql, normalize_range_forms, ) @@ -105,6 +106,10 @@ def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: def query(self, query: Query) -> pd.DataFrame: """Return the flat patch-row relation matching a query.""" + @abc.abstractmethod + def count(self, query: Query) -> int: + """Return how many patches match a query, without projecting rows.""" + @abc.abstractmethod def get_sources(self) -> pd.DataFrame: """Return the sources table.""" @@ -575,6 +580,22 @@ def query(self, query=None) -> pd.DataFrame: df = apply_residuals(df, residuals) return df.reset_index(drop=True) + def count(self, query=None) -> int: + """Count matching patches without projecting or pivoting rows.""" + query = query if query is not None else Query() + queries = [query] if isinstance(query, Query) else list(query) + attr_meta = self._attr_meta() + coord_names = {name for q in queries for name in q.coords} + coord_meta = self._coord_meta(coord_names) if coord_names else pd.DataFrame() + sql, params, residuals = build_count_sql( + queries, self.dialect, attr_meta, coord_meta + ) + if not residuals: + return int(self._fetch_df(sql, params)["n"].iloc[0]) + # A regex residual must inspect string values, so a database count + # cannot resolve it; the full relation already applies the residual. + return len(self.query(queries)) + def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: """Post-process raw SQL output into the flat-relation contract.""" out = df.copy() diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 6b83b9fc4..df5a83835 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -564,7 +564,16 @@ def to_df(self) -> pd.DataFrame: return self._df_cache def __len__(self) -> int: - return len(self.to_df()) + # Count in SQL when the relation is not already realized: coord + # range residuals only drop patches the SQL candidacy already + # excludes and samples/relative residuals never drop patches, so + # the count matches len(to_df()) without projecting or pivoting. + if ( + self._df_cache is not None + and self._df_cache_revision == self._revision.value + ): + return len(self._df_cache) + return self.backend.count(list(self._queries) or None) def get_patch(self, index: int) -> dc.Patch: """Materialize one patch: resolve, then exact two-stage trim.""" diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 4927bd29a..3a669b4cd 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -364,6 +364,25 @@ def build_coord_clause( ) +def _build_where( + queries: list[Query], + dialect: BaseDialect, + attr_meta: pd.DataFrame, + coord_meta: pd.DataFrame, +) -> tuple[_Where, list[tuple[str, re.Pattern]]]: + """Compose the shared WHERE clause and any regex residuals.""" + where = _Where() + residuals: list[tuple[str, re.Pattern]] = [] + for one in queries: + for name, value in one.attrs.items(): + residual = build_attr_clause(where, dialect, attr_meta, name, value) + if residual is not None: + residuals.append((name, residual)) + for name, value in one.coords.items(): + build_coord_clause(where, dialect, coord_meta, name, value) + return where, residuals + + def build_query_sql( query: Query | Sequence[Query], dialect: BaseDialect, @@ -379,15 +398,7 @@ def build_query_sql( re-applied to the resulting dataframe. """ queries = [query] if isinstance(query, Query) else list(query) - where = _Where() - residuals: list[tuple[str, re.Pattern]] = [] - for one in queries: - for name, value in one.attrs.items(): - residual = build_attr_clause(where, dialect, attr_meta, name, value) - if residual is not None: - residuals.append((name, residual)) - for name, value in one.coords.items(): - build_coord_clause(where, dialect, coord_meta, name, value) + where, residuals = _build_where(queries, dialect, attr_meta, coord_meta) # attr columns selected explicitly: `a.*` would duplicate patch_id and # engines disagree on how to dedupe result column names. attr_cols = "".join( @@ -405,6 +416,34 @@ def build_query_sql( return sql, where.params, residuals +def build_count_sql( + query: Query | Sequence[Query], + dialect: BaseDialect, + attr_meta: pd.DataFrame, + coord_meta: pd.DataFrame, +) -> tuple[str, list, list[tuple[str, re.Pattern]]]: + """ + Build a COUNT for one or more AND-composed queries. + + Same WHERE as build_query_sql but no flat projection, coordinate + pivot, or ordering. Returns (sql, params, residuals); a non-empty + residual means the count is not SQL-resolvable (regex must inspect + rows) and the caller must fall back to a projected count. + """ + queries = [query] if isinstance(query, Query) else list(query) + where, residuals = _build_where(queries, dialect, attr_meta, coord_meta) + # the attrs join can stay: it is 1:1 (one attrs row per patch), and a + # WHERE may reference a.. COUNT(p.patch_id) counts patches. + sql = ( + "SELECT COUNT(p.patch_id) AS n " + "FROM patches p " + "JOIN sources s ON s.source_id = p.source_id " + "LEFT JOIN attrs a ON a.patch_id = p.patch_id " + f"WHERE {where.sql}" + ) + return sql, where.params, residuals + + def apply_residuals( df: pd.DataFrame, residuals: list[tuple[str, re.Pattern]] ) -> pd.DataFrame: diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py index c8e41c3eb..713bc529e 100644 --- a/tests/test_io/test_index/test_catalog.py +++ b/tests/test_io/test_index/test_catalog.py @@ -195,6 +195,50 @@ def test_remove(self, patches): catalog.remove([target]) assert len(catalog) == len(patches) - 1 + +class TestCount: + """len(catalog) counts in SQL and agrees with the realized relation.""" + + @pytest.fixture() + def diverse_catalog(self): + """A catalog with heterogeneous attrs and coords.""" + return PatchCatalog.from_patches(list(dc.get_example_spool("diverse_das"))) + + def _selections(self, catalog): + """Views spanning attr, coord range, regex, and chained forms.""" + import re + + df = catalog.to_df() + t0 = df["time_min"].min() + window = (t0, t0 + dc.to_timedelta64(1)) + return [ + catalog, + catalog.select(tag="random"), + catalog.select(time=window), + catalog.select(distance=(0, 50)), + catalog.select(tag=re.compile("rand.*")), # regex residual path + catalog.select(tag="random").select(time=window), + ] + + def test_count_matches_realization(self, diverse_catalog): + """Every view's len equals len(to_df()) (fresh, uncached).""" + for view in self._selections(diverse_catalog): + # a fresh view has no cached relation, so len() counts in SQL + expected = len(view.to_df()) + fresh = view._view(view._queries, view._residuals) + assert len(fresh) == expected + + def test_len_does_not_realize(self, diverse_catalog, monkeypatch): + """A cold len() must not pivot coordinates or fetch the relation.""" + catalog = diverse_catalog.select(network="das2") + fresh = catalog._view(catalog._queries, catalog._residuals) + + def _boom(self): + raise AssertionError("flat relation realized during len()") + + monkeypatch.setattr(type(fresh), "to_df", _boom) + assert isinstance(len(fresh), int) + def test_introspection(self, live_catalog): """Names, sources, and metadata pass through.""" assert "time" in live_catalog.coord_names() From 42e64a4ac096c0fe3f16895cef0c813b65ae9818 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:28:42 +0200 Subject: [PATCH 41/97] Push selected-membership export into SQL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Union of a small selected view called records_from_backend, which fetched every source, patch, attr, coordinate link, and coordinate definition into pandas and only then filtered by patch id. Exporting one patch from a large archive was O(total archive) in memory and latency — prohibitive on the million-source archives the branch targets. Record assembly moves to a backend-independent assemble_source_records that takes already-fetched frames; the fetching becomes an AbstractIndexBackend.export_records(patch_ids) that filters patches first in SQL, then joins only their sources/attrs/links and the referenced coordinate definitions (batched IN clauses for large id sets). This also removes ingest.py's calls to private backend methods. PatchCatalog.union and __getstate__ call export_records; the old records_from_backend is gone. A SQL-trace test asserts exporting one of many patches issues only id-filtered patch queries. --- dascore/io/index/backend.py | 56 ++++++++++++++++++++++++++ dascore/io/index/catalog.py | 8 +--- dascore/io/index/ingest.py | 35 ++++++---------- tests/test_io/test_index/test_union.py | 42 +++++++++++++++++++ 4 files changed, 112 insertions(+), 29 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index aa948677f..b2bdf1363 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -110,6 +110,10 @@ def query(self, query: Query) -> pd.DataFrame: def count(self, query: Query) -> int: """Return how many patches match a query, without projecting rows.""" + @abc.abstractmethod + def export_records(self, patch_ids=None) -> list: + """Reconstruct source records (optionally for a subset of patches).""" + @abc.abstractmethod def get_sources(self) -> pd.DataFrame: """Return the sources table.""" @@ -596,6 +600,58 @@ def count(self, query=None) -> int: # cannot resolve it; the full relation already applies the residual. return len(self.query(queries)) + def _fetch_in(self, base_sql: str, column: str, ids: list) -> pd.DataFrame: + """Fetch ``{base_sql} WHERE {column} IN ids``, batching large sets.""" + if not ids: + return self._fetch_df(f"{base_sql} WHERE 0") + frames = [] + batch = self._in_clause_batch + for start in range(0, len(ids), batch): + chunk = ids[start : start + batch] + marks = ", ".join("?" for _ in chunk) + frames.append( + self._fetch_df(f"{base_sql} WHERE {column} IN ({marks})", chunk) + ) + return pd.concat(frames, ignore_index=True) + + def export_records(self, patch_ids=None) -> list: + """ + Reconstruct source records, filtering by patch id in SQL. + + With patch_ids given, only those patches (and the sources, attrs, + coordinate links, and coordinate definitions they reference) are + fetched — O(selected membership), not O(total archive). The frames + are assembled into the backend-independent transfer format. + """ + from dascore.io.index.ingest import assemble_source_records + + if patch_ids is None: + sources = self._fetch_df("SELECT * FROM sources") + patches = self._fetch_df("SELECT * FROM patches") + attrs = self._fetch_df("SELECT * FROM attrs") + links = self._fetch_df("SELECT * FROM patch_coords") + defs = self._fetch_df("SELECT * FROM coord_defs") + else: + ids = [int(x) for x in patch_ids] + patches = self._fetch_in("SELECT * FROM patches", "patch_id", ids) + if patches.empty: + return [] + source_ids = [int(x) for x in patches["source_id"].unique()] + sources = self._fetch_in("SELECT * FROM sources", "source_id", source_ids) + attrs = self._fetch_in("SELECT * FROM attrs", "patch_id", ids) + links = self._fetch_in("SELECT * FROM patch_coords", "patch_id", ids) + def_ids = ( + [int(x) for x in links["coord_def_id"].unique()] + if not links.empty + else [] + ) + defs = self._fetch_in( + "SELECT * FROM coord_defs", "coord_def_id", def_ids + ) + return assemble_source_records( + sources, patches, attrs, links, defs, self._attr_meta() + ) + def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: """Post-process raw SQL output into the flat-relation contract.""" out = df.copy() diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index df5a83835..a019a45b5 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -355,8 +355,6 @@ def union(cls, catalogs: Sequence[PatchCatalog]) -> PatchCatalog: note this respects row membership, not range trims; re-select on the result for exact envelopes. """ - from dascore.io.index.ingest import records_from_backend - resolver = CompositeResolver() out = cls(resolver=resolver) backend = out.backend @@ -364,7 +362,7 @@ def union(cls, catalogs: Sequence[PatchCatalog]) -> PatchCatalog: catalog, patch_ids = member if isinstance(member, tuple) else (member, None) if patch_ids is None and catalog.is_view: patch_ids = catalog.to_df()["_patch_id"].tolist() - records = records_from_backend(catalog.backend, patch_ids=patch_ids) + records = catalog.backend.export_records(patch_ids=patch_ids) root = getattr(catalog.resolver, "_root", None) if root is not None: records = [_absolutize_record(x, root) for x in records] @@ -424,8 +422,6 @@ def __getstate__(self) -> dict: registry (the store for live patches) pickles with its patches. Directory catalogs rebuild from their index file instead. """ - from dascore.io.index.ingest import records_from_backend - state = dict(self.__dict__) state["_backend"] = None # Live catalogs rebuild from their registry without touching the @@ -437,7 +433,7 @@ def __getstate__(self) -> dict: and not isinstance(self.resolver, LiveResolver) ) if needs_records: - state["_rebuild_records"] = tuple(records_from_backend(self._backend)) + state["_rebuild_records"] = tuple(self._backend.export_records()) return state def _view(self, queries, residuals) -> PatchCatalog: diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index f4e3052d8..4dcbb5af4 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -411,37 +411,26 @@ def _py_scalar(value): return value -def records_from_backend(backend, patch_ids=None) -> list[SourceRecord]: +def assemble_source_records( + sources: pd.DataFrame, + patches: pd.DataFrame, + attrs: pd.DataFrame, + links: pd.DataFrame, + defs: pd.DataFrame, + meta: pd.DataFrame, +) -> list[SourceRecord]: """ - Reconstruct source records from a backend's tables. + Assemble source records from already-fetched index frames. This is the transfer format for merging catalogs: feeding the result to another backend's `write_sources` re-ingests the metadata with fresh ids, coord-def deduplication (def keys are preserved), and - replace-semantics on (base_uri, source_path) identity. - - Parameters - ---------- - backend - The index backend to read. - patch_ids - If not None, only include these patches (and only sources which - still have at least one included patch). + replace-semantics on (base_uri, source_path) identity. The caller + (an index backend's export_records) is responsible for narrowing the + frames — filtering by patch id belongs in SQL, not here. """ - sources = backend._fetch_df("SELECT * FROM sources") if sources.empty: return [] - patches = backend._fetch_df("SELECT * FROM patches") - if patch_ids is not None: - patches = patches[patches["patch_id"].isin(set(patch_ids))] - kept_ids = set(int(x) for x in patches["patch_id"]) - attrs = backend._fetch_df("SELECT * FROM attrs") - links = backend._fetch_df("SELECT * FROM patch_coords") - if patch_ids is not None: # narrow dependent tables to kept patches. - attrs = attrs[attrs["patch_id"].isin(kept_ids)] - links = links[links["patch_id"].isin(kept_ids)] - defs = backend._fetch_df("SELECT * FROM coord_defs") - meta = backend._attr_meta() col_info = { row.column_name: (row.attr_name, row.value_kind, _py_scalar(row.units)) for row in meta.itertuples() diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 9f011240b..439688924 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -207,3 +207,45 @@ def test_remove_updates_live_registry(self): loaded.attr_names() # bootstrap the backend loaded._invalidate() assert len(loaded.to_df()) == 0 + + +class TestExportPushdown: + """Selected-membership export must not scan the whole archive.""" + + def test_export_one_of_many_is_narrow(self): + """Exporting one patch fetches only its own rows, not all sources.""" + patches = [dc.get_example_patch().update_attrs(tag=f"t{i}") for i in range(40)] + catalog = PatchCatalog.from_patches(patches) + catalog.to_df() # bootstrap the backend + backend = catalog.backend + con = backend._con + + fetched_patches = [] + + def _trace(sql): + # Count how many patch rows any SELECT against patches pulls. + if "from patches" in sql.lower() and sql.lower().lstrip().startswith( + "select" + ): + fetched_patches.append(sql) + + target = int(catalog.to_df()["_patch_id"].iloc[0]) + con.set_trace_callback(_trace) + try: + records = backend.export_records(patch_ids=[target]) + finally: + con.set_trace_callback(None) + + # exactly one source/patch comes back... + assert sum(len(r.patches) for r in records) == 1 + # ...and every patches query was id-filtered (no full-table scan). + assert fetched_patches + assert all("patch_id in" in sql.lower() for sql in fetched_patches) + + def test_export_all_matches_full(self): + """export_records() with no ids returns every source, unchanged.""" + patches = [dc.get_example_patch().update_attrs(tag=f"t{i}") for i in range(5)] + catalog = PatchCatalog.from_patches(patches) + catalog.to_df() + records = catalog.backend.export_records() + assert sum(len(r.patches) for r in records) == 5 From 19c8fe42d06d5de7463e200bbd7ce4530d48ce00 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:29:57 +0200 Subject: [PATCH 42/97] Flatten dynamic attrs in one grouped pass _flatten refiltered the whole attr_meta frame for every unique attr name and dropped each typed SQL column from the result one at a time, so a heterogeneous archive with hundreds of attrs paid roughly quadratic metadata scans and whole-frame copies. Group attr_meta once, accumulate the converted series, and drop every typed column in a single operation. Behavior is unchanged. --- dascore/io/index/backend.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index b2bdf1363..166d67527 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -664,18 +664,22 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: for col in ("distance_min", "distance_max", "distance_step"): if col in out: out[col] = pd.to_numeric(out[col]) - # typed attr columns -> original names (coalesce multi-kind attrs) - for name in attr_meta["attr_name"].unique(): + # typed attr columns -> original names (coalesce multi-kind attrs). + # Group the metadata once and drop every typed column in a single + # pass: per-name refiltering plus one-drop-per-column was ~O(A^2) + # metadata scans and frame copies for A dynamic attrs. + cols_to_drop: list[str] = [] + new_columns: dict[str, pd.Series] = {} + for name, rows in attr_meta.groupby("attr_name", sort=False): if name in out.columns: # A dynamic attr restored onto an existing column (e.g. a # coordinate envelope like time_min) would corrupt the # frame; structural columns win. Reserved fixed names are # already refused at ingest. - sanitized = attr_meta.loc[attr_meta["attr_name"] == name, "column_name"] - out = out.drop(columns=[x for x in sanitized if x in out.columns]) + cols_to_drop.extend(c for c in rows["column_name"] if c in out.columns) continue - rows = attr_meta[attr_meta["attr_name"] == name] kinds = set(rows["value_kind"]) + multi_kind = len(rows) > 1 series = None for row in rows.itertuples(): if row.column_name not in out: @@ -687,17 +691,21 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: col = _ns_to_time(col, "timedelta") elif row.value_kind == "bool": col = col.astype("boolean") - if len(rows) > 1: + if multi_kind: # multi-kind attrs coalesce in object space; typed # extension arrays refuse cross-dtype fills col = col.astype(object).where(col.notna(), np.nan) series = col if series is None else series.where(series.notna(), col) - out = out.drop(columns=[row.column_name]) + cols_to_drop.append(row.column_name) if series is not None: if kinds == {"str"}: # flat-contract convention: missing strings are "" series = series.fillna("") - out[name] = series + new_columns[name] = series + if cols_to_drop: + out = out.drop(columns=cols_to_drop) + for name, series in new_columns.items(): + out[name] = series # flat-contract names for source columns renames = { "source_path": "path", From 1b8d984f819b10bf8ea5a0f3e17bb0d414a60e27 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:31:40 +0200 Subject: [PATCH 43/97] Delete sources via FK cascade; stream executemany params Source deletion fetched matching source ids, then issued four DELETEs per id batch (patch_coords, attrs, patches, sources) even though the schema already declares ON DELETE CASCADE on those relations and the connection enables foreign keys. Delete the sources directly by (base_uri, source_path) and let the cascade remove dependents; the intentional coord_defs orphaning is preserved. The cascade test now asserts dependent rows are actually gone, not just filtered from the query. Also stream _executemany's adapted parameters as a generator instead of building a second full list of each already-materialized batch. --- dascore/io/index/backend.py | 31 +++++++------------ dascore/io/index/lite.py | 4 ++- .../test_io/test_index/test_index_contract.py | 15 ++++++++- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 166d67527..8cd476214 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -525,33 +525,24 @@ def write_sources(self, records: list[SourceRecord]) -> None: _in_clause_batch = 5000 def _delete_by_paths(self, source_paths: list[str], base_uri: str = "") -> None: - """Delete sources by (base_uri, source_path) identity.""" + """ + Delete sources by (base_uri, source_path) identity. + + The schema declares sources -> patches -> attrs/patch_coords with + ON DELETE CASCADE and the connection enables foreign keys, so + deleting the sources removes every dependent row. coord_defs are + intentionally left (they may orphan; a rebuild compacts them). + """ if not source_paths: return batch = self._in_clause_batch - ids: list = [] for start in range(0, len(source_paths), batch): chunk = source_paths[start : start + batch] marks = ", ".join("?" for _ in chunk) - found = self._fetch_df( - f"SELECT source_id FROM sources WHERE source_path IN ({marks}) " - "AND base_uri = ?", + self._execute( + f"DELETE FROM sources WHERE source_path IN ({marks}) AND base_uri = ?", [*chunk, base_uri], - )["source_id"].tolist() - ids.extend(found) - for start in range(0, len(ids), batch): - chunk = ids[start : start + batch] - id_marks = ", ".join("?" for _ in chunk) - # coord_defs rows may orphan; harmless, a rebuild compacts them - for sql in ( - f"DELETE FROM patch_coords WHERE patch_id IN " - f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", - f"DELETE FROM attrs WHERE patch_id IN " - f"(SELECT patch_id FROM patches WHERE source_id IN ({id_marks}))", - f"DELETE FROM patches WHERE source_id IN ({id_marks})", - f"DELETE FROM sources WHERE source_id IN ({id_marks})", - ): - self._execute(sql, chunk) + ) def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: """Remove sources (identified by base_uri + path) and dependents.""" diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py index e3a265187..86dc3a290 100644 --- a/dascore/io/index/lite.py +++ b/dascore/io/index/lite.py @@ -115,7 +115,9 @@ def _execute(self, sql: str, params=()) -> None: self._con.execute(sql, _adapt(params)) def _executemany(self, sql: str, seq_of_params) -> None: - self._con.executemany(sql, [_adapt(p) for p in seq_of_params]) + # sqlite3.executemany consumes an iterator, so adapt lazily rather + # than materializing a second copy of each already-built batch. + self._con.executemany(sql, (_adapt(p) for p in seq_of_params)) def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: # numpy_nullable assembly keeps nullable INTEGER columns exact; diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index c85a30564..bdd0105d2 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -394,11 +394,24 @@ def test_replace_source_drops_stale_rows(self, backend): assert "NEW1" in set(df["station"]) def test_delete_cascades(self, backend): - """Delete cascades.""" + """Delete cascades to patches, attrs, and coord links via the FK.""" + before = backend._fetch_df("SELECT patch_id FROM patches") + gone = backend._fetch_df( + "SELECT p.patch_id FROM patches p JOIN sources s " + "ON s.source_id = p.source_id WHERE s.source_path = 'das/file_1.h5'" + )["patch_id"].tolist() + assert gone # the source had patches to cascade-delete backend.delete_sources(["das/file_1.h5"]) df = backend.query() assert len(df) == 3 assert "das/file_1.h5" not in set(df["path"]) + # the deleted source's patches (and their dependents) are gone, + # not merely filtered out of the query. + remaining = set(backend._fetch_df("SELECT patch_id FROM patches")["patch_id"]) + assert remaining == set(before["patch_id"]) - set(gone) + for table in ("attrs", "patch_coords"): + ids = set(backend._fetch_df(f"SELECT patch_id FROM {table}")["patch_id"]) + assert not (ids & set(gone)) def test_reopen_persists(self, backend, tmp_path): """Reopen persists.""" From f720c8a5df2273c91faf17306f7abdd2bb8cc838 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:33:34 +0200 Subject: [PATCH 44/97] Narrow incremental-update change detection to stat columns A directory update fetched the full eight-column sources table just to compare mtime and size, and built whole-archive mtime/size maps for summaries_to_records even when only a handful of paths changed. For a large mostly-unchanged directory those turned a tiny incremental update into another O(total sources) allocation. Add a backend source_stats() projection returning only (source_path, mtime_ns, size_bytes) for change detection, and build the scan stat maps from the changed paths alone. A no-change-update test forbids the wide get_sources() fetch. --- dascore/io/index/backend.py | 12 +++++++++--- dascore/io/index/indexer.py | 13 +++++++++---- tests/test_io/test_index/test_db_dirspool.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 8cd476214..0d073061e 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -118,6 +118,10 @@ def export_records(self, patch_ids=None) -> list: def get_sources(self) -> pd.DataFrame: """Return the sources table.""" + @abc.abstractmethod + def source_stats(self) -> pd.DataFrame: + """Return only (source_path, mtime_ns, size_bytes) for change checks.""" + @abc.abstractmethod def get_metadata(self) -> dict: """Return index-level metadata.""" @@ -636,9 +640,7 @@ def export_records(self, patch_ids=None) -> list: if not links.empty else [] ) - defs = self._fetch_in( - "SELECT * FROM coord_defs", "coord_def_id", def_ids - ) + defs = self._fetch_in("SELECT * FROM coord_defs", "coord_def_id", def_ids) return assemble_source_records( sources, patches, attrs, links, defs, self._attr_meta() ) @@ -845,6 +847,10 @@ def get_sources(self) -> pd.DataFrame: """Return the sources table.""" return self._fetch_df("SELECT * FROM sources") + def source_stats(self) -> pd.DataFrame: + """Return only the columns incremental change detection needs.""" + return self._fetch_df("SELECT source_path, mtime_ns, size_bytes FROM sources") + def get_metadata(self) -> dict: """Return index-level metadata.""" return self._fetch_df("SELECT * FROM meta_data").iloc[0].to_dict() diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index d986d92ea..7955d873e 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -217,7 +217,7 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: if pd.isnull(row.mtime_ns) else (int(row.mtime_ns), int(row.size_bytes)) ) - for row in self._backend.get_sources().itertuples() + for row in self._backend.source_stats().itertuples() } stale = [path for path in stored if path not in files] changed = [ @@ -240,14 +240,19 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: if stale: self._backend.delete_sources(stale) if changed: - scan_paths = [files[rel][2] for rel in changed] + # Only changed paths are rescanned, so the stat maps handed to + # summaries_to_records need only cover them — not the whole + # archive (a large mostly-unchanged directory otherwise built + # full-archive mtime/size maps for a tiny update). + changed_stats = {rel: files[rel] for rel in changed} + scan_paths = [stat[2] for stat in changed_stats.values()] summaries = dc.scan(scan_paths, progress=progress) # scan reports absolute source paths; stat maps use them too records = summaries_to_records( summaries, relative_to=str(self.path), - mtimes_ns={str(p): m for _, (m, _, p) in files.items()}, - sizes_bytes={str(p): s for _, (_, s, p) in files.items()}, + mtimes_ns={str(p): m for (m, _, p) in changed_stats.values()}, + sizes_bytes={str(p): s for (_, s, p) in changed_stats.values()}, ) # Every visited path gets a sources row, even when scanning # produced no patches (e.g. a non-fiber file). Otherwise such diff --git a/tests/test_io/test_index/test_db_dirspool.py b/tests/test_io/test_index/test_db_dirspool.py index db42715ed..3e733574a 100644 --- a/tests/test_io/test_index/test_db_dirspool.py +++ b/tests/test_io/test_index/test_db_dirspool.py @@ -106,3 +106,20 @@ def test_deleted_file_removed(self, fresh): target.unlink() updated = spool.update(progress=None) assert len(updated) == 2 + + def test_no_change_update_uses_narrow_projection(self, fresh): + """A no-op update reads only the stat columns, not the wide table.""" + path, spool = fresh + backend = spool.indexer._backend + + def _boom(self): + raise AssertionError("wide get_sources() during no-change update") + + # A no-change update must not fetch the full sources table. + original = type(backend).get_sources + type(backend).get_sources = _boom + try: + reupdated = spool.update(progress=None) + finally: + type(backend).get_sources = original + assert len(reupdated) == 3 From c71c8bccdec01a8b22d95248c0139e205f270f31 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:36:16 +0200 Subject: [PATCH 45/97] Move relative-select helpers to utils.pd relative_offset and relative_ranges_to_absolute operate only on a dataframe's envelope columns and are used by generic DataFrameSpool selection, but lived in io.index.query, forcing core.spool to import the index query builder for backend-independent relative selection. Move them beside adjust_segments/filter_df in utils.pd; the catalog and spool both import them from there. Restores a clean dependency direction (core -> generic pd utilities, not core -> io backend). The catalog-pushdown test now realizes via get_contents to inspect the composed query, since len() counts in SQL and no longer fetches rows. --- dascore/core/spool.py | 2 +- dascore/io/index/catalog.py | 3 +- dascore/io/index/query.py | 43 ---------------------- dascore/utils/pd.py | 45 ++++++++++++++++++++++- tests/test_core/test_spool_select_spec.py | 4 +- 5 files changed, 49 insertions(+), 48 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index d776960f2..4f5155453 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -1002,7 +1002,7 @@ def _resolve_select_kwargs(self, _attrs, _coords, kwargs) -> dict: def _relative_select_kwargs(self, kwargs: dict) -> dict: """Resolve relative bounds against the spool's global envelopes.""" - from dascore.io.index.query import relative_ranges_to_absolute + from dascore.utils.pd import relative_ranges_to_absolute return relative_ranges_to_absolute(self._df, kwargs) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index a019a45b5..fe910c7ae 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -32,10 +32,9 @@ from dascore.io.index.query import ( InvalidSpoolQueryError, Query, - relative_ranges_to_absolute, ) from dascore.utils.misc import is_memory_uri -from dascore.utils.pd import adjust_segments +from dascore.utils.pd import adjust_segments, relative_ranges_to_absolute class _CanonicalRange: diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 3a669b4cd..0f64a4d37 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -457,49 +457,6 @@ def apply_residuals( return df -def relative_offset(gmin, gmax, value): - """ - Resolve one relative bound against a global [gmin, gmax] envelope. - - Positive offsets measure from the start, negative from the end; - None/Ellipsis bounds stay open. Datetime envelopes take numeric - seconds offsets. - """ - import dascore as dc - - if value is None or value is Ellipsis: - return None - if isinstance(gmin, pd.Timestamp) or isinstance(gmin, np.datetime64): - delta = dc.to_timedelta64(abs(float(value))) - return (gmin + delta) if value >= 0 else (gmax - delta) - return (gmin + value) if value >= 0 else (gmax + value) - - def glob_match(value, pattern: str) -> bool: """Reference glob semantics (used by pandas fallbacks and tests).""" return isinstance(value, str) and fnmatch.fnmatch(value, pattern) - - -def relative_ranges_to_absolute(df, kwargs: dict) -> dict: - """ - Resolve relative (start, stop) ranges against a frame's global envelopes. - - Shared by the dataframe and catalog select paths so the relative-select - contract has exactly one implementation. - """ - out = {} - for name, value in kwargs.items(): - lo_col, hi_col = f"{name}_min", f"{name}_max" - if lo_col not in df.columns or df.empty: - msg = f"Cannot use relative select on {name!r}." - raise InvalidSpoolQueryError(msg) - if not (isinstance(value, tuple) and len(value) == 2): - msg = f"relative=True requires (start, stop) ranges, got {value!r}." - raise InvalidSpoolQueryError(msg) - gmin, gmax = df[lo_col].min(), df[hi_col].max() - lo, hi = value - out[name] = ( - relative_offset(gmin, gmax, lo), - relative_offset(gmin, gmax, hi), - ) - return out diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index 8ce9ce9e0..57de4118a 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -15,7 +15,7 @@ import dascore as dc from dascore.constants import PatchType from dascore.core.attrs import PatchAttrs -from dascore.exceptions import ParameterError +from dascore.exceptions import InvalidSpoolQueryError, ParameterError from dascore.utils.misc import order_range_tuple, sanitize_range_param from dascore.utils.time import to_datetime64, to_timedelta64 @@ -26,6 +26,49 @@ def get_regex(seed_str): return fnmatch.translate(seed_str) # translate to re +def relative_offset(gmin, gmax, value): + """ + Resolve one relative bound against a global [gmin, gmax] envelope. + + Positive offsets measure from the start, negative from the end; + None/Ellipsis bounds stay open. Datetime envelopes take numeric + seconds offsets. + """ + if value is None or value is Ellipsis: + return None + if isinstance(gmin, pd.Timestamp) or isinstance(gmin, np.datetime64): + delta = to_timedelta64(abs(float(value))) + return (gmin + delta) if value >= 0 else (gmax - delta) + return (gmin + value) if value >= 0 else (gmax + value) + + +def relative_ranges_to_absolute(df, kwargs: dict) -> dict: + """ + Resolve relative (start, stop) ranges against a frame's global envelopes. + + Operates only on the dataframe's `{name}_min`/`{name}_max` envelope + columns, so both the generic dataframe select path and the catalog + share one relative-select implementation without either depending on + the index query builder. + """ + out = {} + for name, value in kwargs.items(): + lo_col, hi_col = f"{name}_min", f"{name}_max" + if lo_col not in df.columns or df.empty: + msg = f"Cannot use relative select on {name!r}." + raise InvalidSpoolQueryError(msg) + if not (isinstance(value, tuple) and len(value) == 2): + msg = f"relative=True requires (start, stop) ranges, got {value!r}." + raise InvalidSpoolQueryError(msg) + gmin, gmax = df[lo_col].min(), df[hi_col].max() + lo, hi = value + out[name] = ( + relative_offset(gmin, gmax, lo), + relative_offset(gmin, gmax, hi), + ) + return out + + def _remove_base_path(series: pd.Series, base="") -> pd.Series: """ Ensure paths stored in column name use unix style paths and have base diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 8359fafb7..676de42d7 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -83,7 +83,9 @@ def wrapped(query=None): monkeypatch.setattr(backend, "query", wrapped) selected = spool.select(time=("2020-01-03", "2020-01-04")) assert calls == [] - assert len(selected) + # realizing the relation (get_contents) runs the composed query; + # len() alone counts in SQL and never fetches rows. + selected.get_contents() queries = calls[0] assert isinstance(queries, list) assert queries[0].coords["time"] == ("2020-01-03", "2020-01-04") From 199deb6f6dfa2f9fd33a4e54365b46ff504c7064 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:43:02 +0200 Subject: [PATCH 46/97] Relocate backend-independent helpers out of the IO index package The chunk planner and two path classifiers are backend-independent but lived under io.index / io.indexer, forcing core spool behavior to depend on the IO index package. - io/index/plan.py -> utils/chunk_plan.py (the planner is used by every DataFrameSpool and only imports generic utils). - is_memory_uri and _directory_writable -> utils/paths.py beside the other path classifiers; _directory_writable becomes public directory_writable. - is_memory_uri now matches the exact memorypatch:// / memory:// schemes instead of any string starting with 'memory', so a real file named e.g. memory_notes.h5 is not misclassified. Imports and doc-reference paths updated; behavior is unchanged. --- dascore/core/spool.py | 8 ++--- dascore/io/index/catalog.py | 2 +- dascore/io/index/indexer.py | 5 ++- dascore/io/indexer.py | 15 --------- dascore/utils/chunk.py | 2 +- .../{io/index/plan.py => utils/chunk_plan.py} | 2 +- dascore/utils/misc.py | 11 ------- dascore/utils/patch.py | 2 +- dascore/utils/paths.py | 31 +++++++++++++++++++ tests/test_io/test_index/test_plan.py | 2 +- tests/test_utils/test_chunk.py | 2 +- 11 files changed, 43 insertions(+), 39 deletions(-) rename dascore/{io/index/plan.py => utils/chunk_plan.py} (99%) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 4f5155453..ae640b334 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -807,7 +807,7 @@ def _as_catalog_member(self): def _chunk_working_df(self) -> pd.DataFrame: """Return the source rows the chunk planner consumes.""" - from dascore.io.index.plan import _ensure_patch_id + from dascore.utils.chunk_plan import _ensure_patch_id source = self._source_df working = source.drop(columns=list(self._drop_columns), errors="ignore") @@ -829,7 +829,7 @@ def chunk_plan( """ Return the plan `chunk` would execute, without touching any data. - The returned [`ChunkPlan`](`dascore.io.index.plan.ChunkPlan`) is a + The returned [`ChunkPlan`](`dascore.utils.chunk_plan.ChunkPlan`) is a read-only diagnostic: its `outputs` table describes each patch the chunked spool would contain (envelopes, step, carried attributes), its `members` table shows exactly which slice of which source patch @@ -848,7 +848,7 @@ def chunk_plan( >>> members = plan.members >>> first = members[members["output_id"] == 0] """ - from dascore.io.index.plan import build_chunk_plan + from dascore.utils.chunk_plan import build_chunk_plan return build_chunk_plan( self._chunk_working_df(), @@ -875,7 +875,7 @@ def chunk( **kwargs, ) -> Self: """{doc}""" - from dascore.io.index.plan import build_chunk_plan + from dascore.utils.chunk_plan import build_chunk_plan source = self._source_df working = self._chunk_working_df() diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index fe910c7ae..df3d07a0f 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -33,7 +33,7 @@ InvalidSpoolQueryError, Query, ) -from dascore.utils.misc import is_memory_uri +from dascore.utils.paths import is_memory_uri from dascore.utils.pd import adjust_segments, relative_ranges_to_absolute diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 7955d873e..6adcfdada 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -23,12 +23,11 @@ from dascore.io.index.ingest import SourceRecord, summaries_to_records from dascore.io.indexer import ( AbstractIndexer, - _directory_writable, _get_index_map, _update_index_map, ) from dascore.utils.misc import _iter_filesystem -from dascore.utils.paths import requires_local_directory +from dascore.utils.paths import directory_writable, requires_local_directory # Structural columns the spool machinery must not see: unique-per-patch # values block chunk merge-compatibility grouping, which compares all @@ -115,7 +114,7 @@ def _find_index_path(self, index_path=None) -> Path: # a fresh SQLite index is built in their place. if not self._is_legacy_or_foreign_index(mapped): return mapped - if not _directory_writable(self.path): + if not directory_writable(self.path): name = f"_dascore_index_{abs(hash(self.path))}.sqlite3" index_path = self.index_map_path.parent / name _update_index_map( diff --git a/dascore/io/indexer.py b/dascore/io/indexer.py index 9f703cfa5..7644e9d85 100644 --- a/dascore/io/indexer.py +++ b/dascore/io/indexer.py @@ -11,7 +11,6 @@ import abc import json -import os from contextlib import suppress from functools import cache from pathlib import Path @@ -53,20 +52,6 @@ def _update_index_map(updates, cache_path) -> dict: return data -def _directory_writable(path): - """Return True if the directory is writable else False.""" - name = "._dascore_write_test_delete_me" - path = Path(path) / name - path.parent.mkdir(exist_ok=True, parents=True) - try: - open(path, "w").close() - except (PermissionError, IsADirectoryError): - return False - else: - os.remove(path) - return True - - class AbstractIndexer: """ A base class for indexers. diff --git a/dascore/utils/chunk.py b/dascore/utils/chunk.py index c5ff8d1f3..fa9af91e5 100644 --- a/dascore/utils/chunk.py +++ b/dascore/utils/chunk.py @@ -1,7 +1,7 @@ """Utilities for chunking dataframes. The interval math here is consumed by the chunk planner -(`dascore.io.index.plan`), which replaced the old ChunkManager. +(`dascore.utils.chunk_plan`), which replaced the old ChunkManager. """ from __future__ import annotations diff --git a/dascore/io/index/plan.py b/dascore/utils/chunk_plan.py similarity index 99% rename from dascore/io/index/plan.py rename to dascore/utils/chunk_plan.py index a6a911934..d6edfdee9 100644 --- a/dascore/io/index/plan.py +++ b/dascore/utils/chunk_plan.py @@ -4,7 +4,7 @@ Implements the "Chunking formalities" spec: the planner consumes the catalog's flat relation (one row per patch: `{dim}_min/max/step` envelopes, `_{dim}_def_key` structural identity, attr columns) and produces a -[`ChunkPlan`](`dascore.io.index.plan.ChunkPlan`) — an outputs table (one row +[`ChunkPlan`](`dascore.utils.chunk_plan.ChunkPlan`) — an outputs table (one row per output patch) plus a members table binding each output to trimmed slices of source patches. No patch data is touched; assembly happens later. diff --git a/dascore/utils/misc.py b/dascore/utils/misc.py index bac28c4c9..dd161159c 100644 --- a/dascore/utils/misc.py +++ b/dascore/utils/misc.py @@ -1123,14 +1123,3 @@ def tukey_fence(data, fence_multiplier=1.5) -> np.ndarray: q_upper = np.nanmin([q3 + diff * fence_multiplier, dmax]) lower_and_top = np.asarray([q_lower, q_upper]) return lower_and_top - - -def is_memory_uri(path) -> bool: - """ - Return True if a path is a synthetic in-memory patch identity. - - Live patches are identified by memory:// or memorypatch:// paths - (see `dascore.io.index.catalog`); such paths dispatch to in-memory - registries and are never treated as file names. - """ - return str(path).startswith("memory") diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index 6b830b4ab..3a2fcd0b0 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -42,12 +42,12 @@ _apply_union_indexers, _merge_tuples, get_middle_value, - is_memory_uri, iterate, to_object_array, warn_or_raise, yield_sub_sequences, ) +from dascore.utils.paths import is_memory_uri from dascore.utils.time import to_float attr_type = dict[str, Any] | str | Sequence[str] | None diff --git a/dascore/utils/paths.py b/dascore/utils/paths.py index 24e76374e..b3cc35dd5 100644 --- a/dascore/utils/paths.py +++ b/dascore/utils/paths.py @@ -2,17 +2,48 @@ from __future__ import annotations +import os from pathlib import Path from dascore.compat import UPath from dascore.exceptions import InvalidSpoolError +# Synthetic URI schemes for in-memory patch identities (see +# dascore.io.index.catalog); such paths dispatch to in-memory registries +# and are never treated as file names. +_MEMORY_SCHEMES = ("memorypatch://", "memory://") + def is_pathlike(resource) -> bool: """Return True if resource is supported path-like input.""" return isinstance(resource, str | Path | UPath) +def is_memory_uri(path) -> bool: + """ + Return True if a path is a synthetic in-memory patch identity. + + Matches the exact ``memorypatch://`` / ``memory://`` schemes rather + than any string beginning with "memory", so a real file or directory + named e.g. ``memory_notes.h5`` is not misclassified. + """ + return str(path).startswith(_MEMORY_SCHEMES) + + +def directory_writable(path) -> bool: + """Return True if the directory is writable else False.""" + name = "._dascore_write_test_delete_me" + probe = Path(path) / name + probe.parent.mkdir(exist_ok=True, parents=True) + try: + open(probe, "w").close() + except (PermissionError, IsADirectoryError): + return False + else: + os.remove(probe) + return True + + def coerce_to_upath(resource) -> UPath: """Return a UPath for path-like resources.""" return resource if isinstance(resource, UPath) else UPath(resource) diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index 721129fb8..501da81bc 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -14,7 +14,7 @@ ParameterError, ) from dascore.io.index.catalog import PatchCatalog -from dascore.io.index.plan import ChunkPlan, build_chunk_plan +from dascore.utils.chunk_plan import ChunkPlan, build_chunk_plan from dascore.utils.time import to_timedelta64 ONE_S = np.timedelta64(1, "s") diff --git a/tests/test_utils/test_chunk.py b/tests/test_utils/test_chunk.py index 1b984f3fb..3f12ada00 100644 --- a/tests/test_utils/test_chunk.py +++ b/tests/test_utils/test_chunk.py @@ -10,8 +10,8 @@ import dascore as dc from dascore.exceptions import ChunkError -from dascore.io.index.plan import build_chunk_plan from dascore.utils.chunk import get_intervals +from dascore.utils.chunk_plan import build_chunk_plan from dascore.utils.time import to_timedelta64 STARTTIME = np.datetime64("2020-01-03") From 808c37704f342bd847c8f8527e0d6e47f9ff0f9b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:48:10 +0200 Subject: [PATCH 47/97] Centralize source-patch-id normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Missing source ids arrive as None, empty strings, pandas NaN, and numpy scalars, and were normalized ad hoc in core.summary, io.core, ingest, and the catalog resolver — with pandas NaN being truthy, a plain 'value or ""' silently kept NaN (a bug the catalog resolver already had to special-case). Add core.summary.normalize_source_patch_id as the single normalizer and route every site through it. Also fixes _user_stacklevel in the relocated chunk planner to resolve the dascore package directory from the package itself instead of a hard-coded parents[2], which pointed one level too shallow after the utils/chunk_plan.py move and blamed pytest for the #662 gap warning. --- dascore/core/summary.py | 28 ++++++++++++++++++++++++---- dascore/io/core.py | 9 +++++---- dascore/io/index/catalog.py | 10 +++------- dascore/io/index/ingest.py | 6 +++--- dascore/utils/chunk_plan.py | 4 +++- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/dascore/core/summary.py b/dascore/core/summary.py index 1f96952a6..a71be9540 100644 --- a/dascore/core/summary.py +++ b/dascore/core/summary.py @@ -10,6 +10,7 @@ from typing import Any import numpy as np +import pandas as pd from pydantic import ConfigDict, Field, model_validator import dascore as dc @@ -20,6 +21,27 @@ from dascore.utils.paths import coerce_to_upath, is_pathlike +def normalize_source_patch_id(value: Any) -> str: + """ + Return a source patch id as a clean string ("" when missing). + + Missing ids arrive as None, empty strings, pandas NaN/NaT, or numpy + scalars. pandas NaN is truthy, so a plain ``value or ""`` does not + normalize it — every conversion site must go through this helper to + avoid the NaN-truthiness bug the catalog resolver already had to fix. + """ + if value is None or value == "": + return "" + try: + if pd.isnull(value): + return "" + except (TypeError, ValueError): + pass # non-scalar (e.g. an array): fall through to str() + if hasattr(value, "item"): # numpy scalar -> python scalar + value = value.item() + return str(value) + + def _to_coord_summary(value: Any, dims: tuple[str, ...] = ()) -> CoordSummary: """Normalize a coordinate summary input.""" # Summary inputs can already be normalized, coord-like objects, or exact @@ -169,10 +191,8 @@ def _normalize_source_patch_id( attrs: PatchAttrs, source_patch_id: Any = "" ) -> tuple[PatchAttrs, str]: """Normalize summary and private attr source ids to one value.""" - summary_source_patch_id = ( - "" if source_patch_id in (None, "") else str(source_patch_id) - ) - attrs_source_patch_id = str(attrs.get("_source_patch_id", "") or "") + summary_source_patch_id = normalize_source_patch_id(source_patch_id) + attrs_source_patch_id = normalize_source_patch_id(attrs.get("_source_patch_id", "")) normalized = summary_source_patch_id or attrs_source_patch_id if normalized: attrs = attrs.update(_source_patch_id=normalized) diff --git a/dascore/io/core.py b/dascore/io/core.py index a1f23c155..b14aad5b7 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -32,7 +32,7 @@ from dascore.core.attrs import PatchAttrs, str_validator from dascore.core.coordmanager import CoordManager from dascore.core.spool import DataFrameSpool -from dascore.core.summary import PatchSummary +from dascore.core.summary import PatchSummary, normalize_source_patch_id from dascore.exceptions import ( DependencyError, InvalidFiberFileError, @@ -264,9 +264,9 @@ def _resolve_read_spool(spool, source_patch_id: object = "") -> dc.Patch: patch without preserving that reload metadata on it; only trust that when the patch doesn't claim a different identity. """ - source_patch_id = str(source_patch_id or "") + source_patch_id = normalize_source_patch_id(source_patch_id) if source_patch_id and len(spool) == 1: - found = str(spool[0].attrs.get("_source_patch_id", "") or "") + found = normalize_source_patch_id(spool[0].attrs.get("_source_patch_id", "")) if found in ("", source_patch_id): return spool[0] return _select_patch_from_spool(spool, source_patch_id=source_patch_id) @@ -287,7 +287,8 @@ def _select_patch_from_spool(spool, source_patch_id: object = "") -> dc.Patch: matches = [ patch for patch in spool - if str(patch.attrs.get("_source_patch_id", "") or "") == source_patch_id + if normalize_source_patch_id(patch.attrs.get("_source_patch_id", "")) + == source_patch_id ] if len(matches) == 1: return matches[0] diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index df3d07a0f..bbd5bdbcd 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -26,6 +26,7 @@ import dascore as dc from dascore.constants import PROGRESS_LEVELS +from dascore.core.summary import normalize_source_patch_id from dascore.exceptions import MissingPatchError from dascore.io.index.backend import get_backend, resolve_query from dascore.io.index.ingest import SourceRecord, patch_record @@ -122,13 +123,8 @@ def _canonical_coord_selectors(backend, coords: dict) -> tuple[dict, dict]: def _row_source_patch_id(row: Mapping) -> str: - """Return the row's source_patch_id as a string ("" when missing). - - Rows fetched through pandas represent missing text values as NaN, - which is truthy, so a plain `or ""` does not normalize them. - """ - value = row.get("source_patch_id") - return "" if value is None or pd.isnull(value) else str(value) + """Return the row's source_patch_id as a normalized string.""" + return normalize_source_patch_id(row.get("source_patch_id")) class PatchResolver(abc.ABC): diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 4dcbb5af4..00b3c88f6 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -18,7 +18,7 @@ import numpy as np import pandas as pd -from dascore.core.summary import PatchSummary +from dascore.core.summary import PatchSummary, normalize_source_patch_id from dascore.io.index.schema import KINDS, RESERVED_ATTR_COLUMNS from dascore.units import get_quantity from dascore.utils.time import to_datetime64, to_int, to_timedelta64 @@ -325,7 +325,7 @@ def patch_record(summary: PatchSummary) -> PatchRecord: dist_min, dist_max, dist_step = _envelope(coords, "distance", "num") shape = tuple(int(x) for x in summary.shape) return PatchRecord( - source_patch_id=summary.source_patch_id or "", + source_patch_id=normalize_source_patch_id(summary.source_patch_id), dims=",".join(summary.dims), shape=",".join(str(x) for x in shape), n_dims=len(summary.dims), @@ -490,7 +490,7 @@ def assemble_source_records( ) patch_records.append( PatchRecord( - source_patch_id=_py_scalar(patch.source_patch_id) or "", + source_patch_id=normalize_source_patch_id(patch.source_patch_id), dims=_py_scalar(patch.dims) or "", shape=_py_scalar(patch.shape) or "", n_dims=_py_scalar(patch.n_dims), diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index d6edfdee9..f08ef7106 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -189,7 +189,9 @@ def _user_stacklevel() -> int: """ import inspect - package_dir = str(Path(__file__).resolve().parents[2]) + # The dascore package directory, resolved from the package itself so + # this does not depend on this module's location within it. + package_dir = str(Path(dc.__file__).resolve().parent) # Frames after this helper's own align exactly with warn's numbering: # level 1 is the frame calling warn. for level, frame_info in enumerate(inspect.stack()[1:], start=1): From 10aa5a71df68987a637d9ffe02d67b20412b8820 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:54:30 +0200 Subject: [PATCH 48/97] Read each catalog row's patch through a single dc.read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FileResolver._read tried the recorded FiberIO directly and, whenever that reader returned anything other than a MemorySpool, discarded the result and called dc.read — reading the file a second time. dc.read already skips format probing when the format and version are supplied and returns the reader's spool directly, so the fast path was pure redundancy. Read once through dc.read. Also finishes routing io.core's remaining source-patch-id conversions through normalize_source_patch_id. The load-path tests now assert a single read and that the recorded format/version are forwarded (empty values omitted so dc.read detects them). --- dascore/io/core.py | 13 ++-- dascore/io/index/catalog.py | 26 +++---- tests/test_clients/test_dirspool.py | 117 +++++++++++----------------- 3 files changed, 63 insertions(+), 93 deletions(-) diff --git a/dascore/io/core.py b/dascore/io/core.py index b14aad5b7..ae1c058d9 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -136,9 +136,7 @@ def _make_scan_payload( "dims": tuple(dims), "shape": tuple(shape), "dtype": str(dtype), - "source_patch_id": "" - if source_patch_id in (None, "") - else str(source_patch_id), + "source_patch_id": normalize_source_patch_id(source_patch_id), } @@ -169,7 +167,10 @@ def _scan_payload_to_summary( source_path=source_path, source_format=source_format, source_version=source_version, - source_patch_id=source_patch_id or payload.get("source_patch_id") or "", + source_patch_id=( + normalize_source_patch_id(source_patch_id) + or normalize_source_patch_id(payload.get("source_patch_id")) + ), ) @@ -190,9 +191,7 @@ def _scan_result_to_summary( normalized_source_path = "" if source_path in (None, "") else source_path normalized_source_format = "" if source_format in (None, "") else source_format normalized_source_version = "" if source_version in (None, "") else source_version - summary_source_patch_id = ( - "" if source_patch_id in (None, "") else str(source_patch_id) - ) + summary_source_patch_id = normalize_source_patch_id(source_patch_id) if isinstance(patch_summary, Mapping): return _scan_payload_to_summary( patch_summary, diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index bbd5bdbcd..e795a2a4c 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -190,24 +190,20 @@ def __init__(self, root: Path | str | None = None): self._root = Path(root) if root is not None else None def _read(self, path, row: Mapping, trim: dict, source_patch_id: str): - """Use a known FiberIO directly, falling back to format detection.""" - from dascore.core.spool import MemorySpool + """ + Read one row's patch through dc.read. - file_format = row.get("file_format") - file_version = row.get("file_version") + The recorded format/version are forwarded so dc.read skips format + probing; it reads the file exactly once (an earlier fast path that + called the reader directly re-read the file whenever the reader + returned a non-MemorySpool). + """ id_kwargs = {"source_patch_id": source_patch_id} if source_patch_id else {} - if file_format and file_version: - fiber_io = dc.io.FiberIO.manager.get_fiberio( - format=file_format, version=file_version - ) - spool = fiber_io.read(path, **id_kwargs, **trim) - if isinstance(spool, MemorySpool): - return spool kwargs = {"path": path} - if file_format: - kwargs["file_format"] = file_format - if file_version: - kwargs["file_version"] = file_version + if row.get("file_format"): + kwargs["file_format"] = row["file_format"] + if row.get("file_version"): + kwargs["file_version"] = row["file_version"] return dc.read(**kwargs, **id_kwargs, **trim) def resolve(self, row: Mapping, **trim) -> dc.Patch: diff --git a/tests/test_clients/test_dirspool.py b/tests/test_clients/test_dirspool.py index 6536721b2..f5c52eda9 100644 --- a/tests/test_clients/test_dirspool.py +++ b/tests/test_clients/test_dirspool.py @@ -145,75 +145,54 @@ def test_merge(self, multi_patch_file_spool): class TestLoadPatchFastPath: - """Tests for the direct FiberIO read path owned by FileResolver.""" + """FileResolver reads each row's patch through a single dc.read call.""" - def test_requires_concrete_format_and_version( + def test_forwards_recorded_format_and_version( self, one_directory_spool, monkeypatch ): - """ - Without a concrete format and version the fast path must defer to - dc.read, which detects them from the file; get_fiberio with a None - version would return the newest reader, not the file's version. - """ + """The recorded format/version are forwarded so dc.read skips probing.""" resolver = one_directory_spool._catalog.resolver - sentinel = object() - monkeypatch.setattr( - "dascore.io.index.catalog.dc.read", lambda **kwargs: sentinel - ) - monkeypatch.setattr( - dc.io.FiberIO.manager, - "get_fiberio", - lambda **kwargs: pytest.fail("FiberIO fast path should not run"), - ) - assert resolver._read("path", {"file_format": ""}, {}, "") is sentinel - assert resolver._read("path", {"file_format": "DASDAE"}, {}, "") is sentinel - assert resolver._read("path", {"file_version": "1"}, {}, "") is sentinel - - def test_unusual_fiberio_spool_defers_to_generic_read( - self, one_directory_spool, monkeypatch - ): - """Fast path should defer if the reader returns a non-memory spool.""" - - class _Reader: - def read(self, *args, **kwargs): - return () - - monkeypatch.setattr( - dc.io.FiberIO.manager, - "get_fiberio", - lambda format, version: _Reader(), - ) - row = { - "file_format": "DASDAE", - "file_version": "1", - } + calls = [] + + def _fake_read(**kwargs): + calls.append(kwargs) + return object() + + monkeypatch.setattr("dascore.io.index.catalog.dc.read", _fake_read) + resolver._read("path", {"file_format": "DASDAE", "file_version": "1"}, {}, "") + assert calls[-1]["file_format"] == "DASDAE" + assert calls[-1]["file_version"] == "1" + # empty format/version are simply omitted (dc.read detects them) + resolver._read("path", {"file_format": ""}, {}, "") + assert "file_format" not in calls[-1] + + def test_reads_file_once(self, one_directory_spool, monkeypatch): + """A row's patch is read exactly once regardless of the reader's return.""" + calls = [] + + def _fake_read(**kwargs): + calls.append(kwargs) + return () # an unusual (empty) reader return + + monkeypatch.setattr("dascore.io.index.catalog.dc.read", _fake_read) + row = {"file_format": "DASDAE", "file_version": "1"} resolver = one_directory_spool._catalog.resolver - sentinel = object() - monkeypatch.setattr( - "dascore.io.index.catalog.dc.read", lambda **kwargs: sentinel - ) - assert resolver._read("path", row, {}, "") is sentinel + resolver._read("path", row, {}, "") + assert len(calls) == 1 - def test_multi_patch_resolves_identity_without_second_read( + def test_multi_patch_resolves_identity_with_single_read( self, one_directory_spool, random_patch, monkeypatch ): - """Multi-patch reads resolve source identity from the loaded spool.""" - - class _Reader: - def read(self, *args, **kwargs): - patch_1 = random_patch.update_attrs(_source_patch_id="first") - patch_2 = random_patch.update_attrs(_source_patch_id="second") - return dc.spool([patch_1, patch_2]) - - monkeypatch.setattr( - dc.io.FiberIO.manager, - "get_fiberio", - lambda format, version: _Reader(), - ) - monkeypatch.setattr( - "dascore.io.index.catalog.dc.read", - lambda **kwargs: pytest.fail("must not re-read the file"), - ) + """Multi-patch reads resolve source identity from one dc.read call.""" + patch_1 = random_patch.update_attrs(_source_patch_id="first") + patch_2 = random_patch.update_attrs(_source_patch_id="second") + reads = [] + + def _fake_read(**kwargs): + reads.append(kwargs) + return dc.spool([patch_1, patch_2]) + + monkeypatch.setattr("dascore.io.index.catalog.dc.read", _fake_read) row = { "path": "path", "file_format": "DASDAE", @@ -223,23 +202,19 @@ def read(self, *args, **kwargs): resolver = one_directory_spool._catalog.resolver patch = resolver.resolve(row) assert patch.attrs["_source_patch_id"] == "second" + assert len(reads) == 1 # the file is read exactly once def test_positional_id_reads_whole_source( self, one_directory_spool, random_patch, monkeypatch ): """Positional ids must ignore trim hints; a trimmed read would shift them.""" + patch_2 = random_patch.update_attrs(tag="second") - class _Reader: - def read(self, *args, **kwargs): - assert "time" not in kwargs, "positional ids must read untrimmed" - patch_2 = random_patch.update_attrs(tag="second") - return dc.spool([random_patch, patch_2]) + def _fake_read(**kwargs): + assert "time" not in kwargs, "positional ids must read untrimmed" + return dc.spool([random_patch, patch_2]) - monkeypatch.setattr( - dc.io.FiberIO.manager, - "get_fiberio", - lambda format, version: _Reader(), - ) + monkeypatch.setattr("dascore.io.index.catalog.dc.read", _fake_read) row = { "path": "path", "file_format": "DASDAE", From 8b8ff4aa2a5aec12cda6ce867f8e836118c101ec Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:54:30 +0200 Subject: [PATCH 49/97] Let FileResolver own directory path resolution DirectorySpool._df_to_dict_list prefixed each stored path with the spool root, duplicating the root-relative resolution FileResolver already performs (and does for the direct resolve_row path). Pass the stored relative paths through unchanged; the resolver, whose root is the spool root for both construction paths, resolves them in one place. --- dascore/clients/dirspool.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dascore/clients/dirspool.py b/dascore/clients/dirspool.py index 2b8bc3515..2d69e7647 100644 --- a/dascore/clients/dirspool.py +++ b/dascore/clients/dirspool.py @@ -123,11 +123,11 @@ def _df_to_dict_list(self, df): """ Convert the dataframe to a list of dicts for iteration. - This is significantly faster than iterating rows. + Stored (relative) paths pass through unchanged; the catalog's + FileResolver owns resolving them against the spool root, so path + resolution lives in exactly one place. """ df = df.copy(deep=False).replace("", None) - # note: need to add extra / here since we no longer store it in db. - df["path"] = (str(self.spool_path) + "/") + df["path"] return super()._df_to_dict_list(df) def _load_patch(self, kwargs) -> Self: From 219a72f35bc3f2bb19b6c52cfbfdc7d9bb2ddc62 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:57:49 +0200 Subject: [PATCH 50/97] Share one directory-format scan-unit definition The directory indexer detected directory-format scan units (XMLBinary and similar, read whole rather than by member) with its own private get_format probe, duplicating what dc.scan's traversal decides. Add io.core.is_directory_format as the single definition and route the indexer through it. The two traversal loops themselves stay separate by design: dc.scan resolves each FiberIO lazily and lets the consumer skip a directory's members, while the indexer walks eagerly to collect per-unit mtime/size stats. They serve different needs, but now agree on what a directory scan unit is. --- dascore/io/core.py | 18 ++++++++++++++++++ dascore/io/index/indexer.py | 8 +++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/dascore/io/core.py b/dascore/io/core.py index ae1c058d9..ff71630a8 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -1322,6 +1322,24 @@ def get_format( return out +def is_directory_format(path) -> bool: + """ + Return True if a directory is itself one FiberIO scan unit. + + A directory-format source (e.g. XMLBinary) is read as a whole rather + than by traversing its members. This is the single definition of that + condition; dc.scan's traversal skips such a directory's contents and + the directory indexer treats it as one stat unit. + """ + if not Path(path).is_dir(): + return False + try: + get_format(path) + except Exception: + return False + return True + + def _maybe_split_gapped_patches(spool, fiber_io, split): """Handle patches whose dimensional coords contain gaps before writing.""" from dascore.core.coords import CoordSegmented diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 6adcfdada..8c7b193c0 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -151,11 +151,9 @@ def _rel(self, path: Path) -> str: def _directory_format(self, path: Path) -> bool: """Return True when a directory is itself one FiberIO scan unit.""" - try: - dc.get_format(path) - except Exception: - return False - return True + from dascore.io.core import is_directory_format + + return is_directory_format(path) def _walk(self) -> dict[str, tuple[int, int, Path]]: """ From 2d744d4f1e01a1b3d2c1037f43e0837e17b30aa1 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 18:58:34 +0200 Subject: [PATCH 51/97] Document spool set-semantics contract; fix stale identity note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Patch identity note still said identity was 'lazily minted', which became wrong when identity moved to eager construction, and the executable cell did not assert the eager/copy-order-independent behavior that change guarantees. Rewrite it as an explicit public contract — spools of in-memory patches have set semantics by patch instance identity — with executable cells covering sequence dedup, deepcopy sharing identity, operations minting a new identity, and resolution back to the same object. --- docs/notes/spool_index.qmd | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/notes/spool_index.qmd b/docs/notes/spool_index.qmd index 7b77c5df6..88b41e6f4 100644 --- a/docs/notes/spool_index.qmd +++ b/docs/notes/spool_index.qmd @@ -44,12 +44,29 @@ required = {"_patch_id", "_time_def_key", "dims", "time_min", "time_max", "time_ assert required.issubset(df.columns) ``` -## Patch identity +## Patch identity and spool set semantics -Every in-memory patch has a lazily minted instance identity (shared by copies — patches are immutable — while every patch operation creates a new instance with its own). The identity is the patch's synthetic source path in the index, so identity, deduplication, and resolution all agree: +A spool of in-memory patches has **set semantics by patch identity**: constructing a spool from a sequence keeps one entry per distinct patch instance, and `spool + spool` unions membership. Each patch carries an instance identity minted eagerly at construction; because patches are immutable, copies (including deep copies and unpickled patches) share that identity, while every patch operation produces a new instance with its own. The identity is the patch's synthetic `memorypatch://` source path in the index, so identity, deduplication, and resolution all agree: ```{python} +import copy + patch = dc.get_example_patch() + +# a sequence with the same instance twice is one spool entry +assert len(dc.spool([patch, patch])) == 1 + +# copies share identity regardless of when they are made — identity is +# eager, so there is no access-order dependence +clone = copy.deepcopy(patch) +assert clone._instance_id == patch._instance_id +assert len(dc.spool([patch, clone])) == 1 + +# any operation mints a new, distinct identity +assert len(dc.spool([patch, patch.new()])) == 2 + +# the identity is the row's synthetic path, and it resolves back to the +# very same object cat = PatchCatalog.from_patches([patch, patch]) # one entry, not two row = cat.to_df().iloc[0] assert len(cat.to_df()) == 1 From 53f6acb0ba8107dfa10254bc901565e4bf8fabfb Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 19:01:54 +0200 Subject: [PATCH 52/97] Compute each coord's unit string once during ingest Profiling memory-spool get_contents (dominated by genuine live-patch ingest into the catalog) showed pint quantity->string formatting as a notable slice, and _coord_record str()'d summary.units twice for every numeric coordinate. Compute it once and reuse it. A Quantity-keyed cache would be faster still but is unsafe (1 m == 100 cm hash-equal with different strings), so the remaining per-coordinate conversion is inherent to recording unit strings. --- dascore/io/index/ingest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 00b3c88f6..1c61f02ed 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -250,12 +250,16 @@ def _coord_record(name: str, summary) -> CoordRecord | None: # A range summary contains its complete representation, so recover the # same exact identity a loaded CoordRange would have produced. fingerprint = summary.to_coord().fingerprint() + # str() on a pint Quantity is comparatively expensive; do it once and + # reuse it below (a Quantity-keyed cache is unsafe — 1 m == 100 cm with + # equal hashes but different strings). + units_str = str(summary.units) if summary.units is not None else None common = dict( coord_name=name, dtype=summary.dtype, coord_dims=",".join(summary.dims), length=summary.len, - units=str(summary.units) if summary.units is not None else None, + units=units_str, coord_hash=fingerprint, ) dtype = np.dtype(summary.dtype) if summary.dtype else None @@ -281,8 +285,8 @@ def _coord_record(name: str, summary) -> CoordRecord | None: ) if dtype is not None and np.issubdtype(dtype, np.number): factor = 1.0 - if summary.units is not None: - factor, base = _base_unit_info(str(summary.units)) + if units_str is not None: + factor, base = _base_unit_info(units_str) common["units"] = base step = summary.step return CoordRecord( From ac54b5890ff179c33b2717520ca8b16909ea5ebf Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 19:04:34 +0200 Subject: [PATCH 53/97] Document and centralize the spool catalog/materialized state model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DataFrameSpool implicitly coordinates a two-state machine (catalog- backed vs materialized) through raw _catalog / _catalog_native flags, which each method re-derived — including defensive getattr because the base class had no _catalog default. Give the base class a _catalog = None default, a documented state model in the class docstring, and an _is_catalog_backed() predicate, and route the length, union-member, and dataframe-build branches through it. This is the pragmatic form of the review's state-encapsulation suggestion: one authoritative predicate for the state, without restructuring the public spool classes into separate strategy objects (that larger refactor is better as its own design-led change). Behavior is unchanged. --- dascore/core/spool.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index ae640b334..8ec2f8ebe 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -479,7 +479,21 @@ def viz(self): class DataFrameSpool(BaseSpool): - """An abstract class for spools whose contents are managed by a dataframe.""" + """ + An abstract class for spools whose contents are managed by a dataframe. + + A spool is in one of two internal states: + + - **catalog-backed** (``_catalog_native`` and ``_catalog is not None``): + rows map one-to-one to a ``PatchCatalog`` query, so metadata + operations (length, selection) can stay lazy and push down to the + index. Use ``_is_catalog_backed()`` to test this and + ``_ensure_catalog()`` to enter it without realizing the relation. + - **materialized**: the managed dataframe/instruction frames are the + authoritative contents. Operations that restructure or reorder rows + (chunk, sort, slice) leave catalog-backed mode via + ``new_from_df`` (which clears ``_catalog_native``). + """ # A dataframe which represents contents as they will be output _df: pd.DataFrame = CacheDescriptor("_cache", "_get_df") @@ -494,10 +508,16 @@ class DataFrameSpool(BaseSpool): _drop_columns = ("patch",) # patch-local selections (samples=True) applied as patches load _post_selects: tuple = () + # The catalog backing this spool (None until one is built). + _catalog = None # True while rows directly represent a PatchCatalog query. Operations # which restructure/order rows switch back to the dataframe machinery. _catalog_native = False + def _is_catalog_backed(self) -> bool: + """True when rows map one-to-one to a live catalog query.""" + return self._catalog_native and self._catalog is not None + def _get_df(self): """Function to get the current df.""" @@ -560,9 +580,8 @@ def __len__(self): # it is cached, on the dataframe path, or when constructor # select_kwargs post-filter rows outside the catalog's queries. if ( - self._catalog_native + self._is_catalog_backed() and not self._select_kwargs - and getattr(self, "_catalog", None) is not None and "_df" not in self._cache ): return len(self._catalog) @@ -795,14 +814,13 @@ def _as_catalog_member(self): Restructured rows (e.g. chunked views) no longer map to sources and contribute their materialized patches instead. """ - catalog = getattr(self, "_catalog", None) - if catalog is None: + if self._catalog is None: return super()._as_catalog_member() if self._catalog_native: - return catalog, None + return self._catalog, None df = self._df if "_patch_id" in df.columns: - return catalog, df["_patch_id"].tolist() + return self._catalog, df["_patch_id"].tolist() return super()._as_catalog_member() def _chunk_working_df(self) -> pd.DataFrame: @@ -1188,7 +1206,7 @@ def __init__(self, data: PatchType | Sequence[PatchType] | None = None): def _get_df(self): """Build the managing dataframes from the input patches.""" - if self._catalog is not None and self._catalog_native: + if self._is_catalog_backed(): current = self._catalog.to_df() df, source, instruction = self._get_dummy_dataframes(current) self._source_df = source From 211dccc768fe2f17c1ea03b4631c30401053c6cc Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 19:39:20 +0200 Subject: [PATCH 54/97] Remove dead _CanonicalRange dunders; cover new-code branches _CanonicalRange.__eq__ and __repr__ were never used (only isinstance and for_patch_coord are), and __eq__ needlessly made instances unhashable; remove both. Add tests for the code this review's rounds introduced that lacked coverage: normalize_source_patch_id across every missing form (None, '', NaN/NaT, numpy scalar, non-scalar), _canonical_range's numeric vs non-numeric shapes, boolean-mask coordinate selection, empty export_records, the materialized (dataframe-path) samples/relative/ namespace selects, and union of a sorted catalog-backed member. --- dascore/io/index/catalog.py | 8 --- tests/test_core/test_spool_select_spec.py | 57 +++++++++++++++ .../test_index/test_index_edge_cases.py | 71 ++++++++++++++++++- tests/test_io/test_index/test_union.py | 15 ++++ 4 files changed, 142 insertions(+), 9 deletions(-) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index e795a2a4c..ba9b7d445 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -55,14 +55,6 @@ class _CanonicalRange: def __init__(self, magnitudes: tuple): self.magnitudes = magnitudes - def __repr__(self) -> str: - return f"_CanonicalRange({self.magnitudes!r})" - - def __eq__(self, other) -> bool: - return ( - isinstance(other, _CanonicalRange) and other.magnitudes == self.magnitudes - ) - def for_patch_coord(self, coord) -> tuple: """Return the range in the representation this coord needs.""" from dascore.units import get_quantity diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 676de42d7..81b7cebb3 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -160,6 +160,21 @@ def test_non_coord_raises(self, spool): with pytest.raises(InvalidSpoolQueryError, match="coordinate-only"): spool.select(tag="random", samples=True) + def test_samples_on_materialized_spool(self, spool): + """Samples select works after chunk (the dataframe select path).""" + # chunk first so the derived spool is materialized, not catalog-native + materialized = spool.chunk(time=None) + assert not materialized._catalog_native + out = materialized.select(distance=(0, 10), samples=True) + assert len(out) == len(materialized) + assert len(out[0].get_coord("distance")) == 10 + + def test_non_coord_on_materialized_raises(self, spool): + """The coordinate-only rule also holds on the dataframe path.""" + materialized = spool.chunk(time=None) + with pytest.raises(InvalidSpoolQueryError, match="coordinate-only"): + materialized.select(tag="random", samples=True) + class TestRelative: """relative=True resolves against the spool envelope (#362).""" @@ -186,6 +201,36 @@ def test_namespaced_coord_with_attr(self, spool): assert len(out) assert set(out.get_contents()["tag"]) == {"random"} + def test_relative_on_materialized_spool(self, spool): + """Relative select works after chunk (the dataframe select path).""" + materialized = spool.chunk(time=None) + assert not materialized._catalog_native + gmin = materialized.get_contents()["time_min"].min() + gmax = materialized.get_contents()["time_max"].max() + out = materialized.select(time=(1, -1), relative=True) + merged = out.chunk(time=None)[0] + time = merged.get_coord("time") + assert time.min() >= np.datetime64(gmin) + np.timedelta64(1, "s") + assert time.max() <= np.datetime64(gmax) - np.timedelta64(1, "s") + + +class TestMaterializedNamespaces: + """_attrs/_coords validation on the dataframe (materialized) path.""" + + def test_namespaces_and_unknown_names(self, spool): + """Namespaced selects and unknown-name errors on a chunked spool.""" + materialized = spool.chunk(time=None) + assert not materialized._catalog_native + assert len(materialized.select(_attrs={"tag": "random"})) == len(materialized) + with pytest.raises(InvalidSpoolQueryError, match="not an attribute"): + materialized.select(_attrs={"distance": (0, 10)}) + with pytest.raises(InvalidSpoolQueryError, match="not a coordinate"): + materialized.select(_coords={"tag": "random"}) + with pytest.raises(InvalidSpoolQueryError, match="both"): + materialized.select(tag="random", _attrs={"tag": "random"}) + with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): + materialized.select(nope=1) + class TestExistingBehaviorKept: """The conventional selections still work.""" @@ -313,3 +358,15 @@ def test_chained_views(self, ft_patch): ) assert float(coord.min()) >= 65 assert float(coord.max()) <= 197 + + def test_boolean_mask_selectors(self, ft_patch): + """Boolean masks (array and list) select coordinates patch-locally.""" + coord = ft_patch.get_coord("distance") + mask = np.zeros(len(coord), dtype=bool) + mask[:5] = True + # ndarray mask + got = dc.spool([ft_patch]).select(distance=mask) + assert len(got[0].get_coord("distance")) == 5 + # equivalent list-of-bools mask + got_list = dc.spool([ft_patch]).select(distance=list(mask)) + assert len(got_list[0].get_coord("distance")) == 5 diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 4a602463e..5241afde8 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -31,7 +31,76 @@ summaries_to_records as s2r, ) from dascore.io.index.query import InvalidSpoolQueryError, glob_match -from dascore.units import get_quantity +from dascore.units import get_quantity, m + + +class TestNormalizeSourcePatchId: + """The single source-patch-id normalizer handles every missing form.""" + + def test_missing_forms_become_empty(self): + """None, empty string, and pandas NaN/NaT all normalize to ''.""" + from dascore.core.summary import normalize_source_patch_id + + assert normalize_source_patch_id(None) == "" + assert normalize_source_patch_id("") == "" + assert normalize_source_patch_id(float("nan")) == "" + assert normalize_source_patch_id(np.nan) == "" + assert normalize_source_patch_id(pd.NaT) == "" + + def test_numpy_scalar_becomes_plain_string(self): + """A numpy scalar is unwrapped before stringifying.""" + from dascore.core.summary import normalize_source_patch_id + + assert normalize_source_patch_id(np.int64(42)) == "42" + + def test_plain_values_stringify(self): + """Ordinary ids pass through as strings.""" + from dascore.core.summary import normalize_source_patch_id + + assert normalize_source_patch_id("abc") == "abc" + assert normalize_source_patch_id(7) == "7" + + def test_non_scalar_falls_through(self): + """A value pd.isnull cannot evaluate as a scalar still stringifies.""" + from dascore.core.summary import normalize_source_patch_id + + # pd.isnull on a list returns an array (truth value is ambiguous), + # so the helper must swallow that and fall through to str(). + assert normalize_source_patch_id([1, 2]) == "[1, 2]" + + +class TestCanonicalRange: + """_canonical_range recognizes only numeric ranges.""" + + def test_bare_and_quantity_bounds(self): + """Bare numbers and quantities become SI magnitudes.""" + from dascore.io.index.catalog import _canonical_range + + assert _canonical_range((20, 60)).magnitudes == (20.0, 60.0) + # 20 m .. 60 m -> SI metres + assert _canonical_range((20 * m, 60 * m)).magnitudes == (20.0, 60.0) + + def test_open_bounds_kept(self): + """A half-open numeric range keeps its open end as None.""" + from dascore.io.index.catalog import _canonical_range + + assert _canonical_range((None, 60)).magnitudes == (None, 60.0) + + @pytest.mark.parametrize( + "value", + [ + np.array([True, False]), # boolean mask, not a range + (None, None), # fully open: no numeric content + (True, False), # bool bounds are not numeric ranges + ("a", "b"), # string bounds are not numeric ranges + (1, 2, 3), # wrong arity + ], + ) + def test_non_numeric_ranges_return_none(self, value): + """Anything that is not a bounded numeric range yields None.""" + from dascore.io.index.catalog import _canonical_range + + assert _canonical_range(value) is None @pytest.fixture(scope="module") diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 439688924..8ca5984c0 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -46,6 +46,15 @@ def test_patches_shared_not_copied(self, contiguous_patches): assert any(x is p1 for x in loaded) assert any(x is p2 for x in loaded) + def test_union_of_materialized_member(self): + """A sorted (materialized but catalog-backed) member unions by ids.""" + sp = dc.get_example_spool("random_das") + other = dc.get_example_spool("diverse_das") + materialized = sp.sort("time") # dataframe path, keeps its catalog + assert not materialized._catalog_native + combined = materialized + other + assert len(combined) == len(sp) + len(other) + def test_select_on_union(self): """Selection works over the merged metadata.""" sp1 = dc.get_example_spool("random_das") @@ -249,3 +258,9 @@ def test_export_all_matches_full(self): catalog.to_df() records = catalog.backend.export_records() assert sum(len(r.patches) for r in records) == 5 + + def test_export_empty_patch_ids(self): + """Exporting an empty id set returns no records without querying.""" + catalog = PatchCatalog.from_patches([dc.get_example_patch()]) + catalog.to_df() + assert catalog.backend.export_records(patch_ids=[]) == [] From efebfc2201073cc8b33ff8cee33e380ce87c3674 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 21:49:37 +0200 Subject: [PATCH 55/97] Fix scanless-format spool bug; reach 100% coverage on the branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restoring dev's 100% coverage surfaced a real regression and several dead branches from the index rework. - patches_to_df returned a None patch column for spool input, so dc.spool() — which wraps dc.read's result in a MemorySpool — crashed on access. Embed the patches (the 'patch' column is the point). Adds a dc.spool(pickle) regression test. Removed provably-unreachable code (per review): the chunk planner's empty-members guard (the caller raises ChunkError for too-short partitions and the earliest source is never fully covered) and its 'guard anyway' group/dims branch (partition keys are single-valued), the _flatten structural-collision backstop (all colliding names are reserved at ingest), and AbstractIndexer's dead ensure_updated default. Simplified MemorySpool.__rich__ to drop the unreachable missing-time branch. Two branches keep a pragma with justification: the RemoteCacheError re-raise in format detection (remote-fetch only) and the chunk member searchsorted boundary guard (unreachable with continuity-partitioned sources). Everything else is covered by real tests — pure-helper units, malformed SQLite indexes, unit backfills, mixed-unit rejection, cascade/export edges, materialized-path selects, and more. Touched modules: 100%. --- dascore/core/spool.py | 19 +- dascore/io/core.py | 5 +- dascore/io/index/backend.py | 11 +- dascore/io/indexer.py | 6 +- dascore/utils/chunk_plan.py | 30 +-- dascore/utils/patch.py | 4 + tests/test_core/test_spool.py | 90 +++++++ tests/test_core/test_spool_select_spec.py | 7 + .../test_index/test_index_edge_cases.py | 220 ++++++++++++++++++ tests/test_io/test_index/test_plan.py | 31 +++ tests/test_io/test_index/test_union.py | 27 +++ tests/test_io/test_io_core.py | 35 +++ tests/test_io/test_pickle/test_pickle.py | 12 + tests/test_utils/test_patch_utils.py | 20 +- 14 files changed, 476 insertions(+), 41 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 8ec2f8ebe..1154ec7c3 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -1330,16 +1330,15 @@ def _strip_identity(df): def __rich__(self): base = super().__rich__() df = self._df - if len(df): - t1 = df["time_min"].min() if "time_min" in df.columns else "" - t2 = df["time_min"].max() if "time_min" in df.columns else "" - tmin = get_nice_text(t1) - tmax = get_nice_text(t2) - if t1 != "" and t2 != "": - duration = get_nice_text(t2 - t1) - else: - duration = "" - base += Text(f"\n Time Span: <{duration}> {tmin} to {tmax}") + # time_min is always part of the flat relation, so a non-empty + # spool always has a renderable time span. + if len(df) and "time_min" in df.columns: + t1, t2 = df["time_min"].min(), df["time_min"].max() + duration = get_nice_text(t2 - t1) + base += Text( + f"\n Time Span: <{duration}> " + f"{get_nice_text(t1)} to {get_nice_text(t2)}" + ) return base def _load_patch(self, kwargs) -> Self: diff --git a/dascore/io/core.py b/dascore/io/core.py index ff71630a8..11ce1caaf 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -640,7 +640,10 @@ def _get_format( # raise, in which case the format doesn't belong. func_input = man.get_resource(required_type) format_version = func(func_input, _pre_cast=True) - except RemoteCacheError: + except RemoteCacheError: # pragma: no cover -- remote fetch only + # A remote fetch failure is a real error, not a "wrong + # format" signal, so it must propagate rather than be + # swallowed by the robustness handler below. raise # For robustness, we need to catch everything else here. except Exception: diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 0d073061e..f599f0ee4 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -661,16 +661,13 @@ def _flatten(self, df: pd.DataFrame, attr_meta: pd.DataFrame) -> pd.DataFrame: # Group the metadata once and drop every typed column in a single # pass: per-name refiltering plus one-drop-per-column was ~O(A^2) # metadata scans and frame copies for A dynamic attrs. + # Every name that could collide with a structural or envelope + # column (RESERVED_ATTR_COLUMNS and *_min/_max/_step) is refused at + # ingest, so a dynamic attr name never shadows an existing column + # here. cols_to_drop: list[str] = [] new_columns: dict[str, pd.Series] = {} for name, rows in attr_meta.groupby("attr_name", sort=False): - if name in out.columns: - # A dynamic attr restored onto an existing column (e.g. a - # coordinate envelope like time_min) would corrupt the - # frame; structural columns win. Reserved fixed names are - # already refused at ingest. - cols_to_drop.extend(c for c in rows["column_name"] if c in out.columns) - continue kinds = set(rows["value_kind"]) multi_kind = len(rows) > 1 series = None diff --git a/dascore/io/indexer.py b/dascore/io/indexer.py index 7644e9d85..f965025c2 100644 --- a/dascore/io/indexer.py +++ b/dascore/io/indexer.py @@ -69,12 +69,10 @@ def update(self) -> Self: Resets any previous selection. """ + @abc.abstractmethod def ensure_updated(self) -> bool: """ Run the initial update if the index was never populated. - Return True when an update actually ran. Indexers which track - their initial-population state override this; by default nothing - happens. + Return True when an update actually ran. """ - return False diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index f08ef7106..ac3cf73de 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -226,7 +226,7 @@ def _coord_owner(col: str, coord_names: set[str]) -> str | None: return None -def _police_columns(sub: pd.DataFrame, name, group_attrs, conflict) -> dict: +def _police_columns(sub: pd.DataFrame, name, conflict) -> dict: """ Return the carried column values for one partition (spec 2.5/6.4). @@ -247,10 +247,8 @@ def _police_columns(sub: pd.DataFrame, name, group_attrs, conflict) -> dict: if single: carried[col] = values[0] continue - in_group = col in group_attrs or col == "dims" - if in_group: # partitioning guarantees this; guard anyway - carried[col] = values[0] - continue + # Group attrs and dims are partition keys, so they are always + # single-valued above and never reach the conflict policy here. if owner is not None or conflict == "raise": msg = ( f"Cannot merge on dim {name} because all values for " @@ -286,17 +284,9 @@ def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFra keep = sub[min_name].values <= sub[max_name].values sub = sub[keep] original = original[keep].reset_index(drop=True) - if sub.empty or outputs.empty: - return pd.DataFrame( - columns=[ - "output_id", - "_patch_id", - min_name, - max_name, - step_name, - "_modified", - ] - ) + # sub and outputs are always non-empty here: a partition too short to + # yield an interval raises ChunkError in the caller (and the earliest + # source is never fully covered), so both keep at least one row. steps = sub[step_name].values src1 = sub[min_name].values src2 = sub[max_name].values @@ -322,7 +312,11 @@ def _build_members(sub: pd.DataFrame, outputs: pd.DataFrame, name) -> pd.DataFra continue lo = max(src1[src_num], chu1[out_num]) hi = min(src2[src_num], chu2[out_num]) - if lo > hi: + if lo > hi: # pragma: no cover -- searchsorted boundary guard + # Sources within a partition are continuous (partitioning + # splits on gaps) and start-corrected, so searchsorted does + # not offer a non-overlapping source in practice; this guards + # against a boundary off-by-one rather than a reachable state. continue unchanged = ( lo == orig_min[src_num] @@ -462,7 +456,7 @@ def build_chunk_plan( except ChunkError: # partition too short; skip (D8) continue sub_sorted = sub.sort_values([min_name, "_patch_id"], kind="stable") - carried = _police_columns(sub_sorted, name, params["group"], conflict) + carried = _police_columns(sub_sorted, name, conflict) outputs = pd.DataFrame(start_stop, columns=[min_name, max_name]) outputs[f"{name}_step"] = part_step for col, val in carried.items(): diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index 3a2fcd0b0..feb81c661 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -339,6 +339,10 @@ def patches_to_df( # Handle spool case if hasattr(patches, "get_contents"): df = patches.get_contents() + # get_contents() carries only metadata; embed the patches so the + # flat-dump path can serve them (the "patch" column is the point). + if "patch" not in df.columns: + df = df.assign(patch=to_object_array(list(patches))) elif isinstance(patches, pd.DataFrame): df = patches else: diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 260de7004..5c8e50cc8 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -21,6 +21,7 @@ from dascore.exceptions import ( InvalidSpoolError, MissingOptionalDependencyError, + MissingPatchError, ParameterError, ) from dascore.utils.downloader import fetch @@ -934,3 +935,92 @@ def __init__(self, val): # This should return False at line 127 assert spool1 != spool2 + + +class TestSpoolCoverageEdges: + """Cover remaining spool-machinery branches with real operations.""" + + @pytest.fixture(scope="class") + def many_contiguous(self): + """Twelve contiguous patches (for >10-row merge handling).""" + t0 = np.datetime64("2020-01-01", "ns") + patch = dc.get_example_patch(time_min=t0) + step = patch.get_coord("time").step + out = [patch] + for _ in range(11): + nxt = dc.get_example_patch(time_min=out[-1].get_coord("time").max() + step) + out.append(nxt) + return out + + def test_equality_and_repr(self): + """Spool equality strips synthetic identity; repr shows a time span.""" + patch = dc.get_example_patch() + left, right = dc.spool([patch]), dc.spool([patch]) + left.get_contents() # realize so equality compares built frames + right.get_contents() + assert left == right + assert "Time Span" in left.__rich__().__str__() + + def test_equality_of_empty_spools(self): + """Empty spools (None frames) compare equal via the None-strip path.""" + assert MemorySpool() == MemorySpool() + + def test_repr_without_time_coordinate(self): + """A spool whose patches have no time coord still renders a span line.""" + data = np.random.default_rng().random((6, 4)) + coords = {"distance": np.arange(6), "frequency": np.arange(4.0)} + patch = dc.Patch(data=data, coords=coords, dims=("distance", "frequency")) + rendered = dc.spool([patch]).__rich__().__str__() + assert "Time Span" in rendered + + def test_large_merge_dedups(self, many_contiguous): + """Merging >10 sources into one patch exercises the de-dup branch.""" + merged = dc.spool(many_contiguous).chunk(time=None) + assert len(merged) == 1 + # 12 contiguous patches merge into one continuous coordinate. + assert merged[0].get_coord("time").size == sum( + p.get_coord("time").size for p in many_contiguous + ) + + def test_union_of_scanless_spool(self, tmp_path): + """A scanless (pickle) spool has no catalog; union falls back to + materializing its patches. + """ + dc.get_example_patch().io.write(tmp_path / "a.pkl", "pickle") + pickle_spool = dc.spool(tmp_path / "a.pkl") + assert pickle_spool._catalog is None + combined = pickle_spool + dc.spool([dc.get_example_patch(tag="other")]) + assert len(combined) == 2 + + def test_union_of_chunked_spool(self, many_contiguous): + """A chunked (restructured) spool's rows no longer map to sources.""" + chunked = dc.spool(many_contiguous).chunk(time=None) + assert not chunked._catalog_native + assert "_patch_id" not in chunked._df.columns + combined = chunked + dc.spool([dc.get_example_patch(tag="other")]) + assert len(combined) == 2 + + def test_iteration_skips_unresolvable_patch(self, monkeypatch): + """A patch that fails to resolve is skipped with a #583 warning.""" + # a sorted spool is materialized, so iteration runs through the + # base __iter__ (memory spools have a fast patch-list iterator). + spool = dc.spool([dc.get_example_patch()]).sort("time") + + def _raise(_ind): + raise MissingPatchError("trimmed to nothing") + + monkeypatch.setattr(spool, "_get_patches_from_index", _raise) + with pytest.warns(UserWarning, match="Skipping patch"): + assert list(spool) == [] + + def test_merge_buffer_grows_when_estimate_short(self, many_contiguous, monkeypatch): + """An under-estimated merge buffer is grown to fit (uneven sampling).""" + import dascore.core.spool as spool_mod + + # Force the pre-merge sample estimate to be too small so the + # streaming buffer must grow mid-merge. + monkeypatch.setattr(spool_mod, "_estimate_merge_samples", lambda *a, **k: 1) + merged = dc.spool(many_contiguous).chunk(time=None) + assert merged[0].get_coord("time").size == sum( + p.get_coord("time").size for p in many_contiguous + ) diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 81b7cebb3..3fc23379f 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -222,6 +222,13 @@ def test_namespaces_and_unknown_names(self, spool): materialized = spool.chunk(time=None) assert not materialized._catalog_native assert len(materialized.select(_attrs={"tag": "random"})) == len(materialized) + # a valid _coords range narrows the materialized spool + df = materialized.get_contents() + t0 = df["time_min"].min() + narrowed = materialized.select( + _coords={"time": (t0, t0 + np.timedelta64(2, "s"))} + ) + assert len(narrowed) <= len(materialized) with pytest.raises(InvalidSpoolQueryError, match="not an attribute"): materialized.select(_attrs={"distance": (0, 10)}) with pytest.raises(InvalidSpoolQueryError, match="not a coordinate"): diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 5241afde8..0c7c7fe7a 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -34,6 +34,226 @@ from dascore.units import get_quantity, m +class TestIndexCoverageEdges: + """Remaining index branches covered with real backends/spools.""" + + def test_live_resolver_missing_patch(self): + """A row for a patch absent from the registry raises MissingPatchError.""" + from dascore.exceptions import MissingPatchError + from dascore.io.index.catalog import LiveResolver + + resolver = LiveResolver([dc.get_example_patch()]) + with pytest.raises(MissingPatchError, match="not available"): + resolver.resolve({"path": "memorypatch://not-a-real-id"}) + + def test_mixed_compatible_unit_range(self): + """A coord range mixing compatible units resolves.""" + from dascore.units import get_quantity + + cm = get_quantity("cm") + out = dc.spool([dc.get_example_patch()]).select(distance=(1 * m, 200 * cm)) + assert len(out.get_contents()) == 1 + + def test_backend_range_incompatible_units_raise(self, backend): + """A hand-built coord range mixing incompatible units is rejected. + + The catalog canonicalizes units before the backend, so only a + direct Query reaches the multi-unit compatibility check. + """ + from dascore.units import s + + with pytest.raises(UnitError, match="Cannot convert"): + backend.query(Query(coords={"distance": (1 * m, 2 * s)})) + + def test_export_skips_source_without_patches(self, tmp_path): + """A non-fiber file gets a sources row with no patches; export skips it.""" + dc.get_example_patch().io.write(tmp_path / "a.h5", "dasdae") + (tmp_path / "junk.txt").write_text("not a fiber file") + spool = dc.spool(tmp_path).update(progress=None) + backend = spool._catalog.backend + assert len(backend.get_sources()) == 2 # the h5 and the junk file + records = backend.export_records() + assert sum(len(r.patches) for r in records) == 1 # only the real patch + + def test_attr_meta_units_backfilled(self, tmp_path): + """An attr first seen unitless gets its unit backfilled by a later write.""" + from dascore.core.summary import PatchSummary + + def _summary(gain, path): + return PatchSummary( + attrs={"tag": "t", "gain": gain}, + coords={ + "distance": { + "dtype": "float64", + "min": 0.0, + "max": 10.0, + "step": 1.0, + "units": "m", + "dims": ("distance",), + "len": 11, + } + }, + dims=("distance",), + shape=(11,), + dtype="float64", + source_path=path, + source_format="X", + source_version="1", + ) + + back = get_backend(tmp_path / "i.sqlite3") + try: + back.write_sources(s2r([_summary(5.0, "a.h5")])) # unitless + meta = back._attr_meta() + assert list(meta.loc[meta["attr_name"] == "gain", "units"]) == [None] + back.write_sources(s2r([_summary(5.0 * m, "b.h5")])) # units -> backfill + meta = back._attr_meta() + assert list(meta.loc[meta["attr_name"] == "gain", "units"]) == ["m"] + finally: + back.close() + + def test_reopen_missing_meta_row(self, tmp_path): + """An index whose meta row was lost is rejected on reopen.""" + import sqlite3 + + from dascore.exceptions import InvalidIndexError + + path = tmp_path / "i.sqlite3" + get_backend(path).close() + con = sqlite3.connect(path) + con.execute("DELETE FROM meta_data") + con.commit() + con.close() + with pytest.raises(InvalidIndexError, match="not a valid"): + get_backend(path) + + def test_reopen_table_missing_column(self, tmp_path): + """An index whose table lost a column is rejected on reopen.""" + import sqlite3 + + from dascore.exceptions import InvalidIndexError + + path = tmp_path / "i.sqlite3" + get_backend(path).close() + con = sqlite3.connect(path) + con.execute("ALTER TABLE patches DROP COLUMN n_dims") + con.commit() + con.close() + with pytest.raises(InvalidIndexError, match="missing columns"): + get_backend(path) + + def test_legacy_index_check_unopenable_path(self, tmp_path): + """A path that exists but cannot be opened as a file is not an index.""" + idx = DBDirectoryIndexer(tmp_path) + # opening a directory raises IsADirectoryError (an OSError), which the + # header probe suppresses before concluding it is not a legacy index. + sub = tmp_path / "adir" + sub.mkdir() + assert idx._is_legacy_or_foreign_index(sub) is False + + def test_schema_creation_rolls_back_on_failure(self, tmp_path): + """A failure while creating the schema rolls back and re-raises.""" + from dascore.io.index.lite import SQLiteBackend + + class _BoomBackend(SQLiteBackend): + def _execute(self, sql, params=()): + if "INSERT INTO meta_data" in sql: + raise RuntimeError("boom during schema init") + return super()._execute(sql, params) + + with pytest.raises(RuntimeError, match="boom during schema init"): + _BoomBackend(tmp_path / "i.sqlite3") + + def test_reopen_missing_dynamic_attr_column(self, tmp_path): + """attr_meta referencing an absent attrs column is rejected on reopen.""" + import sqlite3 + + from dascore.exceptions import InvalidIndexError + + path = tmp_path / "i.sqlite3" + get_backend(path).close() + con = sqlite3.connect(path) + con.execute( + "INSERT INTO attr_meta (attr_name, value_kind, column_name, units) " + "VALUES ('ghost', 'num', 'ghost__num', NULL)" + ) + con.commit() + con.close() + with pytest.raises(InvalidIndexError, match="missing dynamic columns"): + get_backend(path) + + +class TestPureHelpers: + """Small pure helpers covered directly (least-contrived form).""" + + def test_is_directory_format_on_file(self, tmp_path): + """A plain file is never a directory scan unit.""" + from dascore.io.core import is_directory_format + + f = tmp_path / "a.txt" + f.write_text("x") + assert is_directory_format(f) is False + + def test_memory_backend_refuses_pickle(self): + """An in-memory backend cannot be pickled (owners serialize rows).""" + import pickle + + from dascore.io.index.lite import SQLiteBackend + + back = SQLiteBackend(":memory:") + try: + with pytest.raises(TypeError, match="cannot be pickled"): + pickle.dumps(back) + finally: + back.close() + + def test_py_scalar_bool_and_int(self): + """_py_scalar unwraps numpy bool/int to plain python scalars.""" + from dascore.io.index.ingest import _py_scalar + + assert _py_scalar(np.bool_(True)) is True + assert _py_scalar(np.int64(5)) == 5 + assert isinstance(_py_scalar(np.int64(5)), int) + + def test_assemble_records_empty_sources(self): + """No sources yields no records.""" + from dascore.io.index.ingest import assemble_source_records + + empty = pd.DataFrame() + assert assemble_source_records(empty, empty, empty, empty, empty, empty) == [] + + def test_units_compatible(self): + """_units_compatible is True for same dimensionality, False otherwise.""" + from dascore.io.index.backend import SQLIndexBackend + + assert SQLIndexBackend._units_compatible("m", "ft") is True + assert SQLIndexBackend._units_compatible("m", "s") is False + + def test_legacy_index_check_missing_path(self, tmp_path): + """A path that does not exist is not a legacy/foreign index.""" + idx = DBDirectoryIndexer(tmp_path) + assert idx._is_legacy_or_foreign_index(tmp_path / "nope.h5") is False + + def test_wrong_arity_coord_query_raises(self, backend): + """A hand-built coord range of the wrong length is rejected.""" + from dascore.exceptions import ParameterError + + with pytest.raises(ParameterError, match="length 2 sequence"): + backend.query(Query(coords={"distance": (1, 2, 3)})) + + def test_to_target_unit_paths(self): + """Quantity on a unitless target raises; on a unit target it converts.""" + from dascore.io.index.query import _to_target_unit + from dascore.units import get_quantity as _gq + + typed = typed_value(5 * m) # a numeric TypedValue carrying units + with pytest.raises(UnitError, match="unitless"): + _to_target_unit(typed, None, "distance") + # converting to a compatible unit returns a plain magnitude + out = _to_target_unit(typed, str(_gq("m").units), "distance") + assert out == pytest.approx(5.0) + + class TestNormalizeSourcePatchId: """The single source-patch-id normalizer handles every missing form.""" diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index 501da81bc..582016987 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -390,3 +390,34 @@ def test_segment_event_time(self): assert len(chunked) > 1 for sub in chunked: assert "event_time" in sub.dims + + +class TestChunkPlanCoverageEdges: + """Remaining chunk-planner branches.""" + + def test_partial_overlap_members(self): + """Overlapping sources drop the covered span (non-overlap skip).""" + import numpy as np + + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) + step = p1.get_coord("time").step + # p2 overlaps p1's tail by 100 samples. + p2 = dc.get_example_patch(time_min=p1.get_coord("time").max() - 100 * step) + merged = dc.spool([p1, p2]).chunk(time=None) + # the overlap is removed, so the merge is shorter than the naive sum. + naive = p1.get_coord("time").size + p2.get_coord("time").size + assert merged[0].get_coord("time").size < naive + + def test_user_stacklevel_fallback(self, monkeypatch): + """With no user frame in the stack, the stacklevel falls back to 1.""" + import inspect as _inspect + + import dascore.utils.chunk_plan as cp + + # Every frame reports a dascore path, so no "user" frame is found. + class _Frame: + filename = cp.__file__ + + monkeypatch.setattr(_inspect, "stack", lambda: [_Frame()] * 3) + assert cp._user_stacklevel() == 1 diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 8ca5984c0..9b88f9ae0 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -264,3 +264,30 @@ def test_export_empty_patch_ids(self): catalog = PatchCatalog.from_patches([dc.get_example_patch()]) catalog.to_df() assert catalog.backend.export_records(patch_ids=[]) == [] + + def test_absolutize_record_passthrough(self): + """A record already carrying an absolute/URI path is returned as-is.""" + from dascore.io.index.catalog import _absolutize_record + from dascore.io.index.ingest import SourceRecord + + rec = SourceRecord( + source_path="/abs/a.h5", source_format="X", format_version="1" + ) + assert _absolutize_record(rec, "/root") is rec + uri = SourceRecord( + source_path="s3://bucket/a.h5", source_format="X", format_version="1" + ) + assert _absolutize_record(uri, "/root") is uri + + def test_dir_union_absolutizes_relative_paths(self, tmp_path): + """A directory member's relative source path is absolutized on union.""" + dc.get_example_patch().io.write(tmp_path / "a.h5", "dasdae") + dir_spool = dc.spool(tmp_path).update(progress=None) + combined = dir_spool + dc.spool([dc.get_example_patch(tag="mem")]) + assert len(combined) == 2 + # the file-backed member still loads (its path was made absolute) + contents = combined.get_contents() + file_row = contents[contents["path"].str.endswith("a.h5")] + assert len(file_row) == 1 + loaded = [p for p in combined] + assert len(loaded) == 2 diff --git a/tests/test_io/test_io_core.py b/tests/test_io/test_io_core.py index b6e47b392..a616163de 100644 --- a/tests/test_io/test_io_core.py +++ b/tests/test_io/test_io_core.py @@ -928,3 +928,38 @@ def test_get_supported_io_table(self): # assert that the length of the DataFrame is not 0 assert len(result_df) > 0 + + +class TestIOCoreCoverageEdges: + """Remaining io.core resolution/robustness branches.""" + + def test_non_unique_patch_resolution_raises(self): + """An unresolvable source id in a multi-patch read raises clearly.""" + from dascore.exceptions import PatchAttributeError + from dascore.io.core import _select_patch_from_spool + + spool = dc.spool([dc.get_example_patch(tag="a"), dc.get_example_patch(tag="b")]) + with pytest.raises(PatchAttributeError, match="uniquely resolved"): + _select_patch_from_spool(spool, source_patch_id="neither-id-nor-index") + + def test_single_patch_resolved_by_name(self): + """A one-patch read resolves when the id matches the patch name.""" + from dascore.io.core import _select_patch_from_spool + + patch = dc.get_example_patch() + spool = dc.spool([patch]) + resolved = _select_patch_from_spool( + spool, source_patch_id=str(patch.get_patch_name()) + ) + assert resolved == patch + + def test_corrupt_file_format_detection_is_robust(self, tmp_path): + """A reader raising during format detection is caught, not propagated.""" + from dascore.exceptions import UnknownFiberFormatError + + # valid HDF5 magic followed by garbage: an HDF5 reader raises while + # probing, which format detection must swallow before giving up. + bad = tmp_path / "bad.h5" + bad.write_bytes(b"\x89HDF\r\n\x1a\n" + b"\x00" * 256) + with pytest.raises(UnknownFiberFormatError): + dc.get_format(bad) diff --git a/tests/test_io/test_pickle/test_pickle.py b/tests/test_io/test_pickle/test_pickle.py index fe81fc808..0678c5785 100644 --- a/tests/test_io/test_pickle/test_pickle.py +++ b/tests/test_io/test_pickle/test_pickle.py @@ -42,6 +42,18 @@ def test_read_pickle(self, pickle_patch_path, random_patch): assert isinstance(out[0], dc.Patch) assert random_patch == out[0] + def test_spool_from_pickle(self, pickle_patch_path, random_patch): + """dc.spool on a scanless format wraps the read spool and serves it. + + PICKLE implements read but not scan, so dc.spool routes through + MemorySpool(dc.read(...)); the wrapped patches must load back. + """ + spool = dc.spool(pickle_patch_path) + assert len(spool) == 1 + assert len(spool.get_contents()) == 1 + assert spool[0] == random_patch + assert next(iter(spool)) == random_patch + def test_file_not_there(self): """Get format should return false if the file doesn't exist.""" parser = PickleIO() diff --git a/tests/test_utils/test_patch_utils.py b/tests/test_utils/test_patch_utils.py index 160f285bf..fecf1ccf2 100644 --- a/tests/test_utils/test_patch_utils.py +++ b/tests/test_utils/test_patch_utils.py @@ -396,10 +396,28 @@ class TestPatchesToDF: """Test for getting metadata from patch into a dataframe.""" def test_spool_input(self, random_spool): - """A spool should return its contents.""" + """A spool should return its contents with its patches embedded.""" df = patches_to_df(random_spool) assert isinstance(df, pd.DataFrame) assert len(df) == len(random_spool) + # the "patch" column carries the actual patches, not None + assert "patch" in df.columns + assert all(isinstance(x, dc.Patch) for x in df["patch"]) + + def test_list_of_patches_input(self, random_spool): + """A plain sequence of patches is scanned and the patches embedded.""" + patches = list(random_spool) + df = patches_to_df(patches) + assert isinstance(df, pd.DataFrame) + assert len(df) == len(patches) + assert list(df["patch"]) == patches + + def test_empty_list_input(self): + """An empty sequence produces an empty frame with the right columns.""" + df = patches_to_df([]) + assert isinstance(df, pd.DataFrame) + assert len(df) == 0 + assert "patch" in df.columns def test_dataframe_input(self, random_spool): """The function should be idempotent.""" From 03e1b95948272618faf662fcd10883d9c2252e7b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 23:18:49 +0200 Subject: [PATCH 56/97] Fix tutorial doc code for the new selector and flat-relation contract The autogenerated doc-code tests exercise the tutorial cells. Two were stale after the index rework: - index.qmd selected on the envelope column 'time_min' (now rejected; the coordinate is 'time') and chunked a shared, heterogeneous download directory into 60 s windows that the example data can't fill. Use a freshly written example directory and data-fitting chunk arguments. - spool.qmd dropped a 'patch' column that get_contents() no longer emits; drop the internal underscore-prefixed columns for display instead. --- docs/index.qmd | 14 ++++++-------- docs/tutorial/spool.qmd | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/index.qmd b/docs/index.qmd index ae2cb4ca3..bc6dea07c 100644 --- a/docs/index.qmd +++ b/docs/index.qmd @@ -45,14 +45,12 @@ patch = spool[0] ```{python} #| output: false import dascore as dc -# Import fetch to read DASCore example files -from dascore.utils.downloader import fetch -# Fetch a sample file path from DASCore (just to get a usable path for the rest of the cell) -directory_path = fetch('terra15_das_1_trimmed.hdf5').parent -# To read a directory of DAS data stored locally on your machine, -# simply replace the above line with: +# Write example DAS files to a local directory just to get a usable path. +# To read a directory of DAS data stored on your machine, +# simply replace the line below with: # directory_path = "/path/to/data/directory/" +directory_path = dc.examples.spool_to_directory(dc.get_example_spool()) spool = ( # Create a spool to interact with directory data @@ -60,9 +58,9 @@ spool = ( # Index the directory contents .update() # Sub-select a specific time range - .select(time_min=('2020-01-01', ...)) + .select(time=('2020-01-01', ...)) # Specify chunk of the output patches - .chunk(time=60, overlap=10) + .chunk(time=2, overlap=0.5) ) ``` diff --git a/docs/tutorial/spool.qmd b/docs/tutorial/spool.qmd index b0f26b5cb..ae7597ff3 100644 --- a/docs/tutorial/spool.qmd +++ b/docs/tutorial/spool.qmd @@ -170,7 +170,7 @@ print(contents) #| echo: false from IPython.display import display -display(contents.drop(columns=['patch'])) +display(contents.drop(columns=[c for c in contents.columns if c.startswith('_')])) ``` The columns returned by `get_contents()` come from the same patch summary metadata exposed by `Patch.summary`, so fields such as `time_min`, `time_max`, and `distance_step` are available without loading the underlying patch data. Source metadata such as `path`, `file_format`, and `source_patch_id` are also available for file-backed spools. From 9c875e97d68880ed80de8e225ac067483ec18805 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 23:34:54 +0200 Subject: [PATCH 57/97] Store index source paths as POSIX-relative (Windows path fix) summaries_to_records computed the stored relative path by string slicing and lstrip('/'), which on Windows left the leading backslash after the root ('\\random.hdf5'). That corrupted path comparisons and, worse, made _absolutize_record's Path(root) / '\\a.h5' reset to the drive root (C:\a.h5), so unioned directory members could not be read. Strip either separator and normalize the stored path to forward slashes. Also make the _absolutize_record passthrough test use an OS-native absolute path (a POSIX-style /abs path is not absolute on Windows). --- dascore/io/index/ingest.py | 6 ++++-- tests/test_io/test_index/test_union.py | 15 +++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 1c61f02ed..25e4996e7 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -386,8 +386,10 @@ def summaries_to_records( store_path = path root = base_uri or relative_to if root and path.startswith(root): - # "." (not "") when the source IS the root (directory units) - store_path = path[len(root) :].lstrip("/") or "." + # Store a POSIX-relative path: strip the root prefix, then drop + # either separator (Windows uses "\") and normalize to "/". "." + # (not "") marks a source that IS the root (directory units). + store_path = path[len(root) :].lstrip("/\\").replace("\\", "/") or "." out.append( SourceRecord( source_path=store_path, diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 9b88f9ae0..6793eb965 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -265,19 +265,22 @@ def test_export_empty_patch_ids(self): catalog.to_df() assert catalog.backend.export_records(patch_ids=[]) == [] - def test_absolutize_record_passthrough(self): + def test_absolutize_record_passthrough(self, tmp_path): """A record already carrying an absolute/URI path is returned as-is.""" + from pathlib import Path + from dascore.io.index.catalog import _absolutize_record from dascore.io.index.ingest import SourceRecord - rec = SourceRecord( - source_path="/abs/a.h5", source_format="X", format_version="1" - ) - assert _absolutize_record(rec, "/root") is rec + # an OS-native absolute path (drive-qualified on Windows) + abs_path = str((tmp_path / "a.h5").resolve()) + assert Path(abs_path).is_absolute() + rec = SourceRecord(source_path=abs_path, source_format="X", format_version="1") + assert _absolutize_record(rec, str(tmp_path)) is rec uri = SourceRecord( source_path="s3://bucket/a.h5", source_format="X", format_version="1" ) - assert _absolutize_record(uri, "/root") is uri + assert _absolutize_record(uri, str(tmp_path)) is uri def test_dir_union_absolutizes_relative_paths(self, tmp_path): """A directory member's relative source path is absolutized on union.""" From 4fcea0bd2eb50796499a6daf02cb05eddd655ac7 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 11 Jul 2026 23:52:30 +0200 Subject: [PATCH 58/97] Normalize index paths to POSIX and release SQLite handles for Windows Two Windows-only problems surfaced once the doc/path fixes let the suite reach the Windows jobs: - PatchSummary.source_path and the spool root are OS-native, so on Windows stored index paths carried backslashes. That broke POSIX-based deletion and comparison (sources not deleted, wrong prefixes) and made _absolutize_record's Path(root) / '\a.h5' reset to the drive root. Store every source path as POSIX in summaries_to_records. - Windows cannot delete a file with an open handle. The diverse directory-spool fixture's index stayed open while its directory was rmtree'd, and test_index_len unlinked an index another fixture held open. Close the backend on fixture teardown and give test_index_len its own directory, closing before the unlink/rebuild. --- dascore/io/index/ingest.py | 18 +++++++++++------- tests/conftest.py | 9 ++++++--- tests/test_clients/test_dirspool.py | 19 ++++++++++++++----- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 25e4996e7..98f2ce656 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -370,9 +370,16 @@ def summaries_to_records( Optional maps of source_path -> stat values. When omitted the caller is responsible for change detection. """ + # Index paths are stored as POSIX so comparison and deletion are + # separator-agnostic across platforms (PatchSummary.source_path and + # the spool root are OS-native, so on Windows they carry backslashes). by_source: dict[str, list[PatchSummary]] = {} for summary in summaries: - by_source.setdefault(str(summary.source_path), []).append(summary) + by_source.setdefault(str(summary.source_path).replace("\\", "/"), []).append( + summary + ) + root = base_uri or relative_to + root_posix = str(root).replace("\\", "/") if root else None out = [] for path, group in by_source.items(): first = group[0] @@ -384,12 +391,9 @@ def summaries_to_records( record = replace(record, source_patch_id=str(num)) patches.append(record) store_path = path - root = base_uri or relative_to - if root and path.startswith(root): - # Store a POSIX-relative path: strip the root prefix, then drop - # either separator (Windows uses "\") and normalize to "/". "." - # (not "") marks a source that IS the root (directory units). - store_path = path[len(root) :].lstrip("/\\").replace("\\", "/") or "." + if root_posix and path.startswith(root_posix): + # "." (not "") marks a source that IS the root (directory units) + store_path = path[len(root_posix) :].lstrip("/") or "." out.append( SourceRecord( source_path=store_path, diff --git a/tests/conftest.py b/tests/conftest.py index fb88b00f9..c6154f2e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -512,15 +512,18 @@ def diverse_spool(): def diverse_directory_spool(diverse_spool_directory): """Save the diverse spool contents to a directory.""" out = dc.spool(diverse_spool_directory).update() - return out + yield out + # release the SQLite index handle so Windows can clean the temp dir + out.indexer.close() @pytest.fixture(scope="class") @register_func(SPOOL_FIXTURES) def basic_file_spool(two_patch_directory): """Return a DAS bank on basic_bank_directory.""" - out = DirectorySpool(two_patch_directory).update() - return out.update() + out = DirectorySpool(two_patch_directory).update().update() + yield out + out.indexer.close() @pytest.fixture(scope="class") diff --git a/tests/test_clients/test_dirspool.py b/tests/test_clients/test_dirspool.py index f5c52eda9..ba1a2a924 100644 --- a/tests/test_clients/test_dirspool.py +++ b/tests/test_clients/test_dirspool.py @@ -293,12 +293,21 @@ def test_index_exists(self, basic_file_spool): """An index should be returned.""" assert basic_file_spool.indexer.index_path.exists() - def test_index_len(self, basic_index_df, two_patch_directory): - """An index should be returned.""" - spool = dc.spool(two_patch_directory) + def test_index_len(self, random_patch, tmp_path): + """Deleting and rebuilding the index reproduces the contents.""" + # own directory so no other spool holds the index file open + dc.write(random_patch, tmp_path / "a.hdf5", "dasdae") + dc.write(random_patch.update_attrs(tag="b"), tmp_path / "b.hdf5", "dasdae") + spool = dc.spool(tmp_path) + spool.get_contents() # build the index + # close the connection so the index file can be replaced (Windows + # cannot delete a file with an open handle), then rebuild fresh. + spool.indexer.close() spool.indexer.index_path.unlink() - df = spool.update().get_contents() - bank_paths = list(Path(two_patch_directory).rglob("*hdf5")) + rebuilt = dc.spool(tmp_path).update() + df = rebuilt.get_contents() + rebuilt.indexer.close() + bank_paths = list(Path(tmp_path).rglob("*hdf5")) assert isinstance(df, pd.DataFrame) assert len(bank_paths) == len(df) From afc1ad6d97bd12a529343e0f8f2259fedc4c221c Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 12 Jul 2026 00:03:38 +0200 Subject: [PATCH 59/97] Keep the OS-native key for stat lookup while storing POSIX paths Normalizing the ingest group key to POSIX broke the mtimes_ns/sizes_bytes lookup on Windows: the caller keys those maps by the OS-native scan path, so the POSIX key missed and mtime was stored as None, making every incremental update rescan. Group by the original path (for the stat lookup) and derive the POSIX path only for storage. Also normalize the source_path in the no-false-negative and stress test assertions, which compared an OS-native PatchSummary path against the now-POSIX index paths. --- dascore/io/index/ingest.py | 18 +++++++++--------- .../test_index/test_heterogeneity_stress.py | 10 +++++++--- .../test_io/test_index/test_index_contract.py | 4 ++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 98f2ce656..4c3faef34 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -370,14 +370,13 @@ def summaries_to_records( Optional maps of source_path -> stat values. When omitted the caller is responsible for change detection. """ - # Index paths are stored as POSIX so comparison and deletion are - # separator-agnostic across platforms (PatchSummary.source_path and - # the spool root are OS-native, so on Windows they carry backslashes). + # Group by the original (OS-native) source path so the mtimes_ns / + # sizes_bytes maps, which the caller keys by that same path, still + # resolve. Index paths themselves are stored as POSIX so comparison + # and deletion are separator-agnostic across platforms. by_source: dict[str, list[PatchSummary]] = {} for summary in summaries: - by_source.setdefault(str(summary.source_path).replace("\\", "/"), []).append( - summary - ) + by_source.setdefault(str(summary.source_path), []).append(summary) root = base_uri or relative_to root_posix = str(root).replace("\\", "/") if root else None out = [] @@ -390,10 +389,11 @@ def summaries_to_records( # positional identity within the source, per design doc record = replace(record, source_patch_id=str(num)) patches.append(record) - store_path = path - if root_posix and path.startswith(root_posix): + posix_path = path.replace("\\", "/") + store_path = posix_path + if root_posix and posix_path.startswith(root_posix): # "." (not "") marks a source that IS the root (directory units) - store_path = path[len(root_posix) :].lstrip("/") or "." + store_path = posix_path[len(root_posix) :].lstrip("/") or "." out.append( SourceRecord( source_path=store_path, diff --git a/tests/test_io/test_index/test_heterogeneity_stress.py b/tests/test_io/test_index/test_heterogeneity_stress.py index bff7903af..c721844dc 100644 --- a/tests/test_io/test_index/test_heterogeneity_stress.py +++ b/tests/test_io/test_index/test_heterogeneity_stress.py @@ -221,7 +221,11 @@ def test_random_numeric_coord_ranges(self, backend, summaries): get_quantity(str(csum.units)).to_base_units().magnitude ) if float(csum.min) * factor <= hi and float(csum.max) * factor >= lo: - assert str(summary.source_path) in got, (name, lo, hi) + assert str(summary.source_path).replace("\\", "/") in got, ( + name, + lo, + hi, + ) def test_random_time_ranges(self, backend, summaries): """Absolute time queries against datetime coords.""" @@ -236,7 +240,7 @@ def test_random_time_ranges(self, backend, summaries): if csum is None or "datetime" not in str(csum.dtype): continue if csum.min <= hi and csum.max >= lo: - assert str(summary.source_path) in got + assert str(summary.source_path).replace("\\", "/") in got def test_attr_equality_roundtrip(self, backend, summaries): """Str attr equality returns every patch carrying that value.""" @@ -252,4 +256,4 @@ def test_attr_equality_roundtrip(self, backend, summaries): continue name, value = next(iter(attrs.items())) got = set(backend.query(Query(attrs={name: value}))["path"]) - assert str(summary.source_path) in got + assert str(summary.source_path).replace("\\", "/") in got diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index bdd0105d2..0e4298cbd 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -356,7 +356,7 @@ def test_random_time_ranges(self, backend): continue overlaps = tcoord.min <= hi and tcoord.max >= lo if overlaps: - assert str(summary.source_path) in result_paths + assert str(summary.source_path).replace("\\", "/") in result_paths def test_random_numeric_ranges(self, backend): """Random numeric ranges.""" @@ -375,7 +375,7 @@ def test_random_numeric_ranges(self, backend): continue scale = factor[str(dcoord.units.units)] if dcoord.units else 1.0 if dcoord.min * scale <= hi and dcoord.max * scale >= lo: - assert str(summary.source_path) in result_paths + assert str(summary.source_path).replace("\\", "/") in result_paths class TestSourceLifecycle: From 0d33f54571c3c66058ff8dfd15f1d6853b9a2447 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 12 Jul 2026 14:10:14 +0200 Subject: [PATCH 60/97] Address review: union leak, empty-spool repr, path robustness CodeRabbit findings on #751, verified and fixed: - Union of a directory spool built with constructor select_kwargs reintroduced the excluded rows (_as_catalog_member treated any catalog-native spool as the whole catalog). Carry the restricted patch ids when select_kwargs are set. - A bare MemorySpool() has no dataframe, so len(), iteration, and repr raised TypeError. Treat a missing frame as an empty spool. - get_patch_names crashed for multi-coordinate naming (list(*coord_fields) unpacks multiple pairs); flatten the min/max fields explicitly. - directory_writable let a read-only mount's OSError (e.g. EROFS from mkdir) escape instead of returning False; guard the whole probe. - relative_ranges_to_absolute read {name}_max without checking it, raising KeyError instead of the documented InvalidSpoolQueryError. - _find_index_path returned the un-absolutized index_path while recording the absolute form in the index map; return the absolute path. Regression tests added for each; touched modules stay at 100% coverage. --- dascore/core/spool.py | 19 ++++++++++++++----- dascore/io/index/indexer.py | 5 +++-- dascore/utils/patch.py | 2 +- dascore/utils/paths.py | 9 +++++---- dascore/utils/pd.py | 2 +- tests/test_core/test_spool.py | 7 +++++++ tests/test_io/test_index/test_union.py | 13 +++++++++++++ tests/test_utils/test_patch_utils.py | 8 ++++++++ tests/test_utils/test_paths.py | 16 ++++++++++++++++ tests/test_utils/test_pd.py | 22 ++++++++++++++++++++++ 10 files changed, 90 insertions(+), 13 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 1154ec7c3..97b5fc302 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -585,9 +585,13 @@ def __len__(self): and "_df" not in self._cache ): return len(self._catalog) - return len(self._df) + df = self._df + # An empty spool with no patches, data, or catalog has no frame. + return 0 if df is None else len(df) def __iter__(self): + if self._df is None: # an empty spool has nothing to yield + return for ind in range(len(self._df)): try: yield self._unbox_patch(self._get_patches_from_index(ind)) @@ -816,7 +820,11 @@ def _as_catalog_member(self): """ if self._catalog is None: return super()._as_catalog_member() - if self._catalog_native: + # A catalog-native spool is the whole catalog only when nothing + # narrows it; constructor select_kwargs (DirectorySpool) restrict + # the visible rows without touching the catalog, so carry only the + # surviving patch ids rather than the entire catalog. + if self._catalog_native and not self._select_kwargs: return self._catalog, None df = self._df if "_patch_id" in df.columns: @@ -1330,9 +1338,10 @@ def _strip_identity(df): def __rich__(self): base = super().__rich__() df = self._df - # time_min is always part of the flat relation, so a non-empty - # spool always has a renderable time span. - if len(df) and "time_min" in df.columns: + # An empty MemorySpool() has no dataframe; otherwise time_min is + # always part of the flat relation, so a non-empty spool has a + # renderable time span. + if df is not None and len(df) and "time_min" in df.columns: t1, t2 = df["time_min"].min(), df["time_min"].max() duration = get_nice_text(t2 - t1) base += Text( diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 8c7b193c0..b35e9c6d3 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -99,9 +99,10 @@ def _find_index_path(self, index_path=None) -> Path: """ map_key = str(self.path) if index_path: - update = {map_key: str(Path(index_path).absolute())} + index_path = Path(index_path).absolute() + update = {map_key: str(index_path)} _update_index_map(update, cache_path=str(self.index_map_path)) - return Path(index_path) + return index_path expected = self.path / self._index_name with suppress(PermissionError): if expected.exists(): diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index feb81c661..e6b75e1fb 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -630,7 +630,7 @@ def _get_filename(path_ser, strip_extension): # Determine the requested fields; absent columns render as empty so # names don't depend on which metadata engine produced the dataframe. coord_fields = zip([f"{x}_min" for x in coords], [f"{x}_max" for x in coords]) - fields = list(attrs) + list(*coord_fields) + fields = list(attrs) + [field for pair in coord_fields for field in pair] sub = df.reindex(columns=fields).pipe(_format_time_columns).fillna("").astype(str) out = f"{prefix}_{sep}" + sub[fields[0]].str.cat(sub[fields[1:]], sep=sep) return out diff --git a/dascore/utils/paths.py b/dascore/utils/paths.py index b3cc35dd5..5ffc2d441 100644 --- a/dascore/utils/paths.py +++ b/dascore/utils/paths.py @@ -34,13 +34,14 @@ def directory_writable(path) -> bool: """Return True if the directory is writable else False.""" name = "._dascore_write_test_delete_me" probe = Path(path) / name - probe.parent.mkdir(exist_ok=True, parents=True) try: + # a read-only mount raises OSError (e.g. EROFS) from mkdir/open; + # the whole probe must be guarded, not just the write. + probe.parent.mkdir(exist_ok=True, parents=True) open(probe, "w").close() - except (PermissionError, IsADirectoryError): + except OSError: return False - else: - os.remove(probe) + os.remove(probe) return True diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index 57de4118a..d8a1835b2 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -54,7 +54,7 @@ def relative_ranges_to_absolute(df, kwargs: dict) -> dict: out = {} for name, value in kwargs.items(): lo_col, hi_col = f"{name}_min", f"{name}_max" - if lo_col not in df.columns or df.empty: + if lo_col not in df.columns or hi_col not in df.columns or df.empty: msg = f"Cannot use relative select on {name!r}." raise InvalidSpoolQueryError(msg) if not (isinstance(value, tuple) and len(value) == 2): diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 5c8e50cc8..090853b91 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -1024,3 +1024,10 @@ def test_merge_buffer_grows_when_estimate_short(self, many_contiguous, monkeypat assert merged[0].get_coord("time").size == sum( p.get_coord("time").size for p in many_contiguous ) + + def test_empty_memory_spool_len_iter_repr(self): + """A bare MemorySpool() (no dataframe) is a valid empty spool.""" + empty = MemorySpool() + assert len(empty) == 0 + assert list(empty) == [] + assert "Spool" in str(empty) diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 6793eb965..0950b2380 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -118,6 +118,19 @@ def test_same_source_dedups(self, dir_spool): combined = dir_spool + dir_spool assert len(combined) == len(dir_spool) + def test_constructor_select_kwargs_restrict_union(self, tmp_path): + """A select_kwargs-restricted directory spool unions only its rows.""" + base = dc.get_example_spool("random_das") + dc.examples.spool_to_directory(base, path=tmp_path) + full = dc.spool(tmp_path).update() + df = full.get_contents().sort_values("time_min") + window = (df["time_min"].iloc[0], df["time_max"].iloc[0]) # first patch + restricted = dc.spool(tmp_path, select_kwargs={"time": window}) + assert 0 < len(restricted) < len(full) + combined = restricted + dc.spool([dc.get_example_patch(tag="mem")]) + # the union must not reintroduce the rows the constructor excluded + assert len(combined) == len(restricted) + 1 + def test_union_preserves_def_keys(self, dir_spool, contiguous_patches): """Coord definitions deduplicate by def key across members.""" _, p2 = contiguous_patches diff --git a/tests/test_utils/test_patch_utils.py b/tests/test_utils/test_patch_utils.py index fecf1ccf2..30c2e99a6 100644 --- a/tests/test_utils/test_patch_utils.py +++ b/tests/test_utils/test_patch_utils.py @@ -857,6 +857,14 @@ def test_path_column_leave_extension(self, random_directory_spool): names = get_patch_names(random_directory_spool, strip_extension=False) assert "." in names.iloc[0] + def test_multiple_coord_fields(self, random_spool): + """Naming on more than one coordinate flattens the min/max fields.""" + # drop path so the coordinate-based naming branch is exercised + df = random_spool.get_contents().drop(columns=["path"], errors="ignore") + names = get_patch_names(df, coords=("time", "distance")) + assert len(names) == len(df) + assert names.str.len().gt(0).all() + class TestSwapKwargsDimToAxis: """Tests for swap_kwargs_dim_to_axis function.""" diff --git a/tests/test_utils/test_paths.py b/tests/test_utils/test_paths.py index 9c52086bd..384eb8d3f 100644 --- a/tests/test_utils/test_paths.py +++ b/tests/test_utils/test_paths.py @@ -11,6 +11,7 @@ from dascore.utils.paths import ( coerce_to_local_path, coerce_to_upath, + directory_writable, get_path_protocol, is_local_path, is_pathlike, @@ -18,6 +19,21 @@ ) +class TestDirectoryWritable: + """directory_writable probes without leaking exceptions.""" + + def test_writable_directory(self, tmp_path): + """A normal writable directory returns True.""" + assert directory_writable(tmp_path) is True + + def test_unwritable_returns_false(self, tmp_path): + """A probe that can't create its parent returns False, not OSError.""" + # a path *under a file* makes mkdir raise NotADirectoryError (OSError) + a_file = tmp_path / "a_file" + a_file.write_text("x") + assert directory_writable(a_file / "sub") is False + + class TestIsPathlike: """Tests for ``is_pathlike``.""" diff --git a/tests/test_utils/test_pd.py b/tests/test_utils/test_pd.py index 4ceffd08c..00a8cd07d 100644 --- a/tests/test_utils/test_pd.py +++ b/tests/test_utils/test_pd.py @@ -464,3 +464,25 @@ def test_raises(self, example_df_2): msg = "Cannot chunk spool or dataframe" with pytest.raises(ParameterError, match=msg): get_interval_columns(example_df_2, "money") + + +class TestRelativeRangesToAbsolute: + """Relative range resolution validates its envelope columns.""" + + def test_missing_max_column_raises_spool_error(self): + """A frame with the min but not the max column raises the doc'd error.""" + from dascore.exceptions import InvalidSpoolQueryError + from dascore.utils.pd import relative_ranges_to_absolute + + df = pd.DataFrame({"time_min": [0.0]}) # no time_max + with pytest.raises(InvalidSpoolQueryError, match="relative select"): + relative_ranges_to_absolute(df, {"time": (1, -1)}) + + def test_non_tuple_value_raises(self): + """A non-(start, stop) relative value raises rather than mis-resolving.""" + from dascore.exceptions import InvalidSpoolQueryError + from dascore.utils.pd import relative_ranges_to_absolute + + df = pd.DataFrame({"time_min": [0.0], "time_max": [1.0]}) + with pytest.raises(InvalidSpoolQueryError, match="requires"): + relative_ranges_to_absolute(df, {"time": 5}) From 4387ed6a7f98c92b60bdb9d6b839f8fff88df36e Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sun, 12 Jul 2026 17:21:03 +0200 Subject: [PATCH 61/97] Address review round 2: probe cleanup, time-less repr, chunk assert - directory_writable: the probe-file os.remove was outside the guard, so a transient delete failure (Windows AV/locking) on a genuinely writable directory raised instead of returning True; suppress OSError on cleanup. - MemorySpool.__rich__ rendered 'Time Span: NaT to NaT' for patches with no time coordinate (the flat relation always has a time_min column, null here); only render the span when time_min is present. - Strengthen the unequal-distance chunk test to assert the distinct distance envelopes are preserved, not merely the row count. --- dascore/core/spool.py | 17 +++++++++-------- dascore/utils/paths.py | 6 +++++- tests/test_core/test_patch_chunk.py | 7 +++++++ tests/test_core/test_spool.py | 9 +++++++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 97b5fc302..19bf1213f 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -1338,16 +1338,17 @@ def _strip_identity(df): def __rich__(self): base = super().__rich__() df = self._df - # An empty MemorySpool() has no dataframe; otherwise time_min is - # always part of the flat relation, so a non-empty spool has a - # renderable time span. + # An empty MemorySpool() has no dataframe, and patches without a + # time coordinate have a null time_min; only render a time span + # when the spool actually carries one. if df is not None and len(df) and "time_min" in df.columns: t1, t2 = df["time_min"].min(), df["time_min"].max() - duration = get_nice_text(t2 - t1) - base += Text( - f"\n Time Span: <{duration}> " - f"{get_nice_text(t1)} to {get_nice_text(t2)}" - ) + if pd.notna(t1) and pd.notna(t2): + duration = get_nice_text(t2 - t1) + base += Text( + f"\n Time Span: <{duration}> " + f"{get_nice_text(t1)} to {get_nice_text(t2)}" + ) return base def _load_patch(self, kwargs) -> Self: diff --git a/dascore/utils/paths.py b/dascore/utils/paths.py index 5ffc2d441..5b330755c 100644 --- a/dascore/utils/paths.py +++ b/dascore/utils/paths.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from contextlib import suppress from pathlib import Path from dascore.compat import UPath @@ -41,7 +42,10 @@ def directory_writable(path) -> bool: open(probe, "w").close() except OSError: return False - os.remove(probe) + # the directory is writable; a transient failure to remove the probe + # (e.g. Windows AV/file-locking) must not flip the result. + with suppress(OSError): + os.remove(probe) return True diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 418e11ad8..7299355af 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -315,6 +315,13 @@ def test_merge_unequal_other(self, distance_adjacent): out = distance_adjacent.chunk(time=...) assert len(out) == len(distance_adjacent) + # the differing distance envelopes are preserved, not merged/duplicated + def _distance_envelopes(spool): + df = spool.get_contents() + return sorted(zip(df["distance_min"], df["distance_max"])) + + assert _distance_envelopes(out) == _distance_envelopes(distance_adjacent) + def test_merge_adjacent(self, adjacent_spool_no_overlap): """Test simple merge of patches.""" len_1 = len(adjacent_spool_no_overlap) diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 090853b91..c1e39ace8 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -966,12 +966,17 @@ def test_equality_of_empty_spools(self): assert MemorySpool() == MemorySpool() def test_repr_without_time_coordinate(self): - """A spool whose patches have no time coord still renders a span line.""" + """A spool whose patches have no time coord omits the time-span line.""" data = np.random.default_rng().random((6, 4)) coords = {"distance": np.arange(6), "frequency": np.arange(4.0)} patch = dc.Patch(data=data, coords=coords, dims=("distance", "frequency")) rendered = dc.spool([patch]).__rich__().__str__() - assert "Time Span" in rendered + assert "Spool" in rendered + assert "Time Span" not in rendered # no time coordinate to summarize + + def test_repr_with_time_coordinate(self): + """A normal spool renders its time span.""" + assert "Time Span" in dc.spool([dc.get_example_patch()]).__rich__().__str__() def test_large_merge_dedups(self, many_contiguous): """Merging >10 sources into one patch exercises the de-dup branch.""" From 14e8eed7556e203c49b666ca3f14cb024e0003c1 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 09:22:20 +0200 Subject: [PATCH 62/97] Clean up residual spool refactor paths --- dascore/io/h5simple/utils.py | 37 ++--------- dascore/utils/chunk.py | 7 --- dascore/utils/pd.py | 42 +------------ dascore/utils/time.py | 23 ------- tests/test_io/test_h5simple/test_h5simple.py | 61 ------------------- .../test_xml_binary/test_xml_binary.py | 8 +++ tests/test_utils/test_misc.py | 11 ++++ tests/test_utils/test_time.py | 26 ++++---- 8 files changed, 39 insertions(+), 176 deletions(-) diff --git a/dascore/io/h5simple/utils.py b/dascore/io/h5simple/utils.py index 9e0a5a4a9..8e1308484 100644 --- a/dascore/io/h5simple/utils.py +++ b/dascore/io/h5simple/utils.py @@ -18,32 +18,6 @@ DEFAULT_ATTRS = frozenset(("CLASS", "PYTABLES_FORMAT_VERSION", "TITLE", "VERSION")) -def _get_root_attrs(h5): - """Return a mapping-like object for root attrs for either HDF5 backend.""" - if hasattr(h5, "root"): - return h5.root._v_attrs - return h5.attrs - - -def _iter_root_arrays(h5): - """Yield ``(name, node)`` pairs for array-like nodes at the HDF5 root.""" - if hasattr(h5, "list_nodes"): - for node in h5.list_nodes("/"): - if hasattr(node, "shape"): - yield node.name, node - return - for name, node in h5.items(): - if hasattr(node, "shape"): - yield name, node - - -def _get_attr_names(attrs): - """Return the set of attribute names from either backend.""" - if hasattr(attrs, "_v_attrnames"): - return set(attrs._v_attrnames) - return set(attrs) - - def _maybe_trim_data(cm, data, kwargs): """Maybe use kwargs to trim data array.""" new_cm, new_data = cm.select(array=data, **kwargs) @@ -52,12 +26,9 @@ def _maybe_trim_data(cm, data, kwargs): def _get_attrs_coords_and_data(h5, snap, fiber_io): """Return attrs, coordinate manager, and data node.""" - attrs = _get_root_attrs(h5) - attr_names = _get_attr_names(attrs) - DEFAULT_ATTRS - attr_dict = { - x: unbyte(attrs[x] if not hasattr(attrs, "_v_attrnames") else getattr(attrs, x)) - for x in attr_names - } + attrs = h5.attrs + attr_names = set(attrs) - DEFAULT_ATTRS + attr_dict = {x: unbyte(attrs[x]) for x in attr_names} attr_dict["file_version"] = fiber_io.version attr_dict["file_format"] = fiber_io.name cm, data = _get_cm_and_data(h5, snap, dims=attr_dict.get("dims")) @@ -124,7 +95,7 @@ def _get_coords_and_dims(data_node, time_node, other_nodes, snap=True, dims=None def _get_cm_and_data(h5, snap=False, dims=None): """Extract coordinate manager and data node.""" - root_nodes = dict(_iter_root_arrays(h5)) + root_nodes = {name: node for name, node in h5.items() if hasattr(node, "shape")} array_names = set(root_nodes) data_node_name = array_names & DATA_ARRAY_NAMES time_node_name = array_names & TIME_ARRAY_NAMES diff --git a/dascore/utils/chunk.py b/dascore/utils/chunk.py index fa9af91e5..dae35aad6 100644 --- a/dascore/utils/chunk.py +++ b/dascore/utils/chunk.py @@ -50,13 +50,6 @@ def get_intervals( ------- A 2D array where first column is start and second column is end. """ - # when length is null just use entire length - if pd.isnull(length): - out = np.asarray([start, stop]) - if is_datetime64(start): - out = to_datetime64(out) - return np.atleast_2d(out) - if is_datetime64(start): # need to ensure we have numpy datetimes, not pandas start, stop = to_datetime64(start), to_datetime64(stop) diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index d8a1835b2..9c70beb06 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -3,7 +3,6 @@ from __future__ import annotations import fnmatch -import os from collections import defaultdict from collections.abc import Collection, Mapping, Sequence from functools import cache @@ -69,18 +68,6 @@ def relative_ranges_to_absolute(df, kwargs: dict) -> dict: return out -def _remove_base_path(series: pd.Series, base="") -> pd.Series: - """ - Ensure paths stored in column name use unix style paths and have base - path removed. - """ - assert not series.empty, "Series must be non-empty" - unix_paths = series.str.replace(os.sep, "/") - unix_base_path = (str(base) + "/").replace(os.sep, "/") - out = unix_paths.str.replace(unix_base_path, "", regex=False) - return out - - def _get_min_max_query(kwargs, df): """ Get a dict of {column_name: Optional[min_val], Optional[max_val]}. @@ -233,7 +220,7 @@ def _convert_times(df, some_dict): return some_dict -def get_interval_columns(df, name, arrays=False): +def get_interval_columns(df, name): """ Return a series of start, stop, step for columns. @@ -243,8 +230,6 @@ def get_interval_columns(df, name, arrays=False): The input dataframe. name The name of the coordinate (eg time). - arrays - If True, return output as numpy arrays, else pandas series. """ names = f"{name}_min", f"{name}_max", f"{name}_step" missing_cols = set(names) - set(df.columns) @@ -256,10 +241,7 @@ def get_interval_columns(df, name, arrays=False): ) raise ParameterError(msg) start, stop, step = df[names[0]], df[names[1]], df[names[2]] - if not arrays: - return start, stop, step - else: - return start.values, stop.values, step.values + return start, stop, step def yield_range_tuple_from_kwargs(df, kwargs) -> tuple[str, slice]: @@ -529,26 +511,6 @@ def _column_or_value(df, col, value): return out -def _instructions_modified(instruct_df, sub_source): - """ - Determine if the instruction df columns are the same as the source. - - This is useful for determining which patches need select arguments. - """ - # Get the source and desired output dfs broadcast together. - names = set(sub_source.columns) & set(instruct_df.columns) - source = sub_source.loc[instruct_df["source_index"].values] - # not_modified = np.ones(len(instruct_df), dtype=bool) - not_modified = ~_column_or_value(source, "_modified", False) - for name in names: - val1, val2 = source[name].values, instruct_df[name].values - eq = val1 == val2 - null = pd.isnull(val1) & pd.isnull(val2) - not_modified &= eq | null - modified = ~not_modified - return modified - - def patch_to_dataframe(patch: PatchType) -> pd.DataFrame: """ Convert a patch to a dataframe. diff --git a/dascore/utils/time.py b/dascore/utils/time.py index 38bf6f088..9f3e7ba0d 100644 --- a/dascore/utils/time.py +++ b/dascore/utils/time.py @@ -9,10 +9,8 @@ import pandas as pd from dascore.constants import ( - LARGEDT64, NUMPY_TIME_UNIT_MAPPING, ONE_SECOND, - SMALLDT64, timeable_types, ) from dascore.exceptions import TimeError @@ -469,24 +467,3 @@ def dtype_time_like(dtype_or_array) -> bool: if is_timedelta or is_datetime: return True return False - - -def get_max_min_times(kwarg_time=None): - """ - Function to get min/max times from a tuple of possible time values. - - If None, return max/min times possible. - """ - # first unpack time from tuples - assert kwarg_time is None or len(kwarg_time) == 2 - time_min, time_max = (None, None) if kwarg_time is None else kwarg_time - # get defaults if starttime or endtime is none - time_min = None if pd.isnull(time_min) else time_min - time_max = None if pd.isnull(time_max) else time_max - time_min = to_datetime64(time_min or SMALLDT64) - time_max = to_datetime64(time_max or LARGEDT64) - if time_min is not None and time_max is not None: - if time_min > time_max: - msg = "time_min cannot be greater than time_max." - raise ValueError(msg) - return time_min, time_max diff --git a/tests/test_io/test_h5simple/test_h5simple.py b/tests/test_io/test_h5simple/test_h5simple.py index 198672765..c86d372d7 100644 --- a/tests/test_io/test_h5simple/test_h5simple.py +++ b/tests/test_io/test_h5simple/test_h5simple.py @@ -5,15 +5,9 @@ import shutil import h5py -import numpy as np import pytest import dascore as dc -from dascore.io.h5simple.utils import ( - _get_attr_names, - _get_root_attrs, - _iter_root_arrays, -) from dascore.utils.downloader import fetch @@ -45,58 +39,3 @@ def test_dims_in_attrs(self, h5simple_with_dim_attrs_path): """Ensure if 'dims' is in attrs it gets used.""" patch = dc.spool(h5simple_with_dim_attrs_path, file_format="h5simple")[0] assert isinstance(patch, dc.Patch) - - -class TestH5SimpleInternalHelpers: - """Direct tests for helper branches that still support PyTables fixtures.""" - - def test_get_root_attrs_supports_pytables(self, tmp_path): - """PyTables handles should expose root attrs through the helper.""" - path = tmp_path / "root_attrs.h5" - tables = pytest.importorskip("tables") - with tables.open_file(path, "w") as h5: - h5.root._v_attrs["dims"] = "distance,time" - attrs = _get_root_attrs(h5) - assert attrs.dims == "distance,time" - - def test_iter_root_arrays_supports_pytables(self, tmp_path): - """PyTables root arrays should still be discoverable by helper code.""" - path = tmp_path / "root_arrays.h5" - tables = pytest.importorskip("tables") - with tables.open_file(path, "w") as h5: - h5.create_array("/", "data", obj=np.arange(3)) - names = [name for name, _node in _iter_root_arrays(h5)] - assert names == ["data"] - - def test_get_attr_names_supports_pytables_attrs(self, tmp_path): - """PyTables attr containers should still expose their stored keys.""" - path = tmp_path / "attr_names.h5" - tables = pytest.importorskip("tables") - with tables.open_file(path, "w") as h5: - h5.root._v_attrs["dims"] = "distance,time" - out = _get_attr_names(h5.root._v_attrs) - assert "dims" in out - - def test_get_root_attrs_supports_h5py(self, tmp_path): - """h5py files should continue to use the attrs mapping directly.""" - path = tmp_path / "h5py_attrs.h5" - with h5py.File(path, "w") as h5: - h5.attrs["dims"] = "distance,time" - attrs = _get_root_attrs(h5) - assert attrs["dims"] == "distance,time" - - def test_iter_root_arrays_supports_h5py(self, tmp_path): - """h5py root arrays should still be discoverable by helper code.""" - path = tmp_path / "h5py_root_arrays.h5" - with h5py.File(path, "w") as h5: - h5.create_dataset("data", data=np.arange(3)) - names = [name for name, _node in _iter_root_arrays(h5)] - assert names == ["data"] - - def test_get_attr_names_supports_h5py_attrs(self, tmp_path): - """h5py attr containers should still expose their stored keys.""" - path = tmp_path / "h5py_attr_names.h5" - with h5py.File(path, "w") as h5: - h5.attrs["dims"] = "distance,time" - out = _get_attr_names(h5.attrs) - assert "dims" in out diff --git a/tests/test_io/test_xml_binary/test_xml_binary.py b/tests/test_io/test_xml_binary/test_xml_binary.py index a3726eb52..dbbed1347 100644 --- a/tests/test_io/test_xml_binary/test_xml_binary.py +++ b/tests/test_io/test_xml_binary/test_xml_binary.py @@ -191,6 +191,14 @@ def test_mtime(self, binary_xml_directory): scan3 = dc.scan(binary_xml_directory, timestamp=mtime - 50) assert len(scan3) == 2 + def test_direct_scan_filters_all_by_mtime(self, binary_xml_directory): + """The FiberIO scan contract returns empty after filtering every file.""" + fiber = XMLBinaryV1() + newest = max( + path.stat().st_mtime for path in binary_xml_directory.glob("*.raw") + ) + assert fiber.scan(binary_xml_directory, timestamp=newest + 1) == [] + def test_remote_directory(self, remote_binary_xml_directory): """Remote XMLBinary directories should be scannable.""" fiber = XMLBinaryV1() diff --git a/tests/test_utils/test_misc.py b/tests/test_utils/test_misc.py index 378db4b2c..7d87e8764 100644 --- a/tests/test_utils/test_misc.py +++ b/tests/test_utils/test_misc.py @@ -384,6 +384,17 @@ def test_empty_sequence_false(self): """An empty set of diffs should not be considered close enough.""" assert not all_diffs_close_enough([]) + @pytest.mark.parametrize( + "diffs", + [ + np.array([np.nan, np.nan]), + np.array(["NaT", "NaT"], dtype="timedelta64[ns]"), + ], + ) + def test_all_null_false(self, diffs): + """Diffs containing only null values are not close enough.""" + assert not all_diffs_close_enough(diffs) + class TestOptionalImport: """Ensure the optional import works.""" diff --git a/tests/test_utils/test_time.py b/tests/test_utils/test_time.py index e98ce0e6f..5bc147da0 100644 --- a/tests/test_utils/test_time.py +++ b/tests/test_utils/test_time.py @@ -13,7 +13,6 @@ from dascore.compat import random_state from dascore.exceptions import TimeError from dascore.utils.time import ( - get_max_min_times, is_datetime64, is_timedelta64, saturate_add, @@ -321,6 +320,13 @@ def test_timedelta(self): assert isinstance(out, np.timedelta64) assert out == to_timedelta64(3600) + def test_series(self): + """A Series converts to timedeltas without losing its index.""" + ser = pd.Series([1.0, 2.0], index=["first", "second"]) + out = to_timedelta64(ser) + expected = pd.Series(to_timedelta64(ser.values), index=ser.index) + pd.testing.assert_series_equal(out, expected) + def test_pandas_string_array(self): """Ensure pandas StringArray converts to timedelta64[ns].""" arr = pd.array(["1s", "2s", None], dtype="string") @@ -424,6 +430,13 @@ def test_datetime64(self): out = to_int(to_datetime64("1970-01-01") + np.timedelta64(1, "ns")) assert out == 1 + def test_series(self): + """A datetime Series converts to integer ns and preserves its index.""" + ser = pd.Series(to_datetime64(["1970-01-01", "2000-01-01"])) + ser.index = ["first", "second"] + out = to_int(ser) + pd.testing.assert_series_equal(out, ser.astype(np.int64)) + def test_timedelta64_array(self): """Ensure int ns is returned for datetime64.""" array = to_datetime64(["2017-01-01", "1970-01-01", "1999-01-01"]) @@ -662,14 +675,3 @@ def test_dtype(self): d2 = np.array([1, 2]).astype("timedelta64[ms]").dtype assert not is_timedelta64(d1) assert is_timedelta64(d2) - - -class TestGetmaxMinTimes: - """Tests for max_min fetching.""" - - def test_raises_bad_value(self): - """Simple test to make sure error is raised if unordered tuple.""" - t1 = to_datetime64("2020-01-01") - t2 = to_datetime64("1994-01-01") - with pytest.raises(ValueError): - get_max_min_times((t1, t2)) From fc1370df2ff86189540a8e76e4af354310663ea1 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 09:22:27 +0200 Subject: [PATCH 63/97] Delay Codecov reporting until eight uploads --- codecov.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/codecov.yml b/codecov.yml index 4d6cf5961..de414ff77 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,11 @@ +codecov: + notify: + after_n_builds: 8 + wait_for_ci: true + +comment: + after_n_builds: 8 + flags: unittests: carryforward: false From 4ea086efe2d60341500d81d90f2993291478af35 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 09:51:58 +0200 Subject: [PATCH 64/97] Address CodeRabbit index review --- dascore/io/core.py | 2 +- dascore/io/index/backend.py | 13 +- dascore/io/index/indexer.py | 47 +++-- dascore/io/index/ingest.py | 38 ++-- dascore/io/index/query.py | 10 +- dascore/utils/chunk_plan.py | 3 + dascore/utils/patch.py | 2 +- dascore/utils/paths.py | 17 +- tests/test_core/test_patch_chunk.py | 2 +- .../test_index/test_index_edge_cases.py | 162 ++++++++++++++++++ tests/test_io/test_index/test_plan.py | 5 + tests/test_io/test_indexer.py | 8 +- tests/test_io/test_io_core.py | 9 + tests/test_utils/test_patch_utils.py | 8 + tests/test_utils/test_paths.py | 7 + 15 files changed, 282 insertions(+), 51 deletions(-) diff --git a/dascore/io/core.py b/dascore/io/core.py index 11ce1caaf..f037a9a1a 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -266,7 +266,7 @@ def _resolve_read_spool(spool, source_patch_id: object = "") -> dc.Patch: source_patch_id = normalize_source_patch_id(source_patch_id) if source_patch_id and len(spool) == 1: found = normalize_source_patch_id(spool[0].attrs.get("_source_patch_id", "")) - if found in ("", source_patch_id): + if found == source_patch_id or (not found and not source_patch_id.isdigit()): return spool[0] return _select_patch_from_spool(spool, source_patch_id=source_patch_id) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index f599f0ee4..e67e2139a 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -217,11 +217,11 @@ def _ensure_schema(self) -> None: "INSERT INTO meta_data VALUES (?, ?, ?, ?)", (WHAT_IS_THIS, INDEX_VERSION, dc.__version__, time.time_ns()), ) + self._commit() except Exception: with suppress(Exception): self._rollback() raise - self._commit() def _validate_schema(self, tables: set[str]) -> None: """Validate an existing index before issuing any DDL or mutation.""" @@ -517,12 +517,12 @@ def write_sources(self, records: list[SourceRecord]) -> None: [(pid, name, dims, def_ids[key]) for pid, name, dims, key in link_rows], ) self._execute("UPDATE meta_data SET last_indexed_ns = ?", (now,)) + self._commit() except Exception: # A failed rollback must not mask the original error. with suppress(Exception): self._rollback() raise - self._commit() # Batch size for IN (...) parameter lists; SQLite caps bound # variables (32766 by default) so large replacements must chunk. @@ -553,10 +553,11 @@ def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: self._begin() try: self._delete_by_paths(source_paths, base_uri=base_uri) + self._commit() except Exception: - self._rollback() + with suppress(Exception): + self._rollback() raise - self._commit() # --- queries ----------------------------------------------------- @@ -915,6 +916,10 @@ def _shape_coord_selector(name: str, value): # accept the same open/slice range forms patch-level select does attrs = {k: normalize_range_forms(v) for k, v in (_attrs or {}).items()} coords = {k: normalize_range_forms(v) for k, v in (_coords or {}).items()} + duplicates = set(attrs) & set(coords) + if duplicates: + names = ", ".join(repr(x) for x in sorted(duplicates)) + raise InvalidSpoolQueryError(f"{names} given in both _attrs and _coords.") known_attrs = backend.attr_names() known_coords = backend.coord_names() for name, value in kwargs.items(): diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index b35e9c6d3..cc40a93b5 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -9,6 +9,7 @@ from __future__ import annotations +import hashlib from contextlib import suppress from pathlib import Path @@ -156,16 +157,42 @@ def _directory_format(self, path: Path) -> bool: return is_directory_format(path) + @staticmethod + def _directory_signature(path: Path) -> tuple[int, int]: + """Return a stable 128-bit manifest signature as two SQLite ints.""" + members = sorted( + ( + sub + for sub in path.rglob("*") + if sub.is_file() and not sub.name.startswith(".") + ), + key=lambda sub: sub.relative_to(path).as_posix(), + ) + digest = hashlib.sha256() + for member in members: + stat = member.stat() + relative = member.relative_to(path).as_posix().encode() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(stat.st_mtime_ns.to_bytes(8, "big", signed=True)) + digest.update(stat.st_size.to_bytes(8, "big")) + fingerprint = digest.digest() + return ( + int.from_bytes(fingerprint[:8], "big", signed=True), + int.from_bytes(fingerprint[8:16], "big", signed=True), + ) + def _walk(self) -> dict[str, tuple[int, int, Path]]: """ Walk the spool directory, honoring directory-format scan units. Maps relative path -> (mtime_ns, size, abs path) for every scan unit. A directory-format unit (e.g. XMLBinary) appears as one - entry keyed by the directory, with aggregate stats — max member - mtime and summed member size — so member modification, addition, - and removal all register as a change. Mirrors the skip protocol - dc.scan uses so members are not offered individually. + entry keyed by the directory, with a 128-bit manifest fingerprint + split across the two integer stat fields. The fingerprint covers + every member's relative path, mtime, and size, so member changes + cannot cancel each other out. Mirrors the skip protocol dc.scan + uses so members are not offered individually. """ files: dict[str, tuple[int, int, Path]] = {} gen = _iter_filesystem(self.path, ext=self.ext, include_directories=True) @@ -183,14 +210,8 @@ def _walk(self) -> dict[str, tuple[int, int, Path]]: if path.is_dir(): if self._directory_format(path): signal = "skip" - max_mtime, total_size = 0, 0 - for sub in path.rglob("*"): - if not sub.is_file() or sub.name.startswith("."): - continue - stat = sub.stat() - max_mtime = max(max_mtime, stat.st_mtime_ns) - total_size += stat.st_size - files[self._rel(path)] = (max_mtime, total_size, path) + signature = self._directory_signature(path) + files[self._rel(path)] = (*signature, path) continue stat = path.stat() files[self._rel(path)] = (stat.st_mtime_ns, stat.st_size, path) @@ -207,7 +228,6 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: units (e.g. XMLBinary) are rescanned whole when any member file changes. """ - self._initial_update_done = True files = self._walk() stored = { row.source_path: ( @@ -270,6 +290,7 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: ) if records: self._backend.write_sources(records) + self._initial_update_done = True return self def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 4c3faef34..fa49ace45 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -13,7 +13,6 @@ import re import warnings from dataclasses import dataclass, field, replace -from functools import cache import numpy as np import pandas as pd @@ -160,10 +159,10 @@ def attr_column_name(name: str, kind: str) -> str: return f"{sanitize_attr_name(name)}__{kind}" -@cache -def _base_unit_info(unit_str: str) -> tuple[float, str]: - """Return (scale factor to SI base, canonical base unit string).""" - quant = get_quantity(unit_str).to_base_units() +def _base_unit_info(value, unit_str: str | None = None) -> tuple[float, str]: + """Return a value's base-unit magnitude and canonical unit string.""" + quant = value if unit_str is None else value * get_quantity(unit_str) + quant = get_quantity(quant).to_base_units() return float(quant.magnitude), str(quant.units) @@ -202,8 +201,8 @@ def typed_value(value) -> TypedValue | None: magnitude = getattr(value, "magnitude", 1) if isinstance(magnitude, np.ndarray): return None # array quantities are not scalar attrs - factor, base = _base_unit_info(str(value.units)) - return TypedValue("num", float(magnitude) * factor, units=base) + magnitude, base = _base_unit_info(value) + return TypedValue("num", magnitude, units=base) if isinstance(value, int | np.integer | float | np.floating): return TypedValue("num", float(value)) if isinstance(value, str): @@ -284,16 +283,22 @@ def _coord_record(name: str, summary) -> CoordRecord | None: **common, ) if dtype is not None and np.issubdtype(dtype, np.number): - factor = 1.0 + min_num = float(summary.min) + max_num = float(summary.max) + step = summary.step + step_num = None if pd.isnull(step) else float(step) if units_str is not None: - factor, base = _base_unit_info(units_str) + min_num, base = _base_unit_info(summary.min, units_str) + max_num, _ = _base_unit_info(summary.max, units_str) + if step_num is not None: + step_end, _ = _base_unit_info(summary.min + summary.step, units_str) + step_num = step_end - min_num common["units"] = base - step = summary.step return CoordRecord( value_kind="num", - min_num=float(summary.min) * factor, - max_num=float(summary.max) * factor, - step_num=None if pd.isnull(step) else float(step) * factor, + min_num=min_num, + max_num=max_num, + step_num=step_num, **common, ) if dtype is not None and (dtype.kind in "US" or dtype == object): @@ -379,6 +384,7 @@ def summaries_to_records( by_source.setdefault(str(summary.source_path), []).append(summary) root = base_uri or relative_to root_posix = str(root).replace("\\", "/") if root else None + root_prefix = root_posix.rstrip("/") if root_posix else None out = [] for path, group in by_source.items(): first = group[0] @@ -391,9 +397,11 @@ def summaries_to_records( patches.append(record) posix_path = path.replace("\\", "/") store_path = posix_path - if root_posix and posix_path.startswith(root_posix): + if root_prefix is not None and ( + posix_path == root_prefix or posix_path.startswith(f"{root_prefix}/") + ): # "." (not "") marks a source that IS the root (directory units) - store_path = posix_path[len(root_posix) :].lstrip("/") or "." + store_path = posix_path[len(root_prefix) :].lstrip("/") or "." out.append( SourceRecord( source_path=store_path, diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 0f64a4d37..b60a55a9b 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -349,10 +349,16 @@ def build_coord_clause( else: conditions.append("cd.units IS NULL") if lo is not None: - conditions.append(f"cd.{max_col} >= ?") + clause = f"cd.{max_col} >= ?" + if compatible_units is not None: + clause = f"(cd.units IS NULL OR {clause})" + conditions.append(clause) params.append(lo) if hi is not None: - conditions.append(f"cd.{min_col} <= ?") + clause = f"cd.{min_col} <= ?" + if compatible_units is not None: + clause = f"(cd.units IS NULL OR {clause})" + conditions.append(clause) params.append(hi) # A semi-join the engine can evaluate once (idx_pcoords_name) beats a # correlated EXISTS probed per patch row (~2.5x on a 200k-source index). diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index ac3cf73de..2a852ed76 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -378,6 +378,9 @@ def build_chunk_plan( if missing_dim not in ("raise", "drop"): msg = f"missing_dim must be 'raise' or 'drop', got {missing_dim!r}" raise ParameterError(msg) + if conflict not in ("drop", "raise", "keep_first"): + msg = "conflict must be 'drop', 'raise', or 'keep_first', " f"got {conflict!r}" + raise ParameterError(msg) min_name, max_name = f"{name}_min", f"{name}_max" if min_name not in df.columns and not df.empty: diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index e6b75e1fb..70ca78941 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -625,7 +625,7 @@ def _get_filename(path_ser, strip_extension): if path_ser is not None: # synthetic in-memory identities are not real file names usable = path_ser.str.len().gt(0) & ~path_ser.map(is_memory_uri) - if usable.any(): + if usable.all(): return _get_filename(df["path"], strip_extension) # Determine the requested fields; absent columns render as empty so # names don't depend on which metadata engine produced the dataframe. diff --git a/dascore/utils/paths.py b/dascore/utils/paths.py index 5b330755c..4708edccf 100644 --- a/dascore/utils/paths.py +++ b/dascore/utils/paths.py @@ -2,8 +2,7 @@ from __future__ import annotations -import os -from contextlib import suppress +import tempfile from pathlib import Path from dascore.compat import UPath @@ -33,19 +32,13 @@ def is_memory_uri(path) -> bool: def directory_writable(path) -> bool: """Return True if the directory is writable else False.""" - name = "._dascore_write_test_delete_me" - probe = Path(path) / name + directory = Path(path) try: - # a read-only mount raises OSError (e.g. EROFS) from mkdir/open; - # the whole probe must be guarded, not just the write. - probe.parent.mkdir(exist_ok=True, parents=True) - open(probe, "w").close() + directory.mkdir(exist_ok=True, parents=True) + with tempfile.NamedTemporaryFile(prefix="._dascore_write_test_", dir=directory): + pass except OSError: return False - # the directory is writable; a transient failure to remove the probe - # (e.g. Windows AV/file-locking) must not flip the result. - with suppress(OSError): - os.remove(probe) return True diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 7299355af..d4eeb7ff1 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -318,7 +318,7 @@ def test_merge_unequal_other(self, distance_adjacent): # the differing distance envelopes are preserved, not merged/duplicated def _distance_envelopes(spool): df = spool.get_contents() - return sorted(zip(df["distance_min"], df["distance_max"])) + return sorted(zip(df["distance_min"], df["distance_max"], strict=True)) assert _distance_envelopes(out) == _distance_envelopes(distance_adjacent) diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 0c7c7fe7a..95e800e64 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os import re import sqlite3 @@ -164,6 +165,24 @@ def _execute(self, sql, params=()): with pytest.raises(RuntimeError, match="boom during schema init"): _BoomBackend(tmp_path / "i.sqlite3") + def test_schema_commit_failure_rolls_back(self, tmp_path): + """A schema commit failure follows the protected rollback path.""" + from dascore.io.index.lite import SQLiteBackend + + class _CommitBoomBackend(SQLiteBackend): + rolled_back = False + + def _commit(self): + raise RuntimeError("boom during schema commit") + + def _rollback(self): + type(self).rolled_back = True + return super()._rollback() + + with pytest.raises(RuntimeError, match="schema commit"): + _CommitBoomBackend(tmp_path / "commit.sqlite3") + assert _CommitBoomBackend.rolled_back + def test_reopen_missing_dynamic_attr_column(self, tmp_path): """attr_meta referencing an absent attrs column is rejected on reopen.""" import sqlite3 @@ -390,6 +409,35 @@ def boom(*args, **kwargs): assert len(back.query()) == before back.close() + @pytest.mark.parametrize("operation", ["write", "delete"]) + def test_commit_failure_rolls_back(self, tmp_path, monkeypatch, operation): + """Write and delete commit failures both release their transaction.""" + back = get_backend(tmp_path / f"{operation}.sqlite3") + records = summaries_to_records(make_summaries()) + initial = records[:1] if operation == "write" else records + back.write_sources(initial) + before = len(back.query()) + original_rollback = back._rollback + rolled_back = [] + + def rollback(): + rolled_back.append(True) + original_rollback() + + def commit_failure(): + raise RuntimeError("simulated commit failure") + + monkeypatch.setattr(back, "_rollback", rollback) + monkeypatch.setattr(back, "_commit", commit_failure) + with pytest.raises(RuntimeError, match="commit failure"): + if operation == "write": + back.write_sources(records[1:2]) + else: + back.delete_sources([records[0].source_path]) + assert rolled_back + assert len(back.query()) == before + back.close() + def test_delete_failure_rolls_back(self, tmp_path): """A failing delete leaves the index unchanged.""" back = get_backend(tmp_path / "delete.sqlite3") @@ -428,6 +476,15 @@ def test_duration_attr_roundtrip(self, backend): class TestResolveQueryErrors: """Explicit-namespace validation.""" + def test_duplicate_explicit_namespace_raises(self, backend): + """A name cannot be supplied in both explicit namespaces.""" + with pytest.raises(InvalidSpoolQueryError, match="both _attrs and _coords"): + resolve_query( + backend, + _attrs={"distance": (0, 1)}, + _coords={"distance": (0, 1)}, + ) + def test_unknown_attr_in_explicit_namespace(self, backend): """Unknown key in _attrs raises.""" with pytest.raises(InvalidSpoolQueryError, match="not an attribute"): @@ -559,6 +616,22 @@ def test_null_unit_coord_defs_stay_candidates(self, tmp_path): assert set(df["path"]) == {"with_units.h5", "no_units.h5"} back.close() + def test_nonoverlapping_unitless_coord_stays_candidate(self, tmp_path): + """Numeric envelopes cannot exclude a unitless quantity candidate.""" + back = get_backend(tmp_path / "nonoverlap-unitless.sqlite3") + back.write_sources( + summaries_to_records( + [ + self._summary("with_units.h5", coord_units="m"), + self._summary("no_units.h5"), + ] + ) + ) + meters = get_quantity("m") + df = back.query(Query(coords={"distance": (250 * meters, 300 * meters)})) + assert list(df["path"]) == ["no_units.h5"] + back.close() + def test_all_unitless_quantity_query_keeps_candidates(self, tmp_path): """Unitless defs cannot be proven incompatible; they stay candidates.""" back = get_backend(tmp_path / "unitless.sqlite3") @@ -680,6 +753,49 @@ def test_float_ns_column_rejected(self): class TestIngestEdges: """typed_value and record-building edge cases.""" + def test_offset_quantity_attr_uses_full_conversion(self): + """Affine quantity attrs are converted with their offset.""" + out = typed_value(get_quantity("0 degC")) + assert out is not None + assert out.value == pytest.approx(273.15) + assert out.units == "K" + + def test_offset_coord_uses_full_conversion(self): + """Affine coord bounds use offsets while steps remain deltas.""" + summary = PatchSummary( + attrs={"tag": "temperature"}, + coords={ + "temperature": { + "dtype": "float64", + "min": 0.0, + "max": 100.0, + "step": 1.0, + "units": "degC", + "dims": ("temperature",), + "len": 101, + } + }, + dims=("temperature",), + shape=(101,), + dtype="float32", + source_path="temperature.h5", + source_format="DASDAE", + source_version="1", + ) + out = _coord_record("temperature", summary.coords["temperature"]) + assert out is not None + assert out.min_num == pytest.approx(273.15) + assert out.max_num == pytest.approx(373.15) + assert out.step_num == pytest.approx(1.0) + assert out.units == "K" + + def test_relative_root_requires_path_boundary(self): + """A similarly prefixed path is not made relative to the root.""" + data = make_summaries()[0].dump_structured() + data["source_path"] = "/data/foobar/file.h5" + record = s2r([PatchSummary(**data)], relative_to="/data/foo")[0] + assert record.source_path == "/data/foobar/file.h5" + def test_plain_array_skipped(self): """Arrays are complex attrs; skipped.""" assert typed_value(np.array([1, 2])) is None @@ -741,6 +857,52 @@ def test_multipatch_source_gets_positional_ids(self): class TestIndexerEdges: """DBDirectoryIndexer edge behavior.""" + def test_failed_initial_update_can_retry(self, tmp_path, monkeypatch): + """A failed first walk does not mark automatic updating complete.""" + indexer = DBDirectoryIndexer(tmp_path) + original_walk = indexer._walk + + def fail_walk(): + raise OSError("simulated walk failure") + + monkeypatch.setattr(indexer, "_walk", fail_walk) + with pytest.raises(OSError, match="walk failure"): + indexer.ensure_updated() + assert not indexer._initial_update_done + monkeypatch.setattr(indexer, "_walk", original_walk) + assert indexer.ensure_updated() + assert indexer._initial_update_done + indexer.close() + + def test_directory_manifest_detects_equal_stat_name_swap( + self, tmp_path, monkeypatch + ): + """Equal-size, equal-mtime member replacement changes the signature.""" + unit = tmp_path / "unit" + unit.mkdir() + old = unit / "old.raw" + old.write_bytes(b"same-size") + stamp = 1_700_000_000_000_000_000 + os.utime(old, ns=(stamp, stamp)) + indexer = DBDirectoryIndexer(tmp_path) + monkeypatch.setattr(indexer, "_directory_format", lambda path: path == unit) + monkeypatch.setattr(dc, "scan", lambda *_args, **_kwargs: []) + indexer.update(progress=None) + before = tuple( + indexer._backend.source_stats().loc[0, ["mtime_ns", "size_bytes"]] + ) + + old.unlink() + new = unit / "new.raw" + new.write_bytes(b"same-size") + os.utime(new, ns=(stamp, stamp)) + indexer.update(progress=None) + after = tuple( + indexer._backend.source_stats().loc[0, ["mtime_ns", "size_bytes"]] + ) + assert before != after + indexer.close() + def test_auto_update_on_first_query(self, tmp_path, random_patch): """A brand-new index triggers one update on first query.""" random_patch.io.write(tmp_path / "one.hdf5", "dasdae") diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index 582016987..0f0385fcc 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -249,6 +249,11 @@ def test_drop(self, conflicted_patches): plan = build_chunk_plan(_flat(conflicted_patches), time=None, conflict="drop") assert "data_units" not in plan.outputs.columns + def test_unknown_policy_raises(self, conflicted_patches): + """A misspelled conflict policy cannot silently behave like drop.""" + with pytest.raises(ParameterError, match="conflict must be"): + build_chunk_plan(_flat(conflicted_patches), time=None, conflict="keep_fist") + class TestGroupParameter: """Group attrs partition instead of raising.""" diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index 4f59798f2..2238327a7 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -21,13 +21,17 @@ @pytest.fixture(scope="class") def basic_indexer(two_patch_directory): """Return an indexer on the basic spool directory.""" - return DBDirectoryIndexer(two_patch_directory).update(progress=None) + indexer = DBDirectoryIndexer(two_patch_directory).update(progress=None) + yield indexer + indexer.close() @pytest.fixture(scope="class") def diverse_indexer(diverse_spool_directory): """Return an indexer on the diverse spool directory.""" - return DBDirectoryIndexer(diverse_spool_directory).update(progress=None) + indexer = DBDirectoryIndexer(diverse_spool_directory).update(progress=None) + yield indexer + indexer.close() @pytest.fixture(scope="class") diff --git a/tests/test_io/test_io_core.py b/tests/test_io/test_io_core.py index a616163de..222f63e17 100644 --- a/tests/test_io/test_io_core.py +++ b/tests/test_io/test_io_core.py @@ -933,6 +933,15 @@ def test_get_supported_io_table(self): class TestIOCoreCoverageEdges: """Remaining io.core resolution/robustness branches.""" + def test_numeric_singleton_without_identity_not_trusted(self): + """A positional ID cannot resolve an anonymous trimmed singleton.""" + from dascore.exceptions import PatchAttributeError + from dascore.io.core import _resolve_read_spool + + spool = dc.spool([dc.get_example_patch()]) + with pytest.raises(PatchAttributeError, match="uniquely resolved"): + _resolve_read_spool(spool, source_patch_id="1") + def test_non_unique_patch_resolution_raises(self): """An unresolvable source id in a multi-patch read raises clearly.""" from dascore.exceptions import PatchAttributeError diff --git a/tests/test_utils/test_patch_utils.py b/tests/test_utils/test_patch_utils.py index 30c2e99a6..6a8e4c5ed 100644 --- a/tests/test_utils/test_patch_utils.py +++ b/tests/test_utils/test_patch_utils.py @@ -857,6 +857,14 @@ def test_path_column_leave_extension(self, random_directory_spool): names = get_patch_names(random_directory_spool, strip_extension=False) assert "." in names.iloc[0] + def test_mixed_path_sources_use_metadata(self, random_spool): + """Mixed real and memory paths consistently use metadata names.""" + df = random_spool.get_contents().iloc[:2].copy() + df["path"] = ["/tmp/real_file.h5", "memory://registry/patch"] + names = get_patch_names(df) + assert names.iloc[0] != "real_file" + assert names.iloc[1] != "patch" + def test_multiple_coord_fields(self, random_spool): """Naming on more than one coordinate flattens the min/max fields.""" # drop path so the coordinate-based naming branch is exercised diff --git a/tests/test_utils/test_paths.py b/tests/test_utils/test_paths.py index 384eb8d3f..e19b1931e 100644 --- a/tests/test_utils/test_paths.py +++ b/tests/test_utils/test_paths.py @@ -33,6 +33,13 @@ def test_unwritable_returns_false(self, tmp_path): a_file.write_text("x") assert directory_writable(a_file / "sub") is False + def test_existing_legacy_probe_is_preserved(self, tmp_path): + """The writability probe never truncates a predictable old sentinel.""" + sentinel = tmp_path / "._dascore_write_test_delete_me" + sentinel.write_text("keep me") + assert directory_writable(tmp_path) is True + assert sentinel.read_text() == "keep me" + class TestIsPathlike: """Tests for ``is_pathlike``.""" From 78bf8bb6d524c1ca28102f574142c37e154c6f1a Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 09:57:42 +0200 Subject: [PATCH 65/97] Strengthen restricted index update test --- tests/test_io/test_indexer.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index 2238327a7..4372a2d32 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -241,12 +241,28 @@ def test_noop_update_rescans_nothing(self, basic_indexer): def test_update_with_specific_paths(self, basic_indexer): """Updating with specific paths restricts the rescan.""" - files = list(basic_indexer.path.rglob("*.hdf5")) - assert len(files) > 0 - updated = basic_indexer.update(paths=[files[0].name], progress=None) - assert len(updated()) >= 1 - updated2 = basic_indexer.update(paths=[str(files[0])], progress=None) - assert len(updated2()) >= 1 + files = sorted(basic_indexer.path.rglob("*.hdf5")) + assert len(files) >= 2 + + def _indexed_times(): + sources = basic_indexer._backend.get_sources().set_index("source_path") + return sources["last_indexed_ns"].to_dict() + + before = _indexed_times() + for path in files[:2]: + stat = path.stat() + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + first, second = (basic_indexer._rel(path) for path in files[:2]) + basic_indexer.update(paths=[files[0].name], progress=None) + after_relative = _indexed_times() + assert after_relative[first] > before[first] + assert after_relative[second] == before[second] + + basic_indexer.update(paths=[str(files[1])], progress=None) + after_absolute = _indexed_times() + assert after_absolute[first] == after_relative[first] + assert after_absolute[second] > after_relative[second] class TestNameResolution: From d4162f919d20f208d1cea9800d3fba81ec9c8c09 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 10:12:21 +0200 Subject: [PATCH 66/97] Publish Codecov after seven uploads --- codecov.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index de414ff77..70d65c692 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,10 +1,9 @@ codecov: notify: - after_n_builds: 8 - wait_for_ci: true + after_n_builds: 7 comment: - after_n_builds: 8 + after_n_builds: 7 flags: unittests: From c0a558d7b47df9be53c50bf5762af7c9c537edf7 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 13:33:44 +0200 Subject: [PATCH 67/97] Address remaining CodeRabbit findings --- dascore/io/index/backend.py | 20 +++++++++++- dascore/io/index/indexer.py | 14 +++++---- dascore/io/index/query.py | 31 ++++++++++++------- dascore/utils/hdf5.py | 3 +- dascore/utils/patch.py | 24 +++++++++----- tests/test_core/test_patch_chunk.py | 28 +++++++++++++++++ tests/test_io/test_index/test_db_dirspool.py | 10 ++---- .../test_io/test_index/test_index_contract.py | 16 +++++++++- .../test_index/test_index_edge_cases.py | 17 +++++++--- tests/test_io/test_indexer.py | 4 ++- tests/test_utils/test_hdf_utils.py | 7 +++++ 11 files changed, 133 insertions(+), 41 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index e67e2139a..4d07c9b87 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -126,6 +126,10 @@ def source_stats(self) -> pd.DataFrame: def get_metadata(self) -> dict: """Return index-level metadata.""" + @abc.abstractmethod + def mark_initial_update_done(self) -> None: + """Persist that the directory index completed its first update.""" + @abc.abstractmethod def attr_names(self) -> set[str]: """Return original attr names known to the index.""" @@ -215,7 +219,7 @@ def _ensure_schema(self) -> None: ) self._execute( "INSERT INTO meta_data VALUES (?, ?, ?, ?)", - (WHAT_IS_THIS, INDEX_VERSION, dc.__version__, time.time_ns()), + (WHAT_IS_THIS, INDEX_VERSION, dc.__version__, 0), ) self._commit() except Exception: @@ -435,6 +439,20 @@ def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: # --- writes ------------------------------------------------------ + def mark_initial_update_done(self) -> None: + """Persist successful completion of a directory index's first update.""" + self._begin() + try: + self._execute( + "UPDATE meta_data SET last_indexed_ns = ?", + (time.time_ns(),), + ) + self._commit() + except Exception: + with suppress(Exception): + self._rollback() + raise + def write_sources(self, records: list[SourceRecord]) -> None: """ Insert or replace sources and all dependent rows, atomically. diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index cc40a93b5..000a619cf 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -62,12 +62,12 @@ def __init__( requires_local_directory(path, label="DBDirectoryIndexer") self.path = Path(path).absolute() self.index_path = Path(self._find_index_path(index_path)) - # A brand-new index triggers one automatic update on first query, - # matching the historic auto-index-on-first-access behavior. - self._initial_update_done = ( - self.index_path.exists() and self.index_path.stat().st_size > 0 - ) self._backend = get_backend(self.index_path) + # Schema creation alone is not a successful directory scan. Read the + # transactional marker so a new process retries an interrupted first + # update instead of trusting a merely nonempty SQLite file. + metadata = self._backend.get_metadata() + self._initial_update_done = bool(metadata["last_indexed_ns"]) @property def _index_name(self) -> str: @@ -290,7 +290,9 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: ) if records: self._backend.write_sources(records) - self._initial_update_done = True + if not self._initial_update_done: + self._backend.mark_initial_update_done() + self._initial_update_done = True return self def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index b60a55a9b..1f1720271 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -117,32 +117,39 @@ def _range_bounds( coerced usable bounds so callers don't coerce twice. """ lo_raw, hi_raw = value - lo = hi = None kind = None typed_values = [] + typed_bounds = [] for raw, side in ((lo_raw, "lo"), (hi_raw, "hi")): if raw is None or raw is Ellipsis: continue typed = _coerce_scalar(raw, target_kinds) typed_values.append(typed) knd = typed.kind - val = ( - typed.value - if target_units is _UNSET - else _to_target_unit(typed, target_units, name) - ) if kind is not None and knd != kind: msg = f"Range bounds {value!r} have mixed kinds ({kind}, {knd})." raise InvalidSpoolQueryError(msg) kind = knd + typed_bounds.append((side, typed)) + if kind is None: + msg = f"Range {value!r} has no usable bounds." + raise InvalidSpoolQueryError(msg) + + # Validate all bound kinds before attempting unit conversion. An + # unsupported but internally consistent kind is a valid no-match query; + # mixed kinds remain an invalid range. + lo = hi = None + for side, typed in typed_bounds: + val = ( + typed.value + if target_units is _UNSET or kind not in target_kinds + else _to_target_unit(typed, target_units, name) + ) if side == "lo": lo = val else: hi = val - if kind is None: - msg = f"Range {value!r} has no usable bounds." - raise InvalidSpoolQueryError(msg) - if lo is not None and hi is not None and lo > hi: + if kind in target_kinds and lo is not None and hi is not None and lo > hi: msg = f"Range {value!r} has lo > hi after coercion." raise InvalidSpoolQueryError(msg) return kind, lo, hi, typed_values @@ -251,6 +258,8 @@ def col(kind): coerced = [_coerce_scalar(v, kinds) for v in value] by_kind: dict[str, list] = {} for typed in coerced: + if typed.kind not in kinds: + continue val = _to_target_unit(typed, units.get(typed.kind), name) by_kind.setdefault(typed.kind, []).append(val) subclauses = [] @@ -274,10 +283,10 @@ def col(kind): return None typed = _coerce_scalar(value, kinds) kind = typed.kind - val = _to_target_unit(typed, units.get(kind), name) if kind not in kinds: where.add("FALSE") return None + val = _to_target_unit(typed, units.get(kind), name) where.add(f"{col(kind)} = ?", val) return None diff --git a/dascore/utils/hdf5.py b/dascore/utils/hdf5.py index bf049131e..4a30476df 100644 --- a/dascore/utils/hdf5.py +++ b/dascore/utils/hdf5.py @@ -167,7 +167,8 @@ def open_h5_resource( handle.close() raise try: - _maybe_make_parent_directory(resource) + if mode != "r": + _maybe_make_parent_directory(resource) return _ManagedH5pyFile(constructor(resource, mode=mode)) except TypeError: msg = f"Couldn't get handle from {resource} using h5py" diff --git a/dascore/utils/patch.py b/dascore/utils/patch.py index 70ca78941..c59e7d130 100644 --- a/dascore/utils/patch.py +++ b/dascore/utils/patch.py @@ -32,7 +32,7 @@ PatchAttributeError, PatchCoordinateError, ) -from dascore.units import get_quantity, is_percent +from dascore.units import convert_units, get_quantity, is_percent from dascore.utils.attrs import combine_patch_attrs from dascore.utils.coordmanager import merge_coord_managers from dascore.utils.deprecate import deprecate @@ -423,13 +423,20 @@ def _get_merge_dim(df) -> str | None: return dims_vary[dims_vary].index[0] -def _middle_step(df, dim): - """Return the middle value of non-null member steps, or None.""" - col = df[f"{dim}_step"].values - valid = col[~pd.isnull(col)] - if not len(valid): +def _middle_step(coords, dim, target_units): + """Return the middle member step expressed in the merged coord's units.""" + steps = [] + for manager in coords: + coord = manager.coord_map[dim] + step = coord.step + if pd.isnull(step): + continue + if target_units is not None and coord.units is not None: + step = convert_units(step, to_units=target_units, from_units=coord.units) + steps.append(step) + if not steps: return None - return get_middle_value(valid) + return get_middle_value(np.asarray(steps)) def _split_coord_merge_kwargs(merge_kwargs) -> tuple[dict, dict]: @@ -465,7 +472,8 @@ def _get_merged_coord( return merge_coord_managers( coords, dim=merge_dim, drop_conflicting=drop_conflicting ) - if snap_coords and (step := _middle_step(df, merge_dim)) is not None: + step = _middle_step(coords, merge_dim, merged.units) + if snap_coords and step is not None: merged = merged.simplify(tolerance * np.abs(step)) # Passing the pre-built dim coord avoids materializing the members' # concatenated values only to discard them. diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index d4eeb7ff1..689f90ae7 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -415,6 +415,34 @@ def test_merge_distance(self, distance_adjacent): assert old_df["distance_min"].min() == new_df["distance_min"].min() assert old_df["distance_max"].max() == new_df["distance_max"].max() + def test_non_si_merge_tolerance_uses_coord_units(self, random_patch): + """Canonical index steps are not interpreted in native coord units.""" + from dascore.utils.patch import _get_merged_coord + + size = len(random_patch.get_coord("distance")) + first = dc.get_coord(data=np.arange(size, dtype=float), units="km") + second = dc.get_coord( + data=np.arange(size, dtype=float) + size + 8, + units="km", + ) + patches = [ + random_patch.update_coords(distance=first), + random_patch.update_coords(distance=second), + ] + # Index summaries store numeric dimension steps in canonical SI. + summaries = pd.DataFrame({"distance_step": [1000.0, 1000.0]}) + manager = _get_merged_coord( + summaries, + "distance", + [patch.coords for patch in patches], + tolerance=1.5, + ) + merged = manager.coord_map["distance"] + assert not merged.evenly_sampled + assert np.array_equal( + merged.values, np.concatenate([first.values, second.values]) + ) + def test_merge_distance_no_order(self, distance_adjacent_no_order): """Ensure distance can be merged with unsorted coords.""" sp = distance_adjacent_no_order.chunk(distance=...) diff --git a/tests/test_io/test_index/test_db_dirspool.py b/tests/test_io/test_index/test_db_dirspool.py index 3e733574a..52cec35bc 100644 --- a/tests/test_io/test_index/test_db_dirspool.py +++ b/tests/test_io/test_index/test_db_dirspool.py @@ -107,7 +107,7 @@ def test_deleted_file_removed(self, fresh): updated = spool.update(progress=None) assert len(updated) == 2 - def test_no_change_update_uses_narrow_projection(self, fresh): + def test_no_change_update_uses_narrow_projection(self, fresh, monkeypatch): """A no-op update reads only the stat columns, not the wide table.""" path, spool = fresh backend = spool.indexer._backend @@ -116,10 +116,6 @@ def _boom(self): raise AssertionError("wide get_sources() during no-change update") # A no-change update must not fetch the full sources table. - original = type(backend).get_sources - type(backend).get_sources = _boom - try: - reupdated = spool.update(progress=None) - finally: - type(backend).get_sources = original + monkeypatch.setattr(type(backend), "get_sources", _boom) + reupdated = spool.update(progress=None) assert len(reupdated) == 3 diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index 0e4298cbd..6437754ca 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -19,6 +19,7 @@ from dascore.io.index import Query, get_backend, summaries_to_records from dascore.io.index.backend import resolve_query from dascore.io.index.query import InvalidSpoolQueryError +from dascore.io.index.schema import INDEX_VERSION from dascore.units import get_quantity @@ -236,6 +237,19 @@ def test_kind_mismatch_matches_nothing(self, backend): df = backend.query(Query(attrs={"station": 5})) assert df.empty + @pytest.mark.parametrize( + "value", + [ + get_quantity("1 m"), + [get_quantity("1 m"), get_quantity("2 m")], + (get_quantity("900 m"), get_quantity("1 km")), + ], + ) + def test_quantity_kind_mismatch_matches_nothing(self, backend, value): + """Quantity forms do not convert against a string-only attribute.""" + df = backend.query(Query(attrs={"station": value})) + assert df.empty + def test_mixed_kind_attr(self, backend): """Mixed kind attr.""" # shot_number exists as num (42) and str ("unknown") @@ -434,7 +448,7 @@ def test_metadata(self, backend): """Metadata.""" meta = backend.get_metadata() assert meta["what_is_this"] == "dascore_spool_index" - assert meta["index_version"] == 2 + assert meta["index_version"] == INDEX_VERSION def test_names(self, backend): """Names.""" diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 95e800e64..5d3d11d57 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -858,9 +858,9 @@ class TestIndexerEdges: """DBDirectoryIndexer edge behavior.""" def test_failed_initial_update_can_retry(self, tmp_path, monkeypatch): - """A failed first walk does not mark automatic updating complete.""" + """A new process retries when the first automatic update failed.""" indexer = DBDirectoryIndexer(tmp_path) - original_walk = indexer._walk + index_path = indexer.index_path def fail_walk(): raise OSError("simulated walk failure") @@ -869,11 +869,18 @@ def fail_walk(): with pytest.raises(OSError, match="walk failure"): indexer.ensure_updated() assert not indexer._initial_update_done - monkeypatch.setattr(indexer, "_walk", original_walk) - assert indexer.ensure_updated() - assert indexer._initial_update_done indexer.close() + retry = DBDirectoryIndexer(tmp_path, index_path=index_path) + assert not retry._initial_update_done + assert retry.ensure_updated() + assert retry._initial_update_done + retry.close() + + reopened = DBDirectoryIndexer(tmp_path, index_path=index_path) + assert reopened._initial_update_done + reopened.close() + def test_directory_manifest_detects_equal_stat_name_swap( self, tmp_path, monkeypatch ): diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index 4372a2d32..b7451aa40 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -44,7 +44,9 @@ def diverse_df(diverse_indexer): def empty_index(tmp_path_factory): """Create an index around an empty directory.""" path = tmp_path_factory.mktemp("index_created_test") - return DBDirectoryIndexer(path).update(progress=None) + indexer = DBDirectoryIndexer(path).update(progress=None) + yield indexer + indexer.close() class TestFindIndex: diff --git a/tests/test_utils/test_hdf_utils.py b/tests/test_utils/test_hdf_utils.py index 7efb1258c..0187e8d67 100644 --- a/tests/test_utils/test_hdf_utils.py +++ b/tests/test_utils/test_hdf_utils.py @@ -45,6 +45,13 @@ def test_reader_get_handle(self, tmp_path): with closing(H5Reader.get_handle(path)) as handle: assert "waveforms" in handle + def test_missing_reader_does_not_create_parent(self, tmp_path): + """Opening a missing file for reading does not mutate the filesystem.""" + path = tmp_path / "missing_parent" / "missing.h5" + with pytest.raises(OSError): + H5Reader.get_handle(path) + assert not path.parent.exists() + class TestGetH5pyFile: """Tests for unwrapping managed h5py handles.""" From 5373df2444fd0454cd1f82943b53132d28757dba Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 13:43:57 +0200 Subject: [PATCH 68/97] Cover index marker rollback --- dascore/io/index/query.py | 2 -- .../test_index/test_index_edge_cases.py | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 1f1720271..7f788c5be 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -265,8 +265,6 @@ def col(kind): subclauses = [] params = [] for kind, vals in by_kind.items(): - if kind not in kinds: - continue marks = ", ".join("?" for _ in vals) subclauses.append(f"{col(kind)} IN ({marks})") params.extend(vals) diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 5d3d11d57..377fa9552 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -438,6 +438,28 @@ def commit_failure(): assert len(back.query()) == before back.close() + def test_marker_commit_failure_rolls_back(self, tmp_path, monkeypatch): + """A marker commit failure restores the initial metadata value.""" + back = get_backend(tmp_path / "marker.sqlite3") + before = back.get_metadata()["last_indexed_ns"] + original_rollback = back._rollback + rolled_back = [] + + def rollback(): + rolled_back.append(True) + original_rollback() + + def commit_failure(): + raise RuntimeError("simulated commit failure") + + monkeypatch.setattr(back, "_rollback", rollback) + monkeypatch.setattr(back, "_commit", commit_failure) + with pytest.raises(RuntimeError, match="commit failure"): + back.mark_initial_update_done() + assert rolled_back + assert back.get_metadata()["last_indexed_ns"] == before + back.close() + def test_delete_failure_rolls_back(self, tmp_path): """A failing delete leaves the index unchanged.""" back = get_backend(tmp_path / "delete.sqlite3") From ef6b44d08cc3651afc06723e4404c05eecf35373 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 21:56:17 +0200 Subject: [PATCH 69/97] Reduce redundancy in index backend and query layer Behavior-preserving cleanup of the new index code: - backend: extract a _transaction() context manager for the four identical begin/commit/rollback blocks, and _placeholders()/ _iter_in_batches() helpers for the repeated IN-clause batching; reuse _fetch_in in _pivot_coords; share the query/count metadata prologue. - query: hoist the shared FROM skeleton and query-normalization out of build_query_sql/build_count_sql. - catalog: drop the residual tuple's always-False 'relative' element. - ingest: merge the datetime/timedelta branches of _coord_record behind a single converter and an early dtype-None guard. - spool: drop the no-op _patch_id reassignment in _chunk_working_df. --- dascore/core/spool.py | 9 +-- dascore/io/index/backend.py | 124 +++++++++++++++++------------------- dascore/io/index/catalog.py | 14 ++-- dascore/io/index/ingest.py | 26 +++----- dascore/io/index/query.py | 33 ++++++---- 5 files changed, 102 insertions(+), 104 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 19bf1213f..489030d39 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -835,10 +835,11 @@ def _chunk_working_df(self) -> pd.DataFrame: """Return the source rows the chunk planner consumes.""" from dascore.utils.chunk_plan import _ensure_patch_id - source = self._source_df - working = source.drop(columns=list(self._drop_columns), errors="ignore") - if "_patch_id" in source.columns: - working = working.assign(_patch_id=source["_patch_id"]) + # _patch_id is never in _drop_columns, so it survives the drop when + # present; _ensure_patch_id supplies a positional fallback otherwise. + working = self._source_df.drop( + columns=list(self._drop_columns), errors="ignore" + ) return _ensure_patch_id(working) def chunk_plan( diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 4d07c9b87..07fd313f8 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -11,7 +11,7 @@ import abc import time import warnings -from contextlib import suppress +from contextlib import contextmanager, suppress from pathlib import Path import numpy as np @@ -191,6 +191,23 @@ def _existing_tables(self) -> set[str]: def _table_columns(self, table: str) -> set[str]: """Return persisted columns for one table.""" + @contextmanager + def _transaction(self): + """ + Run the wrapped body inside one transaction. + + Commits on normal (or early-return) exit; on any error rolls back + without letting a failed rollback mask the original exception. + """ + self._begin() + try: + yield + self._commit() + except Exception: + with suppress(Exception): + self._rollback() + raise + # --- schema ------------------------------------------------------ def _ensure_schema(self) -> None: @@ -198,14 +215,12 @@ def _ensure_schema(self) -> None: if tables: self._validate_schema(tables) return - self._begin() - try: + with self._transaction(): # Another connection may have initialized the file while this # writer waited for BEGIN IMMEDIATE. Re-check under the lock. tables = self._existing_tables() if tables: self._validate_schema(tables) - self._commit() return for name, columns in TABLES.items(): self._execute( @@ -221,11 +236,6 @@ def _ensure_schema(self) -> None: "INSERT INTO meta_data VALUES (?, ?, ?, ?)", (WHAT_IS_THIS, INDEX_VERSION, dc.__version__, 0), ) - self._commit() - except Exception: - with suppress(Exception): - self._rollback() - raise def _validate_schema(self, tables: set[str]) -> None: """Validate an existing index before issuing any DDL or mutation.""" @@ -283,8 +293,7 @@ def _coord_meta(self, names=None) -> pd.DataFrame: params: list = [] if names is not None: params = sorted(names) - marks = ", ".join("?" for _ in params) - sql += f" WHERE pc.coord_name IN ({marks})" + sql += f" WHERE pc.coord_name IN ({self._placeholders(len(params))})" return self._fetch_df(sql, params) def _next_id(self, table: str, column: str) -> int: @@ -385,10 +394,7 @@ def _ensure_coord_defs(self, defs_needed: dict) -> dict[str, int]: """ keys = list(defs_needed) mapping: dict[str, int] = {} - batch = self._in_clause_batch - for start in range(0, len(keys), batch): - chunk = keys[start : start + batch] - marks = ", ".join("?" for _ in chunk) + for chunk, marks in self._iter_in_batches(keys): found = self._fetch_df( f"SELECT def_key, coord_def_id FROM coord_defs " f"WHERE def_key IN ({marks})", @@ -433,7 +439,7 @@ def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: if not rows: return quoted = ", ".join(self.dialect.quote(c) for c in columns) - marks = ", ".join("?" for _ in columns) + marks = self._placeholders(len(columns)) sql = f"INSERT INTO {self.dialect.quote(table)} ({quoted}) VALUES ({marks})" self._executemany(sql, rows) @@ -441,17 +447,11 @@ def _bulk_insert(self, table: str, columns: tuple, rows: list) -> None: def mark_initial_update_done(self) -> None: """Persist successful completion of a directory index's first update.""" - self._begin() - try: + with self._transaction(): self._execute( "UPDATE meta_data SET last_indexed_ns = ?", (time.time_ns(),), ) - self._commit() - except Exception: - with suppress(Exception): - self._rollback() - raise def write_sources(self, records: list[SourceRecord]) -> None: """ @@ -460,8 +460,7 @@ def write_sources(self, records: list[SourceRecord]) -> None: Rows are batched per table (attrs grouped by column signature) so columnar engines aren't punished by row-at-a-time inserts. """ - self._begin() - try: + with self._transaction(): by_base: dict[str, list[str]] = {} for record in records: by_base.setdefault(record.base_uri or "", []).append(record.source_path) @@ -535,17 +534,29 @@ def write_sources(self, records: list[SourceRecord]) -> None: [(pid, name, dims, def_ids[key]) for pid, name, dims, key in link_rows], ) self._execute("UPDATE meta_data SET last_indexed_ns = ?", (now,)) - self._commit() - except Exception: - # A failed rollback must not mask the original error. - with suppress(Exception): - self._rollback() - raise # Batch size for IN (...) parameter lists; SQLite caps bound # variables (32766 by default) so large replacements must chunk. _in_clause_batch = 5000 + @staticmethod + def _placeholders(count: int) -> str: + """Return a comma-separated run of ``count`` ``?`` bind markers.""" + return ", ".join("?" for _ in range(count)) + + def _iter_in_batches(self, items): + """ + Yield ``(chunk, marks)`` for an ``IN (...)`` list. + + Splitting on ``_in_clause_batch`` keeps each statement under + SQLite's bound-variable cap; ``marks`` is the placeholder run for + the chunk. + """ + batch = self._in_clause_batch + for start in range(0, len(items), batch): + chunk = items[start : start + batch] + yield chunk, self._placeholders(len(chunk)) + def _delete_by_paths(self, source_paths: list[str], base_uri: str = "") -> None: """ Delete sources by (base_uri, source_path) identity. @@ -557,10 +568,7 @@ def _delete_by_paths(self, source_paths: list[str], base_uri: str = "") -> None: """ if not source_paths: return - batch = self._in_clause_batch - for start in range(0, len(source_paths), batch): - chunk = source_paths[start : start + batch] - marks = ", ".join("?" for _ in chunk) + for chunk, marks in self._iter_in_batches(source_paths): self._execute( f"DELETE FROM sources WHERE source_path IN ({marks}) AND base_uri = ?", [*chunk, base_uri], @@ -568,26 +576,29 @@ def _delete_by_paths(self, source_paths: list[str], base_uri: str = "") -> None: def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: """Remove sources (identified by base_uri + path) and dependents.""" - self._begin() - try: + with self._transaction(): self._delete_by_paths(source_paths, base_uri=base_uri) - self._commit() - except Exception: - with suppress(Exception): - self._rollback() - raise # --- queries ----------------------------------------------------- - def query(self, query=None) -> pd.DataFrame: - """Return the flat patch-row relation for a query (or several).""" + def _query_context(self, query): + """ + Normalize a query (or several) and fetch the metadata SQL needs. + + Returns ``(queries, attr_meta, coord_meta)``; coord metadata is + only consulted for coord predicates, so the (whole-relation + DISTINCT) scan is skipped for attr-only/empty queries. + """ query = query if query is not None else Query() queries = [query] if isinstance(query, Query) else list(query) attr_meta = self._attr_meta() - # coord metadata is only consulted for coord predicates; skip the - # (whole-relation DISTINCT) scan for attr-only/empty queries. coord_names = {name for q in queries for name in q.coords} coord_meta = self._coord_meta(coord_names) if coord_names else pd.DataFrame() + return queries, attr_meta, coord_meta + + def query(self, query=None) -> pd.DataFrame: + """Return the flat patch-row relation for a query (or several).""" + queries, attr_meta, coord_meta = self._query_context(query) sql, params, residuals = build_query_sql( queries, self.dialect, attr_meta, coord_meta ) @@ -600,11 +611,7 @@ def query(self, query=None) -> pd.DataFrame: def count(self, query=None) -> int: """Count matching patches without projecting or pivoting rows.""" - query = query if query is not None else Query() - queries = [query] if isinstance(query, Query) else list(query) - attr_meta = self._attr_meta() - coord_names = {name for q in queries for name in q.coords} - coord_meta = self._coord_meta(coord_names) if coord_names else pd.DataFrame() + queries, attr_meta, coord_meta = self._query_context(query) sql, params, residuals = build_count_sql( queries, self.dialect, attr_meta, coord_meta ) @@ -619,10 +626,7 @@ def _fetch_in(self, base_sql: str, column: str, ids: list) -> pd.DataFrame: if not ids: return self._fetch_df(f"{base_sql} WHERE 0") frames = [] - batch = self._in_clause_batch - for start in range(0, len(ids), batch): - chunk = ids[start : start + batch] - marks = ", ".join("?" for _ in chunk) + for chunk, marks in self._iter_in_batches(ids): frames.append( self._fetch_df(f"{base_sql} WHERE {column} IN ({marks})", chunk) ) @@ -816,15 +820,7 @@ def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: coords = self._fetch_df(link_sql) coords = coords[coords["patch_id"].isin(set(ids))].reset_index(drop=True) else: - frames = [] - batch = self._in_clause_batch - for start in range(0, len(ids), batch): - chunk = ids[start : start + batch] - marks = ", ".join("?" for _ in chunk) - frames.append( - self._fetch_df(f"{link_sql} WHERE pc.patch_id IN ({marks})", chunk) - ) - coords = pd.concat(frames, ignore_index=True) + coords = self._fetch_in(link_sql, "pc.patch_id", ids) if coords.empty: return out coords = self._add_envelope_objects(coords) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index ba9b7d445..a3e0b615d 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -297,7 +297,7 @@ def __init__( resolver: PatchResolver | None = None, syncer=None, queries: tuple[Query, ...] = (), - residuals: tuple[tuple[dict, bool, bool], ...] = (), + residuals: tuple[tuple[dict, bool], ...] = (), revision: _CatalogRevision | None = None, ): self._backend = backend @@ -480,7 +480,7 @@ def select( f"{sorted(query.attrs)}." ) raise InvalidSpoolQueryError(msg) - residual = (dict(query.coords), True, False) + residual = (dict(query.coords), True) return self._view(self._queries, (*self._residuals, residual)) if relative and query.coords: query = Query( @@ -496,7 +496,7 @@ def select( self.backend, query.coords ) query = Query(attrs=query.attrs, coords=si_coords) - residuals = (*residuals, (residual_coords, False, False)) + residuals = (*residuals, (residual_coords, False)) return self._view((*self._queries, query), residuals) def _relative_to_absolute(self, kwargs: dict) -> dict: @@ -568,7 +568,7 @@ def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Pat hints they only reduce reading, exactness is re-applied above. """ trim_hint = {} - for coords, samples, _ in self._residuals: + for coords, samples in self._residuals: if not samples: # Canonical-SI and quantity bounds stay out of reader # hints: readers take numbers in their native units, so @@ -584,7 +584,7 @@ def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Pat ) trim_hint.update(extra_trim or {}) patch = self.resolver.resolve(row, **trim_hint) - for coords, samples, relative in self._residuals: + for coords, samples in self._residuals: coord_map = patch.coords.coord_map usable = { k: ( @@ -596,7 +596,9 @@ def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Pat if k in coord_map } if usable: - patch = patch.select(**usable, samples=samples, relative=relative) + # residual bounds are already absolute (relative queries + # resolve to absolute before the residual is recorded). + patch = patch.select(**usable, samples=samples, relative=False) return patch def __iter__(self): diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index fa49ace45..98334f452 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -262,27 +262,21 @@ def _coord_record(name: str, summary) -> CoordRecord | None: coord_hash=fingerprint, ) dtype = np.dtype(summary.dtype) if summary.dtype else None - if dtype is not None and np.issubdtype(dtype, np.datetime64): + if dtype is None: + return None # unsupported coord representation: skip, per design + if dtype.kind in "mM": # datetime64 ("M") / timedelta64 ("m") + is_datetime = dtype.kind == "M" + convert = to_datetime64 if is_datetime else to_timedelta64 step = summary.step return CoordRecord( value_kind="time", - is_relative=False, - min_ns=to_int(to_datetime64(summary.min)), - max_ns=to_int(to_datetime64(summary.max)), + is_relative=not is_datetime, + min_ns=to_int(convert(summary.min)), + max_ns=to_int(convert(summary.max)), step_ns=None if pd.isnull(step) else to_int(to_timedelta64(step)), **common, ) - if dtype is not None and np.issubdtype(dtype, np.timedelta64): - step = summary.step - return CoordRecord( - value_kind="time", - is_relative=True, - min_ns=to_int(to_timedelta64(summary.min)), - max_ns=to_int(to_timedelta64(summary.max)), - step_ns=None if pd.isnull(step) else to_int(to_timedelta64(step)), - **common, - ) - if dtype is not None and np.issubdtype(dtype, np.number): + if np.issubdtype(dtype, np.number): min_num = float(summary.min) max_num = float(summary.max) step = summary.step @@ -301,7 +295,7 @@ def _coord_record(name: str, summary) -> CoordRecord | None: step_num=step_num, **common, ) - if dtype is not None and (dtype.kind in "US" or dtype == object): + if dtype.kind in "US" or dtype == object: return CoordRecord( value_kind="str", min_str=str(summary.min), diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 7f788c5be..c74832d04 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -56,6 +56,20 @@ def _is_range(value) -> bool: return isinstance(value, tuple) and len(value) == 2 +# Shared join skeleton for the patch relation. The attrs join is 1:1 (one +# attrs row per patch), so it is safe for both the projection and the count. +_FROM = ( + "FROM patches p " + "JOIN sources s ON s.source_id = p.source_id " + "LEFT JOIN attrs a ON a.patch_id = p.patch_id " +) + + +def _as_query_list(query: Query | Sequence[Query]) -> list[Query]: + """Normalize a single Query or a sequence of them to a list.""" + return [query] if isinstance(query, Query) else list(query) + + def normalize_range_forms(value): """ Normalize the patch-level slice range form to a 2-tuple. @@ -410,7 +424,7 @@ def build_query_sql( where residuals maps attr names to regex patterns that must be re-applied to the resulting dataframe. """ - queries = [query] if isinstance(query, Query) else list(query) + queries = _as_query_list(query) where, residuals = _build_where(queries, dialect, attr_meta, coord_meta) # attr columns selected explicitly: `a.*` would duplicate patch_id and # engines disagree on how to dedupe result column names. @@ -420,9 +434,7 @@ def build_query_sql( sql = ( "SELECT s.source_path, s.base_uri, s.source_format, s.format_version, " f"p.*{attr_cols} " - "FROM patches p " - "JOIN sources s ON s.source_id = p.source_id " - "LEFT JOIN attrs a ON a.patch_id = p.patch_id " + f"{_FROM}" f"WHERE {where.sql} " "ORDER BY p.time_min NULLS LAST, p.patch_id" ) @@ -443,17 +455,10 @@ def build_count_sql( residual means the count is not SQL-resolvable (regex must inspect rows) and the caller must fall back to a projected count. """ - queries = [query] if isinstance(query, Query) else list(query) + queries = _as_query_list(query) where, residuals = _build_where(queries, dialect, attr_meta, coord_meta) - # the attrs join can stay: it is 1:1 (one attrs row per patch), and a - # WHERE may reference a.. COUNT(p.patch_id) counts patches. - sql = ( - "SELECT COUNT(p.patch_id) AS n " - "FROM patches p " - "JOIN sources s ON s.source_id = p.source_id " - "LEFT JOIN attrs a ON a.patch_id = p.patch_id " - f"WHERE {where.sql}" - ) + # COUNT(p.patch_id) counts patches; a WHERE may reference a.. + sql = f"SELECT COUNT(p.patch_id) AS n {_FROM}WHERE {where.sql}" return sql, where.params, residuals From a2cda013982f122433a0c796d702456dbcf2363d Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 22:03:02 +0200 Subject: [PATCH 70/97] Consolidate shared index helpers and spool select plumbing More behavior-preserving cleanup: - Move SPOOL_HIDDEN_COLUMNS to schema.py as the single source of truth; reference it from both the directory indexer and the catalog. - Add utils.misc.is_range and use it for the (start, stop) range check in query, catalog, and utils.pd (dropping four inline copies). - spool: extract _coord_only_kwargs (used twice in _load_trimmed_patch), fold the _attrs/_coords validation loops in _resolve_select_kwargs, share the dummy-dataframe tail in MemorySpool._get_df, and have chunk() delegate plan construction to chunk_plan(). - query: loop over the (lo, hi) bound pair in build_attr_clause and build_coord_clause instead of duplicating the >= / <= blocks. --- dascore/core/spool.py | 72 +++++++++++++++++-------------------- dascore/io/index/catalog.py | 12 ++++--- dascore/io/index/indexer.py | 8 ++--- dascore/io/index/query.py | 33 +++++++---------- dascore/io/index/schema.py | 5 +++ dascore/utils/misc.py | 5 +++ dascore/utils/pd.py | 4 +-- 7 files changed, 66 insertions(+), 73 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 489030d39..3f880811c 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -110,6 +110,15 @@ def _estimate_merge_samples(df, dim) -> int | None: return int(counts.sum()) +def _coord_only_kwargs(patch, kwargs) -> dict: + """Keep only the kwargs naming a dim or coordinate of patch.""" + return { + k: v + for k, v in kwargs.items() + if k in patch.dims or k in patch.coords.coord_map + } + + class BaseSpool(NamespaceOwner, abc.ABC): """Spool Abstract Base Class (ABC) for defining Spool interface.""" @@ -675,21 +684,11 @@ def _load_trimmed_patch(self, patch_kwargs, joined) -> dc.Patch: source_kwargs = kwargs if kwargs.get("_modified") else self._select_kwargs # attr-style entries (e.g. constructor select_kwargs) filter rows # above; only coordinate entries are valid patch selections. - select_kwargs = { - i: v - for i, v in source_kwargs.items() - if i in patch.dims or i in patch.coords.coord_map - } - if select_kwargs: + if select_kwargs := _coord_only_kwargs(patch, source_kwargs): patch = patch.select(**select_kwargs) # patch-local selections (samples=True) recorded by spool.select for post_kwargs, samples in self._post_selects: - usable = { - k: v - for k, v in post_kwargs.items() - if k in patch.dims or k in patch.coords.coord_map - } - if usable: + if usable := _coord_only_kwargs(patch, post_kwargs): patch = patch.select(**usable, samples=samples) return patch @@ -902,12 +901,9 @@ def chunk( **kwargs, ) -> Self: """{doc}""" - from dascore.utils.chunk_plan import build_chunk_plan - source = self._source_df working = self._chunk_working_df() - plan = build_chunk_plan( - working, + plan = self.chunk_plan( overlap=overlap, keep_partial=keep_partial, snap_coords=snap_coords, @@ -1003,16 +999,17 @@ def _resolve_select_kwargs(self, _attrs, _coords, kwargs) -> dict: """ attrs, coords = self._select_namespaces() out = {} - for name, value in (_attrs or {}).items(): - if name not in attrs: - msg = f"{name!r} is not an attribute of this spool." - raise InvalidSpoolQueryError(msg) - out[name] = value - for name, value in (_coords or {}).items(): - if name not in coords: - msg = f"{name!r} is not a coordinate of this spool." - raise InvalidSpoolQueryError(msg) - out[name] = value + + def _add(items, allowed, noun): + for name, value in (items or {}).items(): + if name not in allowed: + raise InvalidSpoolQueryError( + f"{name!r} is not {noun} of this spool." + ) + out[name] = value + + _add(_attrs, attrs, "an attribute") + _add(_coords, coords, "a coordinate") for name, value in kwargs.items(): if name in out: msg = f"{name!r} given as both a bare kwarg and in _attrs/_coords." @@ -1217,19 +1214,16 @@ def _get_df(self): """Build the managing dataframes from the input patches.""" if self._is_catalog_backed(): current = self._catalog.to_df() - df, source, instruction = self._get_dummy_dataframes(current) - self._source_df = source - self._instruction_df = instruction - return df - data = self._patches if self._patches is not None else self._data - if data is None: - return None - if self._patches is not None: - # patch-list spools run on the index catalog: one metadata - # engine (and one select semantics) for every spool type. - current = self._get_catalog().to_df() - else: # spools/dataframes: legacy flat-dump path (patch column) - current = patches_to_df(data) + else: + data = self._patches if self._patches is not None else self._data + if data is None: + return None + if self._patches is not None: + # patch-list spools run on the index catalog: one metadata + # engine (and one select semantics) for every spool type. + current = self._get_catalog().to_df() + else: # spools/dataframes: legacy flat-dump path (patch column) + current = patches_to_df(data) df, source, instruction = self._get_dummy_dataframes(current) self._source_df = source self._instruction_df = instruction diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index a3e0b615d..2537d8a8c 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -34,6 +34,8 @@ InvalidSpoolQueryError, Query, ) +from dascore.io.index.schema import SPOOL_HIDDEN_COLUMNS +from dascore.utils.misc import is_range from dascore.utils.paths import is_memory_uri from dascore.utils.pd import adjust_segments, relative_ranges_to_absolute @@ -68,7 +70,7 @@ def for_patch_coord(self, coord) -> tuple: def _canonical_range(value) -> _CanonicalRange | None: """Return the canonical SI form of a numeric range, or None.""" - if not (isinstance(value, tuple) and len(value) == 2): + if not is_range(value): return None magnitudes = [] for bound in value: @@ -515,9 +517,9 @@ def to_df(self) -> pd.DataFrame: """ if self._df_cache is None or self._df_cache_revision != self._revision.value: df = self.backend.query(list(self._queries) or None) - df = df.drop( - columns=["n_dims", "sample_count_total", "shape"], errors="ignore" - ).rename(columns={"patch_id": "_patch_id"}) + df = df.drop(columns=list(SPOOL_HIDDEN_COLUMNS), errors="ignore").rename( + columns={"patch_id": "_patch_id"} + ) # SQL identifies overlapping source patches. Expose the selected # envelopes, matching spool.get_contents() and the exact trim # applied when each patch is materialized. Each pass copies the @@ -529,7 +531,7 @@ def to_df(self) -> pd.DataFrame: ranges := { name: value for name, value in query.coords.items() - if isinstance(value, tuple) and len(value) == 2 + if is_range(value) } ) ] diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 000a619cf..0db4853d3 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -22,6 +22,7 @@ from dascore.constants import PROGRESS_LEVELS from dascore.io.index.backend import get_backend, resolve_query from dascore.io.index.ingest import SourceRecord, summaries_to_records +from dascore.io.index.schema import SPOOL_HIDDEN_COLUMNS from dascore.io.indexer import ( AbstractIndexer, _get_index_map, @@ -30,11 +31,6 @@ from dascore.utils.misc import _iter_filesystem from dascore.utils.paths import directory_writable, requires_local_directory -# Structural columns the spool machinery must not see: unique-per-patch -# values block chunk merge-compatibility grouping, which compares all -# non-private columns. -_SPOOL_HIDDEN_COLUMNS = ("n_dims", "sample_count_total", "shape") - class DBDirectoryIndexer(AbstractIndexer): """ @@ -305,7 +301,7 @@ def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: self.ensure_updated() query = resolve_query(self._backend, _attrs=_attrs, _coords=_coords, **kwargs) df = self._backend.query(query) - df = df.drop(columns=list(_SPOOL_HIDDEN_COLUMNS), errors="ignore") + df = df.drop(columns=list(SPOOL_HIDDEN_COLUMNS), errors="ignore") return df.rename(columns={"patch_id": "_patch_id"}) __call__ = get_contents diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index c74832d04..54da2e28b 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -22,7 +22,7 @@ from dascore.io.index.dialect import BaseDialect from dascore.io.index.ingest import typed_value from dascore.units import convert_units -from dascore.utils.misc import sanitize_range_param +from dascore.utils.misc import is_range, sanitize_range_param _GLOB_CHARS = frozenset("*?[") _UNSET = object() @@ -51,11 +51,6 @@ def _is_collection(value) -> bool: return isinstance(value, list | tuple | set | frozenset) -def _is_range(value) -> bool: - """True for a 2-tuple range (possibly with open bounds).""" - return isinstance(value, tuple) and len(value) == 2 - - # Shared join skeleton for the patch relation. The attrs join is 1:1 (one # attrs row per patch), so it is safe for both the projection and the count. _FROM = ( @@ -248,7 +243,7 @@ def col(kind): return None where.add(f"{col('str')} IS NOT NULL") return value - if _is_range(value): + if is_range(value): # Attr metadata has one canonical unit per typed column. probe = next( ( @@ -263,10 +258,9 @@ def col(kind): if kind not in kinds: where.add("FALSE") return None - if lo is not None: - where.add(f"{col(kind)} >= ?", lo) - if hi is not None: - where.add(f"{col(kind)} <= ?", hi) + for bound, op in ((lo, ">="), (hi, "<=")): + if bound is not None: + where.add(f"{col(kind)} {op} ?", bound) return None if _is_collection(value): coerced = [_coerce_scalar(v, kinds) for v in value] @@ -323,7 +317,7 @@ def build_coord_clause( raise ParameterError(msg) kinds = set(rows["value_kind"]) or {"time", "num", "str"} typed_values = [] - if _is_range(value): + if is_range(value): kind, lo, hi, typed_values = _range_bounds(value, kinds) elif _is_collection(value) and np.asarray(value).dtype == bool: # boolean masks are patch-local; no index predicate at all, @@ -369,18 +363,15 @@ def build_coord_clause( params.extend(sorted(compatible_units)) else: conditions.append("cd.units IS NULL") - if lo is not None: - clause = f"cd.{max_col} >= ?" - if compatible_units is not None: - clause = f"(cd.units IS NULL OR {clause})" - conditions.append(clause) - params.append(lo) - if hi is not None: - clause = f"cd.{min_col} <= ?" + # lo bounds the coord max (overlap), hi bounds the coord min. + for bound, bound_col, op in ((lo, max_col, ">="), (hi, min_col, "<=")): + if bound is None: + continue + clause = f"cd.{bound_col} {op} ?" if compatible_units is not None: clause = f"(cd.units IS NULL OR {clause})" conditions.append(clause) - params.append(hi) + params.append(bound) # A semi-join the engine can evaluate once (idx_pcoords_name) beats a # correlated EXISTS probed per patch row (~2.5x on a 200k-source index). where.add( diff --git a/dascore/io/index/schema.py b/dascore/io/index/schema.py index f3f943b4a..c90d30965 100644 --- a/dascore/io/index/schema.py +++ b/dascore/io/index/schema.py @@ -209,6 +209,11 @@ } ) +# Structural columns the spool machinery must not see: unique-per-patch +# values block chunk merge-compatibility grouping, which compares all +# non-private columns. +SPOOL_HIDDEN_COLUMNS = ("n_dims", "sample_count_total", "shape") + # Explicit secondary indexes. Every other access path is covered by a # PRIMARY KEY or UNIQUE autoindex above — patch_coords(patch_id, # coord_name), sources(base_uri, source_path), patches(source_id, diff --git a/dascore/utils/misc.py b/dascore/utils/misc.py index dd161159c..876f1a740 100644 --- a/dascore/utils/misc.py +++ b/dascore/utils/misc.py @@ -766,6 +766,11 @@ def _dict_list_diffs(dict_list): return sorted(out) +def is_range(value) -> bool: + """True for a 2-tuple range (a ``(start, stop)`` selector).""" + return isinstance(value, tuple) and len(value) == 2 + + def sanitize_range_param(select) -> tuple: """Given a slice or tuple, check and return slice or tuple.""" # convert ellipses or ellipses values diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index 9c70beb06..cf18dbb00 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -15,7 +15,7 @@ from dascore.constants import PatchType from dascore.core.attrs import PatchAttrs from dascore.exceptions import InvalidSpoolQueryError, ParameterError -from dascore.utils.misc import order_range_tuple, sanitize_range_param +from dascore.utils.misc import is_range, order_range_tuple, sanitize_range_param from dascore.utils.time import to_datetime64, to_timedelta64 @@ -56,7 +56,7 @@ def relative_ranges_to_absolute(df, kwargs: dict) -> dict: if lo_col not in df.columns or hi_col not in df.columns or df.empty: msg = f"Cannot use relative select on {name!r}." raise InvalidSpoolQueryError(msg) - if not (isinstance(value, tuple) and len(value) == 2): + if not is_range(value): msg = f"relative=True requires (start, stop) ranges, got {value!r}." raise InvalidSpoolQueryError(msg) gmin, gmax = df[lo_col].min(), df[hi_col].max() From 67ad515f6bb39be964db6618623eb738284d39ea Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Tue, 14 Jul 2026 22:28:04 +0200 Subject: [PATCH 71/97] Cover unsupported non-null coord dtype in _coord_record The datetime/timedelta branch merge added an early return for a missing dtype, so the final skip branch is now reachable only by a real but unsupported dtype (e.g. bool). Parametrize the existing test over both. --- tests/test_io/test_index/test_index_edge_cases.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 377fa9552..48c7cf506 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -850,11 +850,11 @@ def test_reserved_attr_name_warns(self): records = s2r([summary]) assert "patch_id" not in records[0].patches[0].attrs - def test_unsupported_coord_dtype_skipped(self): - """A coord with no usable dtype produces no record.""" + @pytest.mark.parametrize("dtype", ["", np.dtype(bool)]) + def test_unsupported_coord_dtype_skipped(self, dtype): + """A coord with a missing or unsupported dtype produces no record.""" class _Stub: - dtype = "" dims = ("x",) len = 2 units = None @@ -863,7 +863,11 @@ class _Stub: max = 1 step = None - assert _coord_record("x", _Stub()) is None + # "" exercises the missing-dtype guard; a bool dtype is a real + # dtype that none of the value-kind branches handle. + stub = _Stub() + stub.dtype = dtype + assert _coord_record("x", stub) is None def test_multipatch_source_gets_positional_ids(self): """Multi-patch sources get positional source_patch_ids.""" From 41246019fec97792090f52d4f9a72e7f9a7c862a Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 16 Jul 2026 21:25:09 +0200 Subject: [PATCH 72/97] Trim redundancy from the index package Remove dead and duplicated code found reviewing the branch: - Delete `query.glob_match`, which had no caller outside the test written to cover it. - Delete `DBDirectoryIndexer.get_contents`/`__call__`. Nothing in dascore called it (the catalog owns querying, and calls `ensure_updated` itself); its only consumer was test_indexer.py, which now builds the same frame through a local helper. - Assemble records in `assemble_source_records` from the dataclass fields rather than naming all 28 columns by hand. - Merge `build_query_sql`/`build_count_sql` into one `build_sql`; they differed only in the projection. The backend now shares the query-list normalization instead of repeating it. --- dascore/io/index/backend.py | 15 +++--- dascore/io/index/indexer.py | 18 +------ dascore/io/index/ingest.py | 43 +++++++-------- dascore/io/index/query.py | 52 +++++++------------ .../test_index/test_index_edge_cases.py | 21 ++++---- tests/test_io/test_indexer.py | 40 +++++++++----- 6 files changed, 82 insertions(+), 107 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 07fd313f8..5f15a2724 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -28,9 +28,9 @@ from dascore.io.index.ingest import SourceRecord, attr_column_name from dascore.io.index.query import ( Query, + _as_query_list, apply_residuals, - build_count_sql, - build_query_sql, + build_sql, normalize_range_forms, ) from dascore.io.index.schema import ( @@ -589,8 +589,7 @@ def _query_context(self, query): only consulted for coord predicates, so the (whole-relation DISTINCT) scan is skipped for attr-only/empty queries. """ - query = query if query is not None else Query() - queries = [query] if isinstance(query, Query) else list(query) + queries = _as_query_list(query if query is not None else Query()) attr_meta = self._attr_meta() coord_names = {name for q in queries for name in q.coords} coord_meta = self._coord_meta(coord_names) if coord_names else pd.DataFrame() @@ -599,9 +598,7 @@ def _query_context(self, query): def query(self, query=None) -> pd.DataFrame: """Return the flat patch-row relation for a query (or several).""" queries, attr_meta, coord_meta = self._query_context(query) - sql, params, residuals = build_query_sql( - queries, self.dialect, attr_meta, coord_meta - ) + sql, params, residuals = build_sql(queries, self.dialect, attr_meta, coord_meta) df = self._fetch_df(sql, params) df = self._flatten(df, attr_meta) df = self._pivot_coords(df) @@ -612,8 +609,8 @@ def query(self, query=None) -> pd.DataFrame: def count(self, query=None) -> int: """Count matching patches without projecting or pivoting rows.""" queries, attr_meta, coord_meta = self._query_context(query) - sql, params, residuals = build_count_sql( - queries, self.dialect, attr_meta, coord_meta + sql, params, residuals = build_sql( + queries, self.dialect, attr_meta, coord_meta, count=True ) if not residuals: return int(self._fetch_df(sql, params)["n"].iloc[0]) diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 0db4853d3..7e5c1f5a5 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -20,9 +20,8 @@ from dascore.compat import UPath from dascore.config import config_attr from dascore.constants import PROGRESS_LEVELS -from dascore.io.index.backend import get_backend, resolve_query +from dascore.io.index.backend import get_backend from dascore.io.index.ingest import SourceRecord, summaries_to_records -from dascore.io.index.schema import SPOOL_HIDDEN_COLUMNS from dascore.io.indexer import ( AbstractIndexer, _get_index_map, @@ -291,21 +290,6 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: self._initial_update_done = True return self - def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: - """ - Query the index, returning the spool-facing flat relation. - - Bare kwargs resolve attrs-first then coords; `_attrs`/`_coords` - disambiguate explicitly (see the selector semantics spec). - """ - self.ensure_updated() - query = resolve_query(self._backend, _attrs=_attrs, _coords=_coords, **kwargs) - df = self._backend.query(query) - df = df.drop(columns=list(SPOOL_HIDDEN_COLUMNS), errors="ignore") - return df.rename(columns={"patch_id": "_patch_id"}) - - __call__ = get_contents - def close(self) -> None: """Close the backend.""" self._backend.close() diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 98334f452..5a6afd873 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -12,7 +12,7 @@ import hashlib import re import warnings -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field, fields, replace import numpy as np import pandas as pd @@ -410,6 +410,23 @@ def summaries_to_records( return out +# Record fields read straight off an index row of the same name. The +# remaining fields need per-field handling: a coord's name/dims are +# patch-level (they come from the link row, not the shared definition), +# its hash is stored as "fingerprint", and a patch's id/dims/shape get +# normalized below. +_COORD_DEF_FIELDS = tuple( + f.name + for f in fields(CoordRecord) + if f.name not in ("coord_name", "coord_dims", "coord_hash") +) +_PATCH_ROW_FIELDS = tuple( + f.name + for f in fields(PatchRecord) + if f.name not in ("source_patch_id", "dims", "shape", "attrs", "coords") +) + + def _py_scalar(value): """Convert a fetched cell to the plain python scalar records use.""" if value is None or pd.isnull(value): @@ -483,21 +500,8 @@ def assemble_source_records( CoordRecord( coord_name=link.coord_name, coord_dims=link.coord_dims, - value_kind=cdef.value_kind, - dtype=_py_scalar(cdef.dtype), - length=_py_scalar(cdef.length), - units=_py_scalar(cdef.units), - min_num=_py_scalar(cdef.min_num), - max_num=_py_scalar(cdef.max_num), - step_num=_py_scalar(cdef.step_num), - min_ns=_py_scalar(cdef.min_ns), - max_ns=_py_scalar(cdef.max_ns), - step_ns=_py_scalar(cdef.step_ns), - min_str=_py_scalar(cdef.min_str), - max_str=_py_scalar(cdef.max_str), - is_monotonic=_py_scalar(cdef.is_monotonic), - is_relative=_py_scalar(cdef.is_relative), coord_hash=_py_scalar(cdef.fingerprint), + **{f: _py_scalar(getattr(cdef, f)) for f in _COORD_DEF_FIELDS}, ) ) patch_records.append( @@ -505,16 +509,9 @@ def assemble_source_records( source_patch_id=normalize_source_patch_id(patch.source_patch_id), dims=_py_scalar(patch.dims) or "", shape=_py_scalar(patch.shape) or "", - n_dims=_py_scalar(patch.n_dims), - sample_count_total=_py_scalar(patch.sample_count_total), - time_min=_py_scalar(patch.time_min), - time_max=_py_scalar(patch.time_max), - time_step=_py_scalar(patch.time_step), - distance_min=_py_scalar(patch.distance_min), - distance_max=_py_scalar(patch.distance_max), - distance_step=_py_scalar(patch.distance_step), attrs=typed, coords=tuple(coords), + **{f: _py_scalar(getattr(patch, f)) for f in _PATCH_ROW_FIELDS}, ) ) out.append( diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 54da2e28b..a4da74e50 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -10,7 +10,6 @@ from __future__ import annotations -import fnmatch import re from collections.abc import Sequence from dataclasses import dataclass, field @@ -401,22 +400,33 @@ def _build_where( return where, residuals -def build_query_sql( +def build_sql( query: Query | Sequence[Query], dialect: BaseDialect, attr_meta: pd.DataFrame, coord_meta: pd.DataFrame, + count: bool = False, ) -> tuple[str, list, list[tuple[str, re.Pattern]]]: """ - Build the flat-relation SELECT for one or more AND-composed queries. - - coord_meta must cover every coordinate the queries reference (it may - be empty for attr-only queries). Returns (sql, params, residuals) - where residuals maps attr names to regex patterns that must be - re-applied to the resulting dataframe. + Build SQL for one or more AND-composed queries. + + By default this projects the flat relation; with count=True the same + WHERE is reused for a COUNT with no projection, coordinate pivot, or + ordering. coord_meta must cover every coordinate the queries + reference (it may be empty for attr-only queries). + + Returns (sql, params, residuals), where residuals pairs attr names + with regex patterns that must be re-applied to the resulting + dataframe. For a count a non-empty residual means the count is not + SQL-resolvable (regex must inspect rows) and the caller must fall + back to a projected count. """ queries = _as_query_list(query) where, residuals = _build_where(queries, dialect, attr_meta, coord_meta) + if count: + # COUNT(p.patch_id) counts patches; a WHERE may reference a.. + sql = f"SELECT COUNT(p.patch_id) AS n {_FROM}WHERE {where.sql}" + return sql, where.params, residuals # attr columns selected explicitly: `a.*` would duplicate patch_id and # engines disagree on how to dedupe result column names. attr_cols = "".join( @@ -432,27 +442,6 @@ def build_query_sql( return sql, where.params, residuals -def build_count_sql( - query: Query | Sequence[Query], - dialect: BaseDialect, - attr_meta: pd.DataFrame, - coord_meta: pd.DataFrame, -) -> tuple[str, list, list[tuple[str, re.Pattern]]]: - """ - Build a COUNT for one or more AND-composed queries. - - Same WHERE as build_query_sql but no flat projection, coordinate - pivot, or ordering. Returns (sql, params, residuals); a non-empty - residual means the count is not SQL-resolvable (regex must inspect - rows) and the caller must fall back to a projected count. - """ - queries = _as_query_list(query) - where, residuals = _build_where(queries, dialect, attr_meta, coord_meta) - # COUNT(p.patch_id) counts patches; a WHERE may reference a.. - sql = f"SELECT COUNT(p.patch_id) AS n {_FROM}WHERE {where.sql}" - return sql, where.params, residuals - - def apply_residuals( df: pd.DataFrame, residuals: list[tuple[str, re.Pattern]] ) -> pd.DataFrame: @@ -464,8 +453,3 @@ def apply_residuals( ) df = df[keep] return df - - -def glob_match(value, pattern: str) -> bool: - """Reference glob semantics (used by pandas fallbacks and tests).""" - return isinstance(value, str) and fnmatch.fnmatch(value, pattern) diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 48c7cf506..c1ef27f9c 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -22,6 +22,7 @@ from dascore.exceptions import UnitError from dascore.io.index import Query, get_backend, summaries_to_records from dascore.io.index.backend import _ns_to_time, adapt_params, resolve_query +from dascore.io.index.catalog import PatchCatalog from dascore.io.index.indexer import DBDirectoryIndexer from dascore.io.index.ingest import ( SourceRecord, @@ -31,7 +32,7 @@ from dascore.io.index.ingest import ( summaries_to_records as s2r, ) -from dascore.io.index.query import InvalidSpoolQueryError, glob_match +from dascore.io.index.query import InvalidSpoolQueryError from dascore.units import get_quantity, m @@ -582,11 +583,6 @@ def test_boolean_array_coord_requires_presence_only(self, backend): df = backend.query(Query(coords={"distance": mask})) assert len(df) == 4 # every patch with a distance coord - def test_glob_match_helper(self): - """Reference glob semantics.""" - assert glob_match("STA1", "STA*") - assert not glob_match(5, "STA*") - def test_slice_range_form(self, backend): """Slices resolve to the same range tuples patch selects accept.""" lo = np.datetime64("2024-06-01T00:00:00", "ns") @@ -939,16 +935,18 @@ def test_directory_manifest_detects_equal_stat_name_swap( def test_auto_update_on_first_query(self, tmp_path, random_patch): """A brand-new index triggers one update on first query.""" random_patch.io.write(tmp_path / "one.hdf5", "dasdae") - indexer = DBDirectoryIndexer(tmp_path) - assert len(indexer()) == 1 # no explicit update() call + catalog = PatchCatalog.from_directory(tmp_path) + assert len(catalog) == 1 # no explicit update() call + catalog.close() def test_empty_index_file_updates_on_first_query(self, tmp_path, random_patch): """A pre-created empty SQLite path is still a new index.""" random_patch.io.write(tmp_path / "one.hdf5", "dasdae") index_path = tmp_path / "empty.sqlite3" index_path.touch() - indexer = DBDirectoryIndexer(tmp_path, index_path=index_path) - assert len(indexer()) == 1 + catalog = PatchCatalog.from_directory(tmp_path, index_path=index_path) + assert len(catalog) == 1 + catalog.close() def test_directory_format_unit(self, tmp_path): """Directory-format sources (xml binary) group as one scan unit.""" @@ -971,8 +969,7 @@ def test_directory_format_unit(self, tmp_path): with (sub / name).open("wb") as fi: rand.tofile(fi) indexer = DBDirectoryIndexer(tmp_path).update(progress=None) - df = indexer() - assert len(df) == 2 + assert indexer._backend.count(None) == 2 # unchanged: second update rescans nothing before = indexer._backend.get_sources()["last_indexed_ns"].max() indexer.update(progress=None) diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index b7451aa40..95b80e6f8 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -14,10 +14,26 @@ from dascore.config import set_config from dascore.exceptions import InvalidSpoolError +from dascore.io.index.backend import resolve_query from dascore.io.index.indexer import DBDirectoryIndexer +from dascore.io.index.schema import SPOOL_HIDDEN_COLUMNS from dascore.utils.patch import get_patch_names +def index_contents(indexer, **kwargs) -> pd.DataFrame: + """ + Return an indexer's flat relation, the way a catalog queries it. + + Bare kwargs resolve attrs-first then coords, per the selector + semantics spec. + """ + indexer.ensure_updated() + query = resolve_query(indexer._backend, **kwargs) + df = indexer._backend.query(query) + df = df.drop(columns=list(SPOOL_HIDDEN_COLUMNS), errors="ignore") + return df.rename(columns={"patch_id": "_patch_id"}) + + @pytest.fixture(scope="class") def basic_indexer(two_patch_directory): """Return an indexer on the basic spool directory.""" @@ -37,7 +53,7 @@ def diverse_indexer(diverse_spool_directory): @pytest.fixture(scope="class") def diverse_df(diverse_indexer): """Return the contents of the diverse indexer.""" - return diverse_indexer() + return index_contents(diverse_indexer) @pytest.fixture() @@ -151,7 +167,7 @@ class TestGetContents: def test_get_contents(self, basic_indexer, two_patch_directory): """Ensure contents are returned.""" - out = basic_indexer() + out = index_contents(basic_indexer) files = list(Path(two_patch_directory).rglob("*.hdf5")) assert isinstance(out, pd.DataFrame) assert len(out) == len(files) @@ -163,33 +179,33 @@ def test_filter_time_after(self, diverse_df, diverse_indexer): """Half-open time range keeps every file overlapping it.""" max_starttime = diverse_df["time_min"].max() expected = diverse_df[diverse_df["time_max"] >= max_starttime] - out = diverse_indexer(time=(max_starttime, None)) + out = index_contents(diverse_indexer, time=(max_starttime, None)) assert len(out) == len(expected) def test_filter_time_before(self, diverse_df, diverse_indexer): """Half-open time range keeps every file overlapping it.""" min_endtime = diverse_df["time_max"].min() expected = diverse_df[diverse_df["time_min"] <= min_endtime] - out = diverse_indexer(time=(None, min_endtime)) + out = index_contents(diverse_indexer, time=(None, min_endtime)) assert len(out) == len(expected) def test_filter_station_exact(self, diverse_df, diverse_indexer): """Ensure contents can be filtered on an attr.""" exact_name = diverse_df["station"].unique()[0] - new_df = diverse_indexer(station=exact_name) + new_df = index_contents(diverse_indexer, station=exact_name) assert (new_df["station"] == exact_name).all() def test_filter_isin(self, diverse_df, diverse_indexer): """Ensure contents can be filtered with a collection.""" # empty strings mean "attr missing" and are not queryable (spec). stations = [x for x in diverse_df["station"].unique() if x] - new_df = diverse_indexer(station=stations[:2]) + new_df = index_contents(diverse_indexer, station=stations[:2]) assert set(new_df["station"]) <= set(stations[:2]) assert len(new_df) def test_empty_index(self, empty_index): """An empty index should return an empty dataframe.""" - df = empty_index() + df = index_contents(empty_index) assert df.empty @@ -213,7 +229,7 @@ def test_add_one_patch(self, empty_index, random_patch): path = empty_index.path / get_patch_names(random_patch).iloc[0] random_patch.io.write(path, file_format="dasdae") new_index = empty_index.update(progress=None) - contents = new_index() + contents = index_contents(new_index) assert len(contents) == 1 def test_index_with_bad_file(self, spool_directory_with_non_das_file): @@ -221,7 +237,7 @@ def test_index_with_bad_file(self, spool_directory_with_non_das_file): indexer = DBDirectoryIndexer(spool_directory_with_non_das_file) updated = indexer.update(progress=None) assert isinstance(updated, DBDirectoryIndexer) - assert len(updated()) == 2 + assert len(index_contents(updated)) == 2 def test_removed_file_dropped(self, two_patch_directory, tmp_path_factory): """A deleted file's rows disappear on the next update.""" @@ -230,9 +246,9 @@ def test_removed_file_dropped(self, two_patch_directory, tmp_path_factory): for index in Path(new).glob(".dascore_index*"): index.unlink() indexer = DBDirectoryIndexer(new).update(progress=None) - assert len(indexer()) == 2 + assert len(index_contents(indexer)) == 2 next(iter(Path(new).glob("*.hdf5"))).unlink() - assert len(indexer.update(progress=None)()) == 1 + assert len(index_contents(indexer.update(progress=None))) == 1 def test_noop_update_rescans_nothing(self, basic_indexer): """Unchanged sources are not rescanned.""" @@ -275,4 +291,4 @@ def test_unknown_name_raises(self, basic_indexer): from dascore.io.index.query import InvalidSpoolQueryError with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): - basic_indexer(bad_dimension=(1, 2)) + index_contents(basic_indexer, bad_dimension=(1, 2)) From 65ee5b1ec606741bcaf743bce4e4a31ba188065d Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 16 Jul 2026 21:33:17 +0200 Subject: [PATCH 73/97] Restore the indexer's documented get_contents CI caught that `DBDirectoryIndexer.get_contents` is not dead code: the file_io tutorial teaches it in its own "Directory Indexer" section. It has no caller inside dascore, but it is public surface, so removing it broke the doccode test. Restore the method and the tests that used it. The rest of the cleanup stands (glob_match, record assembly, build_sql). --- dascore/io/index/indexer.py | 18 ++++++++- .../test_index/test_index_edge_cases.py | 14 +++---- tests/test_io/test_indexer.py | 40 ++++++------------- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 7e5c1f5a5..0db4853d3 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -20,8 +20,9 @@ from dascore.compat import UPath from dascore.config import config_attr from dascore.constants import PROGRESS_LEVELS -from dascore.io.index.backend import get_backend +from dascore.io.index.backend import get_backend, resolve_query from dascore.io.index.ingest import SourceRecord, summaries_to_records +from dascore.io.index.schema import SPOOL_HIDDEN_COLUMNS from dascore.io.indexer import ( AbstractIndexer, _get_index_map, @@ -290,6 +291,21 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: self._initial_update_done = True return self + def get_contents(self, _attrs=None, _coords=None, **kwargs) -> pd.DataFrame: + """ + Query the index, returning the spool-facing flat relation. + + Bare kwargs resolve attrs-first then coords; `_attrs`/`_coords` + disambiguate explicitly (see the selector semantics spec). + """ + self.ensure_updated() + query = resolve_query(self._backend, _attrs=_attrs, _coords=_coords, **kwargs) + df = self._backend.query(query) + df = df.drop(columns=list(SPOOL_HIDDEN_COLUMNS), errors="ignore") + return df.rename(columns={"patch_id": "_patch_id"}) + + __call__ = get_contents + def close(self) -> None: """Close the backend.""" self._backend.close() diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index c1ef27f9c..3206b0ea2 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -22,7 +22,6 @@ from dascore.exceptions import UnitError from dascore.io.index import Query, get_backend, summaries_to_records from dascore.io.index.backend import _ns_to_time, adapt_params, resolve_query -from dascore.io.index.catalog import PatchCatalog from dascore.io.index.indexer import DBDirectoryIndexer from dascore.io.index.ingest import ( SourceRecord, @@ -935,18 +934,16 @@ def test_directory_manifest_detects_equal_stat_name_swap( def test_auto_update_on_first_query(self, tmp_path, random_patch): """A brand-new index triggers one update on first query.""" random_patch.io.write(tmp_path / "one.hdf5", "dasdae") - catalog = PatchCatalog.from_directory(tmp_path) - assert len(catalog) == 1 # no explicit update() call - catalog.close() + indexer = DBDirectoryIndexer(tmp_path) + assert len(indexer()) == 1 # no explicit update() call def test_empty_index_file_updates_on_first_query(self, tmp_path, random_patch): """A pre-created empty SQLite path is still a new index.""" random_patch.io.write(tmp_path / "one.hdf5", "dasdae") index_path = tmp_path / "empty.sqlite3" index_path.touch() - catalog = PatchCatalog.from_directory(tmp_path, index_path=index_path) - assert len(catalog) == 1 - catalog.close() + indexer = DBDirectoryIndexer(tmp_path, index_path=index_path) + assert len(indexer()) == 1 def test_directory_format_unit(self, tmp_path): """Directory-format sources (xml binary) group as one scan unit.""" @@ -969,7 +966,8 @@ def test_directory_format_unit(self, tmp_path): with (sub / name).open("wb") as fi: rand.tofile(fi) indexer = DBDirectoryIndexer(tmp_path).update(progress=None) - assert indexer._backend.count(None) == 2 + df = indexer() + assert len(df) == 2 # unchanged: second update rescans nothing before = indexer._backend.get_sources()["last_indexed_ns"].max() indexer.update(progress=None) diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index 95b80e6f8..b7451aa40 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -14,26 +14,10 @@ from dascore.config import set_config from dascore.exceptions import InvalidSpoolError -from dascore.io.index.backend import resolve_query from dascore.io.index.indexer import DBDirectoryIndexer -from dascore.io.index.schema import SPOOL_HIDDEN_COLUMNS from dascore.utils.patch import get_patch_names -def index_contents(indexer, **kwargs) -> pd.DataFrame: - """ - Return an indexer's flat relation, the way a catalog queries it. - - Bare kwargs resolve attrs-first then coords, per the selector - semantics spec. - """ - indexer.ensure_updated() - query = resolve_query(indexer._backend, **kwargs) - df = indexer._backend.query(query) - df = df.drop(columns=list(SPOOL_HIDDEN_COLUMNS), errors="ignore") - return df.rename(columns={"patch_id": "_patch_id"}) - - @pytest.fixture(scope="class") def basic_indexer(two_patch_directory): """Return an indexer on the basic spool directory.""" @@ -53,7 +37,7 @@ def diverse_indexer(diverse_spool_directory): @pytest.fixture(scope="class") def diverse_df(diverse_indexer): """Return the contents of the diverse indexer.""" - return index_contents(diverse_indexer) + return diverse_indexer() @pytest.fixture() @@ -167,7 +151,7 @@ class TestGetContents: def test_get_contents(self, basic_indexer, two_patch_directory): """Ensure contents are returned.""" - out = index_contents(basic_indexer) + out = basic_indexer() files = list(Path(two_patch_directory).rglob("*.hdf5")) assert isinstance(out, pd.DataFrame) assert len(out) == len(files) @@ -179,33 +163,33 @@ def test_filter_time_after(self, diverse_df, diverse_indexer): """Half-open time range keeps every file overlapping it.""" max_starttime = diverse_df["time_min"].max() expected = diverse_df[diverse_df["time_max"] >= max_starttime] - out = index_contents(diverse_indexer, time=(max_starttime, None)) + out = diverse_indexer(time=(max_starttime, None)) assert len(out) == len(expected) def test_filter_time_before(self, diverse_df, diverse_indexer): """Half-open time range keeps every file overlapping it.""" min_endtime = diverse_df["time_max"].min() expected = diverse_df[diverse_df["time_min"] <= min_endtime] - out = index_contents(diverse_indexer, time=(None, min_endtime)) + out = diverse_indexer(time=(None, min_endtime)) assert len(out) == len(expected) def test_filter_station_exact(self, diverse_df, diverse_indexer): """Ensure contents can be filtered on an attr.""" exact_name = diverse_df["station"].unique()[0] - new_df = index_contents(diverse_indexer, station=exact_name) + new_df = diverse_indexer(station=exact_name) assert (new_df["station"] == exact_name).all() def test_filter_isin(self, diverse_df, diverse_indexer): """Ensure contents can be filtered with a collection.""" # empty strings mean "attr missing" and are not queryable (spec). stations = [x for x in diverse_df["station"].unique() if x] - new_df = index_contents(diverse_indexer, station=stations[:2]) + new_df = diverse_indexer(station=stations[:2]) assert set(new_df["station"]) <= set(stations[:2]) assert len(new_df) def test_empty_index(self, empty_index): """An empty index should return an empty dataframe.""" - df = index_contents(empty_index) + df = empty_index() assert df.empty @@ -229,7 +213,7 @@ def test_add_one_patch(self, empty_index, random_patch): path = empty_index.path / get_patch_names(random_patch).iloc[0] random_patch.io.write(path, file_format="dasdae") new_index = empty_index.update(progress=None) - contents = index_contents(new_index) + contents = new_index() assert len(contents) == 1 def test_index_with_bad_file(self, spool_directory_with_non_das_file): @@ -237,7 +221,7 @@ def test_index_with_bad_file(self, spool_directory_with_non_das_file): indexer = DBDirectoryIndexer(spool_directory_with_non_das_file) updated = indexer.update(progress=None) assert isinstance(updated, DBDirectoryIndexer) - assert len(index_contents(updated)) == 2 + assert len(updated()) == 2 def test_removed_file_dropped(self, two_patch_directory, tmp_path_factory): """A deleted file's rows disappear on the next update.""" @@ -246,9 +230,9 @@ def test_removed_file_dropped(self, two_patch_directory, tmp_path_factory): for index in Path(new).glob(".dascore_index*"): index.unlink() indexer = DBDirectoryIndexer(new).update(progress=None) - assert len(index_contents(indexer)) == 2 + assert len(indexer()) == 2 next(iter(Path(new).glob("*.hdf5"))).unlink() - assert len(index_contents(indexer.update(progress=None))) == 1 + assert len(indexer.update(progress=None)()) == 1 def test_noop_update_rescans_nothing(self, basic_indexer): """Unchanged sources are not rescanned.""" @@ -291,4 +275,4 @@ def test_unknown_name_raises(self, basic_indexer): from dascore.io.index.query import InvalidSpoolQueryError with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): - index_contents(basic_indexer, bad_dimension=(1, 2)) + basic_indexer(bad_dimension=(1, 2)) From 4779fccf77a7c1de8bfd86167257c85be4a31ed0 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 16 Jul 2026 22:15:42 +0200 Subject: [PATCH 74/97] Share one selector-name resolver between both select paths `resolve_query` (catalog/SQL) and `DataFrameSpool._resolve_select_kwargs` (dataframe) each implemented the selector spec's name resolution, and had already drifted: - A name given in both `_attrs` and `_coords` raised on the catalog path but was silently accepted on the dataframe path. - A slice selector resolved on the catalog path but raised TypeError on the dataframe path, so `spool.select(time=slice(a, b))` worked until the spool was chunked. Both now resolve names through `resolve_selector_namespaces` in utils.pd, beside `relative_ranges_to_absolute`, which is shared by the two paths for the same reason. It returns the (attrs, coords) split, so `select` no longer re-derives the namespaces to re-split its own kwargs. Only name resolution is shared; applying a predicate stays per-path. --- dascore/core/spool.py | 61 ++++++-------------- dascore/io/index/backend.py | 39 +++---------- dascore/io/index/query.py | 15 +---- dascore/utils/pd.py | 69 +++++++++++++++++++++++ tests/test_core/test_spool_select_spec.py | 20 +++++++ 5 files changed, 114 insertions(+), 90 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 3f880811c..fa2ee37f3 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -63,6 +63,7 @@ filter_df, get_column_names_from_dim, get_dim_names_from_columns, + resolve_selector_namespaces, ) T = TypeVar("T") @@ -989,40 +990,18 @@ def _select_namespaces(self) -> tuple[set[str], set[str]]: } return attrs, coords - def _resolve_select_kwargs(self, _attrs, _coords, kwargs) -> dict: + def _resolve_select_kwargs(self, _attrs, _coords, kwargs) -> tuple[dict, dict]: """ - Validate and merge select kwargs per the selector spec. + Split select kwargs into (attrs, coords) per the selector spec. - Bare names resolve attrs-first, then coords; unknown names raise - (see #435). The _attrs/_coords namespaces validate against their - own side only. + Name resolution is shared with the catalog path, so a name means + the same thing whether or not this spool is catalog-backed; only + how the predicate is applied differs. """ attrs, coords = self._select_namespaces() - out = {} - - def _add(items, allowed, noun): - for name, value in (items or {}).items(): - if name not in allowed: - raise InvalidSpoolQueryError( - f"{name!r} is not {noun} of this spool." - ) - out[name] = value - - _add(_attrs, attrs, "an attribute") - _add(_coords, coords, "a coordinate") - for name, value in kwargs.items(): - if name in out: - msg = f"{name!r} given as both a bare kwarg and in _attrs/_coords." - raise InvalidSpoolQueryError(msg) - if name not in attrs and name not in coords: - msg = ( - f"{name!r} is neither an attribute nor a coordinate of " - f"this spool. Attributes: {sorted(attrs)}; " - f"coordinates: {sorted(coords)}." - ) - raise InvalidSpoolQueryError(msg) - out[name] = value - return out + return resolve_selector_namespaces( + attrs, coords, _attrs=_attrs, _coords=_coords, kwargs=kwargs + ) def _relative_select_kwargs(self, kwargs: dict) -> dict: """Resolve relative bounds against the spool's global envelopes.""" @@ -1064,16 +1043,14 @@ def select( **kwargs, ) return self._new_from_catalog(catalog) - kwargs = self._resolve_select_kwargs(_attrs, _coords, kwargs) + attr_kwargs, coord_kwargs = self._resolve_select_kwargs(_attrs, _coords, kwargs) if samples: # sample indices are patch-local: never filter the spool, # record the selection and apply it as patches load (#447). - _, coords = self._select_namespaces() - non_coords = set(kwargs) - coords - if non_coords: + if attr_kwargs: msg = ( f"samples=True selections are coordinate-only; got " - f"{sorted(non_coords)}." + f"{sorted(attr_kwargs)}." ) raise InvalidSpoolQueryError(msg) new = self.new_from_df( @@ -1081,17 +1058,11 @@ def select( source_df=self._source_df, instruction_df=self._instruction_df, ) - new._post_selects = (*self._post_selects, (kwargs, True)) + new._post_selects = (*self._post_selects, (coord_kwargs, True)) return new - if relative: - _, coords = self._select_namespaces() - coord_kwargs = { - key: value for key, value in kwargs.items() if key in coords - } - attr_kwargs = { - key: value for key, value in kwargs.items() if key not in coords - } - kwargs = {**attr_kwargs, **self._relative_select_kwargs(coord_kwargs)} + if relative and coord_kwargs: + coord_kwargs = self._relative_select_kwargs(coord_kwargs) + kwargs = {**attr_kwargs, **coord_kwargs} filtered_df = adjust_segments(self._df, ignore_bad_kwargs=True, **kwargs) inst = adjust_segments( self._instruction_df, diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 5f15a2724..9fa803d06 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -31,7 +31,6 @@ _as_query_list, apply_residuals, build_sql, - normalize_range_forms, ) from dascore.io.index.schema import ( COORD_DEFS, @@ -46,6 +45,7 @@ WHAT_IS_THIS, ) from dascore.units import convert_units +from dascore.utils.pd import resolve_selector_namespaces # Structural columns whose ns-integer storage maps to pandas time types. _TIME_COLS = {"time_min": "datetime", "time_max": "datetime", "time_step": "timedelta"} @@ -924,36 +924,13 @@ def _shape_coord_selector(name: str, value): ) raise InvalidSpoolQueryError(msg) - # accept the same open/slice range forms patch-level select does - attrs = {k: normalize_range_forms(v) for k, v in (_attrs or {}).items()} - coords = {k: normalize_range_forms(v) for k, v in (_coords or {}).items()} - duplicates = set(attrs) & set(coords) - if duplicates: - names = ", ".join(repr(x) for x in sorted(duplicates)) - raise InvalidSpoolQueryError(f"{names} given in both _attrs and _coords.") - known_attrs = backend.attr_names() - known_coords = backend.coord_names() - for name, value in kwargs.items(): - if name in attrs or name in coords: - msg = f"{name!r} given as both a bare kwarg and in _attrs/_coords." - raise InvalidSpoolQueryError(msg) - value = normalize_range_forms(value) - if name in known_attrs: - attrs[name] = value - elif name in known_coords: - coords[name] = value - else: - msg = ( - f"{name!r} is neither an attribute nor a coordinate of any " - f"patch in this spool." - ) - raise InvalidSpoolQueryError(msg) - for name in attrs: - if name not in known_attrs: - raise InvalidSpoolQueryError(f"{name!r} is not an attribute of this spool.") - for name in coords: - if name not in known_coords: - raise InvalidSpoolQueryError(f"{name!r} is not a coordinate of this spool.") + attrs, coords = resolve_selector_namespaces( + backend.attr_names(), + backend.coord_names(), + _attrs=_attrs, + _coords=_coords, + kwargs=kwargs, + ) coords = {k: _shape_coord_selector(k, v) for k, v in coords.items()} return Query(attrs=_drop_noops(attrs), coords=_drop_noops(coords)) diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index a4da74e50..0d35f288e 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -21,7 +21,7 @@ from dascore.io.index.dialect import BaseDialect from dascore.io.index.ingest import typed_value from dascore.units import convert_units -from dascore.utils.misc import is_range, sanitize_range_param +from dascore.utils.misc import is_range _GLOB_CHARS = frozenset("*?[") _UNSET = object() @@ -64,19 +64,6 @@ def _as_query_list(query: Query | Sequence[Query]) -> list[Query]: return [query] if isinstance(query, Query) else list(query) -def normalize_range_forms(value): - """ - Normalize the patch-level slice range form to a 2-tuple. - - Only slices are converted: bare None/Ellipsis keep their own errors, - and a fully-open range is rejected downstream as having no usable - bounds (per the selector spec). - """ - if isinstance(value, slice): - return sanitize_range_param(value) - return value - - def _coerce_scalar(value, target_kinds: set[str]): """ Coerce a query scalar to (kind, storable value). diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index cf18dbb00..64abae99d 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -68,6 +68,75 @@ def relative_ranges_to_absolute(df, kwargs: dict) -> dict: return out +def normalize_range_forms(value): + """ + Normalize the patch-level slice range form to a 2-tuple. + + Only slices are converted: bare None/Ellipsis keep their own errors, + and a fully-open range is rejected downstream as having no usable + bounds (per the selector spec). + """ + if isinstance(value, slice): + return sanitize_range_param(value) + return value + + +def resolve_selector_namespaces( + known_attrs: Collection[str], + known_coords: Collection[str], + _attrs: Mapping | None = None, + _coords: Mapping | None = None, + kwargs: Mapping | None = None, +) -> tuple[dict, dict]: + """ + Split selector kwargs into (attrs, coords) per the selector spec. + + Bare kwargs resolve against attributes first, then coordinates; + `_attrs`/`_coords` name their namespace explicitly and validate + against that side only. Unknown names, and names supplied in more + than one namespace, raise (see #435). + + Both the catalog (which pushes predicates into SQL) and the generic + dataframe select path resolve names here, so the two agree on which + names are valid, what a bare name means, and which range forms are + accepted — the paths differ only in how they *apply* a predicate. + """ + known_attrs, known_coords = set(known_attrs), set(known_coords) + # A name in both explicit namespaces is a caller error whether or not + # it is valid in either, so this precedes the membership checks. + if duplicates := set(_attrs or {}) & set(_coords or {}): + names = ", ".join(repr(x) for x in sorted(duplicates)) + raise InvalidSpoolQueryError(f"{names} given in both _attrs and _coords.") + attrs: dict = {} + coords: dict = {} + for items, allowed, out, noun in ( + (_attrs, known_attrs, attrs, "an attribute"), + (_coords, known_coords, coords, "a coordinate"), + ): + for name, value in (items or {}).items(): + if name not in allowed: + msg = f"{name!r} is not {noun} of this spool." + raise InvalidSpoolQueryError(msg) + out[name] = normalize_range_forms(value) + for name, value in (kwargs or {}).items(): + if name in attrs or name in coords: + msg = f"{name!r} given as both a bare kwarg and in _attrs/_coords." + raise InvalidSpoolQueryError(msg) + value = normalize_range_forms(value) + if name in known_attrs: + attrs[name] = value + elif name in known_coords: + coords[name] = value + else: + msg = ( + f"{name!r} is neither an attribute nor a coordinate of this " + f"spool. Attributes: {sorted(known_attrs)}; " + f"coordinates: {sorted(known_coords)}." + ) + raise InvalidSpoolQueryError(msg) + return attrs, coords + + def _get_min_max_query(kwargs, df): """ Get a dict of {column_name: Optional[min_val], Optional[max_val]}. diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 3fc23379f..f1f583295 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -238,6 +238,26 @@ def test_namespaces_and_unknown_names(self, spool): with pytest.raises(InvalidSpoolQueryError, match="neither an attribute"): materialized.select(nope=1) + def test_duplicate_namespace_raises(self, spool): + """A name in both explicit namespaces raises on either path.""" + materialized = spool.chunk(time=None) + assert not materialized._catalog_native + for target in (spool, materialized): + with pytest.raises(InvalidSpoolQueryError, match="both _attrs and _coords"): + target.select(_attrs={"time": (None, None)}, _coords={"time": (1, 2)}) + + def test_slice_range_form(self, spool): + """Slice selectors resolve the same on either path (#435 spec).""" + materialized = spool.chunk(time=None) + assert not materialized._catalog_native + t0 = spool.get_contents()["time_min"].min() + window = slice(t0, t0 + np.timedelta64(2, "s")) + for target in (spool, materialized): + sliced = target.select(time=window) + tupled = target.select(time=(window.start, window.stop)) + assert len(sliced) == len(tupled) + assert len(sliced) >= 1 + class TestExistingBehaviorKept: """The conventional selections still work.""" From 15eec34fb20abdecd27601193aa2214c23df1df2 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 17:01:52 +0200 Subject: [PATCH 75/97] Add select-path parity net and make catalog adoption explicit The selector-spec suite now runs every test over four spool states (memory/directory x catalog-native/materialized), so both select implementations must agree; the net immediately caught the two paths raising different messages for scalar relative selects, now unified. _get_catalog is a pure accessor; entering catalog-backed state belongs to _ensure_catalog alone. --- dascore/core/spool.py | 14 ++++++---- dascore/utils/pd.py | 6 ++++- tests/test_core/test_spool_select_spec.py | 33 +++++++++++++++++------ tests/test_utils/test_pd.py | 2 +- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index fa2ee37f3..99df159b6 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -1192,7 +1192,8 @@ def _get_df(self): if self._patches is not None: # patch-list spools run on the index catalog: one metadata # engine (and one select semantics) for every spool type. - current = self._get_catalog().to_df() + self._ensure_catalog() + current = self._catalog.to_df() else: # spools/dataframes: legacy flat-dump path (patch column) current = patches_to_df(data) df, source, instruction = self._get_dummy_dataframes(current) @@ -1201,18 +1202,22 @@ def _get_df(self): return df def _get_catalog(self): - """Get (lazily creating) the catalog for patch-list spools.""" + """Get (lazily creating) the catalog for patch-list spools. + + Pure accessor: never flips ``_catalog_native``; the transition + into catalog-backed state belongs to ``_ensure_catalog``. + """ from dascore.io.index.catalog import PatchCatalog if self._catalog is None: self._catalog = PatchCatalog.from_patches(self._patches) - self._catalog_native = True return self._catalog def _ensure_catalog(self) -> None: """Patch-list spools ingest into a catalog; no flat realization.""" if self._patches is not None and not self._catalog_native: self._get_catalog() + self._catalog_native = True def _as_catalog_member(self): """ @@ -1221,8 +1226,7 @@ def _as_catalog_member(self): Patch-list spools contribute their own (lazily created) catalog, reusing any summaries the patches already computed. """ - if self._catalog is None and self._patches is not None: - self._get_catalog() + self._ensure_catalog() return super()._as_catalog_member() def _get_source_df(self): diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index 64abae99d..803875ea5 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -57,7 +57,11 @@ def relative_ranges_to_absolute(df, kwargs: dict) -> dict: msg = f"Cannot use relative select on {name!r}." raise InvalidSpoolQueryError(msg) if not is_range(value): - msg = f"relative=True requires (start, stop) ranges, got {value!r}." + # same vocabulary as the catalog path's selector shaping + msg = ( + f"relative=True accepts range selectors only (a (start, stop) " + f"tuple or slice), got {value!r}." + ) raise InvalidSpoolQueryError(msg) gmin, gmax = df[lo_col].min(), df[hi_col].max() lo, hi = value diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index f1f583295..6f307f484 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -15,16 +15,31 @@ from dascore.exceptions import InvalidSpoolQueryError -@pytest.fixture(scope="module", params=("memory", "directory")) +@pytest.fixture( + scope="module", + params=("memory", "directory", "memory_df", "directory_df"), +) def spool(request, tmp_path_factory): - """The same patches served by each spool type.""" + """ + The same patches served by each spool type and select-path state. + + The ``*_df`` params force the materialized (dataframe) state via a + content-preserving sort, so every spec test runs over both select + implementations (catalog-native and dataframe) — the parity net for + collapsing the dual-state spool internals. + """ base = dc.get_example_spool("random_das") - if request.param == "memory": - return dc.spool(list(base)) - path = dc.examples.spool_to_directory( - base, path=tmp_path_factory.mktemp("select_spec") - ) - return dc.spool(path).update(progress=None) + if request.param.startswith("memory"): + out = dc.spool(list(base)) + else: + path = dc.examples.spool_to_directory( + base, path=tmp_path_factory.mktemp("select_spec") + ) + out = dc.spool(path).update(progress=None) + if request.param.endswith("_df"): + out = out.sort("time") + assert not out._catalog_native, "sort must yield the materialized state" + return out class TestUnknownNames: @@ -71,6 +86,8 @@ class TestCatalogPushdown: def test_coord_predicate_reaches_backend(self, spool, monkeypatch): """Selection does not query all rows before applying its predicate.""" + if not spool._catalog_native: + pytest.skip("query pushdown only applies to catalog-native spools") catalog = spool._catalog or spool._get_catalog() backend = catalog.backend calls = [] diff --git a/tests/test_utils/test_pd.py b/tests/test_utils/test_pd.py index 00a8cd07d..894719e32 100644 --- a/tests/test_utils/test_pd.py +++ b/tests/test_utils/test_pd.py @@ -484,5 +484,5 @@ def test_non_tuple_value_raises(self): from dascore.utils.pd import relative_ranges_to_absolute df = pd.DataFrame({"time_min": [0.0], "time_max": [1.0]}) - with pytest.raises(InvalidSpoolQueryError, match="requires"): + with pytest.raises(InvalidSpoolQueryError, match="range selectors"): relative_ranges_to_absolute(df, {"time": 5}) From a14b395afc446c0053bf3ac2c7cff2f01250ece8 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 17:17:02 +0200 Subject: [PATCH 76/97] Give the catalog an explicit ordering contract via source ordinals Patch rows now present in (sources.ordinal, patch_id) order instead of riding the flat relation's time sort. Ordinals are assigned at ingest; a replaced source keeps its position while new sources append, which makes catalog unions concatenate and gives duplicate sources dict-merge semantics (first-occurrence position, last-occurrence metadata) with no special-casing. Records export in ordinal order so re-ingest preserves it. The directory syncer renumbers ordinals to time order after each sync, keeping the conventional time-ordered presentation of file archives, while patch-list spools now keep true construction order on every path. INDEX_VERSION bumps to 3; the indexer rebuilds old-version index files automatically since the index is a disposable cache. --- dascore/io/index/backend.py | 54 +++++++++++ dascore/io/index/indexer.py | 15 ++- dascore/io/index/ingest.py | 6 ++ dascore/io/index/query.py | 3 +- dascore/io/index/schema.py | 11 ++- tests/test_io/test_index/test_ordering.py | 111 ++++++++++++++++++++++ 6 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 tests/test_io/test_index/test_ordering.py diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 9fa803d06..5a465a51f 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -464,6 +464,14 @@ def write_sources(self, records: list[SourceRecord]) -> None: by_base: dict[str, list[str]] = {} for record in records: by_base.setdefault(record.base_uri or "", []).append(record.source_path) + # Ordering contract: a replaced source keeps its ordinal + # (first-occurrence position, dict-merge semantics) while new + # sources append after every existing position. Read both + # before the delete below discards them. + kept_ordinals = self._existing_ordinals(by_base) + max_df = self._fetch_df("SELECT max(ordinal) AS m FROM sources") + max_ordinal = max_df["m"].iloc[0] + next_ordinal = 0 if pd.isnull(max_ordinal) else int(max_ordinal) + 1 for base_uri, paths in by_base.items(): self._delete_by_paths(paths, base_uri=base_uri) column_map, skip_units = self._ensure_attr_columns(records) @@ -474,6 +482,11 @@ def write_sources(self, records: list[SourceRecord]) -> None: defs_needed: dict[str, object] = {} attr_groups: dict[tuple[str, ...], list] = {} for record in records: + identity = (record.base_uri or "", record.source_path) + ordinal = kept_ordinals.get(identity) + if ordinal is None: + ordinal = next_ordinal + next_ordinal += 1 source_rows.append( ( source_id, @@ -484,6 +497,7 @@ def write_sources(self, records: list[SourceRecord]) -> None: record.mtime_ns, record.size_bytes, now, + ordinal, ) ) for patch in record.patches: @@ -557,6 +571,46 @@ def _iter_in_batches(self, items): chunk = items[start : start + batch] yield chunk, self._placeholders(len(chunk)) + def _existing_ordinals(self, by_base: dict[str, list[str]]) -> dict: + """Map (base_uri, source_path) -> ordinal for already-stored sources.""" + out: dict[tuple[str, str], int] = {} + for base_uri, paths in by_base.items(): + for chunk, marks in self._iter_in_batches(paths): + df = self._fetch_df( + f"SELECT source_path, ordinal FROM sources " + f"WHERE source_path IN ({marks}) AND base_uri = ?", + [*chunk, base_uri], + ) + for row in df.itertuples(): + if not pd.isnull(row.ordinal): + out[(base_uri, row.source_path)] = int(row.ordinal) + return out + + def renumber_ordinals_by_time(self) -> None: + """ + Renumber source ordinals into time order. + + The directory syncer owns its catalog's presentation order and + calls this after each sync so file archives keep their + conventional time-ordered iteration; sources are ordered by the + earliest patch time (sources without patches last), path as the + deterministic tiebreak. + """ + with self._transaction(): + self._execute( + "UPDATE sources SET ordinal = (" + " SELECT rn - 1 FROM (" + " SELECT s2.source_id AS sid, ROW_NUMBER() OVER (" + " ORDER BY t.min_time IS NULL, t.min_time, s2.source_path" + " ) AS rn" + " FROM sources s2 LEFT JOIN (" + " SELECT source_id, MIN(time_min) AS min_time" + " FROM patches GROUP BY source_id" + " ) t ON t.source_id = s2.source_id" + " ) WHERE sid = sources.source_id" + ")" + ) + def _delete_by_paths(self, source_paths: list[str], base_uri: str = "") -> None: """ Delete sources by (base_uri, source_path) identity. diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index 0db4853d3..c605e0486 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -20,6 +20,7 @@ from dascore.compat import UPath from dascore.config import config_attr from dascore.constants import PROGRESS_LEVELS +from dascore.exceptions import InvalidIndexVersionError from dascore.io.index.backend import get_backend, resolve_query from dascore.io.index.ingest import SourceRecord, summaries_to_records from dascore.io.index.schema import SPOOL_HIDDEN_COLUMNS @@ -58,7 +59,14 @@ def __init__( requires_local_directory(path, label="DBDirectoryIndexer") self.path = Path(path).absolute() self.index_path = Path(self._find_index_path(index_path)) - self._backend = get_backend(self.index_path) + try: + self._backend = get_backend(self.index_path) + except InvalidIndexVersionError: + # The index is a disposable cache and the file already + # identified itself as a dascore spool index of another + # schema version; rebuild it rather than asking the user to. + self.index_path.unlink() + self._backend = get_backend(self.index_path) # Schema creation alone is not a successful directory scan. Read the # transactional marker so a new process retries an interrupted first # update instead of trusting a merely nonempty SQLite file. @@ -286,6 +294,11 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: ) if records: self._backend.write_sources(records) + if stale or changed: + # Directory archives present in time order; ingest assigns + # walk-order ordinals, so each sync renumbers to keep the + # contract (iterate by ordinal) aligned with time. + self._backend.renumber_ordinals_by_time() if not self._initial_update_done: self._backend.mark_initial_update_done() self._initial_update_done = True diff --git a/dascore/io/index/ingest.py b/dascore/io/index/ingest.py index 5a6afd873..bbc245388 100644 --- a/dascore/io/index/ingest.py +++ b/dascore/io/index/ingest.py @@ -460,6 +460,12 @@ def assemble_source_records( """ if sources.empty: return [] + # Records transfer in catalog order: re-ingesting assigns fresh + # sequential ordinals, so record order IS the ordering contract. + if "ordinal" in sources.columns: + sources = sources.sort_values(["ordinal", "source_id"]) + if "patch_id" in patches.columns: + patches = patches.sort_values("patch_id") col_info = { row.column_name: (row.attr_name, row.value_kind, _py_scalar(row.units)) for row in meta.itertuples() diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 0d35f288e..39e6d53ac 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -424,7 +424,8 @@ def build_sql( f"p.*{attr_cols} " f"{_FROM}" f"WHERE {where.sql} " - "ORDER BY p.time_min NULLS LAST, p.patch_id" + # the ordering contract: source ordinal, then file-internal order + "ORDER BY s.ordinal, p.patch_id" ) return sql, where.params, residuals diff --git a/dascore/io/index/schema.py b/dascore/io/index/schema.py index c90d30965..249108568 100644 --- a/dascore/io/index/schema.py +++ b/dascore/io/index/schema.py @@ -11,7 +11,7 @@ from types import MappingProxyType # Version of the index schema, independent of dascore's version. -INDEX_VERSION = 2 +INDEX_VERSION = 3 # Identity string so any tool can sanity-check what it opened. WHAT_IS_THIS = "dascore_spool_index" @@ -47,6 +47,15 @@ "mtime_ns": "int64", "size_bytes": "int64", "last_indexed_ns": "int64", + # The catalog's explicit ordering contract: patch rows present in + # (ordinal, patch_id) order. Assigned at ingest (insertion + # sequence); a replaced source keeps its position while new + # sources append, so merging catalogs concatenates and + # deduplication keeps first-occurrence position with + # last-occurrence metadata (dict-merge semantics). The directory + # syncer renumbers to time order after each sync, preserving the + # conventional time-ordered presentation of file archives. + "ordinal": "int64", } ) diff --git a/tests/test_io/test_index/test_ordering.py b/tests/test_io/test_index/test_ordering.py new file mode 100644 index 000000000..58a29cf5a --- /dev/null +++ b/tests/test_io/test_index/test_ordering.py @@ -0,0 +1,111 @@ +""" +Tests for the catalog ordering contract (source ordinals). + +Patch rows present in (ordinal, patch_id) order: live spools keep +construction order, unions concatenate (dedup keeps first-occurrence +position), and directory archives present in time order (the syncer +renumbers after each sync). +""" + +from __future__ import annotations + +import numpy as np +import pytest + +import dascore as dc + + +@pytest.fixture(scope="module") +def three_patches(): + """Three time-contiguous example patches.""" + t0 = np.datetime64("2020-01-01", "ns") + p1 = dc.get_example_patch(time_min=t0) + time = p1.get_coord("time") + p2 = dc.get_example_patch(time_min=time.max() + time.step) + p3 = dc.get_example_patch(time_min=p2.get_coord("time").max() + time.step) + return p1, p2, p3 + + +class TestLiveSpoolOrder: + """Patch-list spools keep construction order on every path.""" + + def test_out_of_time_order_kept(self, three_patches): + """Construction order wins even when it disagrees with time order.""" + p1, p2, p3 = three_patches + spool = dc.spool([p3, p1, p2]) + # the tuple fast path + loaded = list(spool) + assert loaded[0] is p3 and loaded[1] is p1 and loaded[2] is p2 + # the catalog relation presents the same order + df = spool.get_contents() + expected = [p.get_coord("time").min() for p in (p3, p1, p2)] + assert list(df["time_min"]) == expected + # and indexing after realization still agrees + assert spool[0] is p3 + + def test_selection_preserves_relative_order(self, three_patches): + """A narrowed view keeps the surviving rows in spool order.""" + p1, p2, p3 = three_patches + spool = dc.spool([p3, p1, p2]) + t3 = p3.get_coord("time").min() + t1 = p1.get_coord("time").min() + selected = spool.select(time=(min(t1, t3), None)) + df = selected.get_contents() + assert list(df["time_min"])[:2] == [t3, t1] + + +class TestUnionOrder: + """Combined spools are list concatenation, deduped dict-merge style.""" + + def test_concatenation_order(self, three_patches): + """(a + b) presents a's rows then b's.""" + p1, p2, p3 = three_patches + combined = dc.spool([p3]) + dc.spool([p1, p2]) + loaded = list(combined) + assert loaded[0] is p3 and loaded[1] is p1 and loaded[2] is p2 + + def test_dedup_keeps_first_position(self, three_patches): + """A patch in both members keeps its first position, appears once.""" + p1, p2, p3 = three_patches + combined = dc.spool([p1, p2]) + dc.spool([p2, p3]) + assert len(combined) == 3 + loaded = list(combined) + assert loaded[0] is p1 and loaded[1] is p2 and loaded[2] is p3 + + def test_union_of_union_order(self, three_patches): + """Order survives a second union (export/re-ingest round trip).""" + p1, p2, p3 = three_patches + combined = (dc.spool([p3]) + dc.spool([p2])) + dc.spool([p1]) + loaded = list(combined) + assert loaded[0] is p3 and loaded[1] is p2 and loaded[2] is p1 + + +class TestDirectoryOrder: + """File archives present in time order, maintained across syncs.""" + + def test_time_order_disagrees_with_name_order(self, tmp_path): + """Presentation follows patch time, not file names or walk order.""" + t0 = np.datetime64("2020-01-01", "ns") + early = dc.get_example_patch(time_min=t0) + late = dc.get_example_patch(time_min=t0 + np.timedelta64(3600, "s")) + dc.write(late, tmp_path / "a_late.h5", "dasdae") + dc.write(early, tmp_path / "z_early.h5", "dasdae") + spool = dc.spool(tmp_path).update(progress=None) + df = spool.get_contents() + assert df["time_min"].is_monotonic_increasing + + def test_update_interleaves_new_files_by_time(self, tmp_path): + """A later-indexed but earlier-in-time file sorts into place.""" + t0 = np.datetime64("2020-01-01", "ns") + mid = dc.get_example_patch(time_min=t0 + np.timedelta64(1800, "s")) + late = dc.get_example_patch(time_min=t0 + np.timedelta64(3600, "s")) + dc.write(mid, tmp_path / "mid.h5", "dasdae") + dc.write(late, tmp_path / "late.h5", "dasdae") + dc.spool(tmp_path).update(progress=None) # build the index + early = dc.get_example_patch(time_min=t0) + dc.write(early, tmp_path / "early.h5", "dasdae") + updated = dc.spool(tmp_path).update(progress=None) + df = updated.get_contents() + assert len(df) == 3 + assert df["time_min"].is_monotonic_increasing + assert df["time_min"].iloc[0] == early.get_coord("time").min() From 329afd44de61e204beff67746511ddbae5acf60c Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 17:21:41 +0200 Subject: [PATCH 77/97] Collapse spool live-patch storage into the catalog registry MemorySpool no longer carries a patch tuple or an input blob beside the catalog: the LiveResolver registry (insertion-ordered, identity- deduplicated) is the single store from birth, and the index tables still materialize lazily. The catalog serves len/iteration/indexing straight from the registry while cold (no ingest, no SQL), so patch- list spools keep near-free construction and access without a parallel fast-path representation; the base spool routes identity views through catalog.get_patch, removing the per-class iteration overrides. Legacy dataframe-blob inputs are gone; wrapping another spool realizes its patches into the registry. --- dascore/core/spool.py | 193 ++++++------------ dascore/io/index/catalog.py | 34 +++ tests/test_clients/test_dirspool.py | 18 -- tests/test_core/test_spool.py | 32 +-- tests/test_core/test_spool_select_spec.py | 2 +- tests/test_io/test_index/test_catalog.py | 13 +- .../test_index/test_index_edge_cases.py | 4 +- 7 files changed, 132 insertions(+), 164 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 99df159b6..ea7dda864 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -52,7 +52,6 @@ _spool_up, concatenate_patches, get_patch_names, - patches_to_df, stack_patches, ) from dascore.utils.paths import coerce_to_upath, requires_local_directory @@ -497,12 +496,12 @@ class DataFrameSpool(BaseSpool): - **catalog-backed** (``_catalog_native`` and ``_catalog is not None``): rows map one-to-one to a ``PatchCatalog`` query, so metadata operations (length, selection) can stay lazy and push down to the - index. Use ``_is_catalog_backed()`` to test this and - ``_ensure_catalog()`` to enter it without realizing the relation. + index. Use ``_is_catalog_backed()`` to test this. - **materialized**: the managed dataframe/instruction frames are the authoritative contents. Operations that restructure or reorder rows (chunk, sort, slice) leave catalog-backed mode via - ``new_from_df`` (which clears ``_catalog_native``). + ``new_from_df`` (which clears ``_catalog_native``); the catalog + remains attached for patch resolution. """ # A dataframe which represents contents as they will be output @@ -566,6 +565,20 @@ def _select_from_array(self, array) -> Self: ) return new + def _rows_are_catalog(self) -> bool: + """ + True when patch access can go straight through the catalog. + + Holds for catalog-backed views with no spool-level row filtering + or patch-local selections layered outside the catalog (which + carries its own selection as queries/residuals). + """ + return ( + self._is_catalog_backed() + and not self._select_kwargs + and not self._post_selects + ) + def __getitem__(self, item) -> PatchType | BaseSpool: if isinstance(item, slice): # a slice was used, return a sub-spool new_df = self._df.iloc[item] @@ -579,6 +592,13 @@ def __getitem__(self, item) -> PatchType | BaseSpool: ) elif is_array(item): # An array was passed use np type selection. return self._select_from_array(np.asarray(item)) + elif self._rows_are_catalog() and isinstance(item, int | np.integer): + # catalog rows are 1:1 with patches; skip the instruction join + try: + return self._catalog.get_patch(int(item)) + except IndexError: + msg = f"index of [{item}] is out of bounds for spool." + raise IndexError(msg) from None else: # a single index was used, should return a single patch out = self._unbox_patch(self._get_patches_from_index(item)) return out @@ -600,6 +620,14 @@ def __len__(self): return 0 if df is None else len(df) def __iter__(self): + if self._rows_are_catalog(): + for ind in range(len(self._catalog)): + try: + yield self._catalog.get_patch(ind) + except MissingPatchError as e: + msg = f"Skipping patch at index {ind} (see #583): {e}" + warnings.warn(msg, UserWarning, stacklevel=2) + return if self._df is None: # an empty spool has nothing to yield return for ind in range(len(self._df)): @@ -1009,16 +1037,6 @@ def _relative_select_kwargs(self, kwargs: dict) -> dict: return relative_ranges_to_absolute(self._df, kwargs) - def _ensure_catalog(self) -> None: - """ - Switch to catalog-native mode if this spool supports it. - - Must not realize the flat relation: it exists so selection can - route through the catalog on cold spools while staying lazy. - Dataframe-backed spools (post chunk/sort/slice) stay put. - """ - return - @compose_docstring(doc=BaseSpool.select.__doc__) def select( self, @@ -1031,9 +1049,7 @@ def select( ) -> Self: """{doc}.""" # The catalog path owns the full selector semantics (e.g. unit - # canonicalization); adopt it where possible without realizing - # the flat relation (selection must stay lazy on cold spools). - self._ensure_catalog() + # canonicalization) and stays lazy on cold spools. if self._catalog_native: catalog = self._catalog.select( _attrs=_attrs, @@ -1151,84 +1167,54 @@ class MemorySpool(DataFrameSpool): """ A Spool for storing patches in memory. - When created from patches, the managing dataframes are built lazily - (on first access by an operation which needs them, such as chunk or - select) and simple operations (len, integer access, iteration) are - served straight from the patch tuple. This makes creating a spool - from patches nearly free, which matters when reading many files. + The catalog's live-patch registry is the store from birth: creating + a spool from patches only builds the (insertion-ordered, identity- + deduplicated) registry, so construction stays nearly free; index + tables materialize lazily on the first metadata operation. """ # synthetic catalog identity columns must not join patch kwargs # comparisons or chunk merge-compatibility checks _drop_columns = ("patch", "path", "file_format", "file_version", "source_patch_id") - def __init__(self, data: PatchType | Sequence[PatchType] | None = None): + def __init__(self, data: PatchType | Sequence[PatchType] | Self | None = None): super().__init__() - self._patches: tuple[PatchType, ...] | None = None - self._data = None - self._catalog = None - if data is not None: - if isinstance(data, dc.Patch): - self._patches = (data,) - elif isinstance(data, Sequence) and all( - isinstance(x, dc.Patch) for x in data - ): - # The same patch instance (by lineage: copies share an - # identity) appears once; spools have set semantics for - # identical in-memory patches. - unique = {x._instance_id: x for x in data} - self._patches = tuple(unique.values()) - else: # eg a spool or dataframe; needs the dataframe machinery. - self._data = data + from dascore.io.index.catalog import PatchCatalog - def _get_df(self): - """Build the managing dataframes from the input patches.""" - if self._is_catalog_backed(): - current = self._catalog.to_df() + if isinstance(data, self.__class__): + # copy-construction (the new_from_df convention): share the + # catalog, take fresh derived state + self.__dict__.update(data.__dict__) + self._cache = {} + self._select_kwargs = dict(data._select_kwargs) + self._merge_kwargs = dict(data._merge_kwargs) + return + if data is None: + patches = () + elif isinstance(data, dc.Patch): + patches = (data,) + elif isinstance(data, BaseSpool): + # e.g. wrapping dc.read output; the patches are in memory + patches = tuple(data) + elif isinstance(data, Sequence) and all(isinstance(x, dc.Patch) for x in data): + patches = data else: - data = self._patches if self._patches is not None else self._data - if data is None: - return None - if self._patches is not None: - # patch-list spools run on the index catalog: one metadata - # engine (and one select semantics) for every spool type. - self._ensure_catalog() - current = self._catalog.to_df() - else: # spools/dataframes: legacy flat-dump path (patch column) - current = patches_to_df(data) + msg = ( + "MemorySpool accepts a Patch, a sequence of patches, or a " + f"spool; got {type(data)}." + ) + raise InvalidSpoolError(msg) + self._catalog = PatchCatalog.from_patches(patches) + self._catalog_native = True + + def _get_df(self): + """Realize the flat relation from the catalog.""" + current = self._catalog.to_df() df, source, instruction = self._get_dummy_dataframes(current) self._source_df = source self._instruction_df = instruction return df - def _get_catalog(self): - """Get (lazily creating) the catalog for patch-list spools. - - Pure accessor: never flips ``_catalog_native``; the transition - into catalog-backed state belongs to ``_ensure_catalog``. - """ - from dascore.io.index.catalog import PatchCatalog - - if self._catalog is None: - self._catalog = PatchCatalog.from_patches(self._patches) - return self._catalog - - def _ensure_catalog(self) -> None: - """Patch-list spools ingest into a catalog; no flat realization.""" - if self._patches is not None and not self._catalog_native: - self._get_catalog() - self._catalog_native = True - - def _as_catalog_member(self): - """ - Return (catalog, patch_ids) describing this spool for a union. - - Patch-list spools contribute their own (lazily created) catalog, - reusing any summaries the patches already computed. - """ - self._ensure_catalog() - return super()._as_catalog_member() - def _get_source_df(self): """Build the source df (happens as part of building current df).""" _ = self._df @@ -1239,34 +1225,6 @@ def _get_instruction_df(self): _ = self._df return self._cache.get("_instruction_df") - def __len__(self) -> int: - if self._patches is not None: - return len(self._patches) - return super().__len__() - - def __getitem__(self, item) -> PatchType | BaseSpool: - # Fast path: a spool created directly from patches (which never has - # select kwargs) can serve integer requests from the patch list. - patches = self._patches - if ( - patches is not None - and not self._select_kwargs - and isinstance(item, int | np.integer) - ): - try: - return patches[item] - except IndexError: - msg = f"index of [{item}] is out of bounds for spool." - raise IndexError(msg) from None - return super().__getitem__(item) - - def __iter__(self) -> PatchType: - patches = self._patches - if patches is not None and not self._select_kwargs: - yield from patches - else: - yield from super().__iter__() - def __eq__(self, other) -> bool: """ Equality check which ignores the state of the lazy dataframes. @@ -1299,8 +1257,6 @@ def _strip_identity(df): "_source_df": _strip_identity(self._source_df), "_instruction_df": _strip_identity(self._instruction_df), } - out.pop("_patches", None) - out.pop("_data", None) out.pop("_catalog", None) out.pop("_catalog_native", None) return out @@ -1323,8 +1279,6 @@ def __rich__(self): def _load_patch(self, kwargs) -> Self: """Load the patch into memory.""" - if (patch := kwargs.get("patch")) is not None: - return patch return self._catalog.resolve_row(kwargs) def _new_from_catalog(self, catalog) -> Self: @@ -1334,19 +1288,6 @@ def _new_from_catalog(self, catalog) -> Self: new._catalog_native = True return new - @compose_docstring(doc=DataFrameSpool.new_from_df.__doc__) - def new_from_df(self, *args, **kwargs): - """{doc}.""" - new = super().new_from_df(*args, **kwargs) - # The provided dataframes fully define the new spool; drop the - # construction input so derived spools don't retain their parents. - new._data = None - new._patches = None - # derived spools resolve patches through the shared catalog - new._catalog = self._catalog - new._catalog_native = False - return new - # Add specific implementation of concatenate patches. concatenate = _spool_up(concatenate_patches) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 2537d8a8c..7b96be250 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -310,6 +310,8 @@ def __init__( self._revision = revision or _CatalogRevision() self._df_cache: pd.DataFrame | None = None self._df_cache_revision = -1 + self._live_cache: tuple | None = None + self._live_cache_revision = -1 # Source records for rebuilding an in-memory backend (set by # __getstate__ so pickled catalogs survive losing the connection). self._rebuild_records: tuple = () @@ -436,6 +438,34 @@ def _invalidate(self) -> None: self._revision.value += 1 self._df_cache = None self._df_cache_revision = -1 + self._live_cache = None + self._live_cache_revision = -1 + + def _cold_live_values(self) -> tuple | None: + """ + The patches, in registry (construction) order, when the registry + alone defines contents — a root live catalog whose backend was + never realized. None whenever the registry is not authoritative. + + This keeps len/iteration/indexing on freshly-built patch-list + spools allocation-free: no ingest, no SQL, no flat relation. + """ + cold = ( + self._backend is None + and self._syncer is None + and not self.is_view + and not self._rebuild_records + and isinstance(self.resolver, LiveResolver) + ) + if not cold: + return None + if ( + self._live_cache is None + or self._live_cache_revision != self._revision.value + ): + self._live_cache = tuple(self.resolver.live_entries().values()) + self._live_cache_revision = self._revision.value + return self._live_cache def __deepcopy__(self, memo) -> PatchCatalog: """ @@ -545,6 +575,8 @@ def to_df(self) -> pd.DataFrame: return self._df_cache def __len__(self) -> int: + if (live := self._cold_live_values()) is not None: + return len(live) # Count in SQL when the relation is not already realized: coord # range residuals only drop patches the SQL candidacy already # excludes and samples/relative residuals never drop patches, so @@ -558,6 +590,8 @@ def __len__(self) -> int: def get_patch(self, index: int) -> dc.Patch: """Materialize one patch: resolve, then exact two-stage trim.""" + if (live := self._cold_live_values()) is not None: + return live[index] row = self.to_df().iloc[index].to_dict() return self.resolve_row(row) diff --git a/tests/test_clients/test_dirspool.py b/tests/test_clients/test_dirspool.py index ba1a2a924..de52aca89 100644 --- a/tests/test_clients/test_dirspool.py +++ b/tests/test_clients/test_dirspool.py @@ -3,7 +3,6 @@ from __future__ import annotations from pathlib import Path -from unittest.mock import patch as upatch import numpy as np import pandas as pd @@ -726,23 +725,6 @@ def test_selected_out_distance_shortens_spool(self, dist_differ_spool): """Selecting outside of distance range reduces spool length (#583).""" assert len(dist_differ_spool) == 1 - def test_iteration_unexpected_index_error(self, basic_file_spool): - """ - Ensure unexpected IndexErrors (not #583) are re-raised during iteration. - """ - # TODO this can be deleted once the new indexing is implemented. - # Mock _get_patches_from_index to raise an IndexError with unexpected - # message - with upatch.object( - basic_file_spool, - "_get_patches_from_index", - side_effect=IndexError("unexpected error from pandas"), - ): - # The iteration should re-raise the unexpected IndexError - with pytest.raises(IndexError, match="unexpected error from pandas"): - for _ in basic_file_spool: - pass - def _patch_shape(patch): """Module-level helper (process pools need picklable functions).""" diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index c1e39ace8..082d3a6af 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -142,8 +142,7 @@ def test_input_mutation_does_not_change_spool(self, patch_list): spool = dc.spool(data) data.pop() assert len(spool) == len(patch_list) - # The snapshot itself is immutable. - assert isinstance(spool._patches, tuple) + assert list(spool) == patch_list def test_derived_spools_use_df_machinery(self, patch_list): """Chunked/selected spools must go through the instruction dfs.""" @@ -154,23 +153,29 @@ def test_derived_spools_use_df_machinery(self, patch_list): expected_min = min(x.summary.get_coord_summary("time").min for x in patch_list) assert time_coord.min() == expected_min - def test_derived_spool_does_not_retain_parent(self, patch_list): - """Derived spools must not hold a reference to their parent.""" + def test_derived_spool_shares_only_the_catalog(self, patch_list): + """Derived spools resolve patches through the shared catalog only.""" spool = dc.spool(patch_list) chunked = spool.chunk(time=1) - assert chunked._data is None - assert chunked._patches is None + assert chunked._catalog is spool._catalog + # no other patch containers exist on the instance + assert "_patches" not in chunked.__dict__ + assert "_data" not in chunked.__dict__ def test_single_patch_input_uses_lazy_storage(self, random_patch): - """A single patch should be stored lazily just like a patch sequence.""" + """A single patch lands in the registry without realizing tables.""" spool = MemorySpool(random_patch) - assert spool._patches == (random_patch,) assert len(spool) == 1 + registry = spool._catalog.resolver.live_entries() + assert tuple(registry.values()) == (random_patch,) + # simple access never bootstrapped the index backend + assert spool._catalog._backend is None - def test_empty_memory_spool_has_no_dataframe(self): - """An empty MemorySpool should report no managing dataframe.""" + def test_empty_memory_spool(self): + """An empty MemorySpool is a valid, iterable, zero-length spool.""" spool = MemorySpool() - assert spool._get_df() is None + assert len(spool) == 0 + assert list(spool) == [] def test_instruction_df_builds_from_lazy_patches(self, patch_list): """Lazy patch input should still build instruction dataframes on demand.""" @@ -988,12 +993,11 @@ def test_large_merge_dedups(self, many_contiguous): ) def test_union_of_scanless_spool(self, tmp_path): - """A scanless (pickle) spool has no catalog; union falls back to - materializing its patches. + """A scanless (pickle) spool wraps its read patches in a live + catalog; union shares them like any in-memory member. """ dc.get_example_patch().io.write(tmp_path / "a.pkl", "pickle") pickle_spool = dc.spool(tmp_path / "a.pkl") - assert pickle_spool._catalog is None combined = pickle_spool + dc.spool([dc.get_example_patch(tag="other")]) assert len(combined) == 2 diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 6f307f484..d442f86b1 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -88,7 +88,7 @@ def test_coord_predicate_reaches_backend(self, spool, monkeypatch): """Selection does not query all rows before applying its predicate.""" if not spool._catalog_native: pytest.skip("query pushdown only applies to catalog-native spools") - catalog = spool._catalog or spool._get_catalog() + catalog = spool._catalog backend = catalog.backend calls = [] original = backend.query diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py index 713bc529e..6f77f8cf8 100644 --- a/tests/test_io/test_index/test_catalog.py +++ b/tests/test_io/test_index/test_catalog.py @@ -30,9 +30,16 @@ def test_no_backend_until_needed(self, patches): catalog = PatchCatalog.from_patches(patches) assert catalog._backend is None - def test_first_len_bootstraps(self, live_catalog, patches): - """First metadata op creates the backend and ingests.""" + def test_len_serves_from_registry(self, live_catalog, patches): + """Len (and patch access) never bootstrap a backend.""" assert len(live_catalog) == len(patches) + assert live_catalog.get_patch(0) is patches[0] + assert live_catalog._backend is None + + def test_first_relation_op_bootstraps(self, live_catalog, patches): + """Realizing the flat relation creates the backend and ingests.""" + df = live_catalog.to_df() + assert len(df) == len(patches) assert live_catalog._backend is not None @@ -40,7 +47,7 @@ class TestLiveRoundtrip: """Live patches come back identical.""" def test_iteration_returns_same_patches(self, live_catalog, patches): - """Iterated patches are the registered objects (order: time).""" + """Iterated patches are the registered objects (construction order).""" out = list(live_catalog) assert len(out) == len(patches) starts = [p.get_coord("time").min() for p in out] diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 3206b0ea2..e270e3eb8 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -1269,7 +1269,7 @@ def test_cross_patch_envelope_attr_reserved(self): with pytest.warns(UserWarning, match="reserved attr name 'event_time_min'"): spool = dc.spool([p1, p2]) df = spool.get_contents() - backend = spool._get_catalog().backend + backend = spool._catalog.backend assert "event_time_min" not in backend.attr_names() # the column is the coordinate envelope, not the stray attr value. assert "event_time_min" in df.columns @@ -1281,4 +1281,4 @@ def test_data_units_attr_still_indexed(self): patch = dc.get_example_patch().update_attrs(data_units="strain") spool = dc.spool([patch]) - assert "data_units" in spool._get_catalog().backend.attr_names() + assert "data_units" in spool._catalog.backend.attr_names() From 958917a73ea616dcc1cbd15a4ecd0034d3ef01a6 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 17:27:13 +0200 Subject: [PATCH 78/97] Represent restructured spools as an explicit derived-relation view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chunk, sort, slice, and dataframe-path selects now attach a SpoolView (outputs/members/sources) instead of overwriting the managed-dataframe cache and flipping a mode flag. The state distinction is derived — a spool is catalog-backed exactly when it carries no view — so _catalog_native becomes a read-only property and can no longer drift from the frames it described. The catalog stays attached to planned views purely for patch resolution. --- dascore/clients/dirspool.py | 1 - dascore/clients/filespool.py | 6 +- dascore/core/spool.py | 105 +++++++++++++++++++++++------------ 3 files changed, 74 insertions(+), 38 deletions(-) diff --git a/dascore/clients/dirspool.py b/dascore/clients/dirspool.py index 2d69e7647..94b0d5f88 100644 --- a/dascore/clients/dirspool.py +++ b/dascore/clients/dirspool.py @@ -73,7 +73,6 @@ def __init__( ) self.indexer = self._catalog._syncer assert hasattr(self, "indexer"), "indexer not set." - self._catalog_native = True self._preferred_format = preferred_format def __rich__(self): diff --git a/dascore/clients/filespool.py b/dascore/clients/filespool.py index f388f895f..774541512 100644 --- a/dascore/clients/filespool.py +++ b/dascore/clients/filespool.py @@ -11,7 +11,7 @@ import dascore as dc from dascore.compat import UPath from dascore.constants import PROGRESS_LEVELS, SpoolType -from dascore.core.spool import BaseSpool, DataFrameSpool +from dascore.core.spool import BaseSpool, DataFrameSpool, SpoolView from dascore.io.core import FiberIO from dascore.utils.docs import compose_docstring @@ -56,8 +56,8 @@ def __init__( _format, _version = dc.get_format(path, file_format, file_version) source_df = dc.scan_to_df(path, file_format=_format, file_version=_version) - dfs = self._get_dummy_dataframes(source_df) - self._df, self._source_df, self._instruction_df = dfs + df, source, instruction = self._get_dummy_dataframes(source_df) + self._plan = SpoolView(outputs=df, members=instruction, sources=source) self._file_format = _format self._file_version = _version diff --git a/dascore/core/spool.py b/dascore/core/spool.py index ea7dda864..0f4c53612 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -5,6 +5,7 @@ import abc import warnings from collections.abc import Callable, Generator, Mapping, Sequence +from dataclasses import dataclass from functools import singledispatch from pathlib import Path from typing import ClassVar, Literal, TypeVar @@ -38,7 +39,6 @@ from dascore.utils.docs import compose_docstring from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( - CacheDescriptor, _spool_map, broadcast_for_index, deep_equality_check, @@ -200,7 +200,6 @@ def __add__(self, other) -> BaseSpool: union = PatchCatalog.union(members) new = MemorySpool() new._catalog = union - new._catalog_native = True return new def _as_catalog_member(self): @@ -487,29 +486,39 @@ def viz(self): raise AttributeError(msg) +@dataclass(eq=False, frozen=True) +class SpoolView: + """ + The derived relation a restructured spool presents. + + ``outputs`` are the rows the spool shows (one per patch it yields), + ``members`` bind each output to the source rows that feed it (the + instruction frame), and ``sources`` are those source rows. A spool + without a view presents its catalog's rows directly; operations + that restructure or reorder rows (chunk, sort, slice) attach a view + instead of replacing the backing store. + """ + + outputs: pd.DataFrame + members: pd.DataFrame + sources: pd.DataFrame + + class DataFrameSpool(BaseSpool): """ An abstract class for spools whose contents are managed by a dataframe. - A spool is in one of two internal states: + A spool presents rows from exactly one of two derivations: - - **catalog-backed** (``_catalog_native`` and ``_catalog is not None``): + - **catalog-backed** (``_plan is None`` and a catalog is attached): rows map one-to-one to a ``PatchCatalog`` query, so metadata - operations (length, selection) can stay lazy and push down to the + operations (length, selection) stay lazy and push down to the index. Use ``_is_catalog_backed()`` to test this. - - **materialized**: the managed dataframe/instruction frames are the - authoritative contents. Operations that restructure or reorder rows - (chunk, sort, slice) leave catalog-backed mode via - ``new_from_df`` (which clears ``_catalog_native``); the catalog - remains attached for patch resolution. + - **planned** (``_plan`` is a :class:`SpoolView`): the view's + outputs/members/sources frames are the presented relation. + The catalog remains attached for patch resolution. """ - # A dataframe which represents contents as they will be output - _df: pd.DataFrame = CacheDescriptor("_cache", "_get_df") - # A dataframe which shows patches in the source - _source_df: pd.DataFrame = CacheDescriptor("_cache", "_get_source_df") - # A dataframe of instructions for going from source_df to df - _instruction_df: pd.DataFrame = CacheDescriptor("_cache", "_get_instruction_df") # kwargs for filtering contents _select_kwargs: Mapping | None = FrozenDict() # kwargs for merging patches @@ -519,13 +528,44 @@ class DataFrameSpool(BaseSpool): _post_selects: tuple = () # The catalog backing this spool (None until one is built). _catalog = None - # True while rows directly represent a PatchCatalog query. Operations - # which restructure/order rows switch back to the dataframe machinery. - _catalog_native = False + # The derived relation for restructured views (None = catalog rows). + _plan: SpoolView | None = None def _is_catalog_backed(self) -> bool: """True when rows map one-to-one to a live catalog query.""" - return self._catalog_native and self._catalog is not None + return self._plan is None and self._catalog is not None + + @property + def _catalog_native(self) -> bool: + """Derived state: presented rows are the catalog's own rows.""" + return self._is_catalog_backed() + + @property + def _df(self) -> pd.DataFrame | None: + """The dataframe of contents as they will be output.""" + if self._plan is not None: + return self._plan.outputs + if "_df" not in self._cache: + self._cache["_df"] = self._get_df() + return self._cache["_df"] + + @property + def _source_df(self) -> pd.DataFrame | None: + """The dataframe of source patch rows.""" + if self._plan is not None: + return self._plan.sources + if "_source_df" not in self._cache: + self._cache["_source_df"] = self._get_source_df() + return self._cache["_source_df"] + + @property + def _instruction_df(self) -> pd.DataFrame | None: + """The instructions for going from source_df to df.""" + if self._plan is not None: + return self._plan.members + if "_instruction_df" not in self._cache: + self._cache["_instruction_df"] = self._get_instruction_df() + return self._cache["_instruction_df"] def _get_df(self): """Function to get the current df.""" @@ -985,19 +1025,16 @@ def new_from_df( _, source_, inst_ = self._get_dummy_dataframes(df) source_df = source_df if source_df is not None else source_ instruction_df = instruction_df if instruction_df is not None else inst_ - new._df = df - new._source_df = source_df - new._instruction_df = instruction_df - # Discard stale instruction indices (eg from copied caches). - new._cache.pop("_instruction_indices", None) + # Dataframe-producing operations (chunk, sort, slice) define their + # own row/instruction plan; the catalog stays attached for patch + # resolution but no longer defines the presented rows. + new._plan = SpoolView(outputs=df, members=instruction_df, sources=source_df) + new._cache = {} new._select_kwargs = dict(self._select_kwargs) new._select_kwargs.update(select_kwargs or {}) new._merge_kwargs = dict(self._merge_kwargs) new._merge_kwargs.update(merge_kwargs or {}) new._post_selects = self._post_selects - # Dataframe-producing operations (chunk, sort, slice) define their - # own row/instruction plan and must not bypass it through the catalog. - new._catalog_native = False return new def _select_namespaces(self) -> tuple[set[str], set[str]]: @@ -1100,7 +1137,7 @@ def _new_from_catalog(self, catalog) -> Self: """Create a lazy catalog-native view of this spool.""" new = self.__class__(self) new._catalog = catalog - new._catalog_native = True + new._plan = None new._cache = {} # selection composed into the catalog is dropped, but constructor # select_kwargs (DirectorySpool contract) persist across views. @@ -1205,14 +1242,13 @@ def __init__(self, data: PatchType | Sequence[PatchType] | Self | None = None): ) raise InvalidSpoolError(msg) self._catalog = PatchCatalog.from_patches(patches) - self._catalog_native = True def _get_df(self): """Realize the flat relation from the catalog.""" current = self._catalog.to_df() df, source, instruction = self._get_dummy_dataframes(current) - self._source_df = source - self._instruction_df = instruction + self._cache["_source_df"] = source + self._cache["_instruction_df"] = instruction return df def _get_source_df(self): @@ -1252,13 +1288,15 @@ def _strip_identity(df): out = dict(self.__dict__) # Build (if needed) and compare the dataframes; drop the inputs # they were built from, whose form can differ for equal contents. + # The plan's frames are exposed through the same accessors, so + # planned and identity views with equal contents compare equal. out["_cache"] = { "_df": _strip_identity(self._df), "_source_df": _strip_identity(self._source_df), "_instruction_df": _strip_identity(self._instruction_df), } out.pop("_catalog", None) - out.pop("_catalog_native", None) + out.pop("_plan", None) return out def __rich__(self): @@ -1285,7 +1323,6 @@ def _new_from_catalog(self, catalog) -> Self: """Create a lazy memory-spool view backed by a catalog query.""" new = self.__class__() new._catalog = catalog - new._catalog_native = True return new # Add specific implementation of concatenate patches. From 017e8062496178849cc870a38c55605420c0a4d4 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 17:41:06 +0200 Subject: [PATCH 79/97] Collapse the spool hierarchy into one catalog-backed Spool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MemorySpool, DirectorySpool, and FileSpool are gone (deleted outright, no aliases): every spool is now the concrete Spool under the BaseSpool ABC, and construction is the only difference — dc.spool dispatches to the patch-list constructor, Spool.from_directory, or Spool.from_file (single files now run on a real catalog instead of a bespoke dataframe path, so all three share one select/resolve/chunk engine). Code that discriminated on class switches to the has_live_patches predicate. update() is now case-based on the spool's source: directory spools sync through their indexer, single-file spools rescan (still poking the format's internal index hook), purely in-memory spools are trivially current, and a combined spool with file-backed rows raises instead of silently doing nothing. Equality is over rows, never backends: source identity and provenance columns are stripped (a live spool equals a directory spool over identical contents), order is significant, and the catalog's pending residual selections now participate — two spools differing only by a samples trim no longer compare equal. Open-ended range selectors canonicalize their sentinel (... becomes None) at query shaping. The empty dascore.clients package is removed and its tests move under test_core with the machinery they exercise. --- dascore/__init__.py | 2 +- dascore/clients/__init__.py | 4 - dascore/clients/dirspool.py | 145 ------- dascore/clients/filespool.py | 83 ---- dascore/core/spool.py | 394 ++++++++++++------ dascore/examples.py | 2 +- dascore/io/core.py | 7 +- dascore/io/index/backend.py | 4 +- dascore/io/index/catalog.py | 38 +- dascore/io/index/indexer.py | 2 +- docs/tutorial/file_io.qmd | 6 +- tests/conftest.py | 6 +- tests/test_core/test_coord_segmented.py | 2 +- .../test_directory_spool.py} | 24 +- .../test_file_spool.py} | 26 +- tests/test_core/test_patch_chunk.py | 6 +- tests/test_core/test_spool.py | 27 +- tests/test_core/test_spool_contracts.py | 162 +++++++ tests/test_examples.py | 2 +- tests/test_io/test_index/test_db_dirspool.py | 14 +- .../test_index/test_index_edge_cases.py | 8 +- tests/test_io/test_pickle/test_pickle.py | 2 +- 22 files changed, 542 insertions(+), 424 deletions(-) delete mode 100644 dascore/clients/__init__.py delete mode 100644 dascore/clients/dirspool.py delete mode 100644 dascore/clients/filespool.py rename tests/{test_clients/test_dirspool.py => test_core/test_directory_spool.py} (98%) rename tests/{test_clients/test_filespool.py => test_core/test_file_spool.py} (82%) create mode 100644 tests/test_core/test_spool_contracts.py diff --git a/dascore/__init__.py b/dascore/__init__.py index 6a47de753..dd7f24f73 100644 --- a/dascore/__init__.py +++ b/dascore/__init__.py @@ -8,7 +8,7 @@ from dascore.core.patch import Patch from dascore.core.attrs import PatchAttrs from dascore.core.summary import PatchSummary -from dascore.core.spool import BaseSpool, spool +from dascore.core.spool import BaseSpool, Spool, spool from dascore.core.coordmanager import get_coord_manager, CoordManager from dascore.core.coords import get_coord from dascore.config import DascoreConfig, get_config, reset_config, set_config diff --git a/dascore/clients/__init__.py b/dascore/clients/__init__.py deleted file mode 100644 index 2c6634da4..000000000 --- a/dascore/clients/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -DAS Core module for accessing remote resources. -""" -from __future__ import annotations diff --git a/dascore/clients/dirspool.py b/dascore/clients/dirspool.py deleted file mode 100644 index 94b0d5f88..000000000 --- a/dascore/clients/dirspool.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -A spool for working with file systems. - -The spool uses a database index (sqlite by default) to track files. -""" - -from __future__ import annotations - -import copy -from pathlib import Path - -import pandas as pd -from rich.text import Text -from typing_extensions import Self - -from dascore.compat import UPath -from dascore.constants import PROGRESS_LEVELS -from dascore.core.spool import BaseSpool, DataFrameSpool -from dascore.io.index.catalog import FileResolver, PatchCatalog -from dascore.io.indexer import AbstractIndexer -from dascore.utils.docs import compose_docstring -from dascore.utils.pd import adjust_segments - - -class DirectorySpool(DataFrameSpool): - """ - A spool for interacting with DAS files on disk. - - FileSpool creates and index of all files then allows for simple querying - and bulk processing of the files. - - Parameters - ---------- - base_path - The path to the directory to index. - index_path - The path to the index file containing the contents of the directory. - By default it will be created in the top-level of the data directory. - preferred_format - A string to specify the format of the data. Specifying this parameter - will save time in indexing. - select_kwargs - Dict of keyword arguments to restrict output contents. - """ - - _drop_columns = ("file_format", "file_version", "path", "source_patch_id") - - def __init__( - self, - base_path: str | Path | UPath | Self | AbstractIndexer = ".", - *, - index_path: Path | None = None, - preferred_format: str | None = None, - select_kwargs: dict | None = None, - merge_kwargs: dict | None = None, - ): - super().__init__(select_kwargs=select_kwargs, merge_kwargs=merge_kwargs) - # Init file spool from another file spool - if isinstance(base_path, self.__class__): - self.__dict__.update(copy.deepcopy(base_path.__dict__)) - return - # Init file spool from indexer - elif isinstance(base_path, AbstractIndexer): - self.indexer = base_path - self._catalog = PatchCatalog( - backend=self.indexer._backend, - resolver=FileResolver(root=self.indexer.path), - syncer=self.indexer, - ) - elif isinstance(base_path, Path | str | UPath): - self._catalog = PatchCatalog.from_directory( - base_path, index_path=index_path - ) - self.indexer = self._catalog._syncer - assert hasattr(self, "indexer"), "indexer not set." - self._preferred_format = preferred_format - - def __rich__(self): - """Augment rich string directory spool stuff.""" - base = super().__rich__() - path = self.indexer.path - kwargs = self._select_kwargs - out = base + Text(f"\n Path: {path}") - out += Text(f"\n Select kwargs: {kwargs}") if kwargs else Text("") - return out - - def _get_df(self): - """Get the dataframe of current contents.""" - if not self._select_kwargs: - return self._source_df - # constructor select_kwargs restrict contents (docstring contract) - return adjust_segments( - self._source_df, ignore_bad_kwargs=True, **self._select_kwargs - ) - - def _get_instruction_df(self): - """Return instruction df on how to get from source_df to df.""" - _, _, instruction = self._get_dummy_dataframes(self._df) - return instruction - - def _get_source_df(self): - """Return a dataframe of sources in spool.""" - return self._catalog.to_df().reset_index(drop=True) - - @property - def spool_path(self): - """Return the path in which the spool contents are found.""" - return self.indexer.path - - @compose_docstring(doc=BaseSpool.get_contents.__doc__) - def get_contents(self) -> pd.DataFrame: - """{doc}.""" - return self._df - - @compose_docstring(doc=BaseSpool.update.__doc__) - def update(self, progress: PROGRESS_LEVELS = "standard") -> Self: - """{doc}.""" - self._catalog.update(progress=progress) - return self._new_from_catalog(self._catalog) - - def _df_to_dict_list(self, df): - """ - Convert the dataframe to a list of dicts for iteration. - - Stored (relative) paths pass through unchanged; the catalog's - FileResolver owns resolving them against the spool root, so path - resolution lives in exactly one place. - """ - df = df.copy(deep=False).replace("", None) - return super()._df_to_dict_list(df) - - def _load_patch(self, kwargs) -> Self: - """Given a row from the managed dataframe, return a patch.""" - # Push trims into the reader only when the instruction row narrows - # the source (chunk/select) or constructor select_kwargs restrict - # it; otherwise the whole file is wanted and selection is wasted. - trim = {} - if kwargs.get("_modified") or self._select_kwargs: - merged = {**kwargs, **self._select_kwargs} - trim = { - k: v - for k, v in merged.items() - if k not in self._drop_columns and not k.startswith("_") - } - return self._catalog.resolve_row(kwargs, extra_trim=trim) diff --git a/dascore/clients/filespool.py b/dascore/clients/filespool.py deleted file mode 100644 index 774541512..000000000 --- a/dascore/clients/filespool.py +++ /dev/null @@ -1,83 +0,0 @@ -"""A spool for working with a single file.""" - -from __future__ import annotations - -import copy -from pathlib import Path - -from rich.text import Text -from typing_extensions import Self - -import dascore as dc -from dascore.compat import UPath -from dascore.constants import PROGRESS_LEVELS, SpoolType -from dascore.core.spool import BaseSpool, DataFrameSpool, SpoolView -from dascore.io.core import FiberIO -from dascore.utils.docs import compose_docstring - - -class FileSpool(DataFrameSpool): - """ - A spool for a single file. - - Parameters - ---------- - path - The path to the file. - file_format - The format name, optional. - file_version - The version string of the format, optional. - - Notes - ----- - Some file formats support storing multiple patches, this is most useful - for those formats, but should work on all dascore supported formats. - """ - - _drop_columns = ("source_patch_id",) - - def __init__( - self, - path: str | Path | UPath, - file_format: str | None = None, - file_version: str | None = None, - ): - super().__init__() - # Init file spool from another file spool - if isinstance(path, self.__class__): - self.__dict__.update(copy.deepcopy(path.__dict__)) - return - # Support UPaths, but keep standard Path support here because it is faster - self._path = path if isinstance(path, UPath) else Path(path) - if not self._path.exists() or self._path.is_dir(): - msg = f"{path} does not exist or is a directory" - raise FileNotFoundError(msg) - - _format, _version = dc.get_format(path, file_format, file_version) - source_df = dc.scan_to_df(path, file_format=_format, file_version=_version) - df, source, instruction = self._get_dummy_dataframes(source_df) - self._plan = SpoolView(outputs=df, members=instruction, sources=source) - self._file_format = _format - self._file_version = _version - - def __rich__(self): - """Augment rich string with path.""" - base = super().__rich__() - out = base + Text(f" Path: {self._path}") - return out - - def _load_patch(self, kwargs) -> Self: - """Given a row from the managed dataframe, return a patch.""" - return self._read_and_resolve_patch(dict(kwargs)) - - @compose_docstring(doc=BaseSpool.update.__doc__) - def update(self: SpoolType, progress: PROGRESS_LEVELS = "standard") -> Self: - """ - {doc}. - """ - formatter = FiberIO.manager.get_fiberio( - format=self._file_format, version=self._file_version - ) - getattr(formatter, "index", lambda x: None)(self._path) - return self diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 0f4c53612..63d684daf 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -198,7 +198,7 @@ def __add__(self, other) -> BaseSpool: members = [self._as_catalog_member(), other._as_catalog_member()] union = PatchCatalog.union(members) - new = MemorySpool() + new = Spool() new._catalog = union return new @@ -280,7 +280,7 @@ def chunk( To inspect what a chunk call will do before running it — which output patches it produces and which slice of which source patch feeds each one — use - [`Spool.chunk_plan`](`dascore.core.spool.DataFrameSpool.chunk_plan`), + [`Spool.chunk_plan`](`dascore.core.spool.Spool.chunk_plan`), which takes the same arguments and returns the plan without touching any data. """ @@ -504,16 +504,19 @@ class SpoolView: sources: pd.DataFrame -class DataFrameSpool(BaseSpool): +class Spool(BaseSpool): """ - An abstract class for spools whose contents are managed by a dataframe. + The concrete spool: a `PatchCatalog` plus an optional derived view. - A spool presents rows from exactly one of two derivations: + The catalog is the single store — live patches sit in its resolver + registry, file-backed patches in its index tables — regardless of + how the spool was constructed (patches, a directory, or a single + file). A spool presents rows from exactly one of two derivations: - - **catalog-backed** (``_plan is None`` and a catalog is attached): - rows map one-to-one to a ``PatchCatalog`` query, so metadata - operations (length, selection) stay lazy and push down to the - index. Use ``_is_catalog_backed()`` to test this. + - **catalog-backed** (``_plan is None``): rows map one-to-one to a + ``PatchCatalog`` query, so metadata operations (length, + selection) stay lazy and push down to the index. Use + ``_is_catalog_backed()`` to test this. - **planned** (``_plan`` is a :class:`SpoolView`): the view's outputs/members/sources frames are the presented relation. The catalog remains attached for patch resolution. @@ -523,13 +526,19 @@ class DataFrameSpool(BaseSpool): _select_kwargs: Mapping | None = FrozenDict() # kwargs for merging patches _merge_kwargs: Mapping | None = FrozenDict() - _drop_columns = ("patch",) + # synthetic catalog identity columns must not join patch kwargs + # comparisons or chunk merge-compatibility checks + _drop_columns = ("patch", "path", "file_format", "file_version", "source_patch_id") # patch-local selections (samples=True) applied as patches load _post_selects: tuple = () # The catalog backing this spool (None until one is built). _catalog = None # The derived relation for restructured views (None = catalog rows). _plan: SpoolView | None = None + # single-file provenance (set by from_file; drives update()) + _file_path = None + _file_format = None + _file_version = None def _is_catalog_backed(self) -> bool: """True when rows map one-to-one to a live catalog query.""" @@ -568,21 +577,70 @@ def _instruction_df(self) -> pd.DataFrame | None: return self._cache["_instruction_df"] def _get_df(self): - """Function to get the current df.""" + """Realize the flat relation from the catalog.""" + if self._catalog is None: + return None + current = self._catalog.to_df().reset_index(drop=True) + if self._select_kwargs: + # constructor select_kwargs restrict contents (docstring contract) + current = adjust_segments( + current, ignore_bad_kwargs=True, **self._select_kwargs + ) + df, source, instruction = self._get_dummy_dataframes(current) + self._cache["_source_df"] = source + self._cache["_instruction_df"] = instruction + return df def _get_source_df(self): - """Function to get the current df.""" + """Build the source df (happens as part of building current df).""" + _ = self._df + return self._cache.get("_source_df") def _get_instruction_df(self): - """Function to get the current df.""" + """Build the instruction df (happens as part of building current df).""" + _ = self._df + return self._cache.get("_instruction_df") def __init__( - self, select_kwargs: dict | None = None, merge_kwargs: dict | None = None + self, + data: PatchType | Sequence[PatchType] | BaseSpool | None = None, + select_kwargs: dict | None = None, + merge_kwargs: dict | None = None, ): + from dascore.io.index.catalog import PatchCatalog + self._cache = {} self._select_kwargs = {} if select_kwargs is None else select_kwargs self._merge_kwargs = {} if merge_kwargs is None else merge_kwargs self._post_selects = () + if isinstance(data, Spool): + # copy-construction (the new_from_df convention): share the + # catalog, take fresh derived state + self.__dict__.update(data.__dict__) + self._cache = {} + self._select_kwargs = dict(data._select_kwargs) + if select_kwargs: + self._select_kwargs.update(select_kwargs) + self._merge_kwargs = dict(data._merge_kwargs) + if merge_kwargs: + self._merge_kwargs.update(merge_kwargs) + return + if data is None: + patches = () + elif isinstance(data, dc.Patch): + patches = (data,) + elif isinstance(data, BaseSpool): + # e.g. wrapping dc.read output; the patches are in memory + patches = tuple(data) + elif isinstance(data, Sequence) and all(isinstance(x, dc.Patch) for x in data): + patches = data + else: + msg = ( + "Spool accepts a Patch, a sequence of patches, or a " + f"spool; got {type(data)}." + ) + raise InvalidSpoolError(msg) + self._catalog = PatchCatalog.from_patches(patches) def _select_from_array(self, array) -> Self: """Create new spool with contents changed from array input.""" @@ -860,21 +918,29 @@ def _df_to_dict_list(self, df): """ Convert the dataframe to a list of dicts for iteration. - This is significantly faster than iterating rows. + This is significantly faster than iterating rows. Empty strings + (missing format fields on file rows) normalize to None; stored + relative paths pass through unchanged — the catalog's resolver + owns resolving them against the spool root. """ + df = df.copy(deep=False).replace("", None) return df.to_dict("records") - @abc.abstractmethod def _load_patch(self, kwargs) -> dc.Patch: """Given a row from the managed dataframe, return a patch.""" - - def _read_and_resolve_patch(self, final_kwargs) -> dc.Patch: - """Read patches for one instruction row and resolve to one patch.""" - from dascore.io.core import _resolve_read_spool - - source_patch_id = final_kwargs.get("source_patch_id", "") - spool = dc.read(**final_kwargs) - return _resolve_read_spool(spool, source_patch_id) + # Push trims into the reader only when the instruction row narrows + # the source (chunk/select) or constructor select_kwargs restrict + # it; otherwise the whole source is wanted and selection is wasted. + # Live patches ignore trim hints; exactness is re-applied above. + trim = {} + if kwargs.get("_modified") or self._select_kwargs: + merged = {**kwargs, **self._select_kwargs} + trim = { + k: v + for k, v in merged.items() + if k not in self._drop_columns and not k.startswith("_") + } + return self._catalog.resolve_row(kwargs, extra_trim=trim) def _as_catalog_member(self): """ @@ -889,7 +955,7 @@ def _as_catalog_member(self): if self._catalog is None: return super()._as_catalog_member() # A catalog-native spool is the whole catalog only when nothing - # narrows it; constructor select_kwargs (DirectorySpool) restrict + # narrows it; constructor select_kwargs (directory spools) restrict # the visible rows without touching the catalog, so carry only the # surviving patch ids rather than the entire catalog. if self._catalog_native and not self._select_kwargs: @@ -1140,7 +1206,7 @@ def _new_from_catalog(self, catalog) -> Self: new._plan = None new._cache = {} # selection composed into the catalog is dropped, but constructor - # select_kwargs (DirectorySpool contract) persist across views. + # select_kwargs (directory-spool contract) persist across views. new._select_kwargs = dict(self._select_kwargs) new._post_selects = () return new @@ -1195,71 +1261,135 @@ def split( @compose_docstring(doc=BaseSpool.get_contents.__doc__) def get_contents(self) -> pd.DataFrame: """{doc}.""" + # identity views apply select_kwargs during realization (_get_df) + if self._plan is None: + return self._df return self._df[filter_df(self._df, **self._select_kwargs)] - get_patch_names = get_patch_names - + # --- construction -------------------------------------------------- -class MemorySpool(DataFrameSpool): - """ - A Spool for storing patches in memory. + @classmethod + def from_directory( + cls, + path, + index_path=None, + select_kwargs: dict | None = None, + merge_kwargs: dict | None = None, + ) -> Self: + """ + Create a spool over a directory of fiber files. - The catalog's live-patch registry is the store from birth: creating - a spool from patches only builds the (insertion-ordered, identity- - deduplicated) registry, so construction stays nearly free; index - tables materialize lazily on the first metadata operation. - """ + The directory's index (created/updated via ``update()``) backs + the catalog; ``path`` may also be an existing directory indexer. + """ + from dascore.io.index.catalog import FileResolver, PatchCatalog + from dascore.io.indexer import AbstractIndexer + + out = cls(select_kwargs=select_kwargs, merge_kwargs=merge_kwargs) + if isinstance(path, AbstractIndexer): + out._catalog = PatchCatalog( + backend=path._backend, + resolver=FileResolver(root=path.path), + syncer=path, + ) + else: + out._catalog = PatchCatalog.from_directory(path, index_path=index_path) + return out - # synthetic catalog identity columns must not join patch kwargs - # comparisons or chunk merge-compatibility checks - _drop_columns = ("patch", "path", "file_format", "file_version", "source_patch_id") + @classmethod + def from_file( + cls, + path, + file_format: str | None = None, + file_version: str | None = None, + ) -> Self: + """ + Create a spool over a single (multi-patch capable) fiber file. - def __init__(self, data: PatchType | Sequence[PatchType] | Self | None = None): - super().__init__() + The file is scanned once; patches load lazily per row. + """ + path = path if isinstance(path, UPath) else Path(path) + if not path.exists() or path.is_dir(): + msg = f"{path} does not exist or is a directory" + raise FileNotFoundError(msg) from dascore.io.index.catalog import PatchCatalog - if isinstance(data, self.__class__): - # copy-construction (the new_from_df convention): share the - # catalog, take fresh derived state - self.__dict__.update(data.__dict__) - self._cache = {} - self._select_kwargs = dict(data._select_kwargs) - self._merge_kwargs = dict(data._merge_kwargs) - return - if data is None: - patches = () - elif isinstance(data, dc.Patch): - patches = (data,) - elif isinstance(data, BaseSpool): - # e.g. wrapping dc.read output; the patches are in memory - patches = tuple(data) - elif isinstance(data, Sequence) and all(isinstance(x, dc.Patch) for x in data): - patches = data - else: - msg = ( - "MemorySpool accepts a Patch, a sequence of patches, or a " - f"spool; got {type(data)}." - ) - raise InvalidSpoolError(msg) - self._catalog = PatchCatalog.from_patches(patches) + _format, _version = dc.get_format(path, file_format, file_version) + out = cls() + out._catalog = PatchCatalog.from_file( + path, file_format=_format, file_version=_version + ) + out._file_path = path + out._file_format = _format + out._file_version = _version + return out - def _get_df(self): - """Realize the flat relation from the catalog.""" - current = self._catalog.to_df() - df, source, instruction = self._get_dummy_dataframes(current) - self._cache["_source_df"] = source - self._cache["_instruction_df"] = instruction - return df + # --- capabilities -------------------------------------------------- - def _get_source_df(self): - """Build the source df (happens as part of building current df).""" - _ = self._df - return self._cache.get("_source_df") + @property + def indexer(self): + """The directory syncer, or None for non-directory spools.""" + return None if self._catalog is None else self._catalog._syncer - def _get_instruction_df(self): - """Build the instruction df (happens as part of building current df).""" - _ = self._df - return self._cache.get("_instruction_df") + @property + def spool_path(self): + """Return the path in which the spool contents are found.""" + return self.indexer.path + + @property + def has_live_patches(self) -> bool: + """True when any of this spool's patches live in memory.""" + catalog = self._catalog + return catalog is not None and bool(catalog.resolver.live_entries()) + + def _has_file_rows(self) -> bool: + """True when any catalog row is backed by a file.""" + from dascore.io.index.catalog import LiveResolver + from dascore.utils.paths import is_memory_uri + + if self._catalog is None: + return False + if isinstance(self._catalog.resolver, LiveResolver): + return False + paths = self._catalog.backend.get_sources()["source_path"] + return not paths.map(is_memory_uri).all() + + @compose_docstring(doc=BaseSpool.update.__doc__) + def update(self, progress: PROGRESS_LEVELS = "standard") -> Self: + """ + {doc} + + Update means syncing contents with the backing source: a + directory-backed spool re-indexes its directory, a single-file + spool rescans the file, and a purely in-memory spool is + trivially current (no-op). A spool with file-backed contents + but no update source (e.g. the result of combining spools) + raises — recreate it from its directory instead. + """ + catalog = self._catalog + if catalog is not None and catalog._syncer is not None: + catalog.update(progress=progress) + return self._new_from_catalog(catalog) + if self._file_path is not None: + from dascore.io.core import FiberIO + + formatter = FiberIO.manager.get_fiberio( + format=self._file_format, version=self._file_version + ) + getattr(formatter, "index", lambda _: None)(self._file_path) + return self.from_file( + self._file_path, self._file_format, self._file_version + ) + if not self._has_file_rows(): + return self # in-memory contents are trivially current + msg = ( + "This spool has file-backed contents but no update source " + "(e.g. it combines several spools); recreate it from its " + "directory to pick up new files." + ) + raise InvalidSpoolError(msg) + + # --- equality ------------------------------------------------------ def __eq__(self, other) -> bool: """ @@ -1270,20 +1400,38 @@ def __eq__(self, other) -> bool: """ if self is other: return True - if not isinstance(other, MemorySpool): + if not isinstance(other, Spool): return super().__eq__(other) return deep_equality_check(self._eq_dict(), other._eq_dict()) def _eq_dict(self) -> dict: - """Get a dict for equality checks, normalizing lazy state.""" + """ + Get a dict for equality checks, normalizing lazy state. + + Equality is over rows, never backends: same length and order of + patch rows, row-wise equal semantic columns (source identity + like paths and live-vs-file backing stripped), plus equal + pending residual selections. Whether rows come from a live + registry, an index file, or a plan is invisible; data arrays + are never compared (metadata-level, like everything else here). + """ def _strip_identity(df): - # synthetic per-catalog identities (memory:// paths, ids) are - # not content; equal spools must compare equal without them. - drop = ("path", "_patch_id", "source_patch_id", "file_format") + # synthetic per-catalog identities (memory:// paths, ids) and + # backend provenance (format/version) are not content; equal + # spools must compare equal without them, and column order + # (a construction artifact) must not matter. + drop = ( + "path", + "_patch_id", + "source_patch_id", + "file_format", + "file_version", + ) if df is None: return df - return df.drop(columns=list(drop), errors="ignore") + out = df.drop(columns=list(drop), errors="ignore") + return out[sorted(out.columns)] out = dict(self.__dict__) # Build (if needed) and compare the dataframes; drop the inputs @@ -1295,39 +1443,46 @@ def _strip_identity(df): "_source_df": _strip_identity(self._source_df), "_instruction_df": _strip_identity(self._instruction_df), } - out.pop("_catalog", None) out.pop("_plan", None) + # Backend provenance is not content. + out.pop("_file_path", None) + out.pop("_file_format", None) + out.pop("_file_version", None) + # The catalog object is backend identity, but its composed view + # state is content: residuals (e.g. samples trims) change what + # patches load without changing the visible rows. + catalog = out.pop("_catalog", None) + out["_catalog_residuals"] = None if catalog is None else catalog._residuals return out def __rich__(self): base = super().__rich__() - df = self._df - # An empty MemorySpool() has no dataframe, and patches without a - # time coordinate have a null time_min; only render a time span - # when the spool actually carries one. - if df is not None and len(df) and "time_min" in df.columns: - t1, t2 = df["time_min"].min(), df["time_min"].max() - if pd.notna(t1) and pd.notna(t2): - duration = get_nice_text(t2 - t1) - base += Text( - f"\n Time Span: <{duration}> " - f"{get_nice_text(t1)} to {get_nice_text(t2)}" - ) + indexer = self.indexer + path = getattr(indexer, "path", None) or self._file_path + if path is not None: + base += Text(f"\n Path: {path}") + if self._select_kwargs: + base += Text(f"\n Select kwargs: {self._select_kwargs}") + # Only render a time span when the relation is (or is nearly) + # realized: planned views carry their frames and live spools are + # in memory; a huge directory index is not realized for a repr. + if self._plan is not None or self.has_live_patches: + df = self._df + if df is not None and len(df) and "time_min" in df.columns: + t1, t2 = df["time_min"].min(), df["time_min"].max() + if pd.notna(t1) and pd.notna(t2): + duration = get_nice_text(t2 - t1) + base += Text( + f"\n Time Span: <{duration}> " + f"{get_nice_text(t1)} to {get_nice_text(t2)}" + ) return base - def _load_patch(self, kwargs) -> Self: - """Load the patch into memory.""" - return self._catalog.resolve_row(kwargs) - - def _new_from_catalog(self, catalog) -> Self: - """Create a lazy memory-spool view backed by a catalog query.""" - new = self.__class__() - new._catalog = catalog - return new - # Add specific implementation of concatenate patches. concatenate = _spool_up(concatenate_patches) + get_patch_names = get_patch_names + @singledispatch def spool(obj: path_types | BaseSpool | Sequence[PatchType], **kwargs) -> BaseSpool: @@ -1368,24 +1523,19 @@ def spool(obj: path_types | BaseSpool | Sequence[PatchType], **kwargs) -> BaseSp def _spool_from_str(path, **kwargs): """Get a spool from a path.""" path = coerce_to_upath(path) - # A directory was passed, create Directory Spool + # A directory was passed; index it. if path.is_dir(): requires_local_directory(path, label="Directory spool") - from dascore.clients.dirspool import DirectorySpool - - return DirectorySpool(path, **kwargs) - # A single file was passed. If the file format supports quick scanning - # Return a FileSpool (lazy file reader), else return DirectorySpool. + return Spool.from_directory(path, **kwargs) + # A single file was passed. If the file format supports quick + # scanning build a lazy file-backed spool, else read it into memory. elif path.exists(): # a single file path was passed. _format, _version = dc.get_format(path, **kwargs) formatter = dc.io.FiberIO.manager.get_fiberio(format=_format, version=_version) if formatter.implements_scan: - from dascore.clients.filespool import FileSpool - - return FileSpool(path, _format, _version) - + return Spool.from_file(path, _format, _version) else: - return MemorySpool(dc.read(path, _format, _version)) + return Spool(dc.read(path, _format, _version)) else: msg = ( f"could not get spool from argument: {path}. " @@ -1404,10 +1554,10 @@ def _spool_from_spool(spool, **kwargs): @spool.register(tuple) def _spool_from_patch_list(patch_list, **kwargs): """Return a spool from a sequence of patches.""" - return MemorySpool(patch_list) + return Spool(patch_list) @spool.register(dc.Patch) def _spool_from_patch(patch): """Get a spool from a single patch.""" - return MemorySpool([patch]) + return Spool([patch]) diff --git a/dascore/examples.py b/dascore/examples.py index 0ca2019d8..1eb6ba01b 100644 --- a/dascore/examples.py +++ b/dascore/examples.py @@ -28,7 +28,7 @@ def _load_example_patch_from_file(path: str | Path) -> dc.Patch: - """Load the first patch from an example file without FileSpool indirection.""" + """Load the first patch from an example file without spool indirection.""" with set_config(allow_dasdae_format_unpickle=True): return dc.read(path)[0] diff --git a/dascore/io/core.py b/dascore/io/core.py index f037a9a1a..dcd78be3d 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -31,7 +31,7 @@ ) from dascore.core.attrs import PatchAttrs, str_validator from dascore.core.coordmanager import CoordManager -from dascore.core.spool import DataFrameSpool +from dascore.core.spool import Spool from dascore.core.summary import PatchSummary, normalize_source_patch_id from dascore.exceptions import ( DependencyError, @@ -1008,7 +1008,7 @@ def scan_to_df( """ if isinstance(path, pd.DataFrame): return path - if isinstance(path, DataFrameSpool): + if isinstance(path, Spool): return path.get_contents() info = scan( path=path, @@ -1346,11 +1346,10 @@ def is_directory_format(path) -> bool: def _maybe_split_gapped_patches(spool, fiber_io, split): """Handle patches whose dimensional coords contain gaps before writing.""" from dascore.core.coords import CoordSegmented - from dascore.core.spool import MemorySpool # Only in-memory patches are inspected; file-backed patches always have # contiguous coordinates (gapped patches are never persisted). - if not isinstance(spool, MemorySpool): + if not getattr(spool, "has_live_patches", False): return spool def _has_gaps(patch): diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 5a465a51f..47680e2ad 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -970,7 +970,9 @@ def _shape_coord_selector(name: str, value): if len(value) != 2: msg = f"Coordinate range for {name!r} must be a length 2 sequence." raise ParameterError(msg) - return tuple(value) + # canonicalize the open-end sentinel so equivalent selections + # (None vs ...) stay equivalent downstream (e.g. spool __eq__) + return tuple(None if v is Ellipsis else v for v in value) msg = ( f"Coordinate {name!r} accepts range selectors (a (start, stop) " "tuple or slice, None/... for open ends) or boolean masks; " diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 7b96be250..0a2634066 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -57,6 +57,15 @@ class _CanonicalRange: def __init__(self, magnitudes: tuple): self.magnitudes = magnitudes + def __eq__(self, other) -> bool: + """Value equality so equal selections compare equal (spool __eq__).""" + if not isinstance(other, _CanonicalRange): + return NotImplemented + return self.magnitudes == other.magnitudes + + def __hash__(self) -> int: + return hash(self.magnitudes) + def for_patch_coord(self, coord) -> tuple: """Return the range in the representation this coord needs.""" from dascore.units import get_quantity @@ -190,7 +199,7 @@ def _read(self, path, row: Mapping, trim: dict, source_patch_id: str): The recorded format/version are forwarded so dc.read skips format probing; it reads the file exactly once (an earlier fast path that called the reader directly re-read the file whenever the reader - returned a non-MemorySpool). + returned patches lazily). """ id_kwargs = {"source_patch_id": source_patch_id} if source_patch_id else {} kwargs = {"path": path} @@ -358,6 +367,31 @@ def union(cls, catalogs: Sequence[PatchCatalog]) -> PatchCatalog: out._invalidate() return out + @classmethod + def from_file( + cls, + path: str | Path, + file_format: str | None = None, + file_version: str | None = None, + ) -> PatchCatalog: + """ + Catalog over a single fiber file. + + The file is scanned eagerly (one row per contained patch) into an + in-memory backend; patches load through the file resolver on + demand. There is no syncer — a changed file needs a new catalog. + """ + from dascore.io.index.ingest import summaries_to_records + + summaries = dc.scan( + path, file_format=file_format, file_version=file_version, progress=None + ) + records = summaries_to_records(summaries) + out = cls(resolver=FileResolver()) + out.backend.write_sources(records) + out._invalidate() + return out + @classmethod def from_directory( cls, @@ -471,7 +505,7 @@ def __deepcopy__(self, memo) -> PatchCatalog: """ Derived spools share the catalog (live registry + connection). - DataFrameSpool copies spool state on select/chunk; catalog state + Spool copies its state on select/chunk; catalog state is read-shared, matching the single-writer model. """ return self diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index c605e0486..e0ecb7a61 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -146,7 +146,7 @@ def __deepcopy__(self, memo) -> Self: """ Derived spools share the indexer (and its live DB connection). - DataFrameSpool copies spool state on select/chunk; the index + Spool copies its state on select/chunk; the index connection is read-shared, matching the single-writer model. """ return self diff --git a/docs/tutorial/file_io.qmd b/docs/tutorial/file_io.qmd index 9b3359bda..ba3739fc6 100644 --- a/docs/tutorial/file_io.qmd +++ b/docs/tutorial/file_io.qmd @@ -100,9 +100,9 @@ print(loaded_patch.data.shape) `Patch.attrs` stores non-coordinate metadata only. Coordinate summaries such as `time_min`, `time_max`, and `distance_step` are accessed through [`PatchSummary.get_coord_summary(...)`](`dascore.PatchSummary.get_coord_summary`) or via `patch.summary.get_coord_summary(...)`. ::: -## DirectorySpool +## Directory spools -The [DirectorySpool](`dascore.clients.dirspool.DirectorySpool`) is used to retrieve data from a directory of dascore-readable files. It has the same interface as other spools and is created with the [`dascore.spool`](`dascore.spool`) function. +A spool over a directory of dascore-readable files is created with the [`dascore.spool`](`dascore.spool`) function. It has the same interface as every other spool; the only difference is how it was constructed. For example: @@ -135,7 +135,7 @@ The `Patch.io` namespace also includes functionality for converting `Patch` inst ## Directory Indexer -The `DBDirectoryIndexer` tracks the contents of a directory which contains fiber data. It creates a small, hidden SQLite index named `.dascore_index.sqlite3` at the top of the directory. `DirectorySpool` uses this index internally and pushes metadata selections into SQLite before loading patch data. See the [spool index note](../notes/spool_index.qmd) for the schema and lifecycle. +The `DBDirectoryIndexer` tracks the contents of a directory which contains fiber data. It creates a small, hidden SQLite index named `.dascore_index.sqlite3` at the top of the directory. Directory spools use this index internally and push metadata selections into SQLite before loading patch data. See the [spool index note](../notes/spool_index.qmd) for the schema and lifecycle. ```{python} diff --git a/tests/conftest.py b/tests/conftest.py index c6154f2e5..64e885205 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,11 +16,11 @@ import dascore as dc import dascore.examples as ex -from dascore.clients.dirspool import DirectorySpool from dascore.compat import random_state from dascore.config import set_config from dascore.constants import SpoolType from dascore.core import Patch +from dascore.core.spool import Spool from dascore.examples import get_example_patch from dascore.io.core import read from dascore.utils.coordmanager import merge_coord_managers @@ -497,7 +497,7 @@ def adjacent_spool_no_overlap(random_patch) -> dc.BaseSpool: @register_func(SPOOL_FIXTURES) def one_file_directory_spool(one_file_dir): """Create a directory with a single DAS file.""" - return DirectorySpool(one_file_dir).update() + return Spool.from_directory(one_file_dir).update() @pytest.fixture(scope="class") @@ -521,7 +521,7 @@ def diverse_directory_spool(diverse_spool_directory): @register_func(SPOOL_FIXTURES) def basic_file_spool(two_patch_directory): """Return a DAS bank on basic_bank_directory.""" - out = DirectorySpool(two_patch_directory).update().update() + out = Spool.from_directory(two_patch_directory).update().update() yield out out.indexer.close() diff --git a/tests/test_core/test_coord_segmented.py b/tests/test_core/test_coord_segmented.py index 60e396fbe..36afc3bd3 100644 --- a/tests/test_core/test_coord_segmented.py +++ b/tests/test_core/test_coord_segmented.py @@ -1021,7 +1021,7 @@ def test_write_file_backed_spool_unaffected(self, tmp_path): """Non-memory spools skip the gap inspection (never gapped).""" path1 = dc.write(dc.get_example_patch(), tmp_path / "a.h5", "dasdae") file_spool = dc.spool(path1) - assert not isinstance(file_spool, dc.core.spool.MemorySpool) + assert not file_spool.has_live_patches path2 = dc.write(file_spool, tmp_path / "b.h5", "dasdae") assert path2.exists() diff --git a/tests/test_clients/test_dirspool.py b/tests/test_core/test_directory_spool.py similarity index 98% rename from tests/test_clients/test_dirspool.py rename to tests/test_core/test_directory_spool.py index de52aca89..194995cdc 100644 --- a/tests/test_clients/test_dirspool.py +++ b/tests/test_core/test_directory_spool.py @@ -1,4 +1,4 @@ -"""Tests for FileSpool.""" +"""Tests for directory-backed spools.""" from __future__ import annotations @@ -10,8 +10,8 @@ import dascore as dc import dascore.examples -from dascore.clients.dirspool import DirectorySpool from dascore.constants import ONE_SECOND +from dascore.core.spool import Spool from dascore.exceptions import MissingPatchError, ParameterError from dascore.utils.misc import register_func, suppress_warnings @@ -40,7 +40,7 @@ def dir_spool_index_out_of_order(random_spool, tmp_path_factory): @register_func(DIRECTORY_SPOOLS) def one_directory_spool(one_file_dir): """Create a directory with a single DAS file.""" - spool = DirectorySpool(one_file_dir) + spool = Spool.from_directory(one_file_dir) return spool.update() @@ -98,7 +98,7 @@ class TestDirectorySpoolBasics: def test_isinstance(self, directory_spool): """Simply ensure expected type was returned.""" - assert isinstance(directory_spool, DirectorySpool) + assert isinstance(directory_spool, Spool) def test_selected_str(self, diverse_directory_spool): """Ensure select kwargs show up in str.""" @@ -113,7 +113,7 @@ def test_sorted_multi_patch_uses_source_patch_id(self, tmp_path): patch_2 = dc.get_example_patch() patch_1 = patch_2.update_coords(time=patch_2.coords.get_array("time") + 10) dc.write(dc.spool([patch_1, patch_2]), path / "multi_patch.h5", "dasdae") - spool = DirectorySpool(path).update().sort("time") + spool = Spool.from_directory(path).update().sort("time") patch = spool[0] assert patch.get_coord("time").min() == patch_2.get_coord("time").min() @@ -245,7 +245,7 @@ def first_patch_range(self, random_spool): def test_contents_restricted(self, spool_dir, random_spool, first_patch_range): """Rows outside the requested range must not appear (regression).""" - spool = DirectorySpool( + spool = Spool.from_directory( spool_dir, select_kwargs={"time": first_patch_range} ).update() assert 1 <= len(spool) < len(random_spool) @@ -261,7 +261,7 @@ def test_restriction_survives_select_and_update( self, spool_dir, random_spool, first_patch_range ): """Derived spools keep the constructor restriction.""" - spool = DirectorySpool( + spool = Spool.from_directory( spool_dir, select_kwargs={"time": first_patch_range} ).update() expected = len(spool) @@ -272,10 +272,14 @@ def test_restriction_survives_select_and_update( def test_attr_select_kwargs(self, spool_dir, random_spool): """Attr-valued select_kwargs filter rows and load cleanly.""" - spool = DirectorySpool(spool_dir, select_kwargs={"tag": "random"}).update() + spool = Spool.from_directory( + spool_dir, select_kwargs={"tag": "random"} + ).update() assert len(spool) == len(random_spool) assert isinstance(spool[0], dc.Patch) - empty = DirectorySpool(spool_dir, select_kwargs={"tag": "no_such"}).update() + empty = Spool.from_directory( + spool_dir, select_kwargs={"tag": "no_such"} + ).update() assert len(empty) == 0 @@ -622,7 +626,7 @@ def test_sorted_chunked_selected_spool_can_load_patches( assert all(isinstance(patch, dc.Patch) for patch in chunked) -class TestFileSpoolIntegrations: +class TestFileBackedSpoolIntegrations: """Small integration tests for the file spool.""" @pytest.fixture(scope="class") diff --git a/tests/test_clients/test_filespool.py b/tests/test_core/test_file_spool.py similarity index 82% rename from tests/test_clients/test_filespool.py rename to tests/test_core/test_file_spool.py index 046c4ba6e..786e4c2a5 100644 --- a/tests/test_clients/test_filespool.py +++ b/tests/test_core/test_file_spool.py @@ -6,7 +6,7 @@ from upath import UPath import dascore as dc -from dascore.clients.filespool import FileSpool +from dascore.core.spool import Spool from dascore.exceptions import PatchAttributeError @@ -15,7 +15,7 @@ class TestBasic: def test_type(self, terra15_file_spool, terra15_v5_path): """Ensure a file spool was returned.""" - assert isinstance(terra15_file_spool, FileSpool) + assert isinstance(terra15_file_spool, Spool) assert len(terra15_file_spool) == len(dc.scan_to_df(terra15_v5_path)) def test_get_patch(self, terra15_file_spool): @@ -24,14 +24,14 @@ def test_get_patch(self, terra15_file_spool): assert isinstance(patch, dc.Patch) def test_init_from_filespool(self, terra15_file_spool): - """Ensure FileSpool can init from FileSPool.""" - new = FileSpool(terra15_file_spool) - assert isinstance(new, FileSpool) + """Ensure a spool can copy-construct from another spool.""" + new = Spool(terra15_file_spool) + assert isinstance(new, Spool) def test_str(self, terra15_file_spool): """Ensure file spool works.""" out = str(terra15_file_spool) - assert "FileSpool" in out + assert "Spool" in out def test_update(self, tmp_path_factory, random_patch): """Update should preserve contents even when a format index hook is a no-op.""" @@ -46,16 +46,16 @@ def test_update(self, tmp_path_factory, random_patch): def test_raises_bad_file(self): """Simply ensures a bad file will raise.""" with pytest.raises(FileNotFoundError, match="does not exist"): - FileSpool("/not/a/directory") + Spool.from_file("/not/a/directory") def test_local_upath_file(self, terra15_v5_path): - """Ensure FileSpool accepts local UPath inputs.""" - spool = FileSpool(UPath(terra15_v5_path)) - assert isinstance(spool, FileSpool) + """Ensure from_file accepts local UPath inputs.""" + spool = Spool.from_file(UPath(terra15_v5_path)) + assert isinstance(spool, Spool) assert len(spool) def test_chunk(self, terra15_file_spool): - """Ensure chunking along time axis works with FileSpool.""" + """Ensure chunking along time axis works on a file spool.""" spool = terra15_file_spool time_coord = spool[0].get_coord("time") duration = time_coord.max() - time_coord.min() @@ -71,7 +71,7 @@ def test_sorted_multi_patch_uses_source_patch_id(self, tmp_path): patch_2 = dc.get_example_patch() patch_1 = patch_2.update_coords(time=patch_2.coords.get_array("time") + 10) dc.write(dc.spool([patch_1, patch_2]), path, "dasdae", file_version="1") - spool = FileSpool(path).sort("time") + spool = Spool.from_file(path).sort("time") loaded_patch = spool[0] assert loaded_patch.get_coord("time").min() == patch_2.get_coord("time").min() @@ -80,7 +80,7 @@ def test_multi_patch_without_source_patch_id_raises(self, tmp_path): path = tmp_path / "multi_patch.h5" spool = dc.examples.get_example_spool("random_das", length=2) dc.write(spool, path, "dasdae", file_version="1") - file_spool = FileSpool(path) + file_spool = Spool.from_file(path) kwargs = {"path": str(path), "file_format": "DASDAE", "file_version": "1"} with pytest.raises(PatchAttributeError, match="uniquely resolved"): file_spool._load_patch(kwargs) diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 689f90ae7..43d0f9bd4 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -695,16 +695,16 @@ class TestStreamingMerge: def test_streaming_path_used(self, adjacent_spool_no_overlap, monkeypatch): """Ensure simple merges take the streaming path.""" - from dascore.core.spool import DataFrameSpool + from dascore.core.spool import Spool called = [] - original = DataFrameSpool._merge_patches_streaming + original = Spool._merge_patches_streaming def wrapper(self, *args, **kwargs): called.append(True) return original(self, *args, **kwargs) - monkeypatch.setattr(DataFrameSpool, "_merge_patches_streaming", wrapper) + monkeypatch.setattr(Spool, "_merge_patches_streaming", wrapper) merged = adjacent_spool_no_overlap.chunk(time=None) assert isinstance(merged[0], dc.Patch) assert called diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 082d3a6af..4e42da597 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -11,10 +11,9 @@ import pytest import dascore as dc -from dascore.clients.filespool import FileSpool from dascore.core.spool import ( BaseSpool, - MemorySpool, + Spool, _estimate_merge_samples, _get_varying_dim, ) @@ -90,7 +89,7 @@ def test_viz_raises(self, random_spool): random_spool.viz.waterfall(random_spool) -class TestMemorySpoolLazy: +class TestLiveSpoolLazy: """ Tests for lazy behavior of in-memory spools. @@ -164,7 +163,7 @@ def test_derived_spool_shares_only_the_catalog(self, patch_list): def test_single_patch_input_uses_lazy_storage(self, random_patch): """A single patch lands in the registry without realizing tables.""" - spool = MemorySpool(random_patch) + spool = Spool(random_patch) assert len(spool) == 1 registry = spool._catalog.resolver.live_entries() assert tuple(registry.values()) == (random_patch,) @@ -172,8 +171,8 @@ def test_single_patch_input_uses_lazy_storage(self, random_patch): assert spool._catalog._backend is None def test_empty_memory_spool(self): - """An empty MemorySpool is a valid, iterable, zero-length spool.""" - spool = MemorySpool() + """An empty Spool is a valid, iterable, zero-length spool.""" + spool = Spool() assert len(spool) == 0 assert list(spool) == [] @@ -678,9 +677,9 @@ def test_non_supported_type_raises(self): def test_file_spool(self, random_spool, tmp_path_factory): """ Tests for getting a file spool vs in-memory spool. Basically, - if a format supports scanning a FileSpool is returned. If it doesn't, - all the file contents have to be loaded into memory to scan so a - MemorySpool is just returned. + if a format supports scanning, a lazy file-backed spool is + returned. If it doesn't, all the file contents have to be loaded + into memory, so the spool holds live patches. """ path = tmp_path_factory.mktemp("file_spoolin") dasdae_path = path / "patch.h5" @@ -689,10 +688,10 @@ def test_file_spool(self, random_spool, tmp_path_factory): dc.write(random_spool, pickle_path, "pickle") dasdae_spool = dc.spool(dasdae_path) - assert isinstance(dasdae_spool, FileSpool) + assert not dasdae_spool.has_live_patches pickle_spool = dc.spool(pickle_path) - assert isinstance(pickle_spool, MemorySpool) + assert pickle_spool.has_live_patches class TestSpoolBehaviorOptionalImports: @@ -968,7 +967,7 @@ def test_equality_and_repr(self): def test_equality_of_empty_spools(self): """Empty spools (None frames) compare equal via the None-strip path.""" - assert MemorySpool() == MemorySpool() + assert Spool() == Spool() def test_repr_without_time_coordinate(self): """A spool whose patches have no time coord omits the time-span line.""" @@ -1035,8 +1034,8 @@ def test_merge_buffer_grows_when_estimate_short(self, many_contiguous, monkeypat ) def test_empty_memory_spool_len_iter_repr(self): - """A bare MemorySpool() (no dataframe) is a valid empty spool.""" - empty = MemorySpool() + """A bare Spool() is a valid empty spool.""" + empty = Spool() assert len(empty) == 0 assert list(empty) == [] assert "Spool" in str(empty) diff --git a/tests/test_core/test_spool_contracts.py b/tests/test_core/test_spool_contracts.py new file mode 100644 index 000000000..3bfbc7f97 --- /dev/null +++ b/tests/test_core/test_spool_contracts.py @@ -0,0 +1,162 @@ +""" +Tests for the unified Spool's equality and update contracts. + +Equality is over rows, never backends: order-sensitive, metadata-level, +with pending residual selections included. update() is case-based: +directory spools sync, file spools rescan, in-memory spools are +trivially current, and combined spools with file rows raise. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +import dascore as dc +from dascore.exceptions import InvalidSpoolError + + +@pytest.fixture(scope="module") +def patches(): + """Three contiguous example patches in time order.""" + return list(dc.get_example_spool("random_das")) + + +class TestEqualityContract: + """Equality: rows not backends; order-sensitive; residuals count.""" + + def test_residual_breaks_equality(self, patches): + """A samples residual changes loaded patches, so spools differ.""" + plain = dc.spool(patches) + trimmed = dc.spool(patches).select(distance=(0, 10), samples=True) + # same visible rows (samples never excludes patches) ... + assert len(plain) == len(trimmed) + # ... but not equal spools. + assert plain != trimmed + + def test_equal_selections_compare_equal(self, patches): + """The same selection on equal spools yields equal spools.""" + sel_1 = dc.spool(patches).select(distance=(0, 10)) + sel_2 = dc.spool(patches).select(distance=(0, 10)) + assert sel_1 == sel_2 + + def test_order_matters(self, patches): + """Same patches in a different order are not equal.""" + assert dc.spool(patches) != dc.spool(list(reversed(patches))) + + def test_live_vs_directory_equal(self, patches, tmp_path): + """Identical contents compare equal across backings (rows, not + backends): a live spool equals a directory spool over the same + patches written to disk. + """ + directory = dc.examples.spool_to_directory( + dc.spool(patches), path=tmp_path / "spool_dir" + ) + dir_spool = dc.spool(directory).update(progress=None) + live_spool = dc.spool(patches) + assert live_spool == dir_spool + assert dir_spool == live_spool + + +class TestUpdateContract: + """update() means sync-with-source; the cases differ by source.""" + + def test_live_only_is_noop(self, patches): + """A purely in-memory spool is trivially current.""" + spool = dc.spool(patches) + assert spool.update() is spool + + def test_union_of_live_spools_is_noop(self, patches): + """Combining live spools yields a still-sourceless, current spool.""" + combined = dc.spool(patches[:1]) + dc.spool(patches[1:]) + assert combined.update() is combined + + def test_union_with_file_rows_raises(self, patches, tmp_path): + """A combined spool with file rows has no update source.""" + directory = dc.examples.spool_to_directory( + dc.spool(patches), path=tmp_path / "dir_a" + ) + combined = dc.spool(directory).update(progress=None) + dc.spool(patches[:1]) + with pytest.raises(InvalidSpoolError, match="no update source"): + combined.update() + + def test_directory_update_picks_up_new_files(self, patches, tmp_path): + """The syncer case: new files appear after update().""" + directory = tmp_path / "dir_b" + dc.write(patches[0], directory / "a.h5", "dasdae") + spool = dc.spool(directory).update(progress=None) + assert len(spool) == 1 + dc.write(patches[1], directory / "b.h5", "dasdae") + assert len(spool.update(progress=None)) == 2 + + def test_file_spool_update_refreshes(self, patches, tmp_path): + """The single-file case: update rescans the file.""" + path = tmp_path / "single.h5" + dc.write(patches[0], path, "dasdae") + spool = dc.spool(path) + new = spool.update() + assert len(new) == len(spool) + assert new == spool + + +class TestChunkPlanContract: + """Planned views: collapse semantics and derived state.""" + + def test_replan_collapses(self, patches): + """Re-chunking a planned spool re-plans from members (no nesting).""" + spool = dc.spool(patches) + hourly = spool.chunk(time=2) + merged = hourly.chunk(time=None) + assert len(merged) == 1 + time = merged[0].get_coord("time") + mins = [p.get_coord("time").min() for p in patches] + maxs = [p.get_coord("time").max() for p in patches] + assert time.min() == min(mins) + assert time.max() == max(maxs) + + def test_planned_state_is_derived(self, patches): + """A planned spool reports non-native; identity views native.""" + spool = dc.spool(patches) + assert spool._catalog_native + chunked = spool.chunk(time=2) + assert not chunked._catalog_native + assert chunked._plan is not None + # the flag cannot be assigned; the plan is the state + with pytest.raises(AttributeError): + chunked._catalog_native = True + + +class TestTypeSurface: + """The collapsed hierarchy: BaseSpool ABC with one concrete Spool.""" + + def test_every_spool_is_spool(self, patches, tmp_path): + """All construction paths yield the same concrete class.""" + live = dc.spool(patches) + directory = dc.examples.spool_to_directory( + dc.spool(patches), path=tmp_path / "types_dir" + ) + dir_spool = dc.spool(directory).update(progress=None) + file_path = tmp_path / "one.h5" + dc.write(patches[0], file_path, "dasdae") + file_spool = dc.spool(file_path) + for spool in (live, dir_spool, file_spool, live.chunk(time=2)): + assert type(spool) is dc.Spool + assert isinstance(spool, dc.BaseSpool) + + def test_removed_names_gone(self): + """The old concrete class names are deleted outright.""" + import dascore.core.spool as spool_module + + for name in ("MemorySpool", "DirectorySpool", "FileSpool"): + assert not hasattr(spool_module, name) + with pytest.raises(ImportError): + from dascore.clients.dirspool import DirectorySpool # noqa + + def test_live_patch_predicate(self, patches, tmp_path): + """has_live_patches distinguishes memory content, not class.""" + assert dc.spool(patches).has_live_patches + file_path = tmp_path / "pred.h5" + dc.write(patches[0], file_path, "dasdae") + assert not dc.spool(file_path).has_live_patches + mixed = dc.spool(file_path) + dc.spool(patches[:1]) + assert mixed.has_live_patches diff --git a/tests/test_examples.py b/tests/test_examples.py index 3131de126..e2e0cae52 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -42,7 +42,7 @@ def test_load_example_patch(self, name): assert isinstance(patch, dc.Patch) def test_file_backed_examples_use_direct_read(self, monkeypatch): - """File-backed examples should not depend on FileSpool patch resolution.""" + """File-backed examples should not depend on file-spool patch resolution.""" patch = dc.get_example_patch() def _fetch(_name): diff --git a/tests/test_io/test_index/test_db_dirspool.py b/tests/test_io/test_index/test_db_dirspool.py index 52cec35bc..53a1fc6b8 100644 --- a/tests/test_io/test_index/test_db_dirspool.py +++ b/tests/test_io/test_index/test_db_dirspool.py @@ -1,5 +1,5 @@ """ -Integration tests: DirectorySpool running on the database index. +Integration tests: directory spools running on the database index. Exercises the full path — directory walk, scan, ingest, query, patch loading, and chunk against real files. @@ -11,7 +11,7 @@ import pytest import dascore as dc -from dascore.clients.dirspool import DirectorySpool +from dascore.core.spool import Spool from dascore.examples import spool_to_directory @@ -24,15 +24,15 @@ def spool_directory(tmp_path_factory): @pytest.fixture() def db_spool(spool_directory): - """A DirectorySpool using its SQLite index.""" - spool = DirectorySpool(spool_directory) + """A directory spool using its SQLite index.""" + spool = Spool.from_directory(spool_directory) out = spool.update(progress=None) yield out out.indexer.close() -class TestDBDirectorySpool: - """DirectorySpool wired to the database index.""" +class TestDBDirectorySpools: + """Directory spools wired to the database index.""" def test_length(self, db_spool): """One entry per patch in the source spool.""" @@ -87,7 +87,7 @@ def fresh(self, tmp_path): """A modifiable spool directory and database spool.""" spool = dc.get_example_spool("random_das") path = spool_to_directory(spool, path=tmp_path / "data") - out = DirectorySpool(path).update(progress=None) + out = Spool.from_directory(path).update(progress=None) yield path, out out.indexer.close() diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index e270e3eb8..0e18bc45e 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -977,15 +977,15 @@ def test_directory_format_unit(self, tmp_path): class TestDirSpoolPassthrough: - """DirectorySpool accepts a prebuilt indexer.""" + """Directory spools accept a prebuilt indexer.""" def test_spool_from_indexer(self, tmp_path, random_patch): - """Passing an indexer instance to DirectorySpool works.""" - from dascore.clients.dirspool import DirectorySpool + """Passing an indexer instance to from_directory works.""" + from dascore.core.spool import Spool random_patch.io.write(tmp_path / "one.hdf5", "dasdae") indexer = DBDirectoryIndexer(tmp_path) - spool = DirectorySpool(indexer).update(progress=None) + spool = Spool.from_directory(indexer).update(progress=None) assert len(spool) == 1 diff --git a/tests/test_io/test_pickle/test_pickle.py b/tests/test_io/test_pickle/test_pickle.py index 0678c5785..835d21ba7 100644 --- a/tests/test_io/test_pickle/test_pickle.py +++ b/tests/test_io/test_pickle/test_pickle.py @@ -46,7 +46,7 @@ def test_spool_from_pickle(self, pickle_patch_path, random_patch): """dc.spool on a scanless format wraps the read spool and serves it. PICKLE implements read but not scan, so dc.spool routes through - MemorySpool(dc.read(...)); the wrapped patches must load back. + Spool(dc.read(...)); the wrapped patches must load back. """ spool = dc.spool(pickle_patch_path) assert len(spool) == 1 From 410c2f1b0c9de9c63234e793f318c26829d5bebf Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 17:43:41 +0200 Subject: [PATCH 80/97] Document the ordering contract and unified spool in the notes The spool-index note gains an Ordering section (ordinal contract, union dedup semantics, syncer time renumbering) and records the automatic rebuild of old-version index files; the file IO tutorial drops the removed DirectorySpool class name. --- docs/notes/spool_index.qmd | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/notes/spool_index.qmd b/docs/notes/spool_index.qmd index 88b41e6f4..2e3e18363 100644 --- a/docs/notes/spool_index.qmd +++ b/docs/notes/spool_index.qmd @@ -11,7 +11,7 @@ The schema separates records with different lifetimes and cardinalities. This av | Table | One row per | Purpose | |---|---|---| | `meta_data` | index | Identifies the file and its schema version | -| `sources` | file or directory-format source | Tracks the source path, format, size, and modification time | +| `sources` | file or directory-format source | Tracks the source path, format, size, modification time, and presentation ordinal | | `patches` | patch within a source | Stores patch identity and common time/distance envelopes | | `attrs` | patch | Stores typed attribute values in dynamically added columns | | `attr_meta` | attribute name and value kind | Maps original attribute names to typed storage columns and canonical units | @@ -26,7 +26,11 @@ The last two tables are deliberately separate. Many patches can share a distance The index is an incrementally updated cache. A directory update scans new or changed sources, transactionally replaces their patch rows, and removes rows for deleted sources. Foreign keys cascade source deletion through patches, attributes, and patch-coordinate links. Unreferenced coordinate definitions may remain available for reuse. -The current schema version is validated before any mutation. An unrelated, incomplete, or older prototype database raises an error with instructions to delete and rebuild it; DASCore does not silently repair or migrate it. +The current schema version is validated before any mutation. An unrelated or incomplete database raises an error with instructions to delete and rebuild it; DASCore does not silently repair or migrate it. A file that identifies itself as a DASCore spool index of a different schema version is rebuilt automatically by the directory indexer — the index is a disposable cache whose truth is the files. + +## Ordering + +Patch rows present in `(sources.ordinal, patch_id)` order — the catalog's explicit ordering contract. Ordinals are assigned at ingest: a replaced source keeps its position while new sources append, so merging catalogs concatenates and duplicate sources keep their first-occurrence position with last-occurrence metadata (dict-merge semantics). Spools built from in-memory patches therefore iterate in construction order on every path. The directory indexer renumbers ordinals to time order (earliest patch per source, path as tiebreak) after each sync, so file archives keep their conventional time-ordered presentation, including files added by later updates. SQLite permits concurrent readers and serializes writers. Initialization and updates use an immediate write transaction and a 30-second busy timeout. This relies on correct local-filesystem locking; reliable operation on network filesystems with weak locking is not promised. From d119a4f26061dceb093db3525fc56f69180201fb Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 18:15:09 +0200 Subject: [PATCH 81/97] Finish the spool unification: one selection home, enumerated equality, extracted assembly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Constructor select_kwargs now compose a catalog selection identical to .select(**select_kwargs) — validated eagerly (triggering the initial directory index if absent) — so spool-level selection state is gone entirely: no more triple application through realization, load trims, and content filtering. Spool equality compares an explicitly enumerated semantic state (rows/sources/members plus residuals and policy) instead of __dict__ minus a deny-list, so stray instance attributes can no longer join equality. The member-assembly engine (instruction join, exact trims, streaming merge) moves to dascore/utils/patch_assembly.py beside the chunk planner whose plans it executes, leaving Spool as container API only. The spool notes document the derived-view shape and the plans-never-nest rule. --- dascore/core/spool.py | 381 +++++------------------- dascore/utils/patch_assembly.py | 271 +++++++++++++++++ docs/notes/spool_chunking.qmd | 2 +- docs/notes/spool_selection.qmd | 2 +- tests/test_core/test_patch_chunk.py | 41 ++- tests/test_core/test_spool.py | 176 +++-------- tests/test_core/test_spool_contracts.py | 1 - 7 files changed, 414 insertions(+), 460 deletions(-) create mode 100644 dascore/utils/patch_assembly.py diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 63d684daf..a6a10b1dc 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -28,27 +28,20 @@ timeable_types, ) from dascore.exceptions import ( - CoordMergeError, InvalidSpoolError, InvalidSpoolQueryError, MissingPatchError, ParameterError, ) -from dascore.utils.attrs import combine_patch_attrs from dascore.utils.display import get_dascore_text, get_nice_text from dascore.utils.docs import compose_docstring from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( _spool_map, - broadcast_for_index, deep_equality_check, ) from dascore.utils.namespace import NamespaceOwner from dascore.utils.patch import ( - _force_patch_merge, - _get_merge_dim, - _get_merged_coord, - _split_coord_merge_kwargs, _spool_up, concatenate_patches, get_patch_names, @@ -57,9 +50,7 @@ from dascore.utils.paths import coerce_to_upath, requires_local_directory from dascore.utils.pd import ( _column_or_value, - _convert_min_max_in_kwargs, adjust_segments, - filter_df, get_column_names_from_dim, get_dim_names_from_columns, resolve_selector_namespaces, @@ -68,57 +59,6 @@ T = TypeVar("T") -def _get_varying_dim(df) -> str | None: - """ - Get the single dimension whose range varies across rows of df. - - Returns None when no dimension varies, several do, or the dataframe - doesn't carry range columns for the varying dimension; those cases - need the fully materialized merge to sort out. - """ - dims = get_dim_names_from_columns(df) - varying = [] - for dim in dims: - mins, maxs = df.get(f"{dim}_min"), df.get(f"{dim}_max") - if mins.nunique(dropna=False) > 1 or maxs.nunique(dropna=False) > 1: - varying.append(dim) - return varying[0] if len(varying) == 1 else None - - -def _estimate_merge_samples(df, dim) -> int | None: - """ - Estimate the total number of samples along dim of the merged rows. - - Returns None if the estimate cannot be made (eg unknown steps), in - which case streaming the merge isn't possible. - """ - if dim is None: - return None - cols = [f"{dim}_min", f"{dim}_max", f"{dim}_step"] - if not set(cols).issubset(df.columns): - return None - mins, maxs, steps = (df[x] for x in cols) - if mins.isnull().any() or maxs.isnull().any() or steps.isnull().any(): - return None - ratios = (maxs - mins) / steps - # Degenerate steps (eg 0) make the sample counts meaningless. - if not np.isfinite(ratios.astype(np.float64)).all(): - return None - counts = np.round(ratios).astype(np.int64) + 1 - if (counts < 0).any(): - return None - return int(counts.sum()) - - -def _coord_only_kwargs(patch, kwargs) -> dict: - """Keep only the kwargs naming a dim or coordinate of patch.""" - return { - k: v - for k, v in kwargs.items() - if k in patch.dims or k in patch.coords.coord_map - } - - class BaseSpool(NamespaceOwner, abc.ABC): """Spool Abstract Base Class (ABC) for defining Spool interface.""" @@ -508,10 +448,26 @@ class Spool(BaseSpool): """ The concrete spool: a `PatchCatalog` plus an optional derived view. + Constructed from in-memory patches directly (or via + [`dascore.spool`](`dascore.spool`)), from a directory of files with + [`Spool.from_directory`](`dascore.core.spool.Spool.from_directory`), + or from a single file with + [`Spool.from_file`](`dascore.core.spool.Spool.from_file`). + + Parameters + ---------- + data + A patch, sequence of patches, or another spool whose (in-memory) + patches this spool should hold; None creates an empty spool. + merge_kwargs + Kwargs controlling how member patches merge when assembled. + + Notes + ----- The catalog is the single store — live patches sit in its resolver registry, file-backed patches in its index tables — regardless of - how the spool was constructed (patches, a directory, or a single - file). A spool presents rows from exactly one of two derivations: + how the spool was constructed. A spool presents rows from exactly + one of two derivations: - **catalog-backed** (``_plan is None``): rows map one-to-one to a ``PatchCatalog`` query, so metadata operations (length, @@ -522,8 +478,6 @@ class Spool(BaseSpool): The catalog remains attached for patch resolution. """ - # kwargs for filtering contents - _select_kwargs: Mapping | None = FrozenDict() # kwargs for merging patches _merge_kwargs: Mapping | None = FrozenDict() # synthetic catalog identity columns must not join patch kwargs @@ -581,11 +535,6 @@ def _get_df(self): if self._catalog is None: return None current = self._catalog.to_df().reset_index(drop=True) - if self._select_kwargs: - # constructor select_kwargs restrict contents (docstring contract) - current = adjust_segments( - current, ignore_bad_kwargs=True, **self._select_kwargs - ) df, source, instruction = self._get_dummy_dataframes(current) self._cache["_source_df"] = source self._cache["_instruction_df"] = instruction @@ -604,13 +553,11 @@ def _get_instruction_df(self): def __init__( self, data: PatchType | Sequence[PatchType] | BaseSpool | None = None, - select_kwargs: dict | None = None, merge_kwargs: dict | None = None, ): from dascore.io.index.catalog import PatchCatalog self._cache = {} - self._select_kwargs = {} if select_kwargs is None else select_kwargs self._merge_kwargs = {} if merge_kwargs is None else merge_kwargs self._post_selects = () if isinstance(data, Spool): @@ -618,9 +565,6 @@ def __init__( # catalog, take fresh derived state self.__dict__.update(data.__dict__) self._cache = {} - self._select_kwargs = dict(data._select_kwargs) - if select_kwargs: - self._select_kwargs.update(select_kwargs) self._merge_kwargs = dict(data._merge_kwargs) if merge_kwargs: self._merge_kwargs.update(merge_kwargs) @@ -653,13 +597,11 @@ def _select_from_array(self, array) -> Self: raise ValueError(msg) source = self._source_df inst = self._instruction_df - select_kwargs, merge_kwargs = self._select_kwargs, self._merge_kwargs new = self.new_from_df( df, source_df=source, instruction_df=inst, - select_kwargs=select_kwargs, - merge_kwargs=merge_kwargs, + merge_kwargs=self._merge_kwargs, ) return new @@ -671,11 +613,7 @@ def _rows_are_catalog(self) -> bool: or patch-local selections layered outside the catalog (which carries its own selection as queries/residuals). """ - return ( - self._is_catalog_backed() - and not self._select_kwargs - and not self._post_selects - ) + return self._is_catalog_backed() and not self._post_selects def __getitem__(self, item) -> PatchType | BaseSpool: if isinstance(item, slice): # a slice was used, return a sub-spool @@ -698,20 +636,15 @@ def __getitem__(self, item) -> PatchType | BaseSpool: msg = f"index of [{item}] is out of bounds for spool." raise IndexError(msg) from None else: # a single index was used, should return a single patch - out = self._unbox_patch(self._get_patches_from_index(item)) + out = self._assembler.get_patch(item) return out def __len__(self): # A catalog-native view can count in SQL, skipping the full flat # realization (query + attr expansion + coordinate pivot) a plain # len(self._df) would force. Fall back to the realized frame once - # it is cached, on the dataframe path, or when constructor - # select_kwargs post-filter rows outside the catalog's queries. - if ( - self._is_catalog_backed() - and not self._select_kwargs - and "_df" not in self._cache - ): + # it is cached or on the dataframe path. + if self._is_catalog_backed() and "_df" not in self._cache: return len(self._catalog) df = self._df # An empty spool with no patches, data, or catalog has no frame. @@ -730,165 +663,29 @@ def __iter__(self): return for ind in range(len(self._df)): try: - yield self._unbox_patch(self._get_patches_from_index(ind)) + yield self._assembler.get_patch(ind) except MissingPatchError as e: # The patch couldn't be produced, usually because a # coordinate mismatch trimmed it to nothing (see #583). msg = f"Skipping patch at index {ind} (see #583): {e}" warnings.warn(msg, UserWarning, stacklevel=2) - def _unbox_patch(self, patch_list): - """Unbox a single patch from a patch list, check len.""" - assert len(patch_list) == 1 - return patch_list[0] - - def _get_patches_from_index(self, df_ind): - """Given an index (from current df), return the corresponding patch.""" - source = self._source_df - instruction = self._instruction_df - # handle negative index. - df_ind = df_ind if df_ind >= 0 else len(self._df) + df_ind - try: - inds = self._df.index[df_ind] - except IndexError: - msg = f"index of [{df_ind}] is out of bounds for spool." - raise IndexError(msg) from None - # Group positional instruction rows by current index (and cache) to - # avoid a full instruction df scan for each requested patch. - indices = self._cache.get("_instruction_indices") - if indices is None: - indices = instruction.groupby("current_index").indices - self._cache["_instruction_indices"] = indices - positions = indices.get(inds) - assert positions is not None and len(positions), "no instructions found" - df1 = instruction.iloc[positions] - joined = df1.join(source.drop(columns=df1.columns, errors="ignore")) - # Occasionally, duplicates can creep into the source_df, - # but it costs a bit to check for duplicates, so only check and drop - # duplicates on large joined dataframes where performance might be - # affected. - if len(joined) > 10: - cols = set(joined.columns) - set(self._drop_columns) - joined = joined.drop_duplicates(subset=list(cols), keep="first") - return self._patch_from_instruction_df(joined) - - def _patch_from_instruction_df(self, joined): - """Get the patches joined columns of instruction df.""" - df_dict_list = self._df_to_dict_list(joined) - expected_len = len(joined["current_index"].unique()) - if len(df_dict_list) > expected_len: - # Several sources merge into one patch. When the output size can - # be determined from the instructions, stream the sources into a - # pre-allocated array so they don't all need to be in memory with - # the merged output at once. - merge_dim = _get_varying_dim(joined) - samples = _estimate_merge_samples(joined, merge_dim) - if samples is not None: - patch = self._merge_patches_streaming( - joined, df_dict_list, merge_dim, samples - ) - return [patch] - out = [] - for patch_kwargs in df_dict_list: - patch = self._load_trimmed_patch(patch_kwargs, joined) - # The index doesn't carry all the dimensional info, so get what - # merging needs from the patch coords (cheaper than attr dumps). - info = patch.coords._get_dim_summary() - info["patch"] = patch - out.append(info) - if len(out) > expected_len: - out = _force_patch_merge(out, merge_kwargs=self._merge_kwargs) - return [x["patch"] for x in out] - - def _load_trimmed_patch(self, patch_kwargs, joined) -> dc.Patch: - """Load a single patch and trim it to its instruction range.""" - # convert kwargs to format understood by parser/patch.select - kwargs = _convert_min_max_in_kwargs(patch_kwargs, joined) - patch = self._load_patch(kwargs) - # If the limits of the source patch were not modified, we can just - # use the select kwargs. This is important for missing coordinates - # (NaN values) to not get trimmed out. - source_kwargs = kwargs if kwargs.get("_modified") else self._select_kwargs - # attr-style entries (e.g. constructor select_kwargs) filter rows - # above; only coordinate entries are valid patch selections. - if select_kwargs := _coord_only_kwargs(patch, source_kwargs): - patch = patch.select(**select_kwargs) - # patch-local selections (samples=True) recorded by spool.select - for post_kwargs, samples in self._post_selects: - if usable := _coord_only_kwargs(patch, post_kwargs): - patch = patch.select(**usable, samples=samples) - return patch - - def _merge_patches_streaming(self, joined, df_dict_list, merge_dim, samples): - """ - Merge the patches described by the instructions along merge_dim. + @property + def _assembler(self): + """The (cached per view) executor turning member rows into patches.""" + from dascore.utils.patch_assembly import PatchAssembler - Each patch is copied into a pre-allocated output array as it is - loaded, then released; this avoids holding all source patches and - the merged output in memory at the same time, as concatenating - would. - """ - buffer, offset, axis, dims = None, 0, None, None - coords, attrs, summaries = [], [], [] - for patch_kwargs in df_dict_list: - patch = self._load_trimmed_patch(patch_kwargs, joined) - if dims is None: - dims = patch.dims - axis = patch.get_axis(merge_dim) - elif patch.dims != dims: - patch = patch.transpose(*dims) - data = patch.data - if buffer is None: - shape = list(data.shape) - shape[axis] = samples - buffer = np.empty(shape, dtype=data.dtype) - # Mixed dtypes upcast, mirroring np.concatenate behavior. - dtype = np.result_type(buffer.dtype, data.dtype) - if dtype != buffer.dtype: - buffer = buffer.astype(dtype) - end = offset + data.shape[axis] - if end > buffer.shape[axis]: - # The estimate came up short (eg from slightly uneven - # sampling); grow the buffer to fit. - shape = list(buffer.shape) - shape[axis] = end - new_buffer = np.empty(shape, dtype=buffer.dtype) - head = broadcast_for_index(buffer.ndim, axis, slice(0, offset)) - new_buffer[head] = buffer[head] - buffer = new_buffer - try: - index = broadcast_for_index(buffer.ndim, axis, slice(offset, end)) - buffer[index] = data - except ValueError as e: - msg = ( - f"Cannot merge patches; their shapes are incompatible " - f"along the dimensions not being merged ({merge_dim})." - ) - raise CoordMergeError(msg) from e - offset = end - coords.append(patch.coords) - attrs.append(patch.attrs) - summaries.append(patch.coords._get_dim_summary()) - if offset != buffer.shape[axis]: # over-estimated; trim excess. - buffer = buffer[broadcast_for_index(buffer.ndim, axis, slice(0, offset))] - # Ensure the loaded patches only vary along the expected dimension, - # the same requirement _force_patch_merge enforces. - summary_df = pd.DataFrame(summaries) - found_dim = _get_merge_dim(summary_df) - if found_dim != merge_dim: - msg = ( - f"Cannot merge patches; expected them to vary along " - f"{merge_dim} but found {found_dim}." + if "_assembler" not in self._cache: + self._cache["_assembler"] = PatchAssembler( + df=self._df, + source_df=self._source_df, + instruction_df=self._instruction_df, + load_patch=self._load_patch, + merge_kwargs=self._merge_kwargs, + post_selects=self._post_selects, + drop_columns=self._drop_columns, ) - raise CoordMergeError(msg) - attr_kwargs, coord_kwargs = _split_coord_merge_kwargs(self._merge_kwargs) - conf = attr_kwargs.get("conflicts", None) - drop_conflicting = conf in {"drop", "keep_first"} - new_coord = _get_merged_coord( - summary_df, merge_dim, coords, drop_conflicting, **coord_kwargs - ) - new_attrs = combine_patch_attrs(attrs, **attr_kwargs) - return dc.Patch(data=buffer, coords=new_coord, attrs=new_attrs, dims=list(dims)) + return self._cache["_assembler"] def _get_dummy_dataframes(self, current): """ @@ -914,30 +711,17 @@ def _get_dummy_dataframes(self, current): ) return current, source, instruction - def _df_to_dict_list(self, df): - """ - Convert the dataframe to a list of dicts for iteration. - - This is significantly faster than iterating rows. Empty strings - (missing format fields on file rows) normalize to None; stored - relative paths pass through unchanged — the catalog's resolver - owns resolving them against the spool root. - """ - df = df.copy(deep=False).replace("", None) - return df.to_dict("records") - def _load_patch(self, kwargs) -> dc.Patch: """Given a row from the managed dataframe, return a patch.""" # Push trims into the reader only when the instruction row narrows - # the source (chunk/select) or constructor select_kwargs restrict - # it; otherwise the whole source is wanted and selection is wasted. - # Live patches ignore trim hints; exactness is re-applied above. + # the source (chunk/select); otherwise the whole source is wanted + # and selection is wasted. Live patches ignore trim hints; + # exactness is re-applied above (catalog residuals). trim = {} - if kwargs.get("_modified") or self._select_kwargs: - merged = {**kwargs, **self._select_kwargs} + if kwargs.get("_modified"): trim = { k: v - for k, v in merged.items() + for k, v in kwargs.items() if k not in self._drop_columns and not k.startswith("_") } return self._catalog.resolve_row(kwargs, extra_trim=trim) @@ -954,11 +738,7 @@ def _as_catalog_member(self): """ if self._catalog is None: return super()._as_catalog_member() - # A catalog-native spool is the whole catalog only when nothing - # narrows it; constructor select_kwargs (directory spools) restrict - # the visible rows without touching the catalog, so carry only the - # surviving patch ids rather than the entire catalog. - if self._catalog_native and not self._select_kwargs: + if self._catalog_native: return self._catalog, None df = self._df if "_patch_id" in df.columns: @@ -1082,7 +862,6 @@ def new_from_df( df, source_df=None, instruction_df=None, - select_kwargs=None, merge_kwargs=None, ): """Create a new instance from dataframes.""" @@ -1096,8 +875,6 @@ def new_from_df( # resolution but no longer defines the presented rows. new._plan = SpoolView(outputs=df, members=instruction_df, sources=source_df) new._cache = {} - new._select_kwargs = dict(self._select_kwargs) - new._select_kwargs.update(select_kwargs or {}) new._merge_kwargs = dict(self._merge_kwargs) new._merge_kwargs.update(merge_kwargs or {}) new._post_selects = self._post_selects @@ -1205,9 +982,6 @@ def _new_from_catalog(self, catalog) -> Self: new._catalog = catalog new._plan = None new._cache = {} - # selection composed into the catalog is dropped, but constructor - # select_kwargs (directory-spool contract) persist across views. - new._select_kwargs = dict(self._select_kwargs) new._post_selects = () return new @@ -1261,10 +1035,7 @@ def split( @compose_docstring(doc=BaseSpool.get_contents.__doc__) def get_contents(self) -> pd.DataFrame: """{doc}.""" - # identity views apply select_kwargs during realization (_get_df) - if self._plan is None: - return self._df - return self._df[filter_df(self._df, **self._select_kwargs)] + return self._df # --- construction -------------------------------------------------- @@ -1281,19 +1052,25 @@ def from_directory( The directory's index (created/updated via ``update()``) backs the catalog; ``path`` may also be an existing directory indexer. + ``select_kwargs`` compose a selection into the catalog exactly + like ``.select(**select_kwargs)`` — validating the names + triggers the initial directory index if it doesn't exist yet. """ from dascore.io.index.catalog import FileResolver, PatchCatalog from dascore.io.indexer import AbstractIndexer - out = cls(select_kwargs=select_kwargs, merge_kwargs=merge_kwargs) + out = cls(merge_kwargs=merge_kwargs) if isinstance(path, AbstractIndexer): - out._catalog = PatchCatalog( + catalog = PatchCatalog( backend=path._backend, resolver=FileResolver(root=path.path), syncer=path, ) else: - out._catalog = PatchCatalog.from_directory(path, index_path=index_path) + catalog = PatchCatalog.from_directory(path, index_path=index_path) + if select_kwargs: + catalog = catalog.select(**select_kwargs) + out._catalog = catalog return out @classmethod @@ -1402,18 +1179,20 @@ def __eq__(self, other) -> bool: return True if not isinstance(other, Spool): return super().__eq__(other) - return deep_equality_check(self._eq_dict(), other._eq_dict()) + return deep_equality_check(self._eq_state(), other._eq_state()) - def _eq_dict(self) -> dict: + def _eq_state(self) -> dict: """ - Get a dict for equality checks, normalizing lazy state. + The spool's semantic state, explicitly enumerated for equality. Equality is over rows, never backends: same length and order of patch rows, row-wise equal semantic columns (source identity like paths and live-vs-file backing stripped), plus equal - pending residual selections. Whether rows come from a live - registry, an index file, or a plan is invisible; data arrays - are never compared (metadata-level, like everything else here). + pending residual selections and policy. Whether rows come from + a live registry, an index file, or a plan is invisible; data + arrays are never compared (metadata-level, like everything + here). Because the state is enumerated — never ``__dict__`` — + new instance attributes cannot silently join equality. """ def _strip_identity(df): @@ -1433,27 +1212,21 @@ def _strip_identity(df): out = df.drop(columns=list(drop), errors="ignore") return out[sorted(out.columns)] - out = dict(self.__dict__) - # Build (if needed) and compare the dataframes; drop the inputs - # they were built from, whose form can differ for equal contents. - # The plan's frames are exposed through the same accessors, so - # planned and identity views with equal contents compare equal. - out["_cache"] = { - "_df": _strip_identity(self._df), - "_source_df": _strip_identity(self._source_df), - "_instruction_df": _strip_identity(self._instruction_df), + catalog = self._catalog + return { + # the presented relation (row content and order), plus the + # source rows and member bindings that define patch assembly; + # the plan's frames surface through the same accessors, so + # planned and identity views with equal contents compare equal + "rows": _strip_identity(self._df), + "sources": _strip_identity(self._source_df), + "members": _strip_identity(self._instruction_df), + # residuals (e.g. samples trims) change what patches load + # without changing the visible rows + "residuals": None if catalog is None else catalog._residuals, + "post_selects": self._post_selects, + "merge_kwargs": dict(self._merge_kwargs), } - out.pop("_plan", None) - # Backend provenance is not content. - out.pop("_file_path", None) - out.pop("_file_format", None) - out.pop("_file_version", None) - # The catalog object is backend identity, but its composed view - # state is content: residuals (e.g. samples trims) change what - # patches load without changing the visible rows. - catalog = out.pop("_catalog", None) - out["_catalog_residuals"] = None if catalog is None else catalog._residuals - return out def __rich__(self): base = super().__rich__() @@ -1461,8 +1234,6 @@ def __rich__(self): path = getattr(indexer, "path", None) or self._file_path if path is not None: base += Text(f"\n Path: {path}") - if self._select_kwargs: - base += Text(f"\n Select kwargs: {self._select_kwargs}") # Only render a time span when the relation is (or is nearly) # realized: planned views carry their frames and live spools are # in memory; a huge directory index is not realized for a repr. diff --git a/dascore/utils/patch_assembly.py b/dascore/utils/patch_assembly.py new file mode 100644 index 000000000..17b37ae13 --- /dev/null +++ b/dascore/utils/patch_assembly.py @@ -0,0 +1,271 @@ +""" +Execute spool views: turn member instructions into loaded patches. + +This is the consumer of the members (instruction) table that +`dascore.utils.chunk_plan` produces and every spool view carries: it +joins member rows to their source rows, loads each source patch through +a caller-supplied loader, applies exact trims, and merges multi-member +outputs (streaming into a pre-allocated buffer when the output size is +known). The spool owns *what* rows exist; this module owns *how* a row +becomes a Patch. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd + +import dascore as dc +from dascore.exceptions import CoordMergeError +from dascore.utils.attrs import combine_patch_attrs +from dascore.utils.misc import broadcast_for_index +from dascore.utils.patch import ( + _force_patch_merge, + _get_merge_dim, + _get_merged_coord, + _split_coord_merge_kwargs, +) +from dascore.utils.pd import ( + _convert_min_max_in_kwargs, + get_dim_names_from_columns, +) + + +def _get_varying_dim(df) -> str | None: + """ + Get the single dimension whose range varies across rows of df. + + Returns None when no dimension varies, several do, or the dataframe + doesn't carry range columns for the varying dimension; those cases + need the fully materialized merge to sort out. + """ + dims = get_dim_names_from_columns(df) + varying = [] + for dim in dims: + mins, maxs = df.get(f"{dim}_min"), df.get(f"{dim}_max") + if mins.nunique(dropna=False) > 1 or maxs.nunique(dropna=False) > 1: + varying.append(dim) + return varying[0] if len(varying) == 1 else None + + +def _estimate_merge_samples(df, dim) -> int | None: + """ + Estimate the total number of samples along dim of the merged rows. + + Returns None if the estimate cannot be made (eg unknown steps), in + which case streaming the merge isn't possible. + """ + if dim is None: + return None + cols = [f"{dim}_min", f"{dim}_max", f"{dim}_step"] + if not set(cols).issubset(df.columns): + return None + mins, maxs, steps = (df[x] for x in cols) + if mins.isnull().any() or maxs.isnull().any() or steps.isnull().any(): + return None + ratios = (maxs - mins) / steps + # Degenerate steps (eg 0) make the sample counts meaningless. + if not np.isfinite(ratios.astype(np.float64)).all(): + return None + counts = np.round(ratios).astype(np.int64) + 1 + if (counts < 0).any(): + return None + return int(counts.sum()) + + +def _coord_only_kwargs(patch, kwargs) -> dict: + """Keep only the kwargs naming a dim or coordinate of patch.""" + return { + k: v + for k, v in kwargs.items() + if k in patch.dims or k in patch.coords.coord_map + } + + +@dataclass +class PatchAssembler: + """ + Assemble patches for one spool view. + + The frames define the view (presented rows, member instructions, + source rows); ``load_patch`` resolves one joined row to its source + patch; policy fields carry the merge behavior and patch-local + post-selections. Instances cache the instruction-row index and are + themselves cached per spool view. + """ + + df: pd.DataFrame + source_df: pd.DataFrame + instruction_df: pd.DataFrame + load_patch: Callable[[Mapping], dc.Patch] + merge_kwargs: Mapping + post_selects: tuple = () + drop_columns: tuple = () + _indices: dict | None = field(default=None, repr=False) + + def get_patch(self, df_ind: int) -> dc.Patch: + """Assemble the single patch presented at a row index.""" + patches = self.get_patches_from_index(df_ind) + assert len(patches) == 1 + return patches[0] + + def get_patches_from_index(self, df_ind): + """Given an index (from current df), return the corresponding patch.""" + source = self.source_df + instruction = self.instruction_df + # handle negative index. + df_ind = df_ind if df_ind >= 0 else len(self.df) + df_ind + try: + inds = self.df.index[df_ind] + except IndexError: + msg = f"index of [{df_ind}] is out of bounds for spool." + raise IndexError(msg) from None + # Group positional instruction rows by current index (and cache) to + # avoid a full instruction df scan for each requested patch. + if self._indices is None: + self._indices = instruction.groupby("current_index").indices + positions = self._indices.get(inds) + assert positions is not None and len(positions), "no instructions found" + df1 = instruction.iloc[positions] + joined = df1.join(source.drop(columns=df1.columns, errors="ignore")) + # Occasionally, duplicates can creep into the source_df, + # but it costs a bit to check for duplicates, so only check and drop + # duplicates on large joined dataframes where performance might be + # affected. + if len(joined) > 10: + cols = set(joined.columns) - set(self.drop_columns) + joined = joined.drop_duplicates(subset=list(cols), keep="first") + return self._patch_from_instruction_df(joined) + + def _patch_from_instruction_df(self, joined): + """Get the patches joined columns of instruction df.""" + df_dict_list = self._df_to_dict_list(joined) + expected_len = len(joined["current_index"].unique()) + if len(df_dict_list) > expected_len: + # Several sources merge into one patch. When the output size can + # be determined from the instructions, stream the sources into a + # pre-allocated array so they don't all need to be in memory with + # the merged output at once. + merge_dim = _get_varying_dim(joined) + samples = _estimate_merge_samples(joined, merge_dim) + if samples is not None: + patch = self._merge_patches_streaming( + joined, df_dict_list, merge_dim, samples + ) + return [patch] + out = [] + for patch_kwargs in df_dict_list: + patch = self._load_trimmed_patch(patch_kwargs, joined) + # The index doesn't carry all the dimensional info, so get what + # merging needs from the patch coords (cheaper than attr dumps). + info = patch.coords._get_dim_summary() + info["patch"] = patch + out.append(info) + if len(out) > expected_len: + out = _force_patch_merge(out, merge_kwargs=self.merge_kwargs) + return [x["patch"] for x in out] + + def _load_trimmed_patch(self, patch_kwargs, joined) -> dc.Patch: + """Load a single patch and trim it to its instruction range.""" + # convert kwargs to format understood by parser/patch.select + kwargs = _convert_min_max_in_kwargs(patch_kwargs, joined) + patch = self.load_patch(kwargs) + # If the limits of the source patch were not modified, we can just + # skip selection. This is important for missing coordinates + # (NaN values) to not get trimmed out. + source_kwargs = kwargs if kwargs.get("_modified") else {} + # attr-style entries filter rows above; only coordinate entries + # are valid patch selections. + if select_kwargs := _coord_only_kwargs(patch, source_kwargs): + patch = patch.select(**select_kwargs) + # patch-local selections (samples=True) recorded by spool.select + for post_kwargs, samples in self.post_selects: + if usable := _coord_only_kwargs(patch, post_kwargs): + patch = patch.select(**usable, samples=samples) + return patch + + def _merge_patches_streaming(self, joined, df_dict_list, merge_dim, samples): + """ + Merge the patches described by the instructions along merge_dim. + + Each patch is copied into a pre-allocated output array as it is + loaded, then released; this avoids holding all source patches and + the merged output in memory at the same time, as concatenating + would. + """ + buffer, offset, axis, dims = None, 0, None, None + coords, attrs, summaries = [], [], [] + for patch_kwargs in df_dict_list: + patch = self._load_trimmed_patch(patch_kwargs, joined) + if dims is None: + dims = patch.dims + axis = patch.get_axis(merge_dim) + elif patch.dims != dims: + patch = patch.transpose(*dims) + data = patch.data + if buffer is None: + shape = list(data.shape) + shape[axis] = samples + buffer = np.empty(shape, dtype=data.dtype) + # Mixed dtypes upcast, mirroring np.concatenate behavior. + dtype = np.result_type(buffer.dtype, data.dtype) + if dtype != buffer.dtype: + buffer = buffer.astype(dtype) + end = offset + data.shape[axis] + if end > buffer.shape[axis]: + # The estimate came up short (eg from slightly uneven + # sampling); grow the buffer to fit. + shape = list(buffer.shape) + shape[axis] = end + new_buffer = np.empty(shape, dtype=buffer.dtype) + head = broadcast_for_index(buffer.ndim, axis, slice(0, offset)) + new_buffer[head] = buffer[head] + buffer = new_buffer + try: + index = broadcast_for_index(buffer.ndim, axis, slice(offset, end)) + buffer[index] = data + except ValueError as e: + msg = ( + f"Cannot merge patches; their shapes are incompatible " + f"along the dimensions not being merged ({merge_dim})." + ) + raise CoordMergeError(msg) from e + offset = end + coords.append(patch.coords) + attrs.append(patch.attrs) + summaries.append(patch.coords._get_dim_summary()) + if offset != buffer.shape[axis]: # over-estimated; trim excess. + buffer = buffer[broadcast_for_index(buffer.ndim, axis, slice(0, offset))] + # Ensure the loaded patches only vary along the expected dimension, + # the same requirement _force_patch_merge enforces. + summary_df = pd.DataFrame(summaries) + found_dim = _get_merge_dim(summary_df) + if found_dim != merge_dim: + msg = ( + f"Cannot merge patches; expected them to vary along " + f"{merge_dim} but found {found_dim}." + ) + raise CoordMergeError(msg) + attr_kwargs, coord_kwargs = _split_coord_merge_kwargs(self.merge_kwargs) + conf = attr_kwargs.get("conflicts", None) + drop_conflicting = conf in {"drop", "keep_first"} + new_coord = _get_merged_coord( + summary_df, merge_dim, coords, drop_conflicting, **coord_kwargs + ) + new_attrs = combine_patch_attrs(attrs, **attr_kwargs) + return dc.Patch(data=buffer, coords=new_coord, attrs=new_attrs, dims=list(dims)) + + def _df_to_dict_list(self, df): + """ + Convert the dataframe to a list of dicts for iteration. + + This is significantly faster than iterating rows. Empty strings + (missing format fields on file rows) normalize to None; stored + relative paths pass through unchanged — the catalog's resolver + owns resolving them against the spool root. + """ + df = df.copy(deep=False).replace("", None) + return df.to_dict("records") diff --git a/docs/notes/spool_chunking.qmd b/docs/notes/spool_chunking.qmd index deb38edfb..32caf697e 100644 --- a/docs/notes/spool_chunking.qmd +++ b/docs/notes/spool_chunking.qmd @@ -6,7 +6,7 @@ title: Spool Chunking ## Plans -The planner consumes the spool's flat metadata relation and produces a `ChunkPlan`: an `outputs` table (one row per patch the chunked spool will contain) and a `members` table (which slice of which source patch feeds each output). `Spool.chunk_plan` exposes the plan `chunk` would execute, with the same arguments. +The planner consumes the spool's flat metadata relation and produces a `ChunkPlan`: an `outputs` table (one row per patch the chunked spool will contain) and a `members` table (which slice of which source patch feeds each output). `Spool.chunk_plan` exposes the plan `chunk` would execute, with the same arguments. The chunked spool carries this shape as its derived view, and assembly (`dascore.utils.patch_assembly`) executes it row by row; re-chunking a chunked spool re-plans from the current view's members, so plans never nest. ```{python} import dascore as dc diff --git a/docs/notes/spool_selection.qmd b/docs/notes/spool_selection.qmd index 8754961e6..5b9bd1a6b 100644 --- a/docs/notes/spool_selection.qmd +++ b/docs/notes/spool_selection.qmd @@ -12,7 +12,7 @@ Coordinate predicates select by range — a `(start, stop)` tuple or slice, with Coordinate predicates first select patches whose summary envelopes can overlap the request. The loaded patch is then selected exactly. `samples=True` is always patch-local and therefore never excludes a patch at the index stage. `relative=True` resolves coordinate ranges against the current spool view's global envelope; attribute predicates in the same call remain unchanged. -Operations that create a new row or instruction plan, including chunking, sorting, and slicing, switch that derived spool to dataframe planning. Exact selections already attached to the catalog still apply when source patches are resolved. +Operations that restructure rows — chunking, sorting, slicing — attach a derived view (an outputs/members/sources relation) to the new spool; selection on such a view filters its output rows directly. Exact selections already attached to the catalog still apply when source patches are resolved. Catalog views share their source state. Adding, removing, or rescanning sources invalidates realized metadata so existing views observe the updated catalog under their composed predicates. diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 43d0f9bd4..5a7098407 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -687,6 +687,19 @@ def test_chunk_non_adjacent_within_tolerance_warns(self, random_patch): assert len(out) == 1 +def _bare_assembler(): + """An assembler with no frames, for direct streaming-merge tests.""" + from dascore.utils.patch_assembly import PatchAssembler + + return PatchAssembler( + df=None, + source_df=None, + instruction_df=None, + load_patch=None, + merge_kwargs={}, + ) + + class TestStreamingMerge: """ Tests for the streaming merge path, which copies each patch into a @@ -695,28 +708,28 @@ class TestStreamingMerge: def test_streaming_path_used(self, adjacent_spool_no_overlap, monkeypatch): """Ensure simple merges take the streaming path.""" - from dascore.core.spool import Spool + from dascore.utils.patch_assembly import PatchAssembler called = [] - original = Spool._merge_patches_streaming + original = PatchAssembler._merge_patches_streaming def wrapper(self, *args, **kwargs): called.append(True) return original(self, *args, **kwargs) - monkeypatch.setattr(Spool, "_merge_patches_streaming", wrapper) + monkeypatch.setattr(PatchAssembler, "_merge_patches_streaming", wrapper) merged = adjacent_spool_no_overlap.chunk(time=None) assert isinstance(merged[0], dc.Patch) assert called def test_matches_materialized_merge(self, adjacent_spool_no_overlap, monkeypatch): """Streaming and concatenating merges must produce identical patches.""" - import dascore.core.spool as spool_module + import dascore.utils.patch_assembly as assembly_module streamed = adjacent_spool_no_overlap.chunk(time=None)[0] # Disabling the sample estimate forces the materialized path. monkeypatch.setattr( - spool_module, "_estimate_merge_samples", lambda df, dim: None + assembly_module, "_estimate_merge_samples", lambda df, dim: None ) materialized = adjacent_spool_no_overlap.chunk(time=None)[0] assert np.array_equal(streamed.data, materialized.data) @@ -737,24 +750,24 @@ def test_mixed_dtype_upcast(self, random_patch, small_first): def test_transposes_patch_to_first_patch_dims(self, random_patch, monkeypatch): """Streaming merge should tolerate patches with the same dims reordered.""" - spool = dc.spool([]) + assembler = _bare_assembler() p1 = random_patch.update_attrs(history=[]) time = p1.get_coord("time") p2 = p1.update_coords(time_min=time.max() + time.step).update_attrs(history=[]) p2 = p2.transpose(*reversed(p2.dims)) patches = iter([p1, p2]) monkeypatch.setattr( - spool, "_load_trimmed_patch", lambda patch_kwargs, joined: next(patches) + assembler, "_load_trimmed_patch", lambda patch_kwargs, joined: next(patches) ) time_axis = p1.get_axis("time") samples = p1.data.shape[time_axis] * 2 - out = spool._merge_patches_streaming(None, [{}, {}], "time", samples) + out = assembler._merge_patches_streaming(None, [{}, {}], "time", samples) assert out.dims == p1.dims assert out.data.shape[time_axis] == samples def test_incompatible_shapes_raise_merge_error(self, random_patch, monkeypatch): """Streaming merge should wrap non-merge-dimension shape mismatches.""" - spool = dc.spool([]) + assembler = _bare_assembler() p1 = random_patch.update_attrs(history=[]) time = p1.get_coord("time") p2 = p1.update_coords(time_min=time.max() + time.step).update_attrs(history=[]) @@ -762,16 +775,16 @@ def test_incompatible_shapes_raise_merge_error(self, random_patch, monkeypatch): p2 = p2.select(distance=(None, distance.max() - distance.step)) patches = iter([p1, p2]) monkeypatch.setattr( - spool, "_load_trimmed_patch", lambda patch_kwargs, joined: next(patches) + assembler, "_load_trimmed_patch", lambda patch_kwargs, joined: next(patches) ) msg = "their shapes are incompatible" with pytest.raises(CoordMergeError, match=msg): samples = p1.data.shape[p1.get_axis("time")] * 2 - spool._merge_patches_streaming(None, [{}, {}], "time", samples) + assembler._merge_patches_streaming(None, [{}, {}], "time", samples) def test_unexpected_merge_dimension_raises(self, random_patch, monkeypatch): """Streaming merge should validate the actual varying dimension.""" - spool = dc.spool([]) + assembler = _bare_assembler() p1 = random_patch.update_attrs(history=[]) dist = p1.get_coord("distance") p2 = p1.update_coords(distance_min=dist.max() + dist.step).update_attrs( @@ -779,9 +792,9 @@ def test_unexpected_merge_dimension_raises(self, random_patch, monkeypatch): ) patches = iter([p1, p2]) monkeypatch.setattr( - spool, "_load_trimmed_patch", lambda patch_kwargs, joined: next(patches) + assembler, "_load_trimmed_patch", lambda patch_kwargs, joined: next(patches) ) msg = "expected them to vary along time" with pytest.raises(CoordMergeError, match=msg): samples = p1.data.shape[p1.get_axis("time")] * 2 - spool._merge_patches_streaming(None, [{}, {}], "time", samples) + assembler._merge_patches_streaming(None, [{}, {}], "time", samples) diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 4e42da597..58360000a 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -11,12 +11,7 @@ import pytest import dascore as dc -from dascore.core.spool import ( - BaseSpool, - Spool, - _estimate_merge_samples, - _get_varying_dim, -) +from dascore.core.spool import BaseSpool, Spool from dascore.exceptions import ( InvalidSpoolError, MissingOptionalDependencyError, @@ -24,6 +19,7 @@ ParameterError, ) from dascore.utils.downloader import fetch +from dascore.utils.patch_assembly import _estimate_merge_samples, _get_varying_dim from dascore.utils.time import to_datetime64, to_timedelta64 @@ -229,21 +225,13 @@ def test_eq_self(self, random_spool): """A spool should always eq itself.""" assert random_spool == random_spool - def test_unequal_attr(self, random_spool): - """Simulate some attribute which isn't equal.""" + def test_foreign_attrs_do_not_join_equality(self, random_spool): + """Equality state is enumerated; stray instance attrs are ignored.""" new1 = copy.deepcopy(random_spool) new1.__dict__["bad_attr"] = 1 new2 = copy.deepcopy(random_spool) new2.__dict__["bad_attr"] = 2 - assert new1 != new2 - - def test_unequal_dicts(self, random_spool): - """Simulate some dicts which don't have the same values.""" - new1 = copy.deepcopy(random_spool) - new1.__dict__["bad_attr"] = {1: 2} - new2 = copy.deepcopy(random_spool) - new2.__dict__["bad_attr"] = {2: 3} - assert new1 != new2 + assert new1 == new2 class TestIndexing: @@ -805,140 +793,52 @@ def test_dft_patch_access(self, random_dft_patch): assert isinstance(patch, dc.Patch) -class TestSpoolEquality: - """Tests for spool equality comparisons to ensure 100% coverage.""" - - def test_spool_equality_non_dict_comparison(self, random_spool): - """Test line 107: non-dict comparison in _vals_equal.""" - spool1 = copy.deepcopy(random_spool) - spool2 = copy.deepcopy(random_spool) +class TestDeepEqualityCheck: + """Coverage for deep_equality_check branches (formerly via spool attrs).""" - # Add non-dict values to test the non-dict comparison path - spool1._test_string = "hello" - spool2._test_string = "hello" + def test_non_dict_comparison(self): + """Plain value comparison inside dicts.""" + from dascore.utils.misc import deep_equality_check - # This should be equal - assert spool1 == spool2 - - # Now make them different to test the comparison - spool2._test_string = "world" - - # This should be False - assert spool1 != spool2 + assert deep_equality_check({"a": "hello"}, {"a": "hello"}) + assert not deep_equality_check({"a": "hello"}, {"a": "world"}) - def test_spool_equality_with_objects_having_dict(self, random_spool): - """Test line 127: objects with __dict__ that are not equal.""" + def test_objects_with_dict(self): + """Objects compare via recursive __dict__ comparison.""" + from dascore.utils.misc import deep_equality_check class TestObject: def __init__(self, value): self.value = value - spool1 = copy.deepcopy(random_spool) - spool2 = copy.deepcopy(random_spool) - - # Add objects with __dict__ that have different values - spool1._test_obj = TestObject(1) - spool2._test_obj = TestObject(2) # Different data - - # This should hit line 127 and return False - assert spool1 != spool2 - - def test_spool_equality_with_objects_having_dict_equal(self, random_spool): - """Test objects with __dict__ that are equal via recursive comparison.""" - - class TestObject: - def __init__(self, value): - self.value = value - - spool1 = random_spool - spool2 = copy.deepcopy(random_spool) - - # Add objects with __dict__ that have same internal state - spool1._test_obj = TestObject(42) - spool2._test_obj = TestObject(42) - - # This should be equal via recursive __dict__ comparison - assert spool1 == spool2 - - def test_spool_equality_mixed_types(self): - """Test equality with various mixed data types.""" - # Create simple spools to avoid cache issues - patch = dc.get_example_patch() - spool1 = dc.spool([patch]) - spool2 = dc.spool([patch]) - - # Test with integers (non-dict) - spool1._int_val = 42 - spool2._int_val = 42 - assert spool1 == spool2 - - # Test with lists (non-dict) - spool1._list_val = [1, 2, 3] - spool2._list_val = [1, 2, 3] - assert spool1 == spool2 + assert deep_equality_check({"o": TestObject(42)}, {"o": TestObject(42)}) + assert not deep_equality_check({"o": TestObject(1)}, {"o": TestObject(2)}) - # Test with numpy arrays (non-dict) - spool1._array_val = np.array([1, 2, 3]) - spool2._array_val = np.array([1, 2, 3]) - assert spool1 == spool2 + def test_mixed_types(self): + """Ints, lists, and numpy arrays compare by value.""" + from dascore.utils.misc import deep_equality_check - # Test arrays with different values - spool1._array_val = np.array([1, 2, 3]) - spool2._array_val = np.array([1, 2, 4]) - assert spool1 != spool2 + d1 = {"i": 42, "l": [1, 2, 3], "a": np.array([1, 2, 3])} + d2 = {"i": 42, "l": [1, 2, 3], "a": np.array([1, 2, 3])} + assert deep_equality_check(d1, d2) + d2["a"] = np.array([1, 2, 4]) + assert not deep_equality_check(d1, d2) - def test_spool_equality_with_dataframes(self): - """Test equality with pandas DataFrames (has equals method).""" - # Create simple spools to avoid cache issues - patch = dc.get_example_patch() - spool1 = dc.spool([patch]) - spool2 = dc.spool([patch]) + def test_dataframes(self): + """DataFrames compare via .equals.""" + from dascore.utils.misc import deep_equality_check - # Add DataFrames that should be equal df1 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) df2 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + assert deep_equality_check({"df": df1}, {"df": df2}) + df3 = pd.DataFrame({"a": [1, 2, 4], "b": [4, 5, 6]}) + assert not deep_equality_check({"df": df1}, {"df": df3}) - spool1._test_df = df1 - spool2._test_df = df2 - - # Should be equal via df.equals() - assert spool1 == spool2 - - # Now test with different DataFrames - df3 = pd.DataFrame({"a": [1, 2, 4], "b": [4, 5, 6]}) # Different data - spool2._test_df = df3 - - # Should not be equal - assert spool1 != spool2 - - def test_specific_coverage_lines(self): - """Test to specifically cover lines 107 and 127.""" - # Create minimal spools - patch = dc.get_example_patch() - spool1 = dc.spool([patch]) - spool2 = dc.spool([patch]) - - # Line 107: Non-dict comparison - spool1._string_test = "hello" - spool2._string_test = "hello" - assert spool1 == spool2 - - # Make them different to test line 107 return False - spool2._string_test = "world" - assert spool1 != spool2 - - # Line 127: Objects with __dict__ that are different - class SimpleObj: - def __init__(self, val): - self.val = val - - spool1 = dc.spool([patch]) - spool2 = dc.spool([patch]) - spool1._obj = SimpleObj(1) - spool2._obj = SimpleObj(2) + def test_unequal_sub_dicts(self): + """Nested dicts with different values are unequal.""" + from dascore.utils.misc import deep_equality_check - # This should return False at line 127 - assert spool1 != spool2 + assert not deep_equality_check({"d": {1: 2}}, {"d": {2: 3}}) class TestSpoolCoverageEdges: @@ -1017,17 +917,17 @@ def test_iteration_skips_unresolvable_patch(self, monkeypatch): def _raise(_ind): raise MissingPatchError("trimmed to nothing") - monkeypatch.setattr(spool, "_get_patches_from_index", _raise) + monkeypatch.setattr(spool._assembler, "get_patch", _raise) with pytest.warns(UserWarning, match="Skipping patch"): assert list(spool) == [] def test_merge_buffer_grows_when_estimate_short(self, many_contiguous, monkeypatch): """An under-estimated merge buffer is grown to fit (uneven sampling).""" - import dascore.core.spool as spool_mod + import dascore.utils.patch_assembly as assembly_mod # Force the pre-merge sample estimate to be too small so the # streaming buffer must grow mid-merge. - monkeypatch.setattr(spool_mod, "_estimate_merge_samples", lambda *a, **k: 1) + monkeypatch.setattr(assembly_mod, "_estimate_merge_samples", lambda *a, **k: 1) merged = dc.spool(many_contiguous).chunk(time=None) assert merged[0].get_coord("time").size == sum( p.get_coord("time").size for p in many_contiguous diff --git a/tests/test_core/test_spool_contracts.py b/tests/test_core/test_spool_contracts.py index 3bfbc7f97..1297cf784 100644 --- a/tests/test_core/test_spool_contracts.py +++ b/tests/test_core/test_spool_contracts.py @@ -9,7 +9,6 @@ from __future__ import annotations -import numpy as np import pytest import dascore as dc From 30b5b8eea1bbaeb299abae5edcf6517172647d2b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 18:32:00 +0200 Subject: [PATCH 82/97] Restore full coverage after the unification refactors Six is-None guards from before every spool carried a catalog were dead code; they are removed rather than tested. The reachable branches the refactors left untested get real tests: the BaseSpool update/wrap surface for third-party spools, constructor validation and copy-time merge policy, the catalog fast path's missing-patch skip, assembler negative/out-of-bounds indexing on planned views, canonical-range value semantics, and the automatic rebuild of old-version index files. --- dascore/core/spool.py | 14 +---- tests/test_core/test_spool.py | 72 +++++++++++++++++++++++ tests/test_io/test_index/test_catalog.py | 14 +++++ tests/test_io/test_index/test_ordering.py | 29 +++++++++ 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index a6a10b1dc..e4c90b6a1 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -532,8 +532,6 @@ def _instruction_df(self) -> pd.DataFrame | None: def _get_df(self): """Realize the flat relation from the catalog.""" - if self._catalog is None: - return None current = self._catalog.to_df().reset_index(drop=True) df, source, instruction = self._get_dummy_dataframes(current) self._cache["_source_df"] = source @@ -646,9 +644,7 @@ def __len__(self): # it is cached or on the dataframe path. if self._is_catalog_backed() and "_df" not in self._cache: return len(self._catalog) - df = self._df - # An empty spool with no patches, data, or catalog has no frame. - return 0 if df is None else len(df) + return len(self._df) def __iter__(self): if self._rows_are_catalog(): @@ -659,8 +655,6 @@ def __iter__(self): msg = f"Skipping patch at index {ind} (see #583): {e}" warnings.warn(msg, UserWarning, stacklevel=2) return - if self._df is None: # an empty spool has nothing to yield - return for ind in range(len(self._df)): try: yield self._assembler.get_patch(ind) @@ -736,8 +730,6 @@ def _as_catalog_member(self): Restructured rows (e.g. chunked views) no longer map to sources and contribute their materialized patches instead. """ - if self._catalog is None: - return super()._as_catalog_member() if self._catalog_native: return self._catalog, None df = self._df @@ -1124,8 +1116,6 @@ def _has_file_rows(self) -> bool: from dascore.io.index.catalog import LiveResolver from dascore.utils.paths import is_memory_uri - if self._catalog is None: - return False if isinstance(self._catalog.resolver, LiveResolver): return False paths = self._catalog.backend.get_sources()["source_path"] @@ -1207,8 +1197,6 @@ def _strip_identity(df): "file_format", "file_version", ) - if df is None: - return df out = df.drop(columns=list(drop), errors="ignore") return out[sorted(out.columns)] diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 58360000a..619b85ee0 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -78,6 +78,52 @@ def test_base_concat_raises(self, random_spool): with pytest.raises(NotImplementedError, match=msg): BaseSpool.concatenate(random_spool, time=2) + def test_base_update_returns_self(self, random_spool): + """The BaseSpool update default (for third-party spools) no-ops.""" + assert BaseSpool.update(random_spool) is random_spool + + def test_invalid_input_raises(self): + """A non-patch, non-spool input raises a clear error.""" + with pytest.raises(InvalidSpoolError, match="accepts a Patch"): + Spool(42) + + def test_wraps_third_party_spool(self, random_spool): + """A non-Spool BaseSpool input realizes into the registry.""" + + class MiniSpool(BaseSpool): + """Minimal third-party spool over a patch list.""" + + def __init__(self, patches): + self._patches = list(patches) + + def __getitem__(self, item): + return self._patches[item] + + def __iter__(self): + return iter(self._patches) + + def __len__(self): + return len(self._patches) + + def chunk(self, **kwargs): + raise NotImplementedError + + def select(self, **kwargs): + raise NotImplementedError + + def get_contents(self): + raise NotImplementedError + + patches = list(random_spool) + wrapped = Spool(MiniSpool(patches)) + assert isinstance(wrapped, Spool) + assert list(wrapped) == patches + + def test_copy_construct_merge_kwargs(self, random_spool): + """Copy-construction can override the merge policy.""" + new = Spool(random_spool, merge_kwargs={"conflicts": "drop"}) + assert new._merge_kwargs["conflicts"] == "drop" + def test_viz_raises(self, random_spool): """Ensure Spool.viz raises AttributeError.""" msg = "Apply 'viz' on a Patch object" @@ -177,6 +223,11 @@ def test_instruction_df_builds_from_lazy_patches(self, patch_list): spool = dc.spool(patch_list) assert len(spool._get_instruction_df()) == len(patch_list) + def test_instruction_df_property_cold_access(self, patch_list): + """Accessing the instruction frame first still derives everything.""" + spool = dc.spool(patch_list) + assert len(spool._instruction_df) == len(patch_list) + class TestSpoolHelpers: """Tests for helper functions used by spool implementations.""" @@ -921,6 +972,27 @@ def _raise(_ind): with pytest.warns(UserWarning, match="Skipping patch"): assert list(spool) == [] + def test_catalog_iteration_skips_unresolvable_patch(self, monkeypatch): + """The catalog fast path also skips with a #583 warning.""" + spool = dc.spool([dc.get_example_patch()]) + assert spool._rows_are_catalog() + + def _raise(_ind): + raise MissingPatchError("not available in this session") + + monkeypatch.setattr(spool._catalog, "get_patch", _raise) + with pytest.warns(UserWarning, match="Skipping patch"): + assert list(spool) == [] + + def test_planned_view_negative_and_bad_index(self): + """Assembler indexing handles negatives and raises out-of-bounds.""" + patches = list(dc.get_example_spool(length=2)) + planned = dc.spool(patches).sort("time") + assert planned._plan is not None + assert planned[-1] == planned[len(patches) - 1] + with pytest.raises(IndexError, match="out of bounds"): + _ = planned[len(patches)] + def test_merge_buffer_grows_when_estimate_short(self, many_contiguous, monkeypatch): """An under-estimated merge buffer is grown to fit (uneven sampling).""" import dascore.utils.patch_assembly as assembly_mod diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py index 6f77f8cf8..f39cc0e82 100644 --- a/tests/test_io/test_index/test_catalog.py +++ b/tests/test_io/test_index/test_catalog.py @@ -264,3 +264,17 @@ def test_numeric_relative_offset(self, live_catalog): view = live_catalog.select(distance=(5, -5), relative=True) patch = view.get_patch(0) assert patch.get_coord("distance").min() >= 5 + + +class TestCanonicalRange: + """Value semantics of the deferred canonical-SI range.""" + + def test_eq_and_hash(self): + """Equal magnitudes compare and hash equal; other types don't.""" + from dascore.io.index.catalog import _CanonicalRange + + r1, r2 = _CanonicalRange((1.0, 2.0)), _CanonicalRange((1.0, 2.0)) + assert r1 == r2 + assert hash(r1) == hash(r2) + assert r1 != _CanonicalRange((1.0, 3.0)) + assert r1 != (1.0, 2.0) # non-CanonicalRange comparand diff --git a/tests/test_io/test_index/test_ordering.py b/tests/test_io/test_index/test_ordering.py index 58a29cf5a..0b761f885 100644 --- a/tests/test_io/test_index/test_ordering.py +++ b/tests/test_io/test_index/test_ordering.py @@ -109,3 +109,32 @@ def test_update_interleaves_new_files_by_time(self, tmp_path): assert len(df) == 3 assert df["time_min"].is_monotonic_increasing assert df["time_min"].iloc[0] == early.get_coord("time").min() + + +class TestIndexVersionRebuild: + """Old-version index files rebuild automatically (disposable cache).""" + + def test_version_mismatch_rebuilds(self, tmp_path): + """An index of another schema version is replaced, not fatal.""" + import sqlite3 + + patch = dc.get_example_patch() + dc.write(patch, tmp_path / "a.h5", "dasdae") + spool = dc.spool(tmp_path).update(progress=None) + index_path = spool.indexer.index_path + spool.indexer.close() + # simulate an index written by another (older/newer) schema version + with sqlite3.connect(index_path) as con: + con.execute("UPDATE meta_data SET index_version = 1") + reopened = dc.spool(tmp_path).update(progress=None) + assert len(reopened) == 1 + reopened.indexer.close() + + def test_indexer_deepcopy_shares_instance(self, tmp_path): + """Derived spools share the indexer (and its live DB connection).""" + import copy + + dc.write(dc.get_example_patch(), tmp_path / "a.h5", "dasdae") + spool = dc.spool(tmp_path).update(progress=None) + assert copy.deepcopy(spool.indexer) is spool.indexer + spool.indexer.close() From 841ef781636b4ae3d2cbd57bb47fc0f37b9d4658 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 18:39:49 +0200 Subject: [PATCH 83/97] Close the tampering connection so Windows can rebuild the test index sqlite3's context manager only manages transactions; the open handle made the auto-rebuild unlink fail with WinError 32 on Windows runners. --- tests/test_io/test_index/test_ordering.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_io/test_index/test_ordering.py b/tests/test_io/test_index/test_ordering.py index 0b761f885..e80980210 100644 --- a/tests/test_io/test_index/test_ordering.py +++ b/tests/test_io/test_index/test_ordering.py @@ -123,9 +123,15 @@ def test_version_mismatch_rebuilds(self, tmp_path): spool = dc.spool(tmp_path).update(progress=None) index_path = spool.indexer.index_path spool.indexer.close() - # simulate an index written by another (older/newer) schema version - with sqlite3.connect(index_path) as con: + # simulate an index written by another (older/newer) schema version; + # close the connection explicitly (the sqlite3 context manager only + # manages transactions) or Windows cannot unlink the file below. + con = sqlite3.connect(index_path) + try: con.execute("UPDATE meta_data SET index_version = 1") + con.commit() + finally: + con.close() reopened = dc.spool(tmp_path).update(progress=None) assert len(reopened) == 1 reopened.indexer.close() From fb6a19b80c828b4a7076851335472fc438e1e9e0 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 18:50:11 +0200 Subject: [PATCH 84/97] Let pytest own the diverse spool directory's lifetime The explicit rmtree teardown raced lazily-finalized SQLite index connections on Windows (WinError 32); tmp_path_factory directories need no teardown, matching the sibling directory fixtures. --- tests/conftest.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 64e885205..7d97dc9d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -423,12 +423,15 @@ def two_patch_directory(tmp_path_factory, terra15_das_example_path, random_patch @pytest.fixture(scope="class") -def diverse_spool_directory(diverse_spool): - """Save the diverse spool contents to a directory.""" - out = ex.spool_to_directory(diverse_spool) - yield out - if out.is_dir(): - shutil.rmtree(out) +def diverse_spool_directory(diverse_spool, tmp_path_factory): + """Save the diverse spool contents to a directory. + + Pytest owns the directory's lifetime: an explicit rmtree teardown + raced lazily-finalized SQLite index connections on Windows + (WinError 32), so no teardown here. + """ + out = tmp_path_factory.mktemp("diverse_spool_dir") + return ex.spool_to_directory(diverse_spool, path=out) @pytest.fixture(scope="class") From 6ab3c06e2bb8f016a2f8aafb060b3fc8ffdc076f Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 19:08:41 +0200 Subject: [PATCH 85/97] Cover the last unification fallout outside the audited modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CacheDescriptor lost its only consumer when the managed dataframes became plain properties — deleted. The flat-dump NaT sentinel for datetime coords without a clean step lost its only exercise with the legacy dump path — pinned with a direct test. --- dascore/utils/misc.py | 28 ---------------------------- tests/test_core/test_patch.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/dascore/utils/misc.py b/dascore/utils/misc.py index 876f1a740..78f0d1444 100644 --- a/dascore/utils/misc.py +++ b/dascore/utils/misc.py @@ -432,34 +432,6 @@ def iterate(obj): return obj if isinstance(obj, Iterable) else (obj,) -class CacheDescriptor: - """A descriptor for storing infor in an instance cache (mapping).""" - - def __init__(self, cache_name, func_name, args=None, kwargs=None): - self._cache_name = cache_name - self._func_name = func_name - self._args = () if args is None else args - self._kwargs = {} if kwargs is None else kwargs - - def __set_name__(self, owner, name): - """Method to set the name of the description on the instance.""" - self._name = name - - def __get__(self, instance, owner): - """Get contents of the cache.""" - cache = getattr(instance, self._cache_name) - if self._name not in cache: - func = getattr(instance, self._func_name) - out = func(*self._args, **self._kwargs) - cache[self._name] = out - return cache[self._name] - - def __set__(self, instance, value): - """Set the cache contents.""" - cache = getattr(instance, self._cache_name) - cache[self._name] = value - - def optional_import( package_name: str, on_missing: Literal["raise", "warn", "ignore"] = "raise" ) -> ModuleType | None: diff --git a/tests/test_core/test_patch.py b/tests/test_core/test_patch.py index 00a2ec286..4e4e5a012 100644 --- a/tests/test_core/test_patch.py +++ b/tests/test_core/test_patch.py @@ -486,6 +486,16 @@ def test_flat_dump_dim_tuple_and_exclude(self): assert "distance_min" not in out assert np.isnan(out["time_step"]) + def test_flat_dump_null_datetime_step(self, random_patch): + """A datetime coord without a clean step flat-dumps a NaT sentinel.""" + time = random_patch.coords.get_array("time").copy() + time[-1] += np.timedelta64(1, "s") # break uniformity + patch = random_patch.update_coords(time=time) + out = patch.summary.flat_dump() + step = out["time_step"] + assert isinstance(step, np.timedelta64) + assert pd.isnull(step) + def test_select_from_spool_by_integer_source_patch_id(self, random_patch): """Integer-like source ids should fall back to positional selection.""" spool = dc.spool( From 9b9867ad815528bcb46668d5243a1895391dda66 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 22:38:16 +0200 Subject: [PATCH 86/97] Merge union transfer records per source identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_sources replaces at (base_uri, source_path) grain, so union members exporting different patches of the same multi-patch file overwrote each other — sp[:1] + sp[1:] over a two-patch file dropped a patch. Union now collects every member's records, merges partial records per source (patch union by source_patch_id, first-occurrence position, last-occurrence metadata), and writes once. Resolver absorption is filtered to the live entries a member's rows actually transfer, so unions no longer retain unrelated registry patches. --- dascore/io/index/catalog.py | 55 ++++++++++++++++++++++---- tests/test_io/test_index/test_union.py | 38 ++++++++++++++++++ 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 0a2634066..b75cf1a04 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -18,7 +18,7 @@ import abc from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import numpy as np @@ -245,9 +245,17 @@ def live_entries(self) -> Mapping[str, dc.Patch]: """Return the merged live patch registry.""" return self.live._registry - def absorb(self, resolver: PatchResolver) -> None: - """Take over the live registry entries of another resolver.""" - self.live._registry.update(resolver.live_entries()) + def absorb(self, resolver: PatchResolver, paths=None) -> None: + """ + Take over another resolver's live registry entries. + + ``paths`` restricts absorption to the given synthetic paths + (the entries a transfer actually references); None takes all. + """ + entries = resolver.live_entries() + if paths is not None: + entries = {k: v for k, v in entries.items() if k in paths} + self.live._registry.update(entries) def resolve(self, row: Mapping, **trim) -> dc.Patch: """Dispatch to the live registry or the file reader.""" @@ -276,8 +284,6 @@ def _live_records(registry: Mapping[str, dc.Patch]): def _absolutize_record(record, root): """Return a source record whose relative path is resolved against root.""" - from dataclasses import replace - path = record.source_path if "://" in path or Path(path).is_absolute(): return record @@ -285,6 +291,25 @@ def _absolutize_record(record, root): return replace(record, source_path=resolved, base_uri=None) +def _merge_source_records(existing, new): + """ + Merge two partial records for the same source. + + Union members export only their selected patches, so two members can + hold disjoint (or overlapping) slices of one multi-patch file. The + merged record unions the patch lists by source_patch_id: a patch + keeps its first-occurrence position, a duplicate identity takes the + last occurrence's metadata (dict-merge semantics, matching the + ordering contract), and the source-level metadata (mtime, size) + comes from the last record. + """ + if existing is None: + return new + patches = {p.source_patch_id: p for p in existing.patches} + patches.update({p.source_patch_id: p for p in new.patches}) + return replace(new, patches=tuple(patches.values())) + + @dataclass class _CatalogRevision: """Shared mutation revision for live catalog views.""" @@ -354,6 +379,14 @@ def union(cls, catalogs: Sequence[PatchCatalog]) -> PatchCatalog: resolver = CompositeResolver() out = cls(resolver=resolver) backend = out.backend + # Collect and merge every member's records before writing: + # write_sources replaces at (base_uri, source_path) grain, so + # partial records for the same source — two members selecting + # different patches of one multi-patch file — must merge into a + # complete record or the later write would delete the earlier + # member's patches. Dict insertion order keeps first-occurrence + # position; the merge keeps last-occurrence metadata. + merged: dict[tuple, SourceRecord] = {} for member in catalogs: catalog, patch_ids = member if isinstance(member, tuple) else (member, None) if patch_ids is None and catalog.is_view: @@ -362,8 +395,14 @@ def union(cls, catalogs: Sequence[PatchCatalog]) -> PatchCatalog: root = getattr(catalog.resolver, "_root", None) if root is not None: records = [_absolutize_record(x, root) for x in records] - backend.write_sources(records) - resolver.absorb(catalog.resolver) + for record in records: + identity = (record.base_uri or "", record.source_path) + merged[identity] = _merge_source_records(merged.get(identity), record) + # only the live entries this member actually transfers ride + # along; the rest of the registry stays with its own catalog + member_paths = {record.source_path for record in records} + resolver.absorb(catalog.resolver, paths=member_paths) + backend.write_sources(list(merged.values())) out._invalidate() return out diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 0950b2380..d8346da9b 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -307,3 +307,41 @@ def test_dir_union_absolutizes_relative_paths(self, tmp_path): assert len(file_row) == 1 loaded = [p for p in combined] assert len(loaded) == 2 + + +class TestSameFileUnion: + """Unions of members selecting patches from the same multi-patch file.""" + + @pytest.fixture() + def two_patch_file_spool(self, contiguous_patches, tmp_path): + """A file spool over one file holding two patches.""" + p1, p2 = (x.update_attrs(history=[]) for x in contiguous_patches) + path = tmp_path / "two_patch.h5" + dc.write(dc.spool([p1, p2]), path, "dasdae") + return dc.spool(path) + + def test_disjoint_selections_union(self, two_patch_file_spool): + """Two members holding different patches of one file both survive.""" + sp = two_patch_file_spool + combined = sp[:1] + sp[1:] + assert len(combined) == 2 + for patch in combined: + assert isinstance(patch, dc.Patch) + + def test_overlapping_selections_dedup(self, two_patch_file_spool): + """A patch present in both members appears once (dict-merge).""" + sp = two_patch_file_spool + combined = sp[:2] + sp[1:] + assert len(combined) == 2 + + def test_union_absorbs_only_member_registry_entries(self, contiguous_patches): + """Live entries outside a member's rows don't ride into the union.""" + p1, p2 = contiguous_patches + t1 = p1.get_coord("time") + narrowed = dc.spool([p1, p2]).select(time=(None, t1.max())) + assert len(narrowed) == 1 + other = dc.get_example_patch(time_min="2030-01-01") + combined = narrowed + dc.spool([other]) + registry = combined._catalog.resolver.live_entries() + assert len(registry) == 2 # p1 and other; p2 stayed home + assert len(combined) == 2 From f8858e603e94a9d2b0e9148d3c3313835f220586 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 22:39:33 +0200 Subject: [PATCH 87/97] Serialize views by membership and document the spool migration A catalog view shared its root's whole live registry through pickling, so a one-patch view or split part of an N-patch spool shipped all N data arrays (defeating Spool.map's split-before-serialize strategy). Views now pickle a resolver restricted to the entries their rows reference. The changelog gains the breaking-change migration notes for the class collapse, the root-only update rule, the ordering contract, and the coordinate-mask removal. --- dascore/io/index/catalog.py | 19 ++++++++++++++++ docs/changelog.qmd | 4 ++++ tests/test_io/test_index/test_catalog.py | 28 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index b75cf1a04..42a524776 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -291,6 +291,17 @@ def _absolutize_record(record, root): return replace(record, source_path=resolved, base_uri=None) +def _membership_resolver(resolver: PatchResolver, keep: dict) -> PatchResolver: + """Return a copy of resolver whose live registry holds only `keep`.""" + if isinstance(resolver, LiveResolver): + out = LiveResolver() + out._registry = dict(keep) + return out + out = CompositeResolver() + out.live._registry = dict(keep) + return out + + def _merge_source_records(existing, new): """ Merge two partial records for the same source. @@ -494,6 +505,14 @@ def __getstate__(self) -> dict: ) if needs_records: state["_rebuild_records"] = tuple(self._backend.export_records()) + # A view shares the root's resolver, but must not drag the whole + # live registry across the wire: keep only the entries its rows + # reference (a one-patch view of an N-patch spool serializes one + # patch, not N — the payload Spool.map ships per task). + if self.is_view and self.resolver.live_entries(): + paths = set(self.to_df()["path"].astype(str)) + keep = {k: v for k, v in self.resolver.live_entries().items() if k in paths} + state["resolver"] = _membership_resolver(self.resolver, keep) return state def _view(self, queries, residuals) -> PatchCatalog: diff --git a/docs/changelog.qmd b/docs/changelog.qmd index e5e9abf26..f19b70810 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -4,6 +4,10 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f ## Unreleased API Changes +- **The spool class hierarchy is collapsed into a single concrete class.** `MemorySpool`, `DirectorySpool`, and `FileSpool` are removed (no aliases), along with the `dascore.clients` package; every spool is now `dc.Spool` under the `dc.BaseSpool` ABC. Construct via `dc.spool(...)` (or `Spool.from_directory`/`Spool.from_file`); test for in-memory content with `spool.has_live_patches` instead of `isinstance` checks. The `select_kwargs` constructor parameter for directory spools is removed — use `.select(...)` after `update()`. +- **`Spool.update()` is allowed only on a root spool.** Any derived spool — the result of `select`, slicing, `sort`, `chunk`, or `+` — raises instead of silently refreshing or widening; update the root and re-apply operations (`root = root.update(); view = root.select(...)`). +- **Spools present patches in a defined order.** Patch-list spools keep construction order on every access path; directory spools present in time order (maintained across index updates); combining spools concatenates, with duplicate patches keeping their first position. Spool equality is order-sensitive, metadata-level, and compares contents rather than backing (a live spool can equal a directory spool over identical data). +- **Coordinate boolean masks are no longer accepted by `Spool.select`** (they were only well-defined on spools whose patches share sizes, and never reduced file reads). Use `spool.map(lambda p: p.select(dim=mask))` for per-patch masking; boolean arrays over *patches* (`spool[bool_array]`) still select membership, and `samples=True` index ranges still apply per patch with Python-slice clamping. - Memory and directory spools now share a catalog-backed metadata selection path. Attribute and coordinate candidates are pushed into SQLite lazily, while exact coordinate trimming remains a patch-load operation. - Directory indexes now use the constrained seven-table SQLite schema in `.dascore_index.sqlite3`. Experimental DuckDB and Parquet index backends and the `engine`/`index_engine` selection parameters were removed. Prototype indexes from the earlier schema must be deleted and rebuilt. - Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces, and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py index f39cc0e82..1ce795fec 100644 --- a/tests/test_io/test_index/test_catalog.py +++ b/tests/test_io/test_index/test_catalog.py @@ -278,3 +278,31 @@ def test_eq_and_hash(self): assert hash(r1) == hash(r2) assert r1 != _CanonicalRange((1.0, 3.0)) assert r1 != (1.0, 2.0) # non-CanonicalRange comparand + + +class TestViewSerialization: + """Views serialize only the live entries their rows reference.""" + + def test_view_pickles_membership_only(self): + """A one-patch view of an N-patch live spool ships one patch.""" + import pickle + + base = dc.get_example_patch() + patches = [ + base.update_attrs(tag=str(i)).new( + data=np.random.default_rng(i).random(base.shape) + ) + for i in range(5) + ] + spool = dc.spool(patches) + view = spool.select(tag="0") + assert len(view) == 1 + payload = pickle.dumps(view) + baseline = pickle.dumps(dc.spool([patches[0]])) + assert len(payload) < 2 * len(baseline) + # the round trip serves the right patch from a one-entry registry + loaded = pickle.loads(payload) + assert len(loaded._catalog.resolver.live_entries()) == 1 + assert loaded[0].attrs["tag"] == "0" + # and the root spool's registry is untouched + assert len(spool._catalog.resolver.live_entries()) == 5 From 1fbf95704ce82d78fec6984ddea7d61d3f9d6811 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 17 Jul 2026 22:55:09 +0200 Subject: [PATCH 88/97] Make selection dimension-bearing and sort/slice lazy catalog specs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quantity coordinate queries now reach the SQL builder with their units intact, so candidacy is constrained to dimensionally compatible coordinate definitions (a metre query excludes a seconds coordinate instead of trimming it as 1-2 s, and raises UnitError when nothing stored is compatible); the exact residual carries the query's own base unit for the same reason. Coordinate boolean sample masks are removed from Spool.select per the adjudicated design — they are positional selectors only defined under homogeneity spools never guarantee, and never reduced reads; spool.map with a patch select replaces them. sort, slicing, and array selection on catalog-backed spools become lazy Selection specs instead of materialized frames: sort is an ORDER BY override with the ordinal tiebreak, slices realize an ordered id-membership (ids only, never the flat relation), and subsequent selections compose within the window. split() parts are id windows, so map() ships one member's data per task; in-memory catalogs rebuild membership-restricted content on pickling since their patch ids do not survive re-ingest. Planned-view slicing indexes source rows by unique labels, closing the exponential row-duplication corruption, and assembler indexing rejects out-of-range negative indices instead of wrapping around. --- dascore/core/spool.py | 25 ++- dascore/io/index/backend.py | 73 +++++-- dascore/io/index/catalog.py | 201 ++++++++++++++---- dascore/io/index/query.py | 68 ++++-- dascore/utils/patch_assembly.py | 16 +- tests/test_core/test_spool.py | 17 +- tests/test_core/test_spool_select_spec.py | 124 +++++++++-- .../test_io/test_index/test_index_contract.py | 4 +- .../test_index/test_index_edge_cases.py | 8 +- tests/test_io/test_index/test_union.py | 14 +- 10 files changed, 443 insertions(+), 107 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index e4c90b6a1..19cf7f99f 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -586,13 +586,18 @@ def __init__( def _select_from_array(self, array) -> Self: """Create new spool with contents changed from array input.""" + if not ( + np.issubdtype(array.dtype, np.bool_) + or np.issubdtype(array.dtype, np.integer) + ): + msg = "Only bool or int dtypes are supported for spool array selection." + raise ValueError(msg) + if self._rows_are_catalog(): + return self._new_from_catalog(self._catalog.restrict(array)) if np.issubdtype(array.dtype, np.bool_): # boolean select df = self._df[array] - elif np.issubdtype(array.dtype, np.integer): - df = self._df.iloc[array] else: - msg = "Only bool or int dtypes are supported for spool array selection." - raise ValueError(msg) + df = self._df.iloc[array] source = self._source_df inst = self._instruction_df new = self.new_from_df( @@ -615,10 +620,16 @@ def _rows_are_catalog(self) -> bool: def __getitem__(self, item) -> PatchType | BaseSpool: if isinstance(item, slice): # a slice was used, return a sub-spool + if self._rows_are_catalog(): + # a lazy id-membership window (D2); never realizes the + # flat relation, and keeps split()/map() parts cheap + return self._new_from_catalog(self._catalog.window(item)) new_df = self._df.iloc[item] inst, source = self._instruction_df, self._source_df new_inst = inst[inst["current_index"].isin(new_df.index)] - new_source = source.loc[new_inst.index] + # unique labels only: member rows repeat source labels, and + # label indexing would multiply rows on every slice + new_source = source.loc[new_inst.index.unique()] out = self.new_from_df( df=new_df, instruction_df=new_inst, @@ -980,6 +991,10 @@ def _new_from_catalog(self, catalog) -> Self: @compose_docstring(doc=BaseSpool.sort.__doc__) def sort(self, attribute) -> Self: """{doc}.""" + if self._rows_are_catalog(): + # a lazy ORDER BY spec (D2): no copy, no realization; the + # ordinal contract supplies the deterministic tiebreak + return self._new_from_catalog(self._catalog.order_by(attribute)) df = self._df inst_df = self._instruction_df diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 47680e2ad..daaa011f2 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -103,11 +103,15 @@ def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: """Remove sources (identified by base_uri + path) and dependents.""" @abc.abstractmethod - def query(self, query: Query) -> pd.DataFrame: + def query(self, query: Query, order_by=None, patch_ids=None) -> pd.DataFrame: """Return the flat patch-row relation matching a query.""" @abc.abstractmethod - def count(self, query: Query) -> int: + def query_ids(self, query: Query, order_by=None, patch_ids=None) -> list[int]: + """Return matching patch ids in presentation order.""" + + @abc.abstractmethod + def count(self, query: Query, patch_ids=None) -> int: """Return how many patches match a query, without projecting rows.""" @abc.abstractmethod @@ -649,10 +653,17 @@ def _query_context(self, query): coord_meta = self._coord_meta(coord_names) if coord_names else pd.DataFrame() return queries, attr_meta, coord_meta - def query(self, query=None) -> pd.DataFrame: + def query(self, query=None, order_by=None, patch_ids=None) -> pd.DataFrame: """Return the flat patch-row relation for a query (or several).""" queries, attr_meta, coord_meta = self._query_context(query) - sql, params, residuals = build_sql(queries, self.dialect, attr_meta, coord_meta) + sql, params, residuals = build_sql( + queries, + self.dialect, + attr_meta, + coord_meta, + order_by=order_by, + patch_ids=patch_ids, + ) df = self._fetch_df(sql, params) df = self._flatten(df, attr_meta) df = self._pivot_coords(df) @@ -660,17 +671,40 @@ def query(self, query=None) -> pd.DataFrame: df = apply_residuals(df, residuals) return df.reset_index(drop=True) - def count(self, query=None) -> int: + def query_ids(self, query=None, order_by=None, patch_ids=None) -> list[int]: + """Return matching patch ids in presentation order (ids only).""" + queries, attr_meta, coord_meta = self._query_context(query) + sql, params, residuals = build_sql( + queries, + self.dialect, + attr_meta, + coord_meta, + order_by=order_by, + patch_ids=patch_ids, + ids_only=True, + ) + if residuals: + # regex residuals need string values; realize the relation + df = self.query(queries, order_by=order_by, patch_ids=patch_ids) + return [int(x) for x in df["patch_id"]] + return [int(x) for x in self._fetch_df(sql, params)["patch_id"]] + + def count(self, query=None, patch_ids=None) -> int: """Count matching patches without projecting or pivoting rows.""" queries, attr_meta, coord_meta = self._query_context(query) sql, params, residuals = build_sql( - queries, self.dialect, attr_meta, coord_meta, count=True + queries, + self.dialect, + attr_meta, + coord_meta, + count=True, + patch_ids=patch_ids, ) if not residuals: return int(self._fetch_df(sql, params)["n"].iloc[0]) # A regex residual must inspect string values, so a database count # cannot resolve it; the full relation already applies the residual. - return len(self.query(queries)) + return len(self.query(queries, patch_ids=patch_ids)) def _fetch_in(self, base_sql: str, column: str, ids: list) -> pd.DataFrame: """Fetch ``{base_sql} WHERE {column} IN ids``, batching large sets.""" @@ -955,16 +989,23 @@ def _shape_coord_selector(name: str, value): """ if value is None or value is Ellipsis: return value - if isinstance(value, np.ndarray): - if value.dtype == np.bool_: - return value - elif ( + is_bool_array = (isinstance(value, np.ndarray) and value.dtype == np.bool_) or ( isinstance(value, list) and value and all(isinstance(x, bool | np.bool_) for x in value) - ): - return np.asarray(value, dtype=bool) - elif isinstance(value, tuple | list): + ) + if is_bool_array: + # A sample mask is positional/absolute, so it is only defined + # when every patch shares the coordinate's size — a guarantee + # spools never make — and it can never reduce file reads. + msg = ( + f"Coordinate {name!r} no longer accepts boolean sample " + "masks at the spool level; apply them per patch, e.g. " + "spool.map(lambda p: p.select(...)). Boolean arrays over " + "patches (spool[mask]) still select membership." + ) + raise InvalidSpoolQueryError(msg) + if isinstance(value, tuple | list): # range-like: a (start, stop) pair (2-element list is the # legacy range form). Wrong arity is a malformed range. if len(value) != 2: @@ -975,8 +1016,8 @@ def _shape_coord_selector(name: str, value): return tuple(None if v is Ellipsis else v for v in value) msg = ( f"Coordinate {name!r} accepts range selectors (a (start, stop) " - "tuple or slice, None/... for open ends) or boolean masks; " - f"scalar and membership values are not supported. Got {value!r}." + "tuple or slice, None/... for open ends); scalar, membership, " + f"and boolean-mask values are not supported. Got {value!r}." ) raise InvalidSpoolQueryError(msg) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 42a524776..0a635d6ae 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -45,35 +45,48 @@ class _CanonicalRange: A numeric coordinate range resolved to canonical SI magnitudes. The exact per-patch re-select defers its representation until the - target patch is known: unit-bearing coordinates get quantities in - the canonical unit (`Patch.select` converts them to native units), - unitless coordinates get the bare magnitudes. A single eager form - cannot serve both — raw numbers trim the wrong physical interval - on non-SI patches, quantities break unitless coordinates. + target patch is known: unit-bearing coordinates get quantities + (`Patch.select` converts them to native units), unitless + coordinates get the bare magnitudes. A single eager form cannot + serve both — raw numbers trim the wrong physical interval on non-SI + patches, quantities break unitless coordinates. + + ``units`` records the query's own base unit when the original + bounds carried one, so the residual preserves the query's + dimensionality instead of adopting each patch coordinate's — a + metre query must never trim a seconds coordinate as 1-2 s. """ - __slots__ = ("magnitudes",) + __slots__ = ("magnitudes", "units") - def __init__(self, magnitudes: tuple): + def __init__(self, magnitudes: tuple, units: str | None = None): self.magnitudes = magnitudes + self.units = units def __eq__(self, other) -> bool: """Value equality so equal selections compare equal (spool __eq__).""" if not isinstance(other, _CanonicalRange): return NotImplemented - return self.magnitudes == other.magnitudes + return (self.magnitudes, self.units) == (other.magnitudes, other.units) def __hash__(self) -> int: - return hash(self.magnitudes) + return hash((self.magnitudes, self.units)) def for_patch_coord(self, coord) -> tuple: """Return the range in the representation this coord needs.""" from dascore.units import get_quantity - units = getattr(coord, "units", None) - if units is None: + coord_units = getattr(coord, "units", None) + if coord_units is None: + # unitless coords: bare canonical magnitudes (documented policy) return self.magnitudes - base = get_quantity(str(units)).to_base_units().units + # a unit-bearing query keeps its own dimensionality; a bare + # numeric query means canonical SI in the coord's dimension + base = ( + get_quantity(self.units) + if self.units is not None + else get_quantity(str(coord_units)).to_base_units().units + ) return tuple(None if mag is None else mag * base for mag in self.magnitudes) @@ -82,11 +95,14 @@ def _canonical_range(value) -> _CanonicalRange | None: if not is_range(value): return None magnitudes = [] + units = None for bound in value: if bound is None or bound is Ellipsis: magnitudes.append(None) elif hasattr(bound, "units"): # pint quantity -> SI magnitude - magnitudes.append(float(bound.to_base_units().magnitude)) + base = bound.to_base_units() + magnitudes.append(float(base.magnitude)) + units = str(base.units) elif isinstance(bound, bool | np.bool_): return None elif isinstance(bound, int | float | np.integer | np.floating): @@ -95,34 +111,43 @@ def _canonical_range(value) -> _CanonicalRange | None: return None if all(mag is None for mag in magnitudes): return None - return _CanonicalRange(tuple(magnitudes)) + return _CanonicalRange(tuple(magnitudes), units) + + +def _envelope_range(value): + """Return a range with quantity bounds as SI magnitudes. + + Stored envelope columns are canonical SI, so the presented-envelope + adjustment needs bare magnitudes; non-numeric ranges pass through. + """ + canonical = _canonical_range(value) + return value if canonical is None else canonical.magnitudes def _canonical_coord_selectors(backend, coords: dict) -> tuple[dict, dict]: """ Split coordinate selectors into query-side and residual-side forms. - Numeric coordinate summaries are stored in canonical SI units, so - numeric range bounds resolve to SI magnitudes for the index and - dataframe side: bare numbers are already canonical SI (the index - contract), quantities convert. The residual keeps the range as a - `_CanonicalRange` so each patch decides its own representation at - load time, which keeps mixed unitful/unitless populations correct. + The query side keeps the *original* values: the SQL builder coerces + quantities itself and needs their units to constrain candidacy to + dimensionally compatible coordinate definitions (a metre query must + exclude — or raise on — a seconds coordinate, never trim it). + The residual keeps the range as a `_CanonicalRange` (canonical SI + magnitudes plus the query's base unit) so each patch decides its + own representation at load time, which keeps mixed unitful/unitless + populations correct. Selectors on non-numeric coordinates (time ranges, string ranges) - and boolean masks pass through unchanged. + pass through unchanged. """ meta = backend._coord_meta(set(coords)) numeric = set(meta.loc[meta["value_kind"] == "num", "coord_name"]) - si_coords, residual_coords = {}, {} + query_coords, residual_coords = {}, {} for name, value in coords.items(): canonical = _canonical_range(value) if name in numeric else None - if canonical is None: - si_coords[name] = residual_coords[name] = value - else: - si_coords[name] = canonical.magnitudes - residual_coords[name] = canonical - return si_coords, residual_coords + query_coords[name] = value + residual_coords[name] = value if canonical is None else canonical + return query_coords, residual_coords def _row_source_patch_id(row: Mapping) -> str: @@ -328,6 +353,10 @@ class _CatalogRevision: value: int = 0 +# sentinel: _view keeps the current order/ids spec unless told otherwise +_KEEP = object() + + class PatchCatalog: """ Query-composable metadata catalog over the spool index tables. @@ -346,12 +375,18 @@ def __init__( queries: tuple[Query, ...] = (), residuals: tuple[tuple[dict, bool], ...] = (), revision: _CatalogRevision | None = None, + order: tuple | None = None, + ids: tuple | None = None, ): self._backend = backend self.resolver = resolver self._syncer = syncer self._queries = tuple(queries) self._residuals = tuple(residuals) + # presentation specs (D2): an order override ("attr"|"coord", + # name, ascending) and/or an ordered patch-id membership + self._order = order + self._ids = None if ids is None else tuple(int(x) for x in ids) self._revision = revision or _CatalogRevision() self._df_cache: pd.DataFrame | None = None self._df_cache_revision = -1 @@ -495,6 +530,15 @@ def __getstate__(self) -> dict: """ state = dict(self.__dict__) state["_backend"] = None + # In-memory backends are rebuilt on the other side with FRESH + # patch ids, so a stored id membership would bind to the wrong + # rows. Restrict the rebuilt content to the current membership + # instead (records/registry in presentation order, so re-ingest + # ordinals preserve it) and drop the id spec; the syncer case + # reopens the same database file, where ids stay valid. + rebuilt_membership = self._syncer is None and self._ids is not None + if rebuilt_membership: + state["_ids"] = None # Live catalogs rebuild from their registry without touching the # connection (which may belong to another thread during pickling); # other in-memory catalogs (e.g. unions) capture their rows. @@ -504,18 +548,25 @@ def __getstate__(self) -> dict: and not isinstance(self.resolver, LiveResolver) ) if needs_records: - state["_rebuild_records"] = tuple(self._backend.export_records()) + patch_ids = self._ids if rebuilt_membership else None + state["_rebuild_records"] = tuple( + self._backend.export_records(patch_ids=patch_ids) + ) # A view shares the root's resolver, but must not drag the whole # live registry across the wire: keep only the entries its rows # reference (a one-patch view of an N-patch spool serializes one - # patch, not N — the payload Spool.map ships per task). + # patch, not N — the payload Spool.map ships per task) — in + # presentation order, so a rebuilt registry keeps the view's + # ordering. if self.is_view and self.resolver.live_entries(): - paths = set(self.to_df()["path"].astype(str)) - keep = {k: v for k, v in self.resolver.live_entries().items() if k in paths} + df = self.to_df() + paths = list(dict.fromkeys(df["path"].astype(str))) + entries = self.resolver.live_entries() + keep = {k: entries[k] for k in paths if k in entries} state["resolver"] = _membership_resolver(self.resolver, keep) return state - def _view(self, queries, residuals) -> PatchCatalog: + def _view(self, queries, residuals, order=_KEEP, ids=_KEEP) -> PatchCatalog: out = PatchCatalog( backend=self.backend, resolver=self.resolver, @@ -523,9 +574,68 @@ def _view(self, queries, residuals) -> PatchCatalog: queries=queries, residuals=residuals, revision=self._revision, + order=self._order if order is _KEEP else order, + ids=self._ids if ids is _KEEP else ids, ) return out + def order_by(self, attribute: str, ascending: bool = True) -> PatchCatalog: + """ + Return a view presenting rows ordered by an attribute or coord. + + A lazy presentation spec (D2): realization adds ORDER BY with + the ordinal contract as the deterministic tiebreak; no rows are + copied and no relation is realized here. + """ + name = str(attribute) + coords = self.backend.coord_names() + if name in coords: + spec = ("coord", name, ascending) + elif name in self.backend.attr_names(): + spec = ("attr", name, ascending) + elif name.endswith("_min") and name.removesuffix("_min") in coords: + spec = ("coord", name.removesuffix("_min"), ascending) + else: + msg = "Invalid attribute. Please use a valid attribute such as: 'time'" + raise IndexError(msg) + return self._view(self._queries, self._residuals, order=spec) + + def _ordered_ids(self) -> tuple[int, ...]: + """The view's patch ids in presentation order (ids only, cheap).""" + if self._ids is not None and self._order is None: + return self._ids + return tuple( + self.backend.query_ids( + list(self._queries) or None, + order_by=self._order, + patch_ids=self._ids, + ) + ) + + def window(self, item: slice) -> PatchCatalog: + """ + Return a view restricted to a slice of the presented rows. + + Membership realizes as an ordered id list (ids only — never the + flat relation); subsequent selections compose within the window + per the D2 rules. + """ + ids = self._ordered_ids()[item] + return self._view(self._queries, self._residuals, ids=tuple(ids)) + + def restrict(self, indices) -> PatchCatalog: + """ + Return a view keeping the presented rows an array selects. + + ``indices`` is a boolean mask over rows or an array of integer + positions (order-preserving; duplicate positions collapse to + one row, matching the spool's set semantics). + """ + ids = np.asarray(self._ordered_ids()) + picked = ids[np.asarray(indices)] + deduped = tuple(dict.fromkeys(int(x) for x in picked)) + return self._view(self._queries, self._residuals, ids=deduped) + def _invalidate(self) -> None: self._revision.value += 1 self._df_cache = None @@ -570,8 +680,13 @@ def __deepcopy__(self, memo) -> PatchCatalog: @property def is_view(self) -> bool: - """True when this catalog carries selection state.""" - return bool(self._queries or self._residuals) + """True when this catalog carries selection or presentation state.""" + return bool( + self._queries + or self._residuals + or self._order is not None + or self._ids is not None + ) def _require_root(self, operation: str) -> None: if self.is_view: @@ -638,7 +753,17 @@ def to_df(self) -> pd.DataFrame: compares all non-private columns) is not spuriously blocked. """ if self._df_cache is None or self._df_cache_revision != self._revision.value: - df = self.backend.query(list(self._queries) or None) + df = self.backend.query( + list(self._queries) or None, + order_by=self._order, + patch_ids=self._ids, + ) + if self._ids is not None and self._order is None: + # id membership presents in its own (window/array) order + position = {pid: i for i, pid in enumerate(self._ids)} + df = df.sort_values( + "patch_id", key=lambda s: s.map(position), kind="stable" + ).reset_index(drop=True) df = df.drop(columns=list(SPOOL_HIDDEN_COLUMNS), errors="ignore").rename( columns={"patch_id": "_patch_id"} ) @@ -651,7 +776,7 @@ def to_df(self) -> pd.DataFrame: for query in self._queries if ( ranges := { - name: value + name: _envelope_range(value) for name, value in query.coords.items() if is_range(value) } @@ -678,7 +803,7 @@ def __len__(self) -> int: and self._df_cache_revision == self._revision.value ): return len(self._df_cache) - return self.backend.count(list(self._queries) or None) + return self.backend.count(list(self._queries) or None, patch_ids=self._ids) def get_patch(self, index: int) -> dc.Patch: """Materialize one patch: resolve, then exact two-stage trim.""" diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index 39e6d53ac..fddef889f 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -305,18 +305,12 @@ def build_coord_clause( typed_values = [] if is_range(value): kind, lo, hi, typed_values = _range_bounds(value, kinds) - elif _is_collection(value) and np.asarray(value).dtype == bool: - # boolean masks are patch-local; no index predicate at all, - # but the coord must exist on the patch. - kind = lo = hi = None else: - # Scalars and value membership have no exact patch-level - # meaning; resolve_query rejects them before SQL composition, - # so only a hand-built Query can reach this. - msg = ( - f"Coordinate {name!r} accepts range or boolean-mask " - f"selectors; got {value!r}." - ) + # Scalars, value membership, and boolean masks have no exact + # patch-level meaning spool-wide; resolve_query rejects them + # before SQL composition, so only a hand-built Query can reach + # this. + msg = f"Coordinate {name!r} accepts range selectors; got {value!r}." raise InvalidSpoolQueryError(msg) compatible_units = _compatible_coord_units(rows, typed_values, name) @@ -387,20 +381,49 @@ def _build_where( return where, residuals +def _order_clause(order_by, dialect: BaseDialect, attr_meta: pd.DataFrame) -> str: + """ + Resolve an order spec into an ORDER BY clause. + + ``order_by`` is ``(kind, name, ascending)`` where kind is "attr" + (an attrs-table column ordered by its typed column) or "coord" + (ordered by the coordinate's envelope minimum). The ordinal contract + supplies the deterministic tiebreak. + """ + kind, name, ascending = order_by + direction = "ASC" if ascending else "DESC" + if kind == "coord": + column = f"p.{dialect.quote(f'{name}_min')}" + else: + rows = attr_meta[attr_meta["attr_name"] == name] + columns = [dialect.quote(c) for c in rows["column_name"]] + # an attr observed under several kinds orders by its first column + column = f"a.{columns[0]}" + return f"ORDER BY {column} {direction}, s.ordinal, p.patch_id" + + def build_sql( query: Query | Sequence[Query], dialect: BaseDialect, attr_meta: pd.DataFrame, coord_meta: pd.DataFrame, count: bool = False, + order_by=None, + patch_ids=None, + ids_only: bool = False, ) -> tuple[str, list, list[tuple[str, re.Pattern]]]: """ Build SQL for one or more AND-composed queries. By default this projects the flat relation; with count=True the same WHERE is reused for a COUNT with no projection, coordinate pivot, or - ordering. coord_meta must cover every coordinate the queries - reference (it may be empty for attr-only queries). + ordering; with ids_only=True only ordered patch ids are projected + (the cheap realization slices/windows use). coord_meta must cover + every coordinate the queries reference (it may be empty for + attr-only queries). ``order_by`` overrides the default ordinal + ordering (see `_order_clause`); ``patch_ids`` restricts rows to an + id membership (one JSON parameter, so the SQLite bound-variable cap + does not limit membership size). Returns (sql, params, residuals), where residuals pairs attr names with regex patterns that must be re-applied to the resulting @@ -408,12 +431,28 @@ def build_sql( SQL-resolvable (regex must inspect rows) and the caller must fall back to a projected count. """ + import json + queries = _as_query_list(query) where, residuals = _build_where(queries, dialect, attr_meta, coord_meta) + if patch_ids is not None: + where.add( + "p.patch_id IN (SELECT value FROM json_each(?))", + json.dumps([int(x) for x in patch_ids]), + ) if count: # COUNT(p.patch_id) counts patches; a WHERE may reference a.. sql = f"SELECT COUNT(p.patch_id) AS n {_FROM}WHERE {where.sql}" return sql, where.params, residuals + order = ( + _order_clause(order_by, dialect, attr_meta) + if order_by is not None + # the ordering contract: source ordinal, then file-internal order + else "ORDER BY s.ordinal, p.patch_id" + ) + if ids_only: + sql = f"SELECT p.patch_id {_FROM}WHERE {where.sql} {order}" + return sql, where.params, residuals # attr columns selected explicitly: `a.*` would duplicate patch_id and # engines disagree on how to dedupe result column names. attr_cols = "".join( @@ -424,8 +463,7 @@ def build_sql( f"p.*{attr_cols} " f"{_FROM}" f"WHERE {where.sql} " - # the ordering contract: source ordinal, then file-internal order - "ORDER BY s.ordinal, p.patch_id" + f"{order}" ) return sql, where.params, residuals diff --git a/dascore/utils/patch_assembly.py b/dascore/utils/patch_assembly.py index 17b37ae13..588551411 100644 --- a/dascore/utils/patch_assembly.py +++ b/dascore/utils/patch_assembly.py @@ -116,13 +116,15 @@ def get_patches_from_index(self, df_ind): """Given an index (from current df), return the corresponding patch.""" source = self.source_df instruction = self.instruction_df - # handle negative index. - df_ind = df_ind if df_ind >= 0 else len(self.df) + df_ind - try: - inds = self.df.index[df_ind] - except IndexError: - msg = f"index of [{df_ind}] is out of bounds for spool." - raise IndexError(msg) from None + # handle negative index; a still-negative value after + # normalization is out of bounds and must never wrap around + requested = df_ind + if df_ind < 0: + df_ind = len(self.df) + df_ind + if not 0 <= df_ind < len(self.df): + msg = f"index of [{requested}] is out of bounds for spool." + raise IndexError(msg) + inds = self.df.index[df_ind] # Group positional instruction rows by current index (and cache) to # avoid a full instruction df scan for each requested patch. if self._indices is None: diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 619b85ee0..250e2560b 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -363,8 +363,8 @@ def test_bool_some_true(self, random_spool): bool_array[1] = False out = random_spool[bool_array] assert len(out) == sum(bool_array) - df1 = out.get_contents() - df2 = random_spool.get_contents()[bool_array] + df1 = out.get_contents().reset_index(drop=True) + df2 = random_spool.get_contents()[bool_array].reset_index(drop=True) assert df1.equals(df2) @@ -961,9 +961,11 @@ def test_union_of_chunked_spool(self, many_contiguous): def test_iteration_skips_unresolvable_patch(self, monkeypatch): """A patch that fails to resolve is skipped with a #583 warning.""" - # a sorted spool is materialized, so iteration runs through the - # base __iter__ (memory spools have a fast patch-list iterator). - spool = dc.spool([dc.get_example_patch()]).sort("time") + # force the planned state (sort/slice are lazy catalog specs now) + base = dc.spool([dc.get_example_patch()]) + spool = base.new_from_df( + base._df, source_df=base._source_df, instruction_df=base._instruction_df + ) def _raise(_ind): raise MissingPatchError("trimmed to nothing") @@ -987,7 +989,10 @@ def _raise(_ind): def test_planned_view_negative_and_bad_index(self): """Assembler indexing handles negatives and raises out-of-bounds.""" patches = list(dc.get_example_spool(length=2)) - planned = dc.spool(patches).sort("time") + base = dc.spool(patches) + planned = base.new_from_df( + base._df, source_df=base._source_df, instruction_df=base._instruction_df + ) assert planned._plan is not None assert planned[-1] == planned[len(patches) - 1] with pytest.raises(IndexError, match="out of bounds"): diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index d442f86b1..673cec08d 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -37,8 +37,12 @@ def spool(request, tmp_path_factory): ) out = dc.spool(path).update(progress=None) if request.param.endswith("_df"): - out = out.sort("time") - assert not out._catalog_native, "sort must yield the materialized state" + # force the planned (dataframe) state explicitly: sort/slice are + # lazy catalog specs now, so only a plan materializes the frames + out = out.new_from_df( + out._df, source_df=out._source_df, instruction_df=out._instruction_df + ) + assert not out._catalog_native, "planned state expected" return out @@ -93,9 +97,9 @@ def test_coord_predicate_reaches_backend(self, spool, monkeypatch): calls = [] original = backend.query - def wrapped(query=None): + def wrapped(query=None, **kwargs): calls.append(query) - return original(query) + return original(query, **kwargs) monkeypatch.setattr(backend, "query", wrapped) selected = spool.select(time=("2020-01-03", "2020-01-04")) @@ -403,14 +407,110 @@ def test_chained_views(self, ft_patch): assert float(coord.min()) >= 65 assert float(coord.max()) <= 197 - def test_boolean_mask_selectors(self, ft_patch): - """Boolean masks (array and list) select coordinates patch-locally.""" + def test_boolean_mask_selectors_rejected(self, ft_patch): + """Sample masks are patch-level only; the spool points at map().""" coord = ft_patch.get_coord("distance") mask = np.zeros(len(coord), dtype=bool) mask[:5] = True - # ndarray mask - got = dc.spool([ft_patch]).select(distance=mask) - assert len(got[0].get_coord("distance")) == 5 - # equivalent list-of-bools mask - got_list = dc.spool([ft_patch]).select(distance=list(mask)) - assert len(got_list[0].get_coord("distance")) == 5 + with pytest.raises(InvalidSpoolQueryError, match="boolean sample"): + dc.spool([ft_patch]).select(distance=mask) + with pytest.raises(InvalidSpoolQueryError, match="boolean sample"): + dc.spool([ft_patch]).select(distance=list(mask)) + # the per-patch escape hatch still works + got = ft_patch.select(distance=mask) + assert len(got.get_coord("distance")) == 5 + + +class TestQuantityDimensionality: + """Quantity queries keep their dimensionality end to end (review P1).""" + + @pytest.fixture() + def mixed_unit_spool(self): + """Two patches whose distance coords are metres and seconds.""" + p_m = dc.get_example_patch() + p_s = p_m.update_coords( + distance=p_m.get_coord("distance").set_units("s") + ).update_attrs(history=[]) + return dc.spool([p_m, p_s]) + + def test_incompatible_coord_excluded(self, mixed_unit_spool): + """A metre query never returns (or trims) a seconds coordinate.""" + from dascore.units import get_quantity, m + + out = mixed_unit_spool.select(_coords={"distance": (1 * m, 2 * m)}) + patches = list(out) + assert len(patches) == 1 + units = get_quantity(str(patches[0].get_coord("distance").units)) + assert units.dimensionality == m.dimensionality + + def test_all_incompatible_raises(self): + """A query incompatible with every stored unit raises UnitError.""" + from dascore.exceptions import UnitError + from dascore.units import m + + p_s = dc.get_example_patch().update_coords( + distance=dc.get_example_patch().get_coord("distance").set_units("s") + ) + spool = dc.spool([p_s]) + with pytest.raises(UnitError, match="no units compatible"): + spool.select(_coords={"distance": (1 * m, 2 * m)}).get_contents() + + +class TestLazyOrderAndWindow: + """sort/slice/array selection are lazy Selection specs (D2).""" + + def test_sort_is_lazy_and_ordered(self, tmp_path_factory): + """Sorting composes a spec; realization returns ordered rows.""" + patches = list(dc.get_example_spool("random_das")) + spool = dc.spool(list(reversed(patches))) + out = spool.sort("time") + assert out._catalog_native # no plan materialized + df = out.get_contents() + assert df["time_min"].is_monotonic_increasing + assert list(out) == patches + + def test_slice_is_lazy_window(self): + """Slicing keeps the catalog state and correct membership.""" + patches = list(dc.get_example_spool("random_das")) + spool = dc.spool(patches) + part = spool[1:] + assert part._catalog_native + assert len(part) == len(patches) - 1 + assert list(part) == patches[1:] + + def test_select_after_slice_filters_within_window(self): + """D2 composition: predicates apply inside the window.""" + patches = list(dc.get_example_spool("random_das")) + spool = dc.spool(patches) + t0 = patches[0].get_coord("time").min() + # the first patch is outside the window, so selecting its time + # range inside the window matches nothing + windowed = spool[1:] + assert len(windowed.select(time=(None, t0 + np.timedelta64(1, "s")))) == 0 + + def test_slice_of_slice_composes(self): + """Windows compose arithmetically.""" + patches = list(dc.get_example_spool("random_das")) + part = dc.spool(patches)[1:][1:] + assert list(part) == patches[2:] + + def test_sorted_spool_slice(self): + """A slice of a sorted view respects the sort order.""" + patches = list(dc.get_example_spool("random_das")) + spool = dc.spool(list(reversed(patches))) + first = spool.sort("time")[0:1] + assert list(first) == patches[0:1] + + def test_split_parts_pickle_small(self): + """split() windows keep map() payloads at member size.""" + import pickle + + base = dc.get_example_patch() + rng = np.random.default_rng(0) + patches = [base.new(data=rng.random(base.shape)) for _ in range(5)] + spool = dc.spool(patches) + parts = list(spool.split(size=1)) + assert len(parts) == 5 + payload = len(pickle.dumps(parts[0])) + baseline = len(pickle.dumps(dc.spool([patches[0]]))) + assert payload < 2 * baseline diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index 6437754ca..e5321be45 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -306,13 +306,13 @@ def test_incompatible_quantity_coord_raises(self, backend): def test_scalar_coord_rejected(self, backend): """Scalar coord predicates have no exact patch meaning; rejected.""" - with pytest.raises(InvalidSpoolQueryError, match="range or boolean"): + with pytest.raises(InvalidSpoolQueryError, match="range selectors"): backend.query(Query(coords={"frequency": 100})) def test_array_membership_rejected(self, backend): """Numeric value membership on a coord is rejected, not candidacy.""" values = np.array([10.0, 20.0, 480.0]) - with pytest.raises(InvalidSpoolQueryError, match="range or boolean"): + with pytest.raises(InvalidSpoolQueryError, match="range selectors"): backend.query(Query(coords={"distance": values})) def test_coord_missing_excludes_patch(self, backend): diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 0e18bc45e..982444751 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -576,11 +576,11 @@ def test_glob_on_non_str_attr_empty(self, backend): df = backend.query(Query(attrs={"gauge_length": "1*"})) assert df.empty - def test_boolean_array_coord_requires_presence_only(self, backend): - """Boolean masks are patch-local; index only checks coord presence.""" + def test_boolean_array_coord_rejected(self, backend): + """Boolean sample masks are no longer index predicates.""" mask = np.array([True, False, True]) - df = backend.query(Query(coords={"distance": mask})) - assert len(df) == 4 # every patch with a distance coord + with pytest.raises(InvalidSpoolQueryError, match="range selectors"): + backend.query(Query(coords={"distance": mask})) def test_slice_range_form(self, backend): """Slices resolve to the same range tuples patch selects accept.""" diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index d8346da9b..2219a8a37 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -47,14 +47,24 @@ def test_patches_shared_not_copied(self, contiguous_patches): assert any(x is p2 for x in loaded) def test_union_of_materialized_member(self): - """A sorted (materialized but catalog-backed) member unions by ids.""" + """A planned (materialized but catalog-backed) member unions by ids.""" sp = dc.get_example_spool("random_das") other = dc.get_example_spool("diverse_das") - materialized = sp.sort("time") # dataframe path, keeps its catalog + # force the planned state; sort/slice are lazy specs now + materialized = sp.new_from_df( + sp._df, source_df=sp._source_df, instruction_df=sp._instruction_df + ) assert not materialized._catalog_native combined = materialized + other assert len(combined) == len(sp) + len(other) + def test_union_of_sorted_member(self): + """A lazily sorted member unions by its ordered membership.""" + sp = dc.get_example_spool("random_das") + other = dc.get_example_spool("diverse_das") + combined = sp.sort("time") + other + assert len(combined) == len(sp) + len(other) + def test_select_on_union(self): """Selection works over the merged metadata.""" sp1 = dc.get_example_spool("random_das") From a6c45a067f1c83caa5f61d8418480351b26f6432 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 07:25:44 +0200 Subject: [PATCH 89/97] Make restructured spools derived catalogs; one engine for everything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chunk and concatenate now materialize the current view's membership into a fresh in-memory catalog whose patch rows are the plan outputs; a PlanResolver loads each output's members through the parent's resolver (live registry, files, nested plans route by path scheme) and executes the existing assembly engine, or concatenates in order. Selection, sorting, windowing, union, equality, and serialization on a chunked spool therefore run the identical catalog code path as every other spool — the dataframe execution engine (SpoolView, the planned select branch, the df sort/slice paths, spool-held merge policy and post- selects) is deleted rather than kept in sync. Plans collapse, never nest: re-chunking re-plans from the current view's trimmed members. Virtual outputs carry real coordinate identities (carried fp: def keys survive; the planned dim's range fingerprint is reconstructed exactly). update() implements the adjudicated root-only rule: any operation — select, slice, sort, chunk, concatenate, combine — produces a spool that raises with a re-derive message; directory roots sync, single-file roots rescan, live roots no-op. The selected-file silent-widening bug dies with the rule. from_directory loses select_kwargs/merge_kwargs. concatenate becomes lazy (closing the eager-on-directory-spools memory footgun) and still supports stacking along a new dimension. The planner gains the invariants from review: interval arithmetic uses the absolute step (descending coordinates chunk identically to ascending), every published output must have a member, and patch-local samples windows adjust the working envelopes so same-dimension chunking plans over the truth instead of publishing phantom empty outputs. The shared SQLite backend serializes statement execution behind a lock — split windows iterating in a thread pool interleaved pandas fetches on one connection and corrupted result frames. Equality gains a same- backend/same-spec fast path so comparing large directory spools does not materialize their relations, the sync renumber only rewrites rows whose ordinal changed, and an interrupted initial index renumbers on retry before being marked complete. --- dascore/core/spool.py | 790 +++++++--------------- dascore/io/index/backend.py | 24 +- dascore/io/index/catalog.py | 71 +- dascore/io/index/indexer.py | 8 +- dascore/io/index/lite.py | 20 +- dascore/io/index/planned.py | 387 +++++++++++ dascore/utils/chunk_plan.py | 51 +- docs/notes/spool_chunking.qmd | 2 +- docs/notes/spool_selection.qmd | 4 +- tests/test_core/test_directory_spool.py | 42 +- tests/test_core/test_file_spool.py | 2 +- tests/test_core/test_patch_chunk.py | 7 +- tests/test_core/test_spool.py | 75 +- tests/test_core/test_spool_contracts.py | 39 +- tests/test_core/test_spool_select_spec.py | 40 +- tests/test_io/test_index/test_union.py | 11 +- 16 files changed, 863 insertions(+), 710 deletions(-) create mode 100644 dascore/io/index/planned.py diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 19cf7f99f..1d3d8702a 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -4,8 +4,7 @@ import abc import warnings -from collections.abc import Callable, Generator, Mapping, Sequence -from dataclasses import dataclass +from collections.abc import Callable, Generator, Sequence from functools import singledispatch from pathlib import Path from typing import ClassVar, Literal, TypeVar @@ -29,32 +28,22 @@ ) from dascore.exceptions import ( InvalidSpoolError, - InvalidSpoolQueryError, MissingPatchError, ParameterError, ) from dascore.utils.display import get_dascore_text, get_nice_text from dascore.utils.docs import compose_docstring -from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( _spool_map, deep_equality_check, ) from dascore.utils.namespace import NamespaceOwner from dascore.utils.patch import ( - _spool_up, concatenate_patches, get_patch_names, stack_patches, ) from dascore.utils.paths import coerce_to_upath, requires_local_directory -from dascore.utils.pd import ( - _column_or_value, - adjust_segments, - get_column_names_from_dim, - get_dim_names_from_columns, - resolve_selector_namespaces, -) T = TypeVar("T") @@ -426,27 +415,9 @@ def viz(self): raise AttributeError(msg) -@dataclass(eq=False, frozen=True) -class SpoolView: - """ - The derived relation a restructured spool presents. - - ``outputs`` are the rows the spool shows (one per patch it yields), - ``members`` bind each output to the source rows that feed it (the - instruction frame), and ``sources`` are those source rows. A spool - without a view presents its catalog's rows directly; operations - that restructure or reorder rows (chunk, sort, slice) attach a view - instead of replacing the backing store. - """ - - outputs: pd.DataFrame - members: pd.DataFrame - sources: pd.DataFrame - - class Spool(BaseSpool): """ - The concrete spool: a `PatchCatalog` plus an optional derived view. + The concrete spool: a view over a `PatchCatalog`. Constructed from in-memory patches directly (or via [`dascore.spool`](`dascore.spool`)), from a directory of files with @@ -459,113 +430,36 @@ class Spool(BaseSpool): data A patch, sequence of patches, or another spool whose (in-memory) patches this spool should hold; None creates an empty spool. - merge_kwargs - Kwargs controlling how member patches merge when assembled. Notes ----- - The catalog is the single store — live patches sit in its resolver - registry, file-backed patches in its index tables — regardless of - how the spool was constructed. A spool presents rows from exactly - one of two derivations: - - - **catalog-backed** (``_plan is None``): rows map one-to-one to a - ``PatchCatalog`` query, so metadata operations (length, - selection) stay lazy and push down to the index. Use - ``_is_catalog_backed()`` to test this. - - **planned** (``_plan`` is a :class:`SpoolView`): the view's - outputs/members/sources frames are the presented relation. - The catalog remains attached for patch resolution. + The catalog is the spool's entire state: live patches sit in its + resolver registry, file-backed patches in its index tables, and + restructured views (chunk/concat) are derived in-memory catalogs + whose rows are the plan outputs. Selection, ordering, and windowing + are lazy specs composed on the catalog; one engine serves every + construction path. """ - # kwargs for merging patches - _merge_kwargs: Mapping | None = FrozenDict() # synthetic catalog identity columns must not join patch kwargs # comparisons or chunk merge-compatibility checks _drop_columns = ("patch", "path", "file_format", "file_version", "source_patch_id") - # patch-local selections (samples=True) applied as patches load - _post_selects: tuple = () - # The catalog backing this spool (None until one is built). + # The catalog backing this spool. _catalog = None - # The derived relation for restructured views (None = catalog rows). - _plan: SpoolView | None = None # single-file provenance (set by from_file; drives update()) _file_path = None _file_format = None _file_version = None - def _is_catalog_backed(self) -> bool: - """True when rows map one-to-one to a live catalog query.""" - return self._plan is None and self._catalog is not None - - @property - def _catalog_native(self) -> bool: - """Derived state: presented rows are the catalog's own rows.""" - return self._is_catalog_backed() - - @property - def _df(self) -> pd.DataFrame | None: - """The dataframe of contents as they will be output.""" - if self._plan is not None: - return self._plan.outputs - if "_df" not in self._cache: - self._cache["_df"] = self._get_df() - return self._cache["_df"] - - @property - def _source_df(self) -> pd.DataFrame | None: - """The dataframe of source patch rows.""" - if self._plan is not None: - return self._plan.sources - if "_source_df" not in self._cache: - self._cache["_source_df"] = self._get_source_df() - return self._cache["_source_df"] - - @property - def _instruction_df(self) -> pd.DataFrame | None: - """The instructions for going from source_df to df.""" - if self._plan is not None: - return self._plan.members - if "_instruction_df" not in self._cache: - self._cache["_instruction_df"] = self._get_instruction_df() - return self._cache["_instruction_df"] - - def _get_df(self): - """Realize the flat relation from the catalog.""" - current = self._catalog.to_df().reset_index(drop=True) - df, source, instruction = self._get_dummy_dataframes(current) - self._cache["_source_df"] = source - self._cache["_instruction_df"] = instruction - return df - - def _get_source_df(self): - """Build the source df (happens as part of building current df).""" - _ = self._df - return self._cache.get("_source_df") - - def _get_instruction_df(self): - """Build the instruction df (happens as part of building current df).""" - _ = self._df - return self._cache.get("_instruction_df") - def __init__( self, data: PatchType | Sequence[PatchType] | BaseSpool | None = None, - merge_kwargs: dict | None = None, ): from dascore.io.index.catalog import PatchCatalog - self._cache = {} - self._merge_kwargs = {} if merge_kwargs is None else merge_kwargs - self._post_selects = () if isinstance(data, Spool): - # copy-construction (the new_from_df convention): share the - # catalog, take fresh derived state + # copy-construction: share the catalog and provenance self.__dict__.update(data.__dict__) - self._cache = {} - self._merge_kwargs = dict(data._merge_kwargs) - if merge_kwargs: - self._merge_kwargs.update(merge_kwargs) return if data is None: patches = () @@ -584,180 +478,140 @@ def __init__( raise InvalidSpoolError(msg) self._catalog = PatchCatalog.from_patches(patches) - def _select_from_array(self, array) -> Self: - """Create new spool with contents changed from array input.""" - if not ( - np.issubdtype(array.dtype, np.bool_) - or np.issubdtype(array.dtype, np.integer) - ): - msg = "Only bool or int dtypes are supported for spool array selection." - raise ValueError(msg) - if self._rows_are_catalog(): - return self._new_from_catalog(self._catalog.restrict(array)) - if np.issubdtype(array.dtype, np.bool_): # boolean select - df = self._df[array] - else: - df = self._df.iloc[array] - source = self._source_df - inst = self._instruction_df - new = self.new_from_df( - df, - source_df=source, - instruction_df=inst, - merge_kwargs=self._merge_kwargs, - ) - return new - - def _rows_are_catalog(self) -> bool: - """ - True when patch access can go straight through the catalog. + # --- presented relation -------------------------------------------- - Holds for catalog-backed views with no spool-level row filtering - or patch-local selections layered outside the catalog (which - carries its own selection as queries/residuals). - """ - return self._is_catalog_backed() and not self._post_selects + @property + def _df(self) -> pd.DataFrame: + """The realized flat relation (cached by the catalog).""" + return self._catalog.to_df() - def __getitem__(self, item) -> PatchType | BaseSpool: - if isinstance(item, slice): # a slice was used, return a sub-spool - if self._rows_are_catalog(): - # a lazy id-membership window (D2); never realizes the - # flat relation, and keeps split()/map() parts cheap - return self._new_from_catalog(self._catalog.window(item)) - new_df = self._df.iloc[item] - inst, source = self._instruction_df, self._source_df - new_inst = inst[inst["current_index"].isin(new_df.index)] - # unique labels only: member rows repeat source labels, and - # label indexing would multiply rows on every slice - new_source = source.loc[new_inst.index.unique()] - out = self.new_from_df( - df=new_df, - instruction_df=new_inst, - source_df=new_source, - ) - elif is_array(item): # An array was passed use np type selection. - return self._select_from_array(np.asarray(item)) - elif self._rows_are_catalog() and isinstance(item, int | np.integer): - # catalog rows are 1:1 with patches; skip the instruction join - try: - return self._catalog.get_patch(int(item)) - except IndexError: - msg = f"index of [{item}] is out of bounds for spool." - raise IndexError(msg) from None - else: # a single index was used, should return a single patch - out = self._assembler.get_patch(item) - return out + @compose_docstring(doc=BaseSpool.get_contents.__doc__) + def get_contents(self) -> pd.DataFrame: + """{doc}.""" + return self._df def __len__(self): - # A catalog-native view can count in SQL, skipping the full flat - # realization (query + attr expansion + coordinate pivot) a plain - # len(self._df) would force. Fall back to the realized frame once - # it is cached or on the dataframe path. - if self._is_catalog_backed() and "_df" not in self._cache: - return len(self._catalog) - return len(self._df) + # counting pushes to SQL (or the cold live registry); the flat + # relation is never realized just for a length + return len(self._catalog) + + def __getitem__(self, item) -> PatchType | BaseSpool: + if isinstance(item, slice): + # a lazy id-membership window (D2); never realizes the flat + # relation, and keeps split()/map() parts cheap + return self._new_from_catalog(self._catalog.window(item)) + if is_array(item): + array = np.asarray(item) + if not ( + np.issubdtype(array.dtype, np.bool_) + or np.issubdtype(array.dtype, np.integer) + ): + msg = ( + "Only bool or int dtypes are supported for spool " + "array selection." + ) + raise ValueError(msg) + return self._new_from_catalog(self._catalog.restrict(array)) + try: + return self._catalog.get_patch(int(item)) + except MissingPatchError: + # MissingPatchError subclasses IndexError for backwards + # compatibility; it must never masquerade as out-of-bounds + raise + except IndexError: + msg = f"index of [{item}] is out of bounds for spool." + raise IndexError(msg) from None def __iter__(self): - if self._rows_are_catalog(): - for ind in range(len(self._catalog)): - try: - yield self._catalog.get_patch(ind) - except MissingPatchError as e: - msg = f"Skipping patch at index {ind} (see #583): {e}" - warnings.warn(msg, UserWarning, stacklevel=2) - return - for ind in range(len(self._df)): + for ind in range(len(self._catalog)): try: - yield self._assembler.get_patch(ind) + yield self._catalog.get_patch(ind) except MissingPatchError as e: # The patch couldn't be produced, usually because a # coordinate mismatch trimmed it to nothing (see #583). msg = f"Skipping patch at index {ind} (see #583): {e}" warnings.warn(msg, UserWarning, stacklevel=2) - @property - def _assembler(self): - """The (cached per view) executor turning member rows into patches.""" - from dascore.utils.patch_assembly import PatchAssembler - - if "_assembler" not in self._cache: - self._cache["_assembler"] = PatchAssembler( - df=self._df, - source_df=self._source_df, - instruction_df=self._instruction_df, - load_patch=self._load_patch, - merge_kwargs=self._merge_kwargs, - post_selects=self._post_selects, - drop_columns=self._drop_columns, - ) - return self._cache["_assembler"] - - def _get_dummy_dataframes(self, current): - """ - Return dummy current, source, and instruction dataframes. + # --- selection and presentation specs ------------------------------- - Dummy because the source and current df are the same, so the - instruction df is a straight mapping between the two. - """ - source = current.copy(deep=False) # shallow to not copy patches - dims = get_dim_names_from_columns(source) - cols2keep = get_column_names_from_dim(dims) - instruction = ( - current.copy(deep=False)[cols2keep] - .assign( - source_index=source.index, - # This tracks the current spool row after spool operations. - # It is not the source patch identity within a file. - current_index=source.index, - _modified=lambda x: _column_or_value(x, "_modified", False), - ) - .set_index("source_index") - .sort_values("current_index") + @compose_docstring(doc=BaseSpool.select.__doc__) + def select( + self, + *, + _attrs: dict | None = None, + _coords: dict | None = None, + samples: bool = False, + relative: bool = False, + **kwargs, + ) -> Self: + """{doc}.""" + catalog = self._catalog.select( + _attrs=_attrs, + _coords=_coords, + samples=samples, + relative=relative, + **kwargs, ) - return current, source, instruction - - def _load_patch(self, kwargs) -> dc.Patch: - """Given a row from the managed dataframe, return a patch.""" - # Push trims into the reader only when the instruction row narrows - # the source (chunk/select); otherwise the whole source is wanted - # and selection is wasted. Live patches ignore trim hints; - # exactness is re-applied above (catalog residuals). - trim = {} - if kwargs.get("_modified"): - trim = { - k: v - for k, v in kwargs.items() - if k not in self._drop_columns and not k.startswith("_") - } - return self._catalog.resolve_row(kwargs, extra_trim=trim) + return self._new_from_catalog(catalog) + + @compose_docstring(doc=BaseSpool.sort.__doc__) + def sort(self, attribute) -> Self: + """{doc}.""" + # a lazy ORDER BY spec (D2): no copy, no realization; the + # ordinal contract supplies the deterministic tiebreak + return self._new_from_catalog(self._catalog.order_by(attribute)) + + @compose_docstring(doc=BaseSpool.split.__doc__) + def split( + self, + size: int | None = None, + count: int | None = None, + ) -> Generator[Self, None, None]: + """{doc}.""" + if not ((count is not None) ^ (size is not None)): + msg = "Spool.split requires either spool_count or spool_size." + raise ParameterError(msg) + start = 0 + step = int(np.ceil(len(self) / count if count else size)) + while start < len(self): + yield self[start : start + step] + start += step + + def _new_from_catalog(self, catalog) -> Self: + """Create a spool view over a (possibly derived) catalog.""" + new = self.__class__(self) + new._catalog = catalog + return new def _as_catalog_member(self): + """Return (catalog, patch_ids) describing this spool for a union.""" + return self._catalog, None + + # --- restructuring (materializing) operations ----------------------- + + def _plan_frames(self) -> tuple[pd.DataFrame, pd.DataFrame]: """ - Return (catalog, patch_ids) describing this spool for a union. + Return (source_rows, working) frames for planning. - Catalog-native spools contribute their catalog view directly. - Dataframe-layer selections narrow rows without touching the - catalog, so their membership carries over as patch ids. - Restructured rows (e.g. chunked views) no longer map to sources - and contribute their materialized patches instead. + Plans collapse (never nest): a derived catalog re-plans from its + members — the trimmed source rows — restricted to the outputs + the current view presents. Patch-local samples residuals adjust + the working envelopes so plans reflect the loading truth. """ - if self._catalog_native: - return self._catalog, None - df = self._df - if "_patch_id" in df.columns: - return self._catalog, df["_patch_id"].tolist() - return super()._as_catalog_member() - - def _chunk_working_df(self) -> pd.DataFrame: - """Return the source rows the chunk planner consumes.""" - from dascore.utils.chunk_plan import _ensure_patch_id - - # _patch_id is never in _drop_columns, so it survives the drop when - # present; _ensure_patch_id supplies a positional fallback otherwise. - working = self._source_df.drop( - columns=list(self._drop_columns), errors="ignore" + from dascore.io.index.planned import collapse_working_df + from dascore.utils.chunk_plan import ( + _ensure_patch_id, + samples_adjusted_envelopes, ) - return _ensure_patch_id(working) + + base = collapse_working_df(self._catalog) + if base is None: + base = self._catalog.to_df().reset_index(drop=True) + base = _ensure_patch_id(base) + working = base.drop(columns=list(self._drop_columns), errors="ignore") + working = samples_adjusted_envelopes(working, self._catalog._residuals) + base = base[base["_patch_id"].isin(working["_patch_id"])] + return base.reset_index(drop=True), working.reset_index(drop=True) def chunk_plan( self, @@ -794,8 +648,9 @@ def chunk_plan( """ from dascore.utils.chunk_plan import build_chunk_plan + _, working = self._plan_frames() return build_chunk_plan( - self._chunk_working_df(), + working, overlap=overlap, keep_partial=keep_partial, snap_coords=snap_coords, @@ -819,9 +674,12 @@ def chunk( **kwargs, ) -> Self: """{doc}""" - source = self._source_df - working = self._chunk_working_df() - plan = self.chunk_plan( + from dascore.io.index.planned import derived_catalog + from dascore.utils.chunk_plan import build_chunk_plan + + source_rows, working = self._plan_frames() + plan = build_chunk_plan( + working, overlap=overlap, keep_partial=keep_partial, snap_coords=snap_coords, @@ -836,248 +694,92 @@ def chunk( "snap_coords": snap_coords, "tolerance": tolerance, } - if plan.outputs.empty: - empty = source.iloc[0:0] - return self.new_from_df(empty, merge_kwargs=merge_kwargs) - out_df = plan.outputs.drop(columns=["output_id"]).reset_index(drop=True) - # Instructions bind plan members back to source rows by patch id. - pid_to_index = pd.Series(source.index.values, index=working["_patch_id"].values) - names = [f"{plan.dim}_min", f"{plan.dim}_max", f"{plan.dim}_step"] - instructions = ( - plan.members.assign( - source_index=lambda x: x["_patch_id"].map(pid_to_index), - current_index=lambda x: x["output_id"], - ) - .drop(columns=["output_id", "_patch_id"]) - .loc[:, ["source_index", "current_index", *names, "_modified"]] - .set_index("source_index") - .sort_values("current_index") - ) - return self.new_from_df( - out_df, - source_df=source, - instruction_df=instructions, + catalog = derived_catalog( + source_rows=source_rows, + plan=plan, + parent=self._catalog, merge_kwargs=merge_kwargs, + mode="chunk", + origin_path=self.spool_path, ) + return self._new_from_catalog(catalog) - def new_from_df( - self, - df, - source_df=None, - instruction_df=None, - merge_kwargs=None, - ): - """Create a new instance from dataframes.""" - new = self.__class__(self) - if source_df is None or instruction_df is None: - _, source_, inst_ = self._get_dummy_dataframes(df) - source_df = source_df if source_df is not None else source_ - instruction_df = instruction_df if instruction_df is not None else inst_ - # Dataframe-producing operations (chunk, sort, slice) define their - # own row/instruction plan; the catalog stays attached for patch - # resolution but no longer defines the presented rows. - new._plan = SpoolView(outputs=df, members=instruction_df, sources=source_df) - new._cache = {} - new._merge_kwargs = dict(self._merge_kwargs) - new._merge_kwargs.update(merge_kwargs or {}) - new._post_selects = self._post_selects - return new - - def _select_namespaces(self) -> tuple[set[str], set[str]]: - """Return (attr names, coord names) selectable on this spool.""" - columns = set(self._df.columns) - coords = { - c.removesuffix("_min") - for c in columns - if c.endswith("_min") and f"{c.removesuffix('_min')}_max" in columns - } - skip = set(self._drop_columns) | {"coord_names", "dims"} - attrs = { - c - for c in columns - if not c.startswith("_") - and not c.endswith(("_min", "_max", "_step", "_units")) - and c not in skip - } - return attrs, coords - - def _resolve_select_kwargs(self, _attrs, _coords, kwargs) -> tuple[dict, dict]: - """ - Split select kwargs into (attrs, coords) per the selector spec. - - Name resolution is shared with the catalog path, so a name means - the same thing whether or not this spool is catalog-backed; only - how the predicate is applied differs. - """ - attrs, coords = self._select_namespaces() - return resolve_selector_namespaces( - attrs, coords, _attrs=_attrs, _coords=_coords, kwargs=kwargs - ) - - def _relative_select_kwargs(self, kwargs: dict) -> dict: - """Resolve relative bounds against the spool's global envelopes.""" - from dascore.utils.pd import relative_ranges_to_absolute - - return relative_ranges_to_absolute(self._df, kwargs) + @compose_docstring(desc=concatenate_patches.__doc__) + def concatenate(self, check_behavior: WARN_LEVELS = "warn", **kwargs) -> Self: + """{desc}""" + from dascore.io.index.planned import derived_catalog + from dascore.utils.chunk_plan import ChunkPlan - @compose_docstring(doc=BaseSpool.select.__doc__) - def select( - self, - *, - _attrs: dict | None = None, - _coords: dict | None = None, - samples: bool = False, - relative: bool = False, - **kwargs, - ) -> Self: - """{doc}.""" - # The catalog path owns the full selector semantics (e.g. unit - # canonicalization) and stays lazy on cold spools. - if self._catalog_native: - catalog = self._catalog.select( - _attrs=_attrs, - _coords=_coords, - samples=samples, - relative=relative, - **kwargs, + if len(kwargs) != 1: + msg = ( + "concatenate requires exactly one dimension keyword, " + f"got {sorted(kwargs)}" ) - return self._new_from_catalog(catalog) - attr_kwargs, coord_kwargs = self._resolve_select_kwargs(_attrs, _coords, kwargs) - if samples: - # sample indices are patch-local: never filter the spool, - # record the selection and apply it as patches load (#447). - if attr_kwargs: - msg = ( - f"samples=True selections are coordinate-only; got " - f"{sorted(attr_kwargs)}." - ) - raise InvalidSpoolQueryError(msg) - new = self.new_from_df( - self._df, - source_df=self._source_df, - instruction_df=self._instruction_df, + raise ParameterError(msg) + ((dim, value),) = kwargs.items() + value = None if value is Ellipsis else value + source_rows, working = self._plan_frames() + # a dim absent from the metadata envelopes is legal: concatenate + # can stack patches along a brand-new dimension + has_envelope = f"{dim}_min" in working.columns + count = len(working) if value in (None,) else int(value) + count = max(count, 1) + rows = working.reset_index(drop=True) + member_frames = [] + output_rows = [] + for output_id, start in enumerate(range(0, len(rows), count)): + group_rows = rows.iloc[start : start + count] + members = pd.DataFrame( + { + "output_id": output_id, + "_patch_id": group_rows["_patch_id"].values, + "_modified": False, + } ) - new._post_selects = (*self._post_selects, (coord_kwargs, True)) - return new - if relative and coord_kwargs: - coord_kwargs = self._relative_select_kwargs(coord_kwargs) - kwargs = {**attr_kwargs, **coord_kwargs} - filtered_df = adjust_segments(self._df, ignore_bad_kwargs=True, **kwargs) - inst = adjust_segments( - self._instruction_df, - ignore_bad_kwargs=True, - **kwargs, - ).loc[lambda x: x["current_index"].isin(filtered_df.index)] - source = adjust_segments( - self._source_df.loc[inst.index], ignore_bad_kwargs=True, **kwargs - ) - out = self.new_from_df( - filtered_df, - # Drop rows that are no longer needed. - source_df=source, - instruction_df=inst, + member_frames.append(members) + first = group_rows.iloc[0].to_dict() + if has_envelope: + first[f"{dim}_min"] = group_rows[f"{dim}_min"].min() + first[f"{dim}_max"] = group_rows[f"{dim}_max"].max() + first["output_id"] = output_id + first.pop("_patch_id", None) + output_rows.append(first) + outputs = pd.DataFrame(output_rows) + members = pd.concat(member_frames, ignore_index=True) + plan = ChunkPlan(outputs, members, dim, None, {}) + catalog = derived_catalog( + source_rows=source_rows, + plan=plan, + parent=self._catalog, + merge_kwargs={}, + mode="concat", + check_behavior=check_behavior, + origin_path=self.spool_path, ) - return out - - def _new_from_catalog(self, catalog) -> Self: - """Create a lazy catalog-native view of this spool.""" - new = self.__class__(self) - new._catalog = catalog - new._plan = None - new._cache = {} - new._post_selects = () - return new - - @compose_docstring(doc=BaseSpool.sort.__doc__) - def sort(self, attribute) -> Self: - """{doc}.""" - if self._rows_are_catalog(): - # a lazy ORDER BY spec (D2): no copy, no realization; the - # ordinal contract supplies the deterministic tiebreak - return self._new_from_catalog(self._catalog.order_by(attribute)) - df = self._df - inst_df = self._instruction_df - - # make sure a suitable attribute is entered - attrs = set(df.columns) - if attribute not in attrs: - # make sure we can also cover coordinate names instead of the attribute - if f"{attribute}_min" in attrs: - attribute = f"{attribute}_min" - else: - msg = "Invalid attribute. Please use a valid attribute such as: 'time'" - raise IndexError(msg) - - # get a mapping from the old current index to the sorted ones - sorted_df = df.sort_values(attribute) - sorted_original_indices = sorted_df.index - sorted_df = sorted_df.reset_index(drop=True) - mapper = pd.Series(np.arange(len(sorted_df)), index=sorted_original_indices) - # swap out all the old values with new ones - new_current_index = inst_df["current_index"].map(mapper) - new_instruction_df = inst_df.assign(current_index=new_current_index) - # create new spool from new dataframes - return self.new_from_df( - df=sorted_df, - source_df=self._source_df, - instruction_df=new_instruction_df, - ) - - @compose_docstring(doc=BaseSpool.split.__doc__) - def split( - self, - size: int | None = None, - count: int | None = None, - ) -> Generator[Self, None, None]: - """{doc}.""" - if not ((count is not None) ^ (size is not None)): - msg = "Spool.split requires either spool_count or spool_size." - raise ParameterError(msg) - start = 0 - step = int(np.ceil(len(self) / count if count else size)) - while start < len(self): - yield self[start : start + step] - start += step - - @compose_docstring(doc=BaseSpool.get_contents.__doc__) - def get_contents(self) -> pd.DataFrame: - """{doc}.""" - return self._df + return self._new_from_catalog(catalog) # --- construction -------------------------------------------------- @classmethod - def from_directory( - cls, - path, - index_path=None, - select_kwargs: dict | None = None, - merge_kwargs: dict | None = None, - ) -> Self: + def from_directory(cls, path, index_path=None) -> Self: """ Create a spool over a directory of fiber files. The directory's index (created/updated via ``update()``) backs the catalog; ``path`` may also be an existing directory indexer. - ``select_kwargs`` compose a selection into the catalog exactly - like ``.select(**select_kwargs)`` — validating the names - triggers the initial directory index if it doesn't exist yet. """ from dascore.io.index.catalog import FileResolver, PatchCatalog from dascore.io.indexer import AbstractIndexer - out = cls(merge_kwargs=merge_kwargs) + out = cls() if isinstance(path, AbstractIndexer): - catalog = PatchCatalog( + out._catalog = PatchCatalog( backend=path._backend, resolver=FileResolver(root=path.path), syncer=path, ) else: - catalog = PatchCatalog.from_directory(path, index_path=index_path) - if select_kwargs: - catalog = catalog.select(**select_kwargs) - out._catalog = catalog + out._catalog = PatchCatalog.from_directory(path, index_path=index_path) return out @classmethod @@ -1117,8 +819,13 @@ def indexer(self): @property def spool_path(self): - """Return the path in which the spool contents are found.""" - return self.indexer.path + """The directory or file path this spool derives from, or None.""" + indexer = self.indexer + if indexer is not None: + return indexer.path + if self._file_path is not None: + return self._file_path + return getattr(self._catalog.resolver, "origin_path", None) @property def has_live_patches(self) -> bool: @@ -1126,30 +833,30 @@ def has_live_patches(self) -> bool: catalog = self._catalog return catalog is not None and bool(catalog.resolver.live_entries()) - def _has_file_rows(self) -> bool: - """True when any catalog row is backed by a file.""" - from dascore.io.index.catalog import LiveResolver - from dascore.utils.paths import is_memory_uri - - if isinstance(self._catalog.resolver, LiveResolver): - return False - paths = self._catalog.backend.get_sources()["source_path"] - return not paths.map(is_memory_uri).all() - @compose_docstring(doc=BaseSpool.update.__doc__) def update(self, progress: PROGRESS_LEVELS = "standard") -> Self: """ {doc} - Update means syncing contents with the backing source: a - directory-backed spool re-indexes its directory, a single-file - spool rescans the file, and a purely in-memory spool is - trivially current (no-op). A spool with file-backed contents - but no update source (e.g. the result of combining spools) - raises — recreate it from its directory instead. + Update is allowed only on a root spool — one no operation has + been applied to. Directory roots re-index their directory, + single-file roots rescan the file, and purely in-memory roots + are trivially current (no-op). Any derived spool (the result of + select, slicing, sort, chunk, concatenate, or combining spools) + raises: update the root and re-apply the operations. """ + from dascore.io.index.catalog import LiveResolver + catalog = self._catalog - if catalog is not None and catalog._syncer is not None: + derived_msg = ( + "update() is only allowed on a root spool; this spool is the " + "result of an operation (select/slice/sort/chunk/combine). " + "Update the root spool and re-apply the operations, e.g. " + "root = root.update(); view = root.select(...)." + ) + if catalog is None or catalog.is_view: + raise InvalidSpoolError(derived_msg) + if catalog._syncer is not None: catalog.update(progress=progress) return self._new_from_catalog(catalog) if self._file_path is not None: @@ -1162,28 +869,40 @@ def update(self, progress: PROGRESS_LEVELS = "standard") -> Self: return self.from_file( self._file_path, self._file_format, self._file_version ) - if not self._has_file_rows(): + if isinstance(catalog.resolver, LiveResolver): return self # in-memory contents are trivially current - msg = ( - "This spool has file-backed contents but no update source " - "(e.g. it combines several spools); recreate it from its " - "directory to pick up new files." - ) - raise InvalidSpoolError(msg) + # composite/plan roots are computed spools (unions, chunks) + raise InvalidSpoolError(derived_msg) # --- equality ------------------------------------------------------ def __eq__(self, other) -> bool: """ - Equality check which ignores the state of the lazy dataframes. + Equality check which ignores the state of lazy realization. - The managed dataframes are built and compared directly so that - equality does not depend on whether they were constructed yet. + The flat relations are built and compared directly so that + equality does not depend on whether they were realized yet. """ if self is other: return True if not isinstance(other, Spool): return super().__eq__(other) + # views over the same catalog state are equal without realizing + # the relations (a 200k-row archive must not materialize for ==) + mine, theirs = self._catalog, other._catalog + if ( + mine is not None + and theirs is not None + and ( + mine is theirs + or (mine._backend is not None and mine._backend is theirs._backend) + ) + and mine._queries == theirs._queries + and mine._residuals == theirs._residuals + and mine._order == theirs._order + and mine._ids == theirs._ids + ): + return True return deep_equality_check(self._eq_state(), other._eq_state()) def _eq_state(self) -> dict: @@ -1193,11 +912,11 @@ def _eq_state(self) -> dict: Equality is over rows, never backends: same length and order of patch rows, row-wise equal semantic columns (source identity like paths and live-vs-file backing stripped), plus equal - pending residual selections and policy. Whether rows come from - a live registry, an index file, or a plan is invisible; data - arrays are never compared (metadata-level, like everything - here). Because the state is enumerated — never ``__dict__`` — - new instance attributes cannot silently join equality. + pending residual selections. Whether rows come from a live + registry, an index file, or a plan is invisible; data arrays + are never compared (metadata-level, like everything here). + Because the state is enumerated — never ``__dict__`` — new + instance attributes cannot silently join equality. """ def _strip_identity(df): @@ -1217,30 +936,20 @@ def _strip_identity(df): catalog = self._catalog return { - # the presented relation (row content and order), plus the - # source rows and member bindings that define patch assembly; - # the plan's frames surface through the same accessors, so - # planned and identity views with equal contents compare equal "rows": _strip_identity(self._df), - "sources": _strip_identity(self._source_df), - "members": _strip_identity(self._instruction_df), - # residuals (e.g. samples trims) change what patches load - # without changing the visible rows "residuals": None if catalog is None else catalog._residuals, - "post_selects": self._post_selects, - "merge_kwargs": dict(self._merge_kwargs), } def __rich__(self): base = super().__rich__() - indexer = self.indexer - path = getattr(indexer, "path", None) or self._file_path + path = self.spool_path if path is not None: base += Text(f"\n Path: {path}") - # Only render a time span when the relation is (or is nearly) - # realized: planned views carry their frames and live spools are - # in memory; a huge directory index is not realized for a repr. - if self._plan is not None or self.has_live_patches: + # Only render a time span when realization is cheap: live + # contents, single files, and derived catalogs are in memory; a + # huge directory index is not realized for a repr. + cheap = self.indexer is None + if cheap: df = self._df if df is not None and len(df) and "time_min" in df.columns: t1, t2 = df["time_min"].min(), df["time_min"].max() @@ -1252,9 +961,6 @@ def __rich__(self): ) return base - # Add specific implementation of concatenate patches. - concatenate = _spool_up(concatenate_patches) - get_patch_names = get_patch_names diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index daaa011f2..160e3818c 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -601,17 +601,23 @@ def renumber_ordinals_by_time(self) -> None: deterministic tiebreak. """ with self._transaction(): + # the WHERE clause skips rows whose ordinal is already + # correct, so a one-file sync of a large archive rewrites + # one row instead of churning the whole table through WAL self._execute( + "WITH ranked AS (" + " SELECT s2.source_id AS sid, ROW_NUMBER() OVER (" + " ORDER BY t.min_time IS NULL, t.min_time, s2.source_path" + " ) - 1 AS rn" + " FROM sources s2 LEFT JOIN (" + " SELECT source_id, MIN(time_min) AS min_time" + " FROM patches GROUP BY source_id" + " ) t ON t.source_id = s2.source_id" + ")" "UPDATE sources SET ordinal = (" - " SELECT rn - 1 FROM (" - " SELECT s2.source_id AS sid, ROW_NUMBER() OVER (" - " ORDER BY t.min_time IS NULL, t.min_time, s2.source_path" - " ) AS rn" - " FROM sources s2 LEFT JOIN (" - " SELECT source_id, MIN(time_min) AS min_time" - " FROM patches GROUP BY source_id" - " ) t ON t.source_id = s2.source_id" - " ) WHERE sid = sources.source_id" + " SELECT rn FROM ranked WHERE sid = sources.source_id" + ") WHERE ordinal IS NOT (" + " SELECT rn FROM ranked WHERE sid = sources.source_id" ")" ) diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 0a635d6ae..1b480d5eb 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -155,6 +155,29 @@ def _row_source_patch_id(row: Mapping) -> str: return normalize_source_patch_id(row.get("source_patch_id")) +def apply_exact_residuals(patch: dc.Patch, residuals) -> dc.Patch: + """ + Apply a view's exact residual selections to a loaded patch. + + Shared by catalog row resolution and plan-member loading so the + two-stage select contract has exactly one implementation. + """ + for coords, samples in residuals: + coord_map = patch.coords.coord_map + usable = { + k: ( + v.for_patch_coord(coord_map[k]) if isinstance(v, _CanonicalRange) else v + ) + for k, v in coords.items() + if k in coord_map + } + if usable: + # residual bounds are already absolute (relative queries + # resolve to absolute before the residual is recorded). + patch = patch.select(**usable, samples=samples, relative=False) + return patch + + class PatchResolver(abc.ABC): """Turn one flat-relation row into a Patch.""" @@ -256,23 +279,30 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: class CompositeResolver(PatchResolver): """ - Route rows to a live registry or the filesystem by path scheme. + Route rows to a live registry, a plan, or the filesystem by scheme. - Union catalogs mix file-backed rows (absolute paths) with in-memory - rows (memory:// paths); this resolver dispatches accordingly. + Union catalogs mix file-backed rows (absolute paths), in-memory rows + (memory:// paths), and plan-output rows (plan://token/... paths); + this resolver dispatches accordingly. """ def __init__(self): self.live = LiveResolver() self.file = FileResolver(root=None) + # plan:/// prefix -> the PlanResolver that owns it + self.plans: dict[str, PatchResolver] = {} def live_entries(self) -> Mapping[str, dc.Patch]: """Return the merged live patch registry.""" return self.live._registry + def plan_entries(self) -> Mapping[str, PatchResolver]: + """Return the plan-prefix routing table.""" + return self.plans + def absorb(self, resolver: PatchResolver, paths=None) -> None: """ - Take over another resolver's live registry entries. + Take over another resolver's live and plan entries. ``paths`` restricts absorption to the given synthetic paths (the entries a transfer actually references); None takes all. @@ -281,11 +311,23 @@ def absorb(self, resolver: PatchResolver, paths=None) -> None: if paths is not None: entries = {k: v for k, v in entries.items() if k in paths} self.live._registry.update(entries) + plans = getattr(resolver, "plan_entries", dict)() + if paths is not None: + plans = { + prefix: plan + for prefix, plan in plans.items() + if any(str(p).startswith(prefix) for p in paths) + } + self.plans.update(plans) def resolve(self, row: Mapping, **trim) -> dc.Patch: - """Dispatch to the live registry or the file reader.""" - if is_memory_uri(row.get("path", "")): + """Dispatch by scheme: live registry, plan, or file reader.""" + path = str(row.get("path", "")) + if is_memory_uri(path): return self.live.resolve(row, **trim) + for prefix, plan in self.plans.items(): + if path.startswith(prefix): + return plan.resolve(row, **trim) return self.file.resolve(row, **trim) @@ -837,22 +879,7 @@ def resolve_row(self, row: Mapping, extra_trim: Mapping | None = None) -> dc.Pat ) trim_hint.update(extra_trim or {}) patch = self.resolver.resolve(row, **trim_hint) - for coords, samples in self._residuals: - coord_map = patch.coords.coord_map - usable = { - k: ( - v.for_patch_coord(coord_map[k]) - if isinstance(v, _CanonicalRange) - else v - ) - for k, v in coords.items() - if k in coord_map - } - if usable: - # residual bounds are already absolute (relative queries - # resolve to absolute before the residual is recorded). - patch = patch.select(**usable, samples=samples, relative=False) - return patch + return apply_exact_residuals(patch, self._residuals) def __iter__(self): for index in range(len(self)): diff --git a/dascore/io/index/indexer.py b/dascore/io/index/indexer.py index e0ecb7a61..5bdcc9ad7 100644 --- a/dascore/io/index/indexer.py +++ b/dascore/io/index/indexer.py @@ -294,10 +294,14 @@ def update(self, paths=None, progress: PROGRESS_LEVELS = "standard") -> Self: ) if records: self._backend.write_sources(records) - if stale or changed: + if stale or changed or not self._initial_update_done: # Directory archives present in time order; ingest assigns # walk-order ordinals, so each sync renumbers to keep the - # contract (iterate by ordinal) aligned with time. + # contract (iterate by ordinal) aligned with time. The + # not-yet-marked-done case covers a process killed between + # write_sources committing and this renumber: the retry sees + # no stale/changed files but must still fix walk-order + # ordinals before marking the initial update complete. self._backend.renumber_ordinals_by_time() if not self._initial_update_done: self._backend.mark_initial_update_done() diff --git a/dascore/io/index/lite.py b/dascore/io/index/lite.py index 86dc3a290..f565e3ea5 100644 --- a/dascore/io/index/lite.py +++ b/dascore/io/index/lite.py @@ -3,6 +3,7 @@ from __future__ import annotations import sqlite3 +import threading import weakref from contextlib import suppress from pathlib import Path @@ -84,6 +85,12 @@ def __init__(self, path: str | Path): # cleanup to *its* collection is safe (close() stays idempotent # for explicit use). The finalizer tolerates cross-thread firing. self._finalizer = weakref.finalize(self, _safe_close, self._con) + # Catalog views share this backend across threads (e.g. a + # thread-pool Spool.map over split windows). SQLite serializes + # individual statements, but a pandas fetch spans many cursor + # calls; interleaving them corrupts result frames, so statement + # execution is exclusive per backend. + self._lock = threading.RLock() try: super().__init__() except Exception: @@ -112,21 +119,24 @@ def __setstate__(self, state: dict) -> None: self.__init__(state["_path"]) def _execute(self, sql: str, params=()) -> None: - self._con.execute(sql, _adapt(params)) + with self._lock: + self._con.execute(sql, _adapt(params)) def _executemany(self, sql: str, seq_of_params) -> None: # sqlite3.executemany consumes an iterator, so adapt lazily rather # than materializing a second copy of each already-built batch. - self._con.executemany(sql, (_adapt(p) for p in seq_of_params)) + with self._lock: + self._con.executemany(sql, (_adapt(p) for p in seq_of_params)) def _fetch_df(self, sql: str, params=()) -> pd.DataFrame: # numpy_nullable assembly keeps nullable INTEGER columns exact; # the default path rounds them through float64, corrupting ns # epochs (>2**53). A dtype= hint does NOT prevent that: pandas # builds float64 first and casts after. - df = pd.read_sql_query( - sql, self._con, params=_adapt(params), dtype_backend="numpy_nullable" - ) + with self._lock: + df = pd.read_sql_query( + sql, self._con, params=_adapt(params), dtype_backend="numpy_nullable" + ) return _classic_dtypes(df) def _begin(self) -> None: diff --git a/dascore/io/index/planned.py b/dascore/io/index/planned.py new file mode 100644 index 000000000..a84c6ea6d --- /dev/null +++ b/dascore/io/index/planned.py @@ -0,0 +1,387 @@ +""" +Derived catalogs: chunk/concat plans as first-class catalog rows. + +A restructuring operation materializes the current view's membership +into a fresh in-memory catalog whose *patch rows are the plan outputs*; +a `PlanResolver` turns an output row back into a Patch by loading the +member source patches through the parent's resolver and trimming or +merging them (the existing assembly engine). Every catalog operation — +select, order, window, union, equality, serialization — then runs the +identical code path for planned and identity spools. + +Single-writer rule: derived catalogs are always fresh in-memory +databases; the on-disk index is only ever written by the directory +syncer, and views never write. +""" + +from __future__ import annotations + +import secrets +from collections.abc import Mapping + +import numpy as np +import pandas as pd + +import dascore as dc +from dascore.io.index.backend import get_backend +from dascore.io.index.catalog import ( + CompositeResolver, + PatchCatalog, + PatchResolver, + _row_source_patch_id, + apply_exact_residuals, +) +from dascore.io.index.ingest import ( + CoordRecord, + PatchRecord, + SourceRecord, + typed_value, +) +from dascore.utils.misc import is_range +from dascore.utils.pd import adjust_segments +from dascore.utils.time import to_int + +PLAN_SCHEME = "plan://" +# columns that are structural/positional rather than patch attributes +_NON_ATTR = {"output_id", "dims", "coord_names", "patch"} + + +def _ns(value) -> int | None: + """Convert a datetime/timedelta-like envelope value to ns int.""" + if value is None or pd.isnull(value): + return None + if isinstance(value, pd.Timestamp | pd.Timedelta): + return int(value.value) + return int(to_int(value)) + + +def _num(value) -> float | None: + """Convert a numeric envelope value to float.""" + if value is None or pd.isnull(value): + return None + return float(value) + + +def _coord_record_from_row(row: Mapping, name: str) -> CoordRecord | None: + """ + Build the envelope coord record for one output dimension. + + Delegates to the ingest converter through a range CoordSummary so + virtual outputs carry the same identities real patches would: a + carried ``fp:`` def key survives for non-planned dims, and the + planned dim's range fingerprint is reconstructed exactly. + """ + from dascore.core.coords import CoordSummary + from dascore.io.index.ingest import _coord_record + + lo, hi = row.get(f"{name}_min"), row.get(f"{name}_max") + if lo is None or (pd.isnull(lo) and pd.isnull(hi)): + return None + step = row.get(f"{name}_step") + step = None if step is None or pd.isnull(step) else step + if isinstance(lo, pd.Timestamp): + lo, hi = lo.to_datetime64(), pd.Timestamp(hi).to_datetime64() + dtype = "datetime64[ns]" + elif isinstance(lo, np.datetime64): + dtype = "datetime64[ns]" + elif isinstance(lo, pd.Timedelta | np.timedelta64): + lo, hi = pd.Timedelta(lo).to_timedelta64(), pd.Timedelta(hi).to_timedelta64() + dtype = "timedelta64[ns]" + else: + lo, hi = float(lo), float(hi) + step = None if step is None else abs(float(step)) + dtype = "float64" + if isinstance(step, pd.Timedelta): + step = step.to_timedelta64() + units = row.get(f"{name}_units") + # numeric envelope values are stored canonical-SI; attaching the + # original unit string would make ingest re-convert them + if dtype != "float64" or units == "" or (units is not None and pd.isnull(units)): + units = None + length = None + if step is not None: + try: + length = int(round((hi - lo) / step)) + 1 + except (TypeError, ZeroDivisionError): + length = None + key = row.get(f"_{name}_def_key") + fingerprint = None + if isinstance(key, str) and key.startswith("fp:"): + fingerprint = key[3:] + summary = CoordSummary( + dtype=dtype, + min=lo, + max=hi, + step=step, + units=units, + dims=(name,), + len=length, + fingerprint=fingerprint, + ) + return _coord_record(name, summary) + + +def _output_records(outputs: pd.DataFrame, token: str) -> list[SourceRecord]: + """Convert plan output rows into ingestible source records.""" + records = [] + envelope_suffixes = ("_min", "_max", "_step", "_units") + for row in outputs.to_dict("records"): + output_id = int(row["output_id"]) + dims = str(row.get("dims") or "") + dim_names = [d for d in dims.split(",") if d] + coords = [] + for name in dim_names: + record = _coord_record_from_row(row, name) + if record is not None: + coords.append(record) + attrs = {} + for key, value in row.items(): + if ( + key in _NON_ATTR + or key.startswith("_") + or any(key.endswith(sfx) for sfx in envelope_suffixes) + or value is None + or (np.isscalar(value) and pd.isnull(value)) + ): + continue + typed = typed_value(value) + if typed is not None: + attrs[key] = typed + patch = PatchRecord( + source_patch_id=str(output_id), + dims=dims, + shape="", + n_dims=len(dim_names), + sample_count_total=None, + time_min=_ns(row.get("time_min")), + time_max=_ns(row.get("time_max")), + time_step=_ns(row.get("time_step")), + distance_min=_num(row.get("distance_min")), + distance_max=_num(row.get("distance_max")), + distance_step=_num(row.get("distance_step")), + attrs=attrs, + coords=tuple(coords), + ) + records.append( + SourceRecord( + source_path=f"{PLAN_SCHEME}{token}/{output_id}", + source_format="plan", + format_version="", + patches=(patch,), + ) + ) + return records + + +class PlanResolver(PatchResolver): + """ + Assemble plan-output rows from their member source patches. + + ``member_rows`` carries, per output, the full source row (path, + format, identity, attrs) with the planned dimension's envelope + replaced by the member's trim range; loading goes through ``loader`` + (live registry, files, and nested plan rows), applies the parent + view's residual selections, then trims/merges via the assembly + engine ("chunk" mode) or concatenates in order ("concat" mode). + """ + + def __init__( + self, + *, + token: str, + dim: str, + member_rows: pd.DataFrame, + loader: PatchResolver, + merge_kwargs: Mapping, + parent_residuals: tuple = (), + mode: str = "chunk", + check_behavior: str = "warn", + origin_path=None, + ): + if "output_id" not in member_rows.columns: + msg = "member_rows must carry an output_id column." + raise ValueError(msg) + # plan invariant: outputs without members must never be published + self.token = token + self.dim = dim + self.member_rows = member_rows.reset_index(drop=True) + self.loader = loader + self.merge_kwargs = dict(merge_kwargs) + self.parent_residuals = tuple(parent_residuals) + self.mode = mode + self.check_behavior = check_behavior + # informational only: the directory/file the plan derived from + self.origin_path = origin_path + + def live_entries(self) -> Mapping[str, dc.Patch]: + """Expose the loader's live registry (for absorption/transfer).""" + return self.loader.live_entries() + + def plan_entries(self) -> Mapping[str, PlanResolver]: + """Route plan:// paths with this resolver's token to it.""" + nested = dict(getattr(self.loader, "plan_entries", dict)()) + nested[f"{PLAN_SCHEME}{self.token}/"] = self + return nested + + def _assembler(self): + from dascore.utils.patch_assembly import PatchAssembler + + return PatchAssembler( + df=None, + source_df=None, + instruction_df=None, + load_patch=self._load_member, + merge_kwargs=self.merge_kwargs, + post_selects=(), + drop_columns=( + "patch", + "path", + "file_format", + "file_version", + "source_patch_id", + ), + ) + + def _load_member(self, kwargs: Mapping) -> dc.Patch: + """Load one member source patch, applying parent residuals.""" + trim = {} + if kwargs.get("_modified"): + trim = { + k: v + for k, v in kwargs.items() + if not str(k).startswith("_") + and k not in ("path", "file_format", "file_version", "source_patch_id") + } + patch = self.loader.resolve(kwargs, **trim) + return apply_exact_residuals(patch, self.parent_residuals) + + def resolve(self, row: Mapping, **trim) -> dc.Patch: + """Assemble the output patch a plan row describes.""" + output_id = int(_row_source_patch_id(row)) + members = self.member_rows[self.member_rows["output_id"] == output_id] + assert len(members), "no plan members found for output row" + if self.mode == "concat": + from dascore.utils.patch import concatenate_patches + + patches = [ + self._load_member(kwargs) for kwargs in members.to_dict("records") + ] + out = concatenate_patches( + patches, check_behavior=self.check_behavior, **{self.dim: None} + ) + assert len(out) == 1 + return out[0] + joined = members.assign(current_index=output_id) + patches = self._assembler()._patch_from_instruction_df(joined) + assert len(patches) == 1 + return patches[0] + + +def _residual_ranges(residuals) -> dict: + """Envelope-applicable value ranges from a residual tuple.""" + out = {} + for coords, samples in residuals: + if samples: + continue + for name, value in coords.items(): + magnitudes = getattr(value, "magnitudes", None) + if magnitudes is not None: + out[name] = magnitudes + elif is_range(value) and not any( + hasattr(b, "units") for b in value if b is not None + ): + out[name] = value + return out + + +def derived_catalog( + *, + source_rows: pd.DataFrame, + plan, + parent: PatchCatalog | None, + merge_kwargs: Mapping, + mode: str = "chunk", + check_behavior: str = "warn", + origin_path=None, +) -> PatchCatalog: + """ + Materialize a plan into a fresh in-memory catalog. + + ``source_rows`` are the full member source rows (path/format/ + identity plus envelopes and attrs) keyed by ``_patch_id`` matching + ``plan.members``; ``parent`` supplies the resolver (live registry, + file root, nested plans) and the residual selections its view + carried, which member loading re-applies. + """ + token = secrets.token_hex(8) + name = plan.dim + trims = plan.members + trim_cols = [c for c in trims.columns if c not in ("_patch_id",)] + sources = source_rows.copy(deep=False) + if "_patch_id" not in sources.columns: + from dascore.utils.chunk_plan import _ensure_patch_id + + sources = _ensure_patch_id(sources) + member_rows = trims[["_patch_id", *[c for c in trim_cols]]].merge( + sources.drop(columns=[c for c in trim_cols if c in sources], errors="ignore"), + on="_patch_id", + how="left", + ) + # the member's trimmed range replaces the source envelope for loading + member_rows = member_rows.drop(columns=["_patch_id"]) + parent_residuals = () if parent is None else parent._residuals + # resolve stored-relative paths once; the derived catalog is + # root-independent afterwards + root = getattr(parent.resolver, "_root", None) if parent is not None else None + if root is not None and "path" in member_rows.columns: + member_rows = member_rows.assign( + path=[ + str(p) + if "://" in str(p) or str(p).startswith("/") + else str(root / str(p)) + for p in member_rows["path"] + ] + ) + loader = CompositeResolver() + if parent is not None: + member_paths = set(member_rows.get("path", pd.Series(dtype=str)).astype(str)) + loader.absorb(parent.resolver, paths=member_paths) + resolver = PlanResolver( + token=token, + dim=name, + member_rows=member_rows, + loader=loader, + merge_kwargs=merge_kwargs, + parent_residuals=parent_residuals, + mode=mode, + check_behavior=check_behavior, + origin_path=origin_path, + ) + backend = get_backend(":memory:") + backend.write_sources(_output_records(plan.outputs, token)) + return PatchCatalog(backend=backend, resolver=resolver) + + +def collapse_working_df(catalog: PatchCatalog) -> pd.DataFrame | None: + """ + Return the re-planning frame for a derived catalog, or None. + + Plans collapse (never nest): re-chunking a planned spool plans over + the current view's *members* — the trimmed source rows — restricted + to outputs the view still presents, with the view's value residuals + applied to the envelopes. + """ + resolver = catalog.resolver + if not isinstance(resolver, PlanResolver): + return None + members = resolver.member_rows + if catalog.is_view: + present = { + int(_row_source_patch_id(row)) for row in catalog.to_df().to_dict("records") + } + members = members[members["output_id"].isin(present)] + ranges = _residual_ranges(catalog._residuals) + working = members.drop(columns=["output_id", "_modified"], errors="ignore") + if ranges: + working = adjust_segments(working, ignore_bad_kwargs=True, **ranges) + return working.reset_index(drop=True) diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index 2a852ed76..9ae74af81 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -31,7 +31,7 @@ ParameterError, ) from dascore.utils.chunk import get_intervals -from dascore.utils.misc import get_middle_value +from dascore.utils.misc import get_middle_value, is_range from dascore.utils.pd import _remove_overlaps, get_interval_columns from dascore.utils.time import is_datetime64, is_timedelta64, to_float, to_timedelta64 @@ -95,6 +95,43 @@ def _resolve_group_attrs(group, columns) -> tuple[str, ...]: return tuple(x for x in dc.get_config().groupby_attrs if x in columns) +def samples_adjusted_envelopes(df: pd.DataFrame, residuals) -> pd.DataFrame: + """ + Adjust envelope columns for patch-local samples residuals. + + A ``samples=True`` index window trims each patch at load, so the + planner must consume the trimmed envelopes or it publishes outputs + that lie entirely outside the selected samples (phantom empties). + Only non-negative index windows adjust (Python-slice clamping per + patch); anything else leaves the envelope as a candidacy superset — + exactness is always re-applied at load, and the adjustment only + exists so plans reflect the truth. + """ + + def _usable_index(value) -> bool: + return value is None or (isinstance(value, int | np.integer) and value >= 0) + + df = df.copy(deep=False) + for coords, samples in residuals: + if not samples: + continue + for name, value in coords.items(): + cols = [f"{name}_min", f"{name}_max", f"{name}_step"] + if not set(cols).issubset(df.columns) or not is_range(value): + continue + lo_idx, hi_idx = value + if not (_usable_index(lo_idx) and _usable_index(hi_idx)): + continue + mins, maxs, steps = (df[c] for c in cols) + new_min = mins if lo_idx is None else mins + lo_idx * steps + new_max = maxs if hi_idx is None else mins + hi_idx * steps + df[cols[0]] = new_min.where(new_min <= maxs, other=maxs) + df[cols[1]] = new_max.where(new_max <= maxs, other=maxs) + # rows whose window starts past their end contribute nothing + df = df[df[cols[0]] <= df[cols[1]]] + return df + + def _ensure_patch_id(df: pd.DataFrame) -> pd.DataFrame: """Attach the positional identity fallback for plain dataframes.""" if "_patch_id" in df.columns: @@ -453,7 +490,10 @@ def build_chunk_plan( g_stop, value_c, overlap=overlap_c, - step=part_step, + # interval arithmetic is over (direction-free) envelope + # values; a descending coordinate's negative step would + # invert the final partial interval + step=abs(part_step), keep_partials=keep_partial, ) except ChunkError: # partition too short; skip (D8) @@ -467,9 +507,14 @@ def build_chunk_plan( outputs["output_id"] = np.arange(next_id, next_id + len(outputs)) next_id += len(outputs) members = _build_members(sub, outputs, name) + # Plan invariant: every published output has at least one member. + # An advertised row that cannot assemble is never surfaced as a + # runtime error; it is not surfaced at all. + fed = set(members["output_id"]) if not members.empty else set() + outputs = outputs[outputs["output_id"].isin(fed)] out_frames.append(outputs) member_frames.append(members) - if not out_frames: + if not out_frames or all(x.empty for x in out_frames): msg = "Could not chunk. No segments with sufficient length found." raise ChunkError(msg) outputs = pd.concat(out_frames, ignore_index=True) diff --git a/docs/notes/spool_chunking.qmd b/docs/notes/spool_chunking.qmd index 32caf697e..6ea7f820d 100644 --- a/docs/notes/spool_chunking.qmd +++ b/docs/notes/spool_chunking.qmd @@ -6,7 +6,7 @@ title: Spool Chunking ## Plans -The planner consumes the spool's flat metadata relation and produces a `ChunkPlan`: an `outputs` table (one row per patch the chunked spool will contain) and a `members` table (which slice of which source patch feeds each output). `Spool.chunk_plan` exposes the plan `chunk` would execute, with the same arguments. The chunked spool carries this shape as its derived view, and assembly (`dascore.utils.patch_assembly`) executes it row by row; re-chunking a chunked spool re-plans from the current view's members, so plans never nest. +The planner consumes the spool's flat metadata relation and produces a `ChunkPlan`: an `outputs` table (one row per patch the chunked spool will contain) and a `members` table (which slice of which source patch feeds each output). `Spool.chunk_plan` exposes the plan `chunk` would execute, with the same arguments. The chunked spool *is* a fresh in-memory catalog whose patch rows are the plan outputs; a plan resolver loads each output's members through the parent's resolver and executes the assembly engine (`dascore.utils.patch_assembly`) row by row. Re-chunking a chunked spool re-plans from the current view's members, so plans never nest, and `concatenate` is the same machinery with order-based grouping instead of continuity. ```{python} import dascore as dc diff --git a/docs/notes/spool_selection.qmd b/docs/notes/spool_selection.qmd index 5b9bd1a6b..eff605380 100644 --- a/docs/notes/spool_selection.qmd +++ b/docs/notes/spool_selection.qmd @@ -8,11 +8,11 @@ Bare selector names resolve to attributes first and then coordinates. `_attrs={. Attribute equality, membership, ranges, and glob predicates are evaluated by the index. Regular expressions use a SQL candidate predicate and an exact residual filter; chained regular expressions are combined with AND. Quantities are converted to the canonical unit recorded by the index, and dimensionally incompatible queries raise rather than silently returning incorrect matches. Values stored without units can never be proven incompatible, so they remain candidates for quantity selectors rather than being silently excluded. -Coordinate predicates select by range — a `(start, stop)` tuple or slice, with `None`/`...` for an open end — or by a patch-local boolean mask; scalar and value-membership coordinate selectors have no exact patch-level meaning and are rejected. Numeric coordinate summaries are stored in canonical SI units, so bare numeric range bounds are interpreted as canonical SI regardless of a patch's native coordinate units, and quantities convert. The exact per-patch trim defers its representation until each patch is known, so a mixed archive of unit-bearing and unitless patches is handled correctly in one selection. +Coordinate predicates select by range — a `(start, stop)` tuple or slice, with `None`/`...` for an open end. Scalar, value-membership, and boolean-sample-mask coordinate selectors have no exact patch-level meaning spool-wide and are rejected (apply masks per patch, e.g. `spool.map(lambda p: p.select(...))`; boolean arrays over patches, `spool[mask]`, still select membership). Numeric coordinate summaries are stored in canonical SI units, so bare numeric range bounds are interpreted as canonical SI regardless of a patch's native coordinate units, and quantities convert. The exact per-patch trim defers its representation until each patch is known, so a mixed archive of unit-bearing and unitless patches is handled correctly in one selection. Coordinate predicates first select patches whose summary envelopes can overlap the request. The loaded patch is then selected exactly. `samples=True` is always patch-local and therefore never excludes a patch at the index stage. `relative=True` resolves coordinate ranges against the current spool view's global envelope; attribute predicates in the same call remain unchanged. -Operations that restructure rows — chunking, sorting, slicing — attach a derived view (an outputs/members/sources relation) to the new spool; selection on such a view filters its output rows directly. Exact selections already attached to the catalog still apply when source patches are resolved. +Restructuring operations that create new patch identities (chunking, concatenation) materialize a derived in-memory catalog whose rows are the plan outputs, so selection on a chunked spool runs the identical catalog engine. Sorting, slicing, and array selection never restructure: they compose lazy order and membership specs on the current catalog. Exact selections already attached to a parent view still apply when member source patches load. Catalog views share their source state. Adding, removing, or rescanning sources invalidates realized metadata so existing views observe the updated catalog under their composed predicates. diff --git a/tests/test_core/test_directory_spool.py b/tests/test_core/test_directory_spool.py index 194995cdc..f6736cc12 100644 --- a/tests/test_core/test_directory_spool.py +++ b/tests/test_core/test_directory_spool.py @@ -225,8 +225,8 @@ def _fake_read(**kwargs): assert patch.attrs["tag"] == "second" -class TestSelectKwargs: - """The select_kwargs constructor parameter restricts contents.""" +class TestSelectedDirectorySpools: + """Selection on directory spools (select_kwargs constructor removed).""" @pytest.fixture(scope="class") def spool_dir(self, random_spool, tmp_path_factory): @@ -245,9 +245,7 @@ def first_patch_range(self, random_spool): def test_contents_restricted(self, spool_dir, random_spool, first_patch_range): """Rows outside the requested range must not appear (regression).""" - spool = Spool.from_directory( - spool_dir, select_kwargs={"time": first_patch_range} - ).update() + spool = Spool.from_directory(spool_dir).update().select(time=first_patch_range) assert 1 <= len(spool) < len(random_spool) contents = spool.get_contents() assert (contents["time_min"] <= first_patch_range[1]).all() @@ -257,30 +255,20 @@ def test_contents_restricted(self, spool_dir, random_spool, first_patch_range): assert time.min() >= first_patch_range[0] assert time.max() <= first_patch_range[1] - def test_restriction_survives_select_and_update( + def test_selected_spool_refuses_update( self, spool_dir, random_spool, first_patch_range ): - """Derived spools keep the constructor restriction.""" - spool = Spool.from_directory( - spool_dir, select_kwargs={"time": first_patch_range} - ).update() - expected = len(spool) - assert len(spool.update()) == expected - distance = random_spool[0].get_coord("distance") - sub = spool.select(distance=(distance.min(), distance.max())) - assert len(sub) == expected - - def test_attr_select_kwargs(self, spool_dir, random_spool): - """Attr-valued select_kwargs filter rows and load cleanly.""" - spool = Spool.from_directory( - spool_dir, select_kwargs={"tag": "random"} - ).update() - assert len(spool) == len(random_spool) - assert isinstance(spool[0], dc.Patch) - empty = Spool.from_directory( - spool_dir, select_kwargs={"tag": "no_such"} - ).update() - assert len(empty) == 0 + """D1: any operation severs update().""" + from dascore.exceptions import InvalidSpoolError + + spool = Spool.from_directory(spool_dir).update().select(time=first_patch_range) + with pytest.raises(InvalidSpoolError, match="root spool"): + spool.update() + + def test_select_kwargs_parameter_removed(self, spool_dir): + """The constructor no longer accepts select_kwargs.""" + with pytest.raises(TypeError, match="select_kwargs"): + Spool.from_directory(spool_dir, select_kwargs={"tag": "x"}) class TestDirectoryIndex: diff --git a/tests/test_core/test_file_spool.py b/tests/test_core/test_file_spool.py index 786e4c2a5..6aa3d836c 100644 --- a/tests/test_core/test_file_spool.py +++ b/tests/test_core/test_file_spool.py @@ -83,4 +83,4 @@ def test_multi_patch_without_source_patch_id_raises(self, tmp_path): file_spool = Spool.from_file(path) kwargs = {"path": str(path), "file_format": "DASDAE", "file_version": "1"} with pytest.raises(PatchAttributeError, match="uniquely resolved"): - file_spool._load_patch(kwargs) + file_spool._catalog.resolver.resolve(kwargs) diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 5a7098407..d62d5c72f 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -81,8 +81,11 @@ def test_patches_match_df_contents(self, random_spool): new_content = new_spool.get_contents() # these should be (nearly) identical. common = set(chunk_df.columns) & set(new_content.columns) - # len fields may differ by ±1 between summary-based and data-based counts - skip = {"history"} | {c for c in common if c.endswith("_len")} + # len fields may differ by ±1 between summary-based and data-based + # counts; identity/provenance columns legitimately differ between + # plan rows and re-scanned live patches + skip = {"history", "path", "file_format", "file_version", "source_patch_id"} + skip |= {c for c in common if c.endswith("_len")} cols = sorted(common - skip) comp1, comp2 = chunk_df[cols], new_content[cols] equal_cols = (comp1 == comp2) | (pd.isnull(comp1) & pd.isnull(comp2)) diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 250e2560b..13b275f25 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -119,11 +119,6 @@ def get_contents(self): assert isinstance(wrapped, Spool) assert list(wrapped) == patches - def test_copy_construct_merge_kwargs(self, random_spool): - """Copy-construction can override the merge policy.""" - new = Spool(random_spool, merge_kwargs={"conflicts": "drop"}) - assert new._merge_kwargs["conflicts"] == "drop" - def test_viz_raises(self, random_spool): """Ensure Spool.viz raises AttributeError.""" msg = "Apply 'viz' on a Patch object" @@ -151,7 +146,7 @@ def test_simple_access_builds_no_dataframes(self, patch_list): assert spool[0] == patch_list[0] assert spool[-1] == patch_list[-1] assert list(spool) == patch_list - assert "_df" not in spool._cache + assert spool._catalog._backend is None def test_out_of_bounds_raises(self, patch_list): """The fast path must raise the same IndexError as the df path.""" @@ -159,14 +154,14 @@ def test_out_of_bounds_raises(self, patch_list): match = "out of bounds for spool" with pytest.raises(IndexError, match=match): _ = spool[len(patch_list)] - assert "_df" not in spool._cache + assert spool._catalog._backend is None def test_access_unchanged_after_df_built(self, patch_list): """Patch access must return the same thing before/after df built.""" spool = dc.spool(patch_list) lazy_patches = list(spool) - _ = spool.get_contents() # forces the dataframes to build - assert "_df" in spool._cache + _ = spool.get_contents() # forces the flat relation to build + assert spool._catalog._backend is not None assert list(spool) == lazy_patches assert spool[0] == lazy_patches[0] @@ -194,11 +189,17 @@ def test_derived_spools_use_df_machinery(self, patch_list): expected_min = min(x.summary.get_coord_summary("time").min for x in patch_list) assert time_coord.min() == expected_min - def test_derived_spool_shares_only_the_catalog(self, patch_list): - """Derived spools resolve patches through the shared catalog only.""" + def test_derived_spool_is_own_catalog(self, patch_list): + """A chunked spool is a fresh derived catalog sharing patches.""" + from dascore.io.index.planned import PlanResolver + spool = dc.spool(patch_list) chunked = spool.chunk(time=1) - assert chunked._catalog is spool._catalog + assert chunked._catalog is not spool._catalog + assert isinstance(chunked._catalog.resolver, PlanResolver) + # member loading shares the parent's live patches, not copies + registry = chunked._catalog.resolver.live_entries() + assert {id(p) for p in registry.values()} <= {id(p) for p in patch_list} # no other patch containers exist on the instance assert "_patches" not in chunked.__dict__ assert "_data" not in chunked.__dict__ @@ -218,16 +219,6 @@ def test_empty_memory_spool(self): assert len(spool) == 0 assert list(spool) == [] - def test_instruction_df_builds_from_lazy_patches(self, patch_list): - """Lazy patch input should still build instruction dataframes on demand.""" - spool = dc.spool(patch_list) - assert len(spool._get_instruction_df()) == len(patch_list) - - def test_instruction_df_property_cold_access(self, patch_list): - """Accessing the instruction frame first still derives everything.""" - spool = dc.spool(patch_list) - assert len(spool._instruction_df) == len(patch_list) - class TestSpoolHelpers: """Tests for helper functions used by spool implementations.""" @@ -952,32 +943,18 @@ def test_union_of_scanless_spool(self, tmp_path): assert len(combined) == 2 def test_union_of_chunked_spool(self, many_contiguous): - """A chunked (restructured) spool's rows no longer map to sources.""" + """A chunked spool is a derived catalog; unions compose it.""" + from dascore.io.index.planned import PlanResolver + chunked = dc.spool(many_contiguous).chunk(time=None) - assert not chunked._catalog_native - assert "_patch_id" not in chunked._df.columns + assert isinstance(chunked._catalog.resolver, PlanResolver) combined = chunked + dc.spool([dc.get_example_patch(tag="other")]) assert len(combined) == 2 + assert all(isinstance(p, dc.Patch) for p in combined) def test_iteration_skips_unresolvable_patch(self, monkeypatch): """A patch that fails to resolve is skipped with a #583 warning.""" - # force the planned state (sort/slice are lazy catalog specs now) - base = dc.spool([dc.get_example_patch()]) - spool = base.new_from_df( - base._df, source_df=base._source_df, instruction_df=base._instruction_df - ) - - def _raise(_ind): - raise MissingPatchError("trimmed to nothing") - - monkeypatch.setattr(spool._assembler, "get_patch", _raise) - with pytest.warns(UserWarning, match="Skipping patch"): - assert list(spool) == [] - - def test_catalog_iteration_skips_unresolvable_patch(self, monkeypatch): - """The catalog fast path also skips with a #583 warning.""" spool = dc.spool([dc.get_example_patch()]) - assert spool._rows_are_catalog() def _raise(_ind): raise MissingPatchError("not available in this session") @@ -986,17 +963,15 @@ def _raise(_ind): with pytest.warns(UserWarning, match="Skipping patch"): assert list(spool) == [] - def test_planned_view_negative_and_bad_index(self): - """Assembler indexing handles negatives and raises out-of-bounds.""" + def test_derived_negative_and_bad_index(self): + """Derived-catalog indexing handles negatives, raises out-of-bounds.""" patches = list(dc.get_example_spool(length=2)) - base = dc.spool(patches) - planned = base.new_from_df( - base._df, source_df=base._source_df, instruction_df=base._instruction_df - ) - assert planned._plan is not None - assert planned[-1] == planned[len(patches) - 1] + derived = dc.spool(patches).concatenate(time=1) + assert derived[-1] == derived[len(patches) - 1] + with pytest.raises(IndexError, match="out of bounds"): + _ = derived[len(patches)] with pytest.raises(IndexError, match="out of bounds"): - _ = planned[len(patches)] + _ = derived[-len(patches) - 1] def test_merge_buffer_grows_when_estimate_short(self, many_contiguous, monkeypatch): """An under-estimated merge buffer is grown to fit (uneven sampling).""" diff --git a/tests/test_core/test_spool_contracts.py b/tests/test_core/test_spool_contracts.py index 1297cf784..3cb56fc02 100644 --- a/tests/test_core/test_spool_contracts.py +++ b/tests/test_core/test_spool_contracts.py @@ -65,20 +65,37 @@ def test_live_only_is_noop(self, patches): spool = dc.spool(patches) assert spool.update() is spool - def test_union_of_live_spools_is_noop(self, patches): - """Combining live spools yields a still-sourceless, current spool.""" + def test_union_raises(self, patches): + """Combining spools is a computation; the result cannot update.""" combined = dc.spool(patches[:1]) + dc.spool(patches[1:]) - assert combined.update() is combined + with pytest.raises(InvalidSpoolError, match="root spool"): + combined.update() def test_union_with_file_rows_raises(self, patches, tmp_path): - """A combined spool with file rows has no update source.""" + """A combined spool with file rows cannot update either.""" directory = dc.examples.spool_to_directory( dc.spool(patches), path=tmp_path / "dir_a" ) combined = dc.spool(directory).update(progress=None) + dc.spool(patches[:1]) - with pytest.raises(InvalidSpoolError, match="no update source"): + with pytest.raises(InvalidSpoolError, match="root spool"): combined.update() + def test_selected_spool_raises(self, patches): + """Any operation severs update: a selected spool refuses it.""" + selected = dc.spool(patches).select(tag="random") + with pytest.raises(InvalidSpoolError, match="root spool"): + selected.update() + + def test_selected_file_spool_raises(self, patches, tmp_path): + """A selected single-file spool refuses update instead of + silently widening back to the whole file (review P1). + """ + path = tmp_path / "sel_file.h5" + dc.write(patches[0], path, "dasdae") + selected = dc.spool(path).select(distance=(0, 10), samples=True) + with pytest.raises(InvalidSpoolError, match="root spool"): + selected.update() + def test_directory_update_picks_up_new_files(self, patches, tmp_path): """The syncer case: new files appear after update().""" directory = tmp_path / "dir_b" @@ -114,15 +131,13 @@ def test_replan_collapses(self, patches): assert time.max() == max(maxs) def test_planned_state_is_derived(self, patches): - """A planned spool reports non-native; identity views native.""" + """A chunked spool is a fresh derived catalog, not a mode flag.""" + from dascore.io.index.planned import PlanResolver + spool = dc.spool(patches) - assert spool._catalog_native chunked = spool.chunk(time=2) - assert not chunked._catalog_native - assert chunked._plan is not None - # the flag cannot be assigned; the plan is the state - with pytest.raises(AttributeError): - chunked._catalog_native = True + assert chunked._catalog is not spool._catalog + assert isinstance(chunked._catalog.resolver, PlanResolver) class TestTypeSurface: diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 673cec08d..6ccf71bed 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -17,16 +17,16 @@ @pytest.fixture( scope="module", - params=("memory", "directory", "memory_df", "directory_df"), + params=("memory", "directory", "memory_derived", "directory_derived"), ) def spool(request, tmp_path_factory): """ - The same patches served by each spool type and select-path state. + The same patches served by each spool type and catalog state. - The ``*_df`` params force the materialized (dataframe) state via a - content-preserving sort, so every spec test runs over both select - implementations (catalog-native and dataframe) — the parity net for - collapsing the dual-state spool internals. + The ``*_derived`` params run every spec test over a derived + (plan-backed) catalog via a content-preserving concatenate — the + parity net proving one selector engine serves identity and + restructured spools alike. """ base = dc.get_example_spool("random_das") if request.param.startswith("memory"): @@ -36,13 +36,11 @@ def spool(request, tmp_path_factory): base, path=tmp_path_factory.mktemp("select_spec") ) out = dc.spool(path).update(progress=None) - if request.param.endswith("_df"): - # force the planned (dataframe) state explicitly: sort/slice are - # lazy catalog specs now, so only a plan materializes the frames - out = out.new_from_df( - out._df, source_df=out._source_df, instruction_df=out._instruction_df - ) - assert not out._catalog_native, "planned state expected" + if request.param.endswith("_derived"): + from dascore.io.index.planned import PlanResolver + + out = out.concatenate(time=1) + assert isinstance(out._catalog.resolver, PlanResolver) return out @@ -90,8 +88,6 @@ class TestCatalogPushdown: def test_coord_predicate_reaches_backend(self, spool, monkeypatch): """Selection does not query all rows before applying its predicate.""" - if not spool._catalog_native: - pytest.skip("query pushdown only applies to catalog-native spools") catalog = spool._catalog backend = catalog.backend calls = [] @@ -145,7 +141,7 @@ def test_cold_directory_select(self, tmp_path_factory, forbid_realization): fresh = dc.spool(path) forbid_realization() selected = fresh.select(time=("2020-01-03", "2020-01-04")) - assert selected._catalog_native + assert selected._catalog.is_view def test_cold_memory_select(self, forbid_realization): """A fresh patch-list spool selects via the catalog, lazily.""" @@ -153,7 +149,7 @@ def test_cold_memory_select(self, forbid_realization): forbid_realization() fresh = dc.spool(patches) selected = fresh.select(tag="random") - assert selected._catalog_native + assert selected._catalog.is_view class TestSamples: @@ -183,9 +179,7 @@ def test_non_coord_raises(self, spool): def test_samples_on_materialized_spool(self, spool): """Samples select works after chunk (the dataframe select path).""" - # chunk first so the derived spool is materialized, not catalog-native materialized = spool.chunk(time=None) - assert not materialized._catalog_native out = materialized.select(distance=(0, 10), samples=True) assert len(out) == len(materialized) assert len(out[0].get_coord("distance")) == 10 @@ -225,7 +219,6 @@ def test_namespaced_coord_with_attr(self, spool): def test_relative_on_materialized_spool(self, spool): """Relative select works after chunk (the dataframe select path).""" materialized = spool.chunk(time=None) - assert not materialized._catalog_native gmin = materialized.get_contents()["time_min"].min() gmax = materialized.get_contents()["time_max"].max() out = materialized.select(time=(1, -1), relative=True) @@ -241,7 +234,6 @@ class TestMaterializedNamespaces: def test_namespaces_and_unknown_names(self, spool): """Namespaced selects and unknown-name errors on a chunked spool.""" materialized = spool.chunk(time=None) - assert not materialized._catalog_native assert len(materialized.select(_attrs={"tag": "random"})) == len(materialized) # a valid _coords range narrows the materialized spool df = materialized.get_contents() @@ -262,7 +254,6 @@ def test_namespaces_and_unknown_names(self, spool): def test_duplicate_namespace_raises(self, spool): """A name in both explicit namespaces raises on either path.""" materialized = spool.chunk(time=None) - assert not materialized._catalog_native for target in (spool, materialized): with pytest.raises(InvalidSpoolQueryError, match="both _attrs and _coords"): target.select(_attrs={"time": (None, None)}, _coords={"time": (1, 2)}) @@ -270,7 +261,6 @@ def test_duplicate_namespace_raises(self, spool): def test_slice_range_form(self, spool): """Slice selectors resolve the same on either path (#435 spec).""" materialized = spool.chunk(time=None) - assert not materialized._catalog_native t0 = spool.get_contents()["time_min"].min() window = slice(t0, t0 + np.timedelta64(2, "s")) for target in (spool, materialized): @@ -464,7 +454,7 @@ def test_sort_is_lazy_and_ordered(self, tmp_path_factory): patches = list(dc.get_example_spool("random_das")) spool = dc.spool(list(reversed(patches))) out = spool.sort("time") - assert out._catalog_native # no plan materialized + assert not out._catalog.is_view or out._catalog._order is not None df = out.get_contents() assert df["time_min"].is_monotonic_increasing assert list(out) == patches @@ -474,7 +464,7 @@ def test_slice_is_lazy_window(self): patches = list(dc.get_example_spool("random_das")) spool = dc.spool(patches) part = spool[1:] - assert part._catalog_native + assert part._catalog._ids is not None # lazy id membership assert len(part) == len(patches) - 1 assert list(part) == patches[1:] diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 2219a8a37..36f0a33f4 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -50,11 +50,8 @@ def test_union_of_materialized_member(self): """A planned (materialized but catalog-backed) member unions by ids.""" sp = dc.get_example_spool("random_das") other = dc.get_example_spool("diverse_das") - # force the planned state; sort/slice are lazy specs now - materialized = sp.new_from_df( - sp._df, source_df=sp._source_df, instruction_df=sp._instruction_df - ) - assert not materialized._catalog_native + # a content-preserving derived catalog (concat groups of one) + materialized = sp.concatenate(time=1) combined = materialized + other assert len(combined) == len(sp) + len(other) @@ -129,13 +126,13 @@ def test_same_source_dedups(self, dir_spool): assert len(combined) == len(dir_spool) def test_constructor_select_kwargs_restrict_union(self, tmp_path): - """A select_kwargs-restricted directory spool unions only its rows.""" + """A selection-restricted directory spool unions only its rows.""" base = dc.get_example_spool("random_das") dc.examples.spool_to_directory(base, path=tmp_path) full = dc.spool(tmp_path).update() df = full.get_contents().sort_values("time_min") window = (df["time_min"].iloc[0], df["time_max"].iloc[0]) # first patch - restricted = dc.spool(tmp_path, select_kwargs={"time": window}) + restricted = full.select(time=window) assert 0 < len(restricted) < len(full) combined = restricted + dc.spool([dc.get_example_patch(tag="mem")]) # the union must not reintroduce the rows the constructor excluded From 2066e987e29800a58d97a03fb215786cb2e6c631 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 07:38:30 +0200 Subject: [PATCH 90/97] Close the review coverage: tests for every surviving branch, dead code removed The PatchAssembler loses its orphaned indexing front end and post- select plumbing (the plan resolver only consumes the merge internals), get_column_names_from_dim loses its last caller, and the planned- catalog converters simplify to the pandas scalar forms that actually reach them. New tests pin the operation-order compositions the review demanded: collapse with value and quantity residuals, regex+window+ attr-sort chains, attr membership arrays, envelope-column sort names, membership-restricted pickling of union views, third-party BaseSpool members, negative samples windows, and complete-envelope-overlap merges. Touched modules are back to 100% coverage locally. --- dascore/io/index/planned.py | 33 +--- dascore/utils/patch_assembly.py | 59 +----- dascore/utils/pd.py | 10 - tests/test_core/test_patch_chunk.py | 8 +- tests/test_io/test_index/test_planned.py | 224 +++++++++++++++++++++++ tests/test_utils/test_patch_utils.py | 19 ++ 6 files changed, 260 insertions(+), 93 deletions(-) create mode 100644 tests/test_io/test_index/test_planned.py diff --git a/dascore/io/index/planned.py b/dascore/io/index/planned.py index a84c6ea6d..1f6fd5363 100644 --- a/dascore/io/index/planned.py +++ b/dascore/io/index/planned.py @@ -39,7 +39,6 @@ ) from dascore.utils.misc import is_range from dascore.utils.pd import adjust_segments -from dascore.utils.time import to_int PLAN_SCHEME = "plan://" # columns that are structural/positional rather than patch attributes @@ -50,9 +49,9 @@ def _ns(value) -> int | None: """Convert a datetime/timedelta-like envelope value to ns int.""" if value is None or pd.isnull(value): return None - if isinstance(value, pd.Timestamp | pd.Timedelta): - return int(value.value) - return int(to_int(value)) + if isinstance(value, pd.Timedelta | np.timedelta64): + return int(pd.Timedelta(value).value) + return int(pd.Timestamp(value).value) def _num(value) -> float | None: @@ -79,10 +78,8 @@ def _coord_record_from_row(row: Mapping, name: str) -> CoordRecord | None: return None step = row.get(f"{name}_step") step = None if step is None or pd.isnull(step) else step - if isinstance(lo, pd.Timestamp): - lo, hi = lo.to_datetime64(), pd.Timestamp(hi).to_datetime64() - dtype = "datetime64[ns]" - elif isinstance(lo, np.datetime64): + if isinstance(lo, pd.Timestamp | np.datetime64): + lo, hi = pd.Timestamp(lo).to_datetime64(), pd.Timestamp(hi).to_datetime64() dtype = "datetime64[ns]" elif isinstance(lo, pd.Timedelta | np.timedelta64): lo, hi = pd.Timedelta(lo).to_timedelta64(), pd.Timedelta(hi).to_timedelta64() @@ -93,6 +90,10 @@ def _coord_record_from_row(row: Mapping, name: str) -> CoordRecord | None: dtype = "float64" if isinstance(step, pd.Timedelta): step = step.to_timedelta64() + if step is not None and not step: + # a degenerate (zero) step is not a range; drop it rather than + # letting range reconstruction divide by it + step = None units = row.get(f"{name}_units") # numeric envelope values are stored canonical-SI; attaching the # original unit string would make ingest re-convert them @@ -100,10 +101,7 @@ def _coord_record_from_row(row: Mapping, name: str) -> CoordRecord | None: units = None length = None if step is not None: - try: - length = int(round((hi - lo) / step)) + 1 - except (TypeError, ZeroDivisionError): - length = None + length = int(round((hi - lo) / step)) + 1 key = row.get(f"_{name}_def_key") fingerprint = None if isinstance(key, str) and key.startswith("fp:"): @@ -227,19 +225,8 @@ def _assembler(self): from dascore.utils.patch_assembly import PatchAssembler return PatchAssembler( - df=None, - source_df=None, - instruction_df=None, load_patch=self._load_member, merge_kwargs=self.merge_kwargs, - post_selects=(), - drop_columns=( - "patch", - "path", - "file_format", - "file_version", - "source_patch_id", - ), ) def _load_member(self, kwargs: Mapping) -> dc.Patch: diff --git a/dascore/utils/patch_assembly.py b/dascore/utils/patch_assembly.py index 588551411..8d56b9a9e 100644 --- a/dascore/utils/patch_assembly.py +++ b/dascore/utils/patch_assembly.py @@ -13,7 +13,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass import numpy as np import pandas as pd @@ -88,59 +88,16 @@ def _coord_only_kwargs(patch, kwargs) -> dict: @dataclass class PatchAssembler: """ - Assemble patches for one spool view. + Assemble output patches from joined member rows. - The frames define the view (presented rows, member instructions, - source rows); ``load_patch`` resolves one joined row to its source - patch; policy fields carry the merge behavior and patch-local - post-selections. Instances cache the instruction-row index and are - themselves cached per spool view. + ``load_patch`` resolves one member row to its source patch (residual + selections included); ``merge_kwargs`` carries the merge behavior. + The plan resolver hands this the joined member frame for one output + at a time. """ - df: pd.DataFrame - source_df: pd.DataFrame - instruction_df: pd.DataFrame load_patch: Callable[[Mapping], dc.Patch] merge_kwargs: Mapping - post_selects: tuple = () - drop_columns: tuple = () - _indices: dict | None = field(default=None, repr=False) - - def get_patch(self, df_ind: int) -> dc.Patch: - """Assemble the single patch presented at a row index.""" - patches = self.get_patches_from_index(df_ind) - assert len(patches) == 1 - return patches[0] - - def get_patches_from_index(self, df_ind): - """Given an index (from current df), return the corresponding patch.""" - source = self.source_df - instruction = self.instruction_df - # handle negative index; a still-negative value after - # normalization is out of bounds and must never wrap around - requested = df_ind - if df_ind < 0: - df_ind = len(self.df) + df_ind - if not 0 <= df_ind < len(self.df): - msg = f"index of [{requested}] is out of bounds for spool." - raise IndexError(msg) - inds = self.df.index[df_ind] - # Group positional instruction rows by current index (and cache) to - # avoid a full instruction df scan for each requested patch. - if self._indices is None: - self._indices = instruction.groupby("current_index").indices - positions = self._indices.get(inds) - assert positions is not None and len(positions), "no instructions found" - df1 = instruction.iloc[positions] - joined = df1.join(source.drop(columns=df1.columns, errors="ignore")) - # Occasionally, duplicates can creep into the source_df, - # but it costs a bit to check for duplicates, so only check and drop - # duplicates on large joined dataframes where performance might be - # affected. - if len(joined) > 10: - cols = set(joined.columns) - set(self.drop_columns) - joined = joined.drop_duplicates(subset=list(cols), keep="first") - return self._patch_from_instruction_df(joined) def _patch_from_instruction_df(self, joined): """Get the patches joined columns of instruction df.""" @@ -183,10 +140,6 @@ def _load_trimmed_patch(self, patch_kwargs, joined) -> dc.Patch: # are valid patch selections. if select_kwargs := _coord_only_kwargs(patch, source_kwargs): patch = patch.select(**select_kwargs) - # patch-local selections (samples=True) recorded by spool.select - for post_kwargs, samples in self.post_selects: - if usable := _coord_only_kwargs(patch, post_kwargs): - patch = patch.select(**usable, samples=samples) return patch def _merge_patches_streaming(self, joined, df_dict_list, merge_dim, samples): diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index 803875ea5..b4da3c0b7 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -466,16 +466,6 @@ def get_dim_names_from_columns(df: pd.DataFrame) -> list[str]: return sorted(out) -def get_column_names_from_dim(dims: Sequence[str]) -> list: - """Get column names from a sequence of dimensions.""" - out = [] - for name in dims: - out.append(f"{name}_min") - out.append(f"{name}_max") - out.append(f"{name}_step") - return out - - def fill_defaults_from_pydantic(df, base_model: type[BaseModel]): """ Fill missing columns in dataframe with defaults from base_model. diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index d62d5c72f..6e39e7aac 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -694,13 +694,7 @@ def _bare_assembler(): """An assembler with no frames, for direct streaming-merge tests.""" from dascore.utils.patch_assembly import PatchAssembler - return PatchAssembler( - df=None, - source_df=None, - instruction_df=None, - load_patch=None, - merge_kwargs={}, - ) + return PatchAssembler(load_patch=None, merge_kwargs={}) class TestStreamingMerge: diff --git a/tests/test_io/test_index/test_planned.py b/tests/test_io/test_index/test_planned.py new file mode 100644 index 000000000..d4a3d5f1d --- /dev/null +++ b/tests/test_io/test_index/test_planned.py @@ -0,0 +1,224 @@ +""" +Tests for derived catalogs (plan-as-catalog) and coverage of their edges. +""" + +from __future__ import annotations + +import re + +import numpy as np +import pandas as pd +import pytest + +import dascore as dc +from dascore.exceptions import ParameterError +from dascore.io.index.planned import ( + PlanResolver, + _coord_record_from_row, + _ns, + derived_catalog, +) + + +@pytest.fixture(scope="module") +def patches(): + """Three contiguous example patches.""" + return list(dc.get_example_spool("random_das")) + + +class TestHelpers: + """Unit coverage for the conversion helpers.""" + + def test_ns_forms(self): + """All datetime/timedelta forms convert to the same ns.""" + ts = pd.Timestamp("2020-01-01") + assert _ns(ts) == _ns(ts.to_datetime64()) == ts.value + td = pd.Timedelta(seconds=1) + assert _ns(td) == _ns(td.to_timedelta64()) == td.value + assert _ns(None) is None + + def test_coord_record_numpy_datetimes(self): + """np.datetime64 envelope values build the same record.""" + lo = np.datetime64("2020-01-01", "ns") + hi = np.datetime64("2020-01-02", "ns") + row = {"time_min": lo, "time_max": hi, "time_step": np.timedelta64(1, "s")} + record = _coord_record_from_row(row, "time") + assert record.value_kind == "time" + assert record.min_ns == _ns(lo) + + def test_coord_record_zero_step_length(self): + """A degenerate step leaves length unknown instead of raising.""" + row = {"time_min": 0.0, "time_max": 1.0, "time_step": 0.0} + record = _coord_record_from_row(row, "time") + assert record.length is None + + def test_plan_resolver_requires_output_id(self): + """member_rows without output_id is a construction error.""" + with pytest.raises(ValueError, match="output_id"): + PlanResolver( + token="x", + dim="time", + member_rows=pd.DataFrame({"path": []}), + loader=None, + merge_kwargs={}, + ) + + def test_derived_catalog_adds_patch_ids(self, patches): + """source_rows without _patch_id get positional ids.""" + from dascore.utils.chunk_plan import ChunkPlan + + spool = dc.spool(patches) + rows = spool.get_contents().drop(columns=["_patch_id"], errors="ignore") + rows = rows.reset_index(drop=True) + members = pd.DataFrame( + {"output_id": [0], "_patch_id": [0], "_modified": [False]} + ) + outputs = rows.iloc[:1].assign(output_id=0) + plan = ChunkPlan(outputs, members, "time", None, {}) + catalog = derived_catalog( + source_rows=rows, + plan=plan, + parent=spool._catalog, + merge_kwargs={}, + mode="concat", + ) + assert len(catalog) == 1 + + +class TestDerivedComposition: + """Operation-order coverage over derived catalogs.""" + + def test_collapse_with_value_residual(self, patches): + """Chunk of a selected chunked spool re-plans from trimmed members.""" + t0 = patches[0].get_coord("time").min() + t1 = patches[1].get_coord("time").max() + chunked = dc.spool(patches).chunk(time=2) + selected = chunked.select(time=(t0, t1)) + merged = selected.chunk(time=None) + assert len(merged) >= 1 + out = merged[0] + assert out.get_coord("time").min() >= t0 + + def test_sort_by_attr_on_windowed_regex_view(self, patches): + """Regex selection + window + attr sort compose through SQL.""" + tagged = [ + p.update_attrs(tag=f"t{num}", history=[]) for num, p in enumerate(patches) + ] + spool = dc.spool(tagged) + view = spool.select(tag=re.compile("t[0-9]"))[1:] + out = view.sort("tag") + tags = [p.attrs["tag"] for p in out] + assert tags == sorted(tags) + + def test_attr_membership_array(self, patches): + """Attr membership with a numpy array of values selects rows.""" + tagged = [ + p.update_attrs(tag=f"t{num}", history=[]) for num, p in enumerate(patches) + ] + spool = dc.spool(tagged) + out = spool.select(tag=np.array(["t0", "t2"])) + assert len(out) == 2 + + def test_sort_by_envelope_column_name(self, patches): + """Sort accepts the explicit `{dim}_min` column form.""" + spool = dc.spool(list(reversed(patches))) + out = spool.sort("time_min") + assert out.get_contents()["time_min"].is_monotonic_increasing + + def test_concatenate_requires_one_kwarg(self, patches): + """Concatenate validates its dimension keyword.""" + with pytest.raises(ParameterError, match="exactly one dimension"): + dc.spool(patches).concatenate(time=None, distance=None) + + def test_union_view_of_live_spools_pickles_composite(self, patches): + """A selected union pickles a membership-restricted composite.""" + import pickle + + t0 = patches[0].get_coord("time") + combined = dc.spool(patches[:2]) + dc.spool(patches[2:]) + view = combined.select(time=(None, t0.max())) + assert len(view) == 1 + loaded = pickle.loads(pickle.dumps(view)) + assert len(loaded) == 1 + assert isinstance(loaded[0], dc.Patch) + + def test_missing_live_patch_getitem(self, patches): + """A missing registry entry surfaces as MissingPatchError, not + out-of-bounds. + """ + from dascore.exceptions import MissingPatchError + + spool = dc.spool(patches[:1]) + _ = spool.get_contents() # realize rows + spool._catalog.resolver._registry.clear() + with pytest.raises(MissingPatchError, match="not available"): + spool[0] + + def test_union_with_third_party_spool(self, patches): + """The BaseSpool fallback materializes third-party members.""" + from dascore.core.spool import BaseSpool + + class MiniSpool(BaseSpool): + def __init__(self, inner): + self._inner = list(inner) + + def __getitem__(self, item): + return self._inner[item] + + def __iter__(self): + return iter(self._inner) + + def __len__(self): + return len(self._inner) + + def chunk(self, **kwargs): + raise NotImplementedError + + def select(self, **kwargs): + raise NotImplementedError + + def get_contents(self): + raise NotImplementedError + + combined = dc.spool(patches[:1]) + MiniSpool(patches[1:]) + assert len(combined) == len(patches) + + def test_samples_negative_index_skips_envelope_adjust(self, patches): + """Negative samples windows stay candidacy supersets (no crash).""" + spool = dc.spool(patches[:1]).select(time=(0, -10), samples=True) + merged = spool.chunk(time=None) + assert len(merged) == 1 + + def test_complete_overlap_merge(self): + """Two identical-envelope patches merge by keeping the first.""" + patch = dc.get_example_patch() + twin = patch.new() + merged = dc.spool([patch, twin]).chunk(time=None) + assert len(merged) == 1 + assert isinstance(merged[0], dc.Patch) + + +class TestRemainingEdges: + """Direct coverage of defensive/rare branches.""" + + def test_collapse_with_quantity_residual(self, patches): + """A quantity-selected chunked view re-chunks without applying + unit-bearing bounds to envelopes (they stay load residuals). + """ + from dascore.units import m + + chunked = dc.spool(patches).chunk(time=2) + selected = chunked.select(_coords={"distance": (0 * m, 10 * m)}) + merged = selected.chunk(time=None) + assert len(merged) == 1 + coord = merged[0].get_coord("distance") + assert float(coord.max()) <= 10 + + def test_samples_adjust_skips_missing_columns(self): + """Residuals naming absent envelope columns pass through.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + df = pd.DataFrame({"time_min": [0.0], "time_max": [1.0]}) + residuals = (({"depth": (0, 5)}, True),) + out = samples_adjusted_envelopes(df, residuals) + assert out.equals(df) diff --git a/tests/test_utils/test_patch_utils.py b/tests/test_utils/test_patch_utils.py index 6a8e4c5ed..b19f0ece5 100644 --- a/tests/test_utils/test_patch_utils.py +++ b/tests/test_utils/test_patch_utils.py @@ -1133,3 +1133,22 @@ def test_none_overlap_matches_default(self, random_patch): random_patch, distance=self.window * step, overlap=None ) assert out == (self.window, random_patch.get_axis("distance"), None) + + +class TestForcePatchMergeOverlap: + """_force_patch_merge tolerates complete-envelope overlap (keep first).""" + + def test_complete_overlap_keeps_first(self, random_patch): + """Identical envelopes merge to the first patch.""" + from dascore.utils.patch import _force_patch_merge + + twin = random_patch.new() + infos = [] + for patch in (random_patch, twin): + info = patch.coords._get_dim_summary() + info["patch"] = patch + info["dims"] = ",".join(patch.dims) + infos.append(info) + out = _force_patch_merge(infos, merge_kwargs={}) + assert len(out) == 1 + assert out[0]["patch"] is random_patch From 5c4270e1ecbd96cebed0e106d1ec8ddb8942e1bf Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 10:56:20 +0200 Subject: [PATCH 91/97] Fix the third-round review findings: unions keep contents, catalogs keep every coordinate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combining spools now preserves each operand's current contents: __add__ auto-materializes operands carrying union-lossy lazy state (coordinate and samples residual trims, sort specs) into identity-plan derived catalogs — table work only, no patch loads — while membership-style state still unions by rows so identity dedup keeps working. Derived catalogs record every coordinate of their members (numeric, time, and string), aggregated from the member source rows, with def-key identity kept only when values provably survive assembly; identity claims are likewise dropped for any coordinate riding a residual-trimmed dim. Sorting accepts any known coordinate (non-hot names order by their coord_defs envelope minimum through a correlated subquery), samples envelopes use the last included index and respect orientation, sampling groups compare magnitudes against a stable anchor (descending contiguous patches now merge; tolerance chains can no longer drift), concatenating an empty spool returns an empty spool, and the removed PyTables reader/writer aliases are documented in the changelog. The _attrs/_coords namespaces additionally accept a name or collection of names tagging bare kwargs. --- dascore/core/spool.py | 78 +++++++++++-- dascore/io/index/backend.py | 21 +++- dascore/io/index/planned.py | 129 +++++++++++++++++++++- dascore/io/index/query.py | 47 +++++--- dascore/utils/chunk_plan.py | 69 +++++++++--- dascore/utils/hdf5.py | 4 - dascore/utils/pd.py | 31 +++++- docs/changelog.qmd | 5 +- docs/notes/spool_selection.qmd | 2 +- tests/test_core/test_patch_chunk.py | 20 ++++ tests/test_core/test_spool.py | 11 ++ tests/test_core/test_spool_select_spec.py | 50 +++++++++ tests/test_io/test_index/test_ordering.py | 52 +++++++++ tests/test_io/test_index/test_plan.py | 108 ++++++++++++++++++ tests/test_io/test_index/test_planned.py | 94 ++++++++++++++++ tests/test_io/test_index/test_union.py | 103 ++++++++++++++++- 16 files changed, 768 insertions(+), 56 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 1d3d8702a..c95ea8bba 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -226,11 +226,13 @@ def select(self, **kwargs) -> Self: Parameters ---------- _attrs - A dict of attribute selections; names validate as attributes - only (disambiguates names shared with coordinates). + Attribute selections: a dict of ``name -> selector`` (the + general form — required when a name cannot be a Python + keyword) or a name/collection of names tagging bare kwargs + as attributes (disambiguates names shared with coordinates). _coords - A dict of coordinate selections; names validate as - coordinates only. + Coordinate selections; same forms as ``_attrs``, validating + names as coordinates only. samples If True, selections are coordinate-only and given in sample indices; they never exclude patches, but are applied to each @@ -584,8 +586,61 @@ def _new_from_catalog(self, catalog) -> Self: return new def _as_catalog_member(self): - """Return (catalog, patch_ids) describing this spool for a union.""" - return self._catalog, None + """ + Return (catalog, patch_ids) describing this spool for a union. + + Row membership (attr predicates, windows, id arrays) survives a + table union as-is, but residual trims and order specs live + Python-side and would silently vanish; a spool carrying those + first bakes them into a derived catalog (tables only — no patch + data is loaded). + """ + catalog = self._catalog + if catalog._residuals or catalog._order is not None: + return self._materialize_lossy(), None + return catalog, None + + def _materialize_lossy(self): + """ + Bake residual trims and presentation order into a derived catalog. + + An identity plan over the view's presented rows: one output per + row (in presentation order, so ordinals record the order spec), + with trimmed envelopes as the output envelopes and the trims + themselves re-applied at load through the plan resolver. + """ + from dascore.io.index.planned import derived_catalog + from dascore.utils.chunk_plan import ( + _SOURCE_COLUMNS, + ChunkPlan, + samples_adjusted_envelopes, + ) + + rows = self._df.reset_index(drop=True) + working = samples_adjusted_envelopes(rows, self._catalog._residuals) + working = working.reset_index(drop=True) + ids = np.arange(len(working), dtype=np.int64) + # outputs are not file rows: source bookkeeping stays on the + # members (where loading needs it), never on the derived rows + outputs = working.drop( + columns=["_patch_id", *_SOURCE_COLUMNS], errors="ignore" + ).assign(output_id=ids) + members = pd.DataFrame( + { + "output_id": ids, + "_patch_id": working.get("_patch_id", pd.Series(dtype=object)).values, + "_modified": False, + } + ) + plan = ChunkPlan(outputs, members, "", None, {}) + return derived_catalog( + source_rows=working, + plan=plan, + parent=self._catalog, + merge_kwargs={}, + mode="identity", + origin_path=self.spool_path, + ) # --- restructuring (materializing) operations ----------------------- @@ -745,7 +800,16 @@ def concatenate(self, check_behavior: WARN_LEVELS = "warn", **kwargs) -> Self: first.pop("_patch_id", None) output_rows.append(first) outputs = pd.DataFrame(output_rows) - members = pd.concat(member_frames, ignore_index=True) + if member_frames: + members = pd.concat(member_frames, ignore_index=True) + else: # nothing to concatenate: an empty spool stays empty + members = pd.DataFrame( + { + "output_id": pd.Series(dtype=np.int64), + "_patch_id": pd.Series(dtype=object), + "_modified": pd.Series(dtype=bool), + } + ) plan = ChunkPlan(outputs, members, dim, None, {}) catalog = derived_catalog( source_rows=source_rows, diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 160e3818c..d114b8fac 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -645,23 +645,26 @@ def delete_sources(self, source_paths: list[str], base_uri: str = "") -> None: # --- queries ----------------------------------------------------- - def _query_context(self, query): + def _query_context(self, query, order_by=None): """ Normalize a query (or several) and fetch the metadata SQL needs. Returns ``(queries, attr_meta, coord_meta)``; coord metadata is - only consulted for coord predicates, so the (whole-relation - DISTINCT) scan is skipped for attr-only/empty queries. + only consulted for coord predicates and coord ordering, so the + (whole-relation DISTINCT) scan is skipped for attr-only/empty + queries. """ queries = _as_query_list(query if query is not None else Query()) attr_meta = self._attr_meta() coord_names = {name for q in queries for name in q.coords} + if order_by is not None and order_by[0] == "coord": + coord_names.add(order_by[1]) coord_meta = self._coord_meta(coord_names) if coord_names else pd.DataFrame() return queries, attr_meta, coord_meta def query(self, query=None, order_by=None, patch_ids=None) -> pd.DataFrame: """Return the flat patch-row relation for a query (or several).""" - queries, attr_meta, coord_meta = self._query_context(query) + queries, attr_meta, coord_meta = self._query_context(query, order_by=order_by) sql, params, residuals = build_sql( queries, self.dialect, @@ -679,7 +682,7 @@ def query(self, query=None, order_by=None, patch_ids=None) -> pd.DataFrame: def query_ids(self, query=None, order_by=None, patch_ids=None) -> list[int]: """Return matching patch ids in presentation order (ids only).""" - queries, attr_meta, coord_meta = self._query_context(query) + queries, attr_meta, coord_meta = self._query_context(query, order_by=order_by) sql, params, residuals = build_sql( queries, self.dialect, @@ -967,6 +970,14 @@ def coord_names(self) -> set[str]: df = self._fetch_df("SELECT DISTINCT coord_name FROM patch_coords") return set(df["coord_name"]) + def coord_dims_map(self) -> dict[str, str]: + """Return each coord name's dims string (first observed wins).""" + df = self._fetch_df("SELECT DISTINCT coord_name, coord_dims FROM patch_coords") + out: dict[str, str] = {} + for name, dims in zip(df["coord_name"], df["coord_dims"]): + out.setdefault(str(name), str(dims)) + return out + def resolve_query( backend: AbstractIndexBackend, _attrs=None, _coords=None, **kwargs diff --git a/dascore/io/index/planned.py b/dascore/io/index/planned.py index 1f6fd5363..5e962a89b 100644 --- a/dascore/io/index/planned.py +++ b/dascore/io/index/planned.py @@ -61,23 +61,45 @@ def _num(value) -> float | None: return float(value) -def _coord_record_from_row(row: Mapping, name: str) -> CoordRecord | None: +def _coord_record_from_row( + row: Mapping, name: str, dims: tuple[str, ...] | None = None +) -> CoordRecord | None: """ - Build the envelope coord record for one output dimension. + Build the envelope coord record for one output coordinate. Delegates to the ingest converter through a range CoordSummary so virtual outputs carry the same identities real patches would: a carried ``fp:`` def key survives for non-planned dims, and the - planned dim's range fingerprint is reconstructed exactly. + planned dim's range fingerprint is reconstructed exactly. ``dims`` + names the dimensions the coordinate rides (itself by default). """ from dascore.core.coords import CoordSummary from dascore.io.index.ingest import _coord_record + dims = (name,) if dims is None else dims lo, hi = row.get(f"{name}_min"), row.get(f"{name}_max") if lo is None or (pd.isnull(lo) and pd.isnull(hi)): return None step = row.get(f"{name}_step") step = None if step is None or pd.isnull(step) else step + if isinstance(lo, str): + # string coords have no range representation; store the + # lexicographic envelope directly + key = row.get(f"_{name}_def_key") + fingerprint = ( + key[3:] if isinstance(key, str) and key.startswith("fp:") else None + ) + return CoordRecord( + coord_name=name, + value_kind="str", + dtype="str", + coord_dims=",".join(dims), + length=None, + units=None, + min_str=str(lo), + max_str=None if hi is None or pd.isnull(hi) else str(hi), + coord_hash=fingerprint, + ) if isinstance(lo, pd.Timestamp | np.datetime64): lo, hi = pd.Timestamp(lo).to_datetime64(), pd.Timestamp(hi).to_datetime64() dtype = "datetime64[ns]" @@ -112,16 +134,82 @@ def _coord_record_from_row(row: Mapping, name: str) -> CoordRecord | None: max=hi, step=step, units=units, - dims=(name,), + dims=dims, len=length, fingerprint=fingerprint, ) return _coord_record(name, summary) -def _output_records(outputs: pd.DataFrame, token: str) -> list[SourceRecord]: +def _aux_coord_info( + source_rows: pd.DataFrame, + members: pd.DataFrame, + plan_dim: str, + coord_dims_map: Mapping[str, str], + trimmed_dims: frozenset[str] = frozenset(), +) -> dict[int, dict[str, dict]]: + """ + Aggregate per-output envelope info for auxiliary coordinates. + + Aggregated from the *member source rows* (authoritative, unlike the + planner's carried columns). Structural identity (def key and step, + which permit fingerprint claims) is kept only when every member + shares one def key and the values provably survive assembly: a + coordinate riding the planned dimension is trimmed/merged with it, + so only a lone unmodified member keeps identity there. Envelopes + always aggregate — the catalog contract is candidacy, with exact + values re-established at load. + """ + out: dict[int, dict[str, dict]] = {} + if not len(members) or not coord_dims_map: + return out + cols = [c for c in ("output_id", "_patch_id", "_modified") if c in members.columns] + joined = members[cols].merge(source_rows, on="_patch_id", how="left") + for name, dims_str in coord_dims_map.items(): + cmin, cmax = f"{name}_min", f"{name}_max" + if cmin not in joined.columns: + continue + dims = tuple(d for d in str(dims_str).split(",") if d) + rides = plan_dim in dims + # a residual selection trims the dims it rides at load, changing + # the values of every coordinate on those dims + trimmed = bool(set(dims) & trimmed_dims) + key_col, step_col = f"_{name}_def_key", f"{name}_step" + for output_id, sub in joined.groupby("output_id"): + lo, hi = sub[cmin].min(), sub[cmax].max() + if pd.isnull(lo) and pd.isnull(hi): + continue + keys = set(sub[key_col].dropna()) if key_col in sub.columns else set() + modified = bool(sub["_modified"].any()) if "_modified" in sub else False + keep = ( + len(keys) == 1 + and not trimmed + and (not rides or (len(sub) == 1 and not modified)) + ) + steps = ( + set(sub[step_col].dropna()) + if keep and step_col in sub.columns + else set() + ) + info = { + cmin: lo, + cmax: hi, + step_col: steps.pop() if len(steps) == 1 else None, + key_col: keys.pop() if keep else None, + "dims": dims, + } + out.setdefault(int(output_id), {})[name] = info + return out + + +def _output_records( + outputs: pd.DataFrame, + token: str, + aux_info: Mapping[int, Mapping[str, Mapping]] | None = None, +) -> list[SourceRecord]: """Convert plan output rows into ingestible source records.""" records = [] + aux_info = aux_info or {} envelope_suffixes = ("_min", "_max", "_step", "_units") for row in outputs.to_dict("records"): output_id = int(row["output_id"]) @@ -132,6 +220,14 @@ def _output_records(outputs: pd.DataFrame, token: str) -> list[SourceRecord]: record = _coord_record_from_row(row, name) if record is not None: coords.append(record) + # auxiliary (non-dimension) coordinates remain on the assembled + # patches, so the catalog must keep describing them + for name, info in aux_info.get(output_id, {}).items(): + if name in dim_names: + continue + record = _coord_record_from_row(info, name, dims=info["dims"]) + if record is not None: + coords.append(record) attrs = {} for key, value in row.items(): if ( @@ -247,6 +343,10 @@ def resolve(self, row: Mapping, **trim) -> dc.Patch: output_id = int(_row_source_patch_id(row)) members = self.member_rows[self.member_rows["output_id"] == output_id] assert len(members), "no plan members found for output row" + if self.mode == "identity": + # one untouched member per output; residuals apply at load + assert len(members) == 1 + return self._load_member(members.iloc[0].to_dict()) if self.mode == "concat": from dascore.utils.patch import concatenate_patches @@ -345,7 +445,24 @@ def derived_catalog( origin_path=origin_path, ) backend = get_backend(":memory:") - backend.write_sources(_output_records(plan.outputs, token)) + coord_dims_map = {} if parent is None else parent.backend.coord_dims_map() + # residual selections trim at load; identity claims (def keys) for + # coordinates on the trimmed dims would describe the untrimmed values + residual_names = {n for coords, _ in parent_residuals for n in coords} + trimmed_dims = frozenset( + d for n in residual_names for d in str(coord_dims_map.get(n, n)).split(",") if d + ) + outputs = plan.outputs + stale_keys = [ + f"_{c}_def_key" + for c, dims_str in coord_dims_map.items() + if set(str(dims_str).split(",")) & trimmed_dims + and f"_{c}_def_key" in outputs.columns + ] + if stale_keys: + outputs = outputs.drop(columns=stale_keys) + aux_info = _aux_coord_info(sources, trims, name, coord_dims_map, trimmed_dims) + backend.write_sources(_output_records(outputs, token, aux_info=aux_info)) return PatchCatalog(backend=backend, resolver=resolver) diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index fddef889f..d3eadd184 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -381,25 +381,46 @@ def _build_where( return where, residuals -def _order_clause(order_by, dialect: BaseDialect, attr_meta: pd.DataFrame) -> str: +# typed coord_defs envelope-minimum column per value kind +_COORD_MIN_COLUMNS = {"num": "min_num", "time": "min_ns", "str": "min_str"} +# the two conventional dims cached as columns on the patches table +_HOT_COORDS = ("time", "distance") + + +def _order_clause( + order_by, dialect: BaseDialect, attr_meta: pd.DataFrame, coord_meta: pd.DataFrame +) -> tuple[str, list]: """ - Resolve an order spec into an ORDER BY clause. + Resolve an order spec into an ORDER BY clause and its parameters. ``order_by`` is ``(kind, name, ascending)`` where kind is "attr" (an attrs-table column ordered by its typed column) or "coord" - (ordered by the coordinate's envelope minimum). The ordinal contract - supplies the deterministic tiebreak. + (ordered by the coordinate's envelope minimum — the hot patches + column when cached, otherwise the linked coord_defs typed minimum). + The ordinal contract supplies the deterministic tiebreak. """ kind, name, ascending = order_by direction = "ASC" if ascending else "DESC" - if kind == "coord": + params: list = [] + if kind == "coord" and name in _HOT_COORDS: column = f"p.{dialect.quote(f'{name}_min')}" + elif kind == "coord": + rows = coord_meta[coord_meta["coord_name"] == name] + # a coord observed under several kinds orders by its first kind + value_kind = str(rows["value_kind"].iloc[0]) + min_col = _COORD_MIN_COLUMNS[value_kind] + column = ( + f"(SELECT cd.{min_col} FROM patch_coords pc " + "JOIN coord_defs cd ON cd.coord_def_id = pc.coord_def_id " + "WHERE pc.patch_id = p.patch_id AND pc.coord_name = ?)" + ) + params.append(name) else: rows = attr_meta[attr_meta["attr_name"] == name] columns = [dialect.quote(c) for c in rows["column_name"]] # an attr observed under several kinds orders by its first column column = f"a.{columns[0]}" - return f"ORDER BY {column} {direction}, s.ordinal, p.patch_id" + return f"ORDER BY {column} {direction}, s.ordinal, p.patch_id", params def build_sql( @@ -444,15 +465,15 @@ def build_sql( # COUNT(p.patch_id) counts patches; a WHERE may reference a.. sql = f"SELECT COUNT(p.patch_id) AS n {_FROM}WHERE {where.sql}" return sql, where.params, residuals - order = ( - _order_clause(order_by, dialect, attr_meta) - if order_by is not None + if order_by is not None: + order, order_params = _order_clause(order_by, dialect, attr_meta, coord_meta) + else: # the ordering contract: source ordinal, then file-internal order - else "ORDER BY s.ordinal, p.patch_id" - ) + order, order_params = "ORDER BY s.ordinal, p.patch_id", [] + params = [*where.params, *order_params] if ids_only: sql = f"SELECT p.patch_id {_FROM}WHERE {where.sql} {order}" - return sql, where.params, residuals + return sql, params, residuals # attr columns selected explicitly: `a.*` would duplicate patch_id and # engines disagree on how to dedupe result column names. attr_cols = "".join( @@ -465,7 +486,7 @@ def build_sql( f"WHERE {where.sql} " f"{order}" ) - return sql, where.params, residuals + return sql, params, residuals def apply_residuals( diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index 9ae74af81..66b486392 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -123,12 +123,27 @@ def _usable_index(value) -> bool: if not (_usable_index(lo_idx) and _usable_index(hi_idx)): continue mins, maxs, steps = (df[c] for c in cols) - new_min = mins if lo_idx is None else mins + lo_idx * steps - new_max = maxs if hi_idx is None else mins + hi_idx * steps - df[cols[0]] = new_min.where(new_min <= maxs, other=maxs) - df[cols[1]] = new_max.where(new_max <= maxs, other=maxs) - # rows whose window starts past their end contribute nothing - df = df[df[cols[0]] <= df[cols[1]]] + # Positions are patch-local sample indices with a stop-exclusive + # hi, so the last included position is hi - 1. Sample 0 sits at + # the envelope min for ascending coords and at the max for + # descending ones. + abs_steps = steps.abs() + descending = to_float(steps.values) < 0 + lo_off = None if lo_idx is None else lo_idx * abs_steps + hi_off = None if hi_idx is None else (hi_idx - 1) * abs_steps + new_min = mins if lo_off is None else mins + lo_off + new_max = maxs if hi_off is None else mins + hi_off + desc_min = maxs if hi_off is None else maxs - hi_off + desc_max = maxs if lo_off is None else maxs - lo_off + new_min = new_min.where(~descending, other=desc_min) + new_max = new_max.where(~descending, other=desc_max) + # rows whose window is empty or lies entirely outside the + # patch contribute nothing; test before clipping so such + # windows are not resurrected as one-sample envelopes + keep = (new_min <= new_max) & (new_min <= maxs) & (new_max >= mins) + df[cols[0]] = new_min.clip(lower=mins, upper=maxs) + df[cols[1]] = new_max.clip(lower=mins, upper=maxs) + df = df[keep] return df @@ -150,16 +165,36 @@ def _dim_def_key_columns(df: pd.DataFrame, name: str) -> list[str]: def _sampling_group(step: pd.Series, tolerance: float) -> pd.Series: - """Label rows whose steps are within relative tolerance (spec 2.3).""" + """ + Label rows whose steps are within relative tolerance (spec 2.3). + + Steps group by orientation (sign) first, then by magnitude against a + stable group anchor: a group opens at its smallest magnitude and + admits members up to ``anchor * (1 + tolerance)``, so a chain of + individually-close steps can never drift a group's endpoints past + the tolerance. Unknown (NaN) steps share one group. + """ col = to_float(step.values) - order = np.argsort(col) - sorted_col = col[order] - prev = np.roll(sorted_col, 1) - with np.errstate(invalid="ignore", divide="ignore"): - diff = (sorted_col - prev) / sorted_col - out_of_threshold = diff > tolerance - group = np.cumsum(out_of_threshold) - return pd.Series(group[np.argsort(order)], index=step.index) + sign = np.sign(col) + mag = np.abs(col) + # orientation-major, magnitude-minor; NaNs sort to the end of both keys + order = np.lexsort((mag, sign)) + sorted_sign, sorted_mag = sign[order], mag[order] + labels = np.zeros(len(col), dtype=np.int64) + label, i, n = 0, 0, len(col) + while i < n: + if np.isnan(sorted_mag[i]): + # NaN keys sort last, so everything from here on is unknown + j = n + else: + block_end = np.searchsorted(sorted_sign, sorted_sign[i], side="right") + bound = sorted_mag[i] * (1 + tolerance) + j = np.searchsorted(sorted_mag[:block_end], bound, side="right") + j = max(j, i + 1) + labels[order[i:j]] = label + label += 1 + i = j + return pd.Series(labels, index=step.index) def _continuity_group(start, stop, step, tolerance) -> pd.Series: @@ -167,7 +202,9 @@ def _continuity_group(start, stop, step, tolerance) -> pd.Series: args = np.argsort(start.to_numpy()) start_sorted = start.iloc[args] stop_sorted = stop.iloc[args] - step_sorted = step.iloc[args] + # envelopes are value-ordered regardless of coordinate orientation, + # so the continuity margin uses the step magnitude + step_sorted = step.iloc[args].abs() stop_cum_max = stop_sorted.cummax() end_markers = stop_cum_max.shift() + step_sorted * tolerance has_gap = start_sorted > end_markers diff --git a/dascore/utils/hdf5.py b/dascore/utils/hdf5.py index 4a30476df..cd5182669 100644 --- a/dascore/utils/hdf5.py +++ b/dascore/utils/hdf5.py @@ -314,10 +314,6 @@ def get_handle(cls, resource): return super().get_handle(resource) -# These are left here for backward compatibility, but should not be -# used in new code. - - def unpack_scalar_h5_dataset(dataset): """ Unpack a scalar H5Py dataset. diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index b4da3c0b7..bac6d5b59 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -97,14 +97,41 @@ def resolve_selector_namespaces( Bare kwargs resolve against attributes first, then coordinates; `_attrs`/`_coords` name their namespace explicitly and validate - against that side only. Unknown names, and names supplied in more - than one namespace, raise (see #435). + against that side only. Each accepts either a mapping of + ``name -> selector`` (the fully general form — required when a name + cannot be a Python keyword, e.g. it collides with a select parameter + or is not an identifier) or a name/collection of names tagging which + *bare kwargs* to interpret in that namespace. Unknown names, and + names supplied in more than one namespace, raise (see #435). Both the catalog (which pushes predicates into SQL) and the generic dataframe select path resolve names here, so the two agree on which names are valid, what a bare name means, and which range forms are accepted — the paths differ only in how they *apply* a predicate. """ + + def _tag_form(spec, kwargs, label): + """Normalize a tag-form spec (names of bare kwargs) to a dict.""" + if spec is None or isinstance(spec, Mapping): + return spec, kwargs + names = [spec] if isinstance(spec, str) else list(spec) + if not all(isinstance(n, str) for n in names): + msg = ( + f"{label} must be a mapping of name -> selector, or a " + "name/collection of names tagging bare keyword arguments." + ) + raise InvalidSpoolQueryError(msg) + kwargs = dict(kwargs or {}) + out = {} + for n in names: + if n not in kwargs: + msg = f"{label}={n!r} names no bare keyword argument." + raise InvalidSpoolQueryError(msg) + out[n] = kwargs.pop(n) + return out, kwargs + + _attrs, kwargs = _tag_form(_attrs, kwargs, "_attrs") + _coords, kwargs = _tag_form(_coords, kwargs, "_coords") known_attrs, known_coords = set(known_attrs), set(known_coords) # A name in both explicit namespaces is a caller error whether or not # it is valid in either, so this precedes the membership checks. diff --git a/docs/changelog.qmd b/docs/changelog.qmd index f19b70810..8315c7118 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -10,8 +10,11 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f - **Coordinate boolean masks are no longer accepted by `Spool.select`** (they were only well-defined on spools whose patches share sizes, and never reduced file reads). Use `spool.map(lambda p: p.select(dim=mask))` for per-patch masking; boolean arrays over *patches* (`spool[bool_array]`) still select membership, and `samples=True` index ranges still apply per patch with Python-slice clamping. - Memory and directory spools now share a catalog-backed metadata selection path. Attribute and coordinate candidates are pushed into SQLite lazily, while exact coordinate trimming remains a patch-load operation. - Directory indexes now use the constrained seven-table SQLite schema in `.dascore_index.sqlite3`. Experimental DuckDB and Parquet index backends and the `engine`/`index_engine` selection parameters were removed. Prototype indexes from the earlier schema must be deleted and rebuilt. -- Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces, and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. +- Combining spools (`a + b`) now preserves each operand's full current contents: pending coordinate/samples trims and sort order are baked into the union (as new patch identities backed by the same lazy loading) instead of being silently dropped. Membership-style state (attribute selections, slices, patch-id arrays) still unions by rows, so identity-based deduplication keeps working there. +- `Spool.sort(...)` accepts any coordinate of the spool (including renamed and auxiliary coordinates), not just `time`/`distance`; chunk/concatenate outputs keep describing non-dimension coordinates so they remain selectable and sortable; contiguous descending-coordinate patches now merge under `chunk(dim=None)`; and `concatenate` on an empty spool returns an empty spool. +- Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces (as `name -> selector` mappings, or as a name/collection of names tagging bare keyword arguments), and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. - The legacy PyTables directory index has been replaced by SQLite. SQLite supports concurrent readers and one serialized writer on local filesystems; network filesystems with unreliable locking are not supported. +- With the PyTables dependency removed, `dascore.utils.hdf5` no longer provides `PyTablesReader`, `PyTablesWriter`, or their `HDF5Reader`/`HDF5Writer` aliases. Use `H5Reader`/`H5Writer` (h5py-based) instead. - DASCore file I/O now accepts `UPath` resources across the main read, scan, spool, and write workflows. Remote file backends such as `memory://` can be used directly for supported formats, and remote directory-based formats such as `XMLBinary` now work when the backend supports listing and file reads. See the file I/O and spool tutorials for examples and current limitations. - `dc.scan(...)` now returns [`PatchSummary`](`dascore.PatchSummary`) objects rather than `PatchAttrs`. - Scan results carry metadata, coordinates, and source information without loading data. File-backed summaries contain enough source information for lazy reloads. diff --git a/docs/notes/spool_selection.qmd b/docs/notes/spool_selection.qmd index eff605380..97bfafff6 100644 --- a/docs/notes/spool_selection.qmd +++ b/docs/notes/spool_selection.qmd @@ -4,7 +4,7 @@ title: Spool Selection `Spool.select` uses one selector model for memory and directory spools. Patch-list and directory spools compose selections in a `PatchCatalog`; ordinary metadata predicates are pushed into SQLite and remain lazy until contents, length, indexing, or iteration requires rows. -Bare selector names resolve to attributes first and then coordinates. `_attrs={...}` and `_coords={...}` provide explicit namespaces when needed. Unknown names raise immediately instead of being ignored. +Bare selector names resolve to attributes first and then coordinates. `_attrs` and `_coords` provide explicit namespaces when needed: either a `name -> selector` mapping (the fully general form, required when a name cannot be a Python keyword) or a name/collection of names tagging which bare keyword arguments belong to that namespace (e.g. `select(sensor=(1, 10), _coords="sensor")`). Unknown names raise immediately instead of being ignored. Attribute equality, membership, ranges, and glob predicates are evaluated by the index. Regular expressions use a SQL candidate predicate and an exact residual filter; chained regular expressions are combined with AND. Quantities are converted to the canonical unit recorded by the index, and dimensionally incompatible queries raise rather than silently returning incorrect matches. Values stored without units can never be proven incompatible, so they remain candidates for quantity selectors rather than being silently excluded. diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 6e39e7aac..f2b2ffb02 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -795,3 +795,23 @@ def test_unexpected_merge_dimension_raises(self, random_patch, monkeypatch): with pytest.raises(CoordMergeError, match=msg): samples = p1.data.shape[p1.get_axis("time")] * 2 assembler._merge_patches_streaming(None, [{}, {}], "time", samples) + + +class TestDescendingChunk: + """Public chunk behavior for descending coordinates (2026-07-18 F5).""" + + def test_contiguous_descending_patches_merge(self): + """Two contiguous descending patches chunk into one patch.""" + p = dc.get_example_patch() + flipped = p.flip("time") + t = p.get_coord("time") + span = t.max() - t.min() + t.step + shifted = flipped.update_coords(time=flipped.get_coord("time").data + span) + merged = dc.spool([shifted, flipped]).chunk(time=None, conflict="drop") + assert len(merged) == 1 + patch = merged[0] + time = patch.get_coord("time") + assert time.reverse_sorted + n_time = p.shape[p.get_axis("time")] + assert patch.shape[patch.get_axis("time")] == 2 * n_time + assert time.min() == t.min() diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 13b275f25..4c2c4e57b 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -991,3 +991,14 @@ def test_empty_memory_spool_len_iter_repr(self): assert len(empty) == 0 assert list(empty) == [] assert "Spool" in str(empty) + + +class TestEmptyConcatenate: + """Concatenating an empty spool returns an empty spool (F7).""" + + @pytest.mark.parametrize("kwargs", [{"time": None}, {"time": 2}, {"new_dim": None}]) + def test_empty_returns_empty(self, kwargs): + """Empty in, empty out — matching chunk's behavior.""" + out = dc.spool([]).concatenate(**kwargs) + assert len(out) == 0 + assert list(out) == [] diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 6ccf71bed..90f7c3ac6 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -504,3 +504,53 @@ def test_split_parts_pickle_small(self): payload = len(pickle.dumps(parts[0])) baseline = len(pickle.dumps(dc.spool([patches[0]]))) assert payload < 2 * baseline + + +class TestNamespaceTagForm: + """_attrs/_coords accept names of bare kwargs (tag form).""" + + @pytest.fixture() + def sensor_spool(self): + """One patch with an aux coord whose name is not an attr.""" + patch = dc.get_example_patch() + n = patch.shape[patch.get_axis("distance")] + return dc.spool( + [patch.update_coords(sensor=("distance", np.arange(n, dtype=float)))] + ) + + def test_coords_tag_string(self, sensor_spool): + """A single name tags one bare kwarg as a coordinate.""" + out = sensor_spool.select(sensor=(10, 20), _coords="sensor") + coord = out[0].get_coord("sensor") + assert coord.min() == 10.0 + assert coord.max() == 20.0 + + def test_coords_tag_collection(self, sensor_spool): + """A collection of names tags several bare kwargs.""" + out = sensor_spool.select(sensor=(10, 20), _coords=["sensor"]) + assert len(out) == 1 + + def test_attrs_tag_string(self, sensor_spool): + """The attr side accepts the same tag form.""" + out = sensor_spool.select(tag="random", _attrs="tag") + assert len(out) == 1 + + def test_dict_form_unchanged(self, sensor_spool): + """The general mapping form keeps working.""" + out = sensor_spool.select(_coords={"sensor": (10, 20)}) + assert len(out) == 1 + + def test_tag_without_kwarg_raises(self, sensor_spool): + """Tagging a name with no matching bare kwarg is an error.""" + with pytest.raises(InvalidSpoolQueryError, match="names no bare keyword"): + sensor_spool.select(_coords="sensor") + + def test_non_string_tag_raises(self, sensor_spool): + """Tag collections must contain strings.""" + with pytest.raises(InvalidSpoolQueryError, match="mapping of name"): + sensor_spool.select(sensor=(1, 2), _coords=[3]) + + def test_tagged_name_validates_namespace(self, sensor_spool): + """A tagged name must belong to the claimed namespace.""" + with pytest.raises(InvalidSpoolQueryError, match="not an attribute"): + sensor_spool.select(sensor=(1, 2), _attrs="sensor") diff --git a/tests/test_io/test_index/test_ordering.py b/tests/test_io/test_index/test_ordering.py index e80980210..6ac8eb92d 100644 --- a/tests/test_io/test_index/test_ordering.py +++ b/tests/test_io/test_index/test_ordering.py @@ -144,3 +144,55 @@ def test_indexer_deepcopy_shares_instance(self, tmp_path): spool = dc.spool(tmp_path).update(progress=None) assert copy.deepcopy(spool.indexer) is spool.indexer spool.indexer.close() + + +class TestSortNonHotCoords: + """Sorting by coords without patches-table columns (2026-07-18 F3).""" + + @pytest.fixture() + def renamed_spool(self): + """Two patches whose time coord is renamed (not a hot column).""" + p = dc.get_example_patch().rename_coords(time="event_time") + t = p.get_coord("event_time") + span = t.max() - t.min() + t.step + p2 = p.update_coords(event_time=t.data + span) + return dc.spool([p2, p]) # deliberately out of order + + @pytest.mark.parametrize("key", ["event_time", "event_time_min"]) + def test_sort_renamed_datetime_coord(self, renamed_spool, key): + """A renamed datetime coord sorts through coord_defs.""" + srt = renamed_spool.sort(key) + mins = [x.get_coord("event_time").min() for x in srt] + assert mins == sorted(mins) + # the realized relation agrees + contents = srt.get_contents() + assert contents["event_time_min"].is_monotonic_increasing + + def test_sort_non_hot_numeric_coord(self): + """A numeric aux coord sorts through coord_defs.""" + p = dc.get_example_patch() + n = p.shape[p.get_axis("distance")] + lo = p.update_coords(sensor=("distance", np.arange(n, dtype=float))) + hi = p.update_coords(sensor=("distance", np.arange(n, dtype=float) + 1000)) + srt = dc.spool([hi, lo]).sort("sensor") + mins = [x.get_coord("sensor").min() for x in srt] + assert mins == sorted(mins) + + def test_sort_string_coord(self): + """A string coord sorts lexicographically through coord_defs.""" + p = dc.get_example_patch() + n = p.shape[p.get_axis("distance")] + pa = p.update_coords(station=("distance", np.array(["a"] * n))) + pb = p.update_coords(station=("distance", np.array(["b"] * n))) + srt = dc.spool([pb, pa]).sort("station") + firsts = [x.get_coord("station").values[0] for x in srt] + assert firsts == ["a", "b"] + + def test_hot_coords_still_sort(self): + """time/distance keep the cached patches-column path.""" + p = dc.get_example_patch() + t = p.get_coord("time") + p2 = p.update_coords(time_min=t.max() + t.step) + srt = dc.spool([p2, p]).sort("time") + mins = [x.get_coord("time").min() for x in srt] + assert mins == sorted(mins) diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index 0f0385fcc..b6652ba54 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -426,3 +426,111 @@ class _Frame: monkeypatch.setattr(_inspect, "stack", lambda: [_Frame()] * 3) assert cp._user_stacklevel() == 1 + + +class TestSamplingGroups: + """Sampling-group partitioning invariants (2026-07-18 F5).""" + + def test_close_steps_group(self): + """Steps within tolerance of the anchor share a group.""" + from dascore.utils.chunk_plan import _sampling_group + + labels = _sampling_group(pd.Series([1.0, 1.04]), 0.05) + assert labels.nunique() == 1 + + def test_chain_does_not_drift_past_tolerance(self): + """Adjacent-close steps cannot chain past the group anchor.""" + from dascore.utils.chunk_plan import _sampling_group + + labels = _sampling_group(pd.Series([1.00, 1.04, 1.08]), 0.05) + assert list(labels) == [0, 0, 1] + + def test_negative_steps_group_by_magnitude(self): + """Widely different negative steps never share a group.""" + from dascore.utils.chunk_plan import _sampling_group + + labels = _sampling_group(pd.Series([-1.0, -5.0]), 0.05) + assert labels.nunique() == 2 + + def test_mixed_orientation_never_groups(self): + """Equal magnitudes with opposite signs stay separate.""" + from dascore.utils.chunk_plan import _sampling_group + + labels = _sampling_group(pd.Series([1.0, -1.0]), 0.05) + assert labels.nunique() == 2 + + def test_unknown_steps_share_one_group(self): + """NaN steps keep their historical single-group behavior.""" + from dascore.utils.chunk_plan import _sampling_group + + labels = _sampling_group(pd.Series([1.0, np.nan, np.nan]), 0.05) + assert labels.nunique() == 2 + assert labels.iloc[1] == labels.iloc[2] + + def test_descending_contiguous_merges(self): + """Contiguous descending patches produce a single merge output.""" + p = dc.get_example_patch() + flipped = p.flip("time") + t = p.get_coord("time") + span = t.max() - t.min() + t.step + shifted = flipped.update_coords(time=flipped.get_coord("time").data + span) + plan = dc.spool([shifted, flipped]).chunk_plan(time=None) + assert len(plan.outputs) == 1 + assert len(plan.members) == 2 + + +class TestSamplesAdjustedEnvelopes: + """Samples residual envelope adjustment (2026-07-18 F4).""" + + @staticmethod + def _frame(): + """One ascending row: 10 samples at step 1 spanning [0, 9].""" + return pd.DataFrame({"time_min": [0.0], "time_max": [9.0], "time_step": [1.0]}) + + def test_exclusive_stop(self): + """The stop index is exclusive: (0, 5) ends at sample 4.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + out = samples_adjusted_envelopes(self._frame(), (({"time": (0, 5)}, True),)) + assert out["time_max"].iloc[0] == 4.0 + + def test_empty_window_drops_row(self): + """A zero-length window contributes nothing.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + out = samples_adjusted_envelopes(self._frame(), (({"time": (3, 3)}, True),)) + assert len(out) == 0 + + def test_start_past_end_drops_row(self): + """A window beyond the patch contributes nothing.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + out = samples_adjusted_envelopes(self._frame(), (({"time": (50, 60)}, True),)) + assert len(out) == 0 + + def test_stop_clamps(self): + """A stop past the end clamps to the envelope max.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + out = samples_adjusted_envelopes(self._frame(), (({"time": (2, 99)}, True),)) + assert out["time_min"].iloc[0] == 2.0 + assert out["time_max"].iloc[0] == 9.0 + + def test_descending_orientation(self): + """On a descending coord, sample 0 sits at the envelope max.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + df = pd.DataFrame({"time_min": [0.0], "time_max": [9.0], "time_step": [-1.0]}) + out = samples_adjusted_envelopes(df, (({"time": (0, 5)}, True),)) + assert out["time_max"].iloc[0] == 9.0 + assert out["time_min"].iloc[0] == 5.0 + + def test_public_same_dim_envelope_matches_patch(self): + """The chunked catalog's envelope matches the loaded patch (F4).""" + p = dc.get_example_patch() + out = dc.spool([p]).select(time=(0, 10), samples=True).chunk(time=None) + patch = out[0] + want = pd.Timestamp(patch.get_coord("time").max()) + got = out.get_contents()["time_max"].iloc[0] + assert got == want + assert patch.shape[patch.get_axis("time")] == 10 diff --git a/tests/test_io/test_index/test_planned.py b/tests/test_io/test_index/test_planned.py index d4a3d5f1d..80370ae8f 100644 --- a/tests/test_io/test_index/test_planned.py +++ b/tests/test_io/test_index/test_planned.py @@ -222,3 +222,97 @@ def test_samples_adjust_skips_missing_columns(self): residuals = (({"depth": (0, 5)}, True),) out = samples_adjusted_envelopes(df, residuals) assert out.equals(df) + + +class TestAuxiliaryCoords: + """Derived catalogs keep non-dimension coords (2026-07-18 F2).""" + + @pytest.fixture() + def sensor_spool(self): + """Two contiguous patches carrying an aux coord on distance.""" + p = dc.get_example_patch() + sensor = np.arange(p.shape[p.get_axis("distance")], dtype=float) + p = p.update_coords(sensor=("distance", sensor)) + t = p.get_coord("time") + p2 = p.update_coords(time_min=t.max() + t.step) + return dc.spool([p, p2]) + + @pytest.mark.parametrize("op", ["chunk", "concatenate"]) + def test_aux_coord_survives(self, sensor_spool, op): + """Chunk and concat outputs keep describing the aux coord.""" + if op == "chunk": + derived = sensor_spool.chunk(time=None, conflict="drop") + else: + derived = sensor_spool.concatenate(time=None) + contents = derived.get_contents() + assert "sensor_min" in contents.columns + assert "sensor_max" in contents.columns + # and it stays selectable + out = derived.select(sensor=(10, 20)) + assert len(out) == 1 + coord = out[0].get_coord("sensor") + assert coord.min() == 10.0 + assert coord.max() == 20.0 + + def test_aux_identity_preserved_when_unchanged(self, sensor_spool): + """Members sharing one def key off the planned dim keep identity.""" + derived = sensor_spool.chunk(time=None, conflict="drop") + source_key = sensor_spool._catalog.to_df()["_sensor_def_key"].iloc[0] + derived_key = derived._catalog.to_df()["_sensor_def_key"].iloc[0] + assert derived_key == source_key + assert str(derived_key).startswith("fp:") + + def test_aux_identity_dropped_when_riding_trimmed_dim(self, sensor_spool): + """A residual trim on distance voids sensor's identity claim.""" + d = sensor_spool[0].get_coord("distance") + lo, hi = d.min() + 5 * d.step, d.min() + 50 * d.step + selected = sensor_spool.select(distance=(lo, hi)) + derived = selected.chunk(time=None, conflict="drop") + key = derived._catalog.to_df()["_sensor_def_key"].iloc[0] + assert not str(key).startswith("fp:") + # loading still yields the trimmed coord + assert derived[0].get_coord("sensor").min() == 5.0 + + def test_string_aux_coord(self): + """String-valued aux coords survive with a lexicographic envelope.""" + p = dc.get_example_patch() + n = p.shape[p.get_axis("distance")] + labels = np.array([f"s{i:03d}" for i in range(n)]) + p = p.update_coords(station=("distance", labels)) + derived = dc.spool([p]).chunk(time=None) + contents = derived.get_contents() + assert contents["station_min"].iloc[0] == "s000" + assert contents["station_max"].iloc[0] == f"s{n - 1:03d}" + assert "station" in derived[0].coords.coord_map + + +class TestAuxInfoEdges: + """Edge branches of the aux-coord aggregation helpers.""" + + def test_coord_record_missing_envelope_returns_none(self): + """A row without envelope values yields no coord record.""" + from dascore.io.index.planned import _coord_record_from_row + + assert _coord_record_from_row({}, "time") is None + + def test_absent_envelope_columns_skipped(self): + """A mapped coord with no envelope columns contributes nothing.""" + from dascore.io.index.planned import _aux_coord_info + + members = pd.DataFrame( + {"output_id": [0], "_patch_id": [1], "_modified": [False]} + ) + sources = pd.DataFrame({"_patch_id": [1]}) + assert _aux_coord_info(sources, members, "time", {"ghost": "distance"}) == {} + + def test_all_null_group_skipped(self): + """An output whose members carry no values for a coord is skipped.""" + from dascore.io.index.planned import _aux_coord_info + + members = pd.DataFrame( + {"output_id": [0], "_patch_id": [1], "_modified": [False]} + ) + sources = pd.DataFrame( + {"_patch_id": [1], "sensor_min": [np.nan], "sensor_max": [np.nan]} + ) + assert _aux_coord_info(sources, members, "time", {"sensor": "distance"}) == {} diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 36f0a33f4..3a9ae3187 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -349,6 +349,107 @@ def test_union_absorbs_only_member_registry_entries(self, contiguous_patches): assert len(narrowed) == 1 other = dc.get_example_patch(time_min="2030-01-01") combined = narrowed + dc.spool([other]) + # the trimmed operand materializes into a plan, so its live + # member rides inside the plan's loader, not the top registry registry = combined._catalog.resolver.live_entries() - assert len(registry) == 2 # p1 and other; p2 stayed home + assert len(registry) == 1 # other + plans = combined._catalog.resolver.plan_entries() + nested_live = {k for p in plans.values() for k in p.live_entries()} + assert len(nested_live) == 1 # p1; p2 stayed home assert len(combined) == 2 + + +class TestLossyStateUnion: + """Residual trims and order specs survive combining (2026-07-18 F1).""" + + def test_value_trim_survives_union(self): + """A coordinate-range selection's trim is baked in, not dropped.""" + p = dc.get_example_patch() + t = p.get_coord("time") + lo, hi = t.min() + 10 * t.step, t.min() + 20 * t.step + selected = dc.spool([p]).select(time=(lo, hi)) + combined = selected + dc.spool([]) + assert len(combined) == 1 + got, want = combined[0], selected[0] + assert got.shape == want.shape + assert got.get_coord("time").min() == want.get_coord("time").min() + assert got.get_coord("time").max() == want.get_coord("time").max() + + def test_samples_trim_survives_union(self): + """A samples window's trim is baked in, not dropped.""" + p = dc.get_example_patch() + selected = dc.spool([p]).select(time=(0, 10), samples=True) + combined = selected + dc.spool([]) + assert combined[0].shape == selected[0].shape + + def test_file_backed_value_trim_survives_union(self, dir_spool): + """The same guarantee holds for file-backed catalogs.""" + patch = dir_spool[0] + t = patch.get_coord("time") + lo, hi = t.min() + 10 * t.step, t.min() + 20 * t.step + selected = dir_spool.select(time=(lo, hi)) + combined = selected + dc.spool([]) + assert combined[0].shape == selected[0].shape + + def test_sort_order_survives_union(self): + """A sort spec bakes into ordinals instead of silently reverting.""" + p = dc.get_example_patch() + t = p.get_coord("time") + early = p.update_attrs(tag="early") + late = p.update_coords(time_min=t.max() + t.step).update_attrs(tag="late") + srt = dc.spool([late, early]).sort("time") + combined = srt + dc.spool([]) + assert [x.attrs.tag for x in combined] == [x.attrs.tag for x in srt] + + def test_membership_selections_still_dedup(self): + """Row-membership state unions as rows: identity dedup preserved.""" + p = dc.get_example_patch() + spool = dc.spool([p.update_attrs(tag=f"t{i}") for i in range(4)]) + combined = spool.select(tag="t2") + spool + assert len(combined) == 4 + + def test_windows_still_union_by_rows(self): + """Slice windows survive as membership without materializing.""" + p = dc.get_example_patch() + spool = dc.spool([p.update_attrs(tag=f"t{i}") for i in range(4)]) + combined = spool[1:3] + dc.spool([]) + assert [x.attrs.tag for x in combined] == ["t1", "t2"] + + def test_two_selected_operands(self): + """Different trims on each operand both survive as new contents.""" + p = dc.get_example_patch() + t = p.get_coord("time") + lo = t.min() + 10 * t.step + a = dc.spool([p]).select(time=(lo, t.min() + 20 * t.step)) + b = dc.spool([p]).select(time=(t.min(), lo)) + combined = a + b + assert len(combined) == 2 + assert {x.shape for x in combined} == {a[0].shape, b[0].shape} + + def test_sorted_derived_union(self): + """A derived (chunked) catalog with an order spec also survives.""" + p = dc.get_example_patch() + t = p.get_coord("time") + early = p.update_attrs(tag="z_early") + # the gap keeps two outputs; distinct tags survive unmerged + late = p.update_coords(time_min=t.max() + 10 * t.step).update_attrs( + tag="a_late" + ) + chunked = dc.spool([early, late]).chunk(time=None) + srt = chunked.sort("tag") + want = [x.attrs.tag for x in srt] + assert want == ["a_late", "z_early"] # tag order != time order + combined = srt + dc.spool([]) + assert [x.attrs.tag for x in combined] == want + + def test_combined_pickles(self): + """A union holding a materialized operand round-trips pickling.""" + import pickle + + p = dc.get_example_patch() + t = p.get_coord("time") + lo, hi = t.min() + 10 * t.step, t.min() + 20 * t.step + combined = dc.spool([p]).select(time=(lo, hi)) + dc.spool([]) + loaded = pickle.loads(pickle.dumps(combined)) + assert len(loaded) == 1 + assert loaded[0].shape == combined[0].shape From d04f3e93ad5330a8ce1dc4d77b1dd6f52880f25e Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 13:18:18 +0200 Subject: [PATCH 92/97] Partition chunks by the chunked dim's units; convert compatible spellings at merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The planner grouped patches by raw SI envelope magnitudes with no knowledge of the chunked dimension's units, so a metre patch and a seconds patch with contiguous magnitudes planned into one output whose assembly failed only at patch access. The canonical (base) unit now rides the flat relation as a private per-coord column and joins the sampling partition: incompatible dimensionality — and unitful next to unitless, which assembly refuses too — can never share an output. Compatible spellings of one dimensionality (metres with feet) still plan together and now genuinely merge: assembly converts each member's merge-dim units to the first member's, and the raw-concatenation merge fallback reattaches the verified common unit it previously dropped. Derived catalogs carry the canonical units on their coordinate definitions, so unit metadata survives restructuring. --- dascore/io/index/backend.py | 6 ++- dascore/io/index/planned.py | 13 +++-- dascore/utils/chunk_plan.py | 13 +++++ dascore/utils/coordmanager.py | 11 ++++- dascore/utils/patch_assembly.py | 35 ++++++++++++- docs/changelog.qmd | 1 + tests/test_core/test_patch_chunk.py | 54 +++++++++++++++++++++ tests/test_utils/test_coordmanager_utils.py | 19 ++++++++ 8 files changed, 144 insertions(+), 8 deletions(-) diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index d114b8fac..8daeb9748 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -901,7 +901,7 @@ def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: ids = out["patch_id"].tolist() link_sql = ( "SELECT pc.patch_id, pc.coord_name, cd.def_key, cd.fingerprint, " - "cd.value_kind, cd.is_relative, cd.min_num, cd.max_num, " + "cd.value_kind, cd.is_relative, cd.units, cd.min_num, cd.max_num, " "cd.step_num, cd.min_ns, cd.max_ns, cd.step_ns, " "cd.min_str, cd.max_str " "FROM patch_coords pc " @@ -926,7 +926,11 @@ def _pivot_coords(self, out: pd.DataFrame) -> pd.DataFrame: mins = dict(zip(pids, group["_env_min"])) maxs = dict(zip(pids, group["_env_max"])) steps = dict(zip(pids, group["_env_step"])) + units = dict(zip(pids, group["units"])) out[f"_{name}_def_key"] = out["patch_id"].map(keys) + # canonical (base) units: numeric envelopes are stored SI, so + # this is the dimensionality marker chunk partitioning needs + out[f"_{name}_units"] = out["patch_id"].map(units) kinds = set(group["value_kind"]) # time/distance envelopes already live on patches... if name in ("time", "distance"): diff --git a/dascore/io/index/planned.py b/dascore/io/index/planned.py index 5e962a89b..4d1ddde52 100644 --- a/dascore/io/index/planned.py +++ b/dascore/io/index/planned.py @@ -116,10 +116,12 @@ def _coord_record_from_row( # a degenerate (zero) step is not a range; drop it rather than # letting range reconstruction divide by it step = None - units = row.get(f"{name}_units") - # numeric envelope values are stored canonical-SI; attaching the - # original unit string would make ingest re-convert them - if dtype != "float64" or units == "" or (units is not None and pd.isnull(units)): + # numeric envelope values are stored canonical-SI, so only the + # canonical (base) unit carried by the pivot may be attached — + # ingest's re-conversion is then the identity (time kinds attach + # without conversion) + units = row.get(f"_{name}_units") + if units == "" or (units is not None and pd.isnull(units)): units = None length = None if step is not None: @@ -191,11 +193,14 @@ def _aux_coord_info( if keep and step_col in sub.columns else set() ) + unit_col = f"_{name}_units" + units = set(sub[unit_col].dropna()) if unit_col in sub.columns else set() info = { cmin: lo, cmax: hi, step_col: steps.pop() if len(steps) == 1 else None, key_col: keys.pop() if keep else None, + unit_col: units.pop() if len(units) == 1 else None, "dims": dims, } out.setdefault(int(output_id), {})[name] = info diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index 66b486392..409197a52 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -234,6 +234,13 @@ def _partition( # (spec 2.2). Non-dimensional coordinate conflicts are policed at # assembly per the `conflict` argument, never partitioned on. cols += [x for x in _dim_def_key_columns(df, name) if x in df.columns] + # The chunked dim's canonical (base) units partition too: envelopes + # are SI magnitudes, so without this a metre patch and a second + # patch with contiguous magnitudes would plan into one unmergeable + # output. Unitless (NULL) stays its own group — assembly cannot + # merge unitless with unitful coordinates either. + if (unit_col := f"_{name}_units") in df.columns: + cols.append(unit_col) base = ( df.groupby(cols, dropna=False, sort=False).ngroup() if cols @@ -337,6 +344,12 @@ def _police_columns(sub: pd.DataFrame, name, conflict) -> dict: for col in _dim_def_key_columns(sub, name): if col in sub.columns: carried[col] = sub[col].iloc[0] + # Canonical units carry for every dimension, the chunked one included + # (partition-constant: units are a sampling-partition component). + for coord in coord_names: + col = f"_{coord}_units" + if col in sub.columns: + carried[col] = sub[col].iloc[0] return carried diff --git a/dascore/utils/coordmanager.py b/dascore/utils/coordmanager.py index dc3a1d419..9c8a60a13 100644 --- a/dascore/utils/coordmanager.py +++ b/dascore/utils/coordmanager.py @@ -135,7 +135,16 @@ def _get_merged_coords(managers, coords_to_merge): data = [x.data for x in snap_coords] dims = managers[0].dim_map[dim] new_data = np.concatenate(data, axis=axis) - out[coord_name] = (dims, new_data) + # raw value concatenation loses the coord's units; reattach + # the (verified common) units so the merge stays unit-true + common_units = next(iter(units)) + if common_units is not None: + from dascore.core.coords import get_coord + + coord = get_coord(data=new_data, units=common_units) + out[coord_name] = (dims, coord) + else: + out[coord_name] = (dims, new_data) return out def _get_new_coords(managers) -> dict[str, tuple[tuple[str, ...], ArrayLike]]: diff --git a/dascore/utils/patch_assembly.py b/dascore/utils/patch_assembly.py index 8d56b9a9e..94b8814b8 100644 --- a/dascore/utils/patch_assembly.py +++ b/dascore/utils/patch_assembly.py @@ -76,6 +76,32 @@ def _estimate_merge_samples(df, dim) -> int | None: return int(counts.sum()) +def _match_merge_units(patch, merge_dim, target_units): + """ + Convert a member's merge-dim units to the first member's. + + The planner groups by SI-canonical envelopes, so one output may mix + unit spellings of one dimensionality (metres with feet); merging + requires a single spelling, and the first member's wins. Returns + (patch, target_units); incompatible or missing units pass through + for the merge itself to police. + """ + from dascore.exceptions import UnitError + + if merge_dim is None or merge_dim not in getattr(patch.coords, "coord_map", {}): + return patch, target_units + units = patch.coords.coord_map[merge_dim].units + if target_units is None: + return patch, units + if units is None or units == target_units: + return patch, target_units + try: + patch = patch.convert_units(**{merge_dim: target_units}) + except UnitError: # incompatible dimensionality: merge will raise + return patch, target_units + return patch, target_units + + def _coord_only_kwargs(patch, kwargs) -> dict: """Keep only the kwargs naming a dim or coordinate of patch.""" return { @@ -103,12 +129,13 @@ def _patch_from_instruction_df(self, joined): """Get the patches joined columns of instruction df.""" df_dict_list = self._df_to_dict_list(joined) expected_len = len(joined["current_index"].unique()) - if len(df_dict_list) > expected_len: + merging = len(df_dict_list) > expected_len + merge_dim = _get_varying_dim(joined) if merging else None + if merging: # Several sources merge into one patch. When the output size can # be determined from the instructions, stream the sources into a # pre-allocated array so they don't all need to be in memory with # the merged output at once. - merge_dim = _get_varying_dim(joined) samples = _estimate_merge_samples(joined, merge_dim) if samples is not None: patch = self._merge_patches_streaming( @@ -116,8 +143,10 @@ def _patch_from_instruction_df(self, joined): ) return [patch] out = [] + target_units = None for patch_kwargs in df_dict_list: patch = self._load_trimmed_patch(patch_kwargs, joined) + patch, target_units = _match_merge_units(patch, merge_dim, target_units) # The index doesn't carry all the dimensional info, so get what # merging needs from the patch coords (cheaper than attr dumps). info = patch.coords._get_dim_summary() @@ -153,8 +182,10 @@ def _merge_patches_streaming(self, joined, df_dict_list, merge_dim, samples): """ buffer, offset, axis, dims = None, 0, None, None coords, attrs, summaries = [], [], [] + target_units = None for patch_kwargs in df_dict_list: patch = self._load_trimmed_patch(patch_kwargs, joined) + patch, target_units = _match_merge_units(patch, merge_dim, target_units) if dims is None: dims = patch.dims axis = patch.get_axis(merge_dim) diff --git a/docs/changelog.qmd b/docs/changelog.qmd index 8315c7118..1eaf8cfc2 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -12,6 +12,7 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f - Directory indexes now use the constrained seven-table SQLite schema in `.dascore_index.sqlite3`. Experimental DuckDB and Parquet index backends and the `engine`/`index_engine` selection parameters were removed. Prototype indexes from the earlier schema must be deleted and rebuilt. - Combining spools (`a + b`) now preserves each operand's full current contents: pending coordinate/samples trims and sort order are baked into the union (as new patch identities backed by the same lazy loading) instead of being silently dropped. Membership-style state (attribute selections, slices, patch-id arrays) still unions by rows, so identity-based deduplication keeps working there. - `Spool.sort(...)` accepts any coordinate of the spool (including renamed and auxiliary coordinates), not just `time`/`distance`; chunk/concatenate outputs keep describing non-dimension coordinates so they remain selectable and sortable; contiguous descending-coordinate patches now merge under `chunk(dim=None)`; and `concatenate` on an empty spool returns an empty spool. +- Chunk partitioning is unit-aware along the chunked dimension: patches whose coordinate units have different dimensionality (or unitful vs unitless) can never plan into one output — they were previously grouped by raw SI magnitude and failed only at load. Compatible unit spellings (e.g. metres and feet) now merge correctly, converted to the first member's units. - Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces (as `name -> selector` mappings, or as a name/collection of names tagging bare keyword arguments), and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. - The legacy PyTables directory index has been replaced by SQLite. SQLite supports concurrent readers and one serialized writer on local filesystems; network filesystems with unreliable locking are not supported. - With the PyTables dependency removed, `dascore.utils.hdf5` no longer provides `PyTablesReader`, `PyTablesWriter`, or their `HDF5Reader`/`HDF5Writer` aliases. Use `H5Reader`/`H5Writer` (h5py-based) instead. diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index f2b2ffb02..4c7d3ccee 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -815,3 +815,57 @@ def test_contiguous_descending_patches_merge(self): n_time = p.shape[p.get_axis("time")] assert patch.shape[patch.get_axis("time")] == 2 * n_time assert time.min() == t.min() + + +class TestMixedUnitChunk: + """Chunk partitioning and merging across unit differences.""" + + @staticmethod + def _shifted(patch, units=None): + """The example patch shifted to be distance-contiguous, in units.""" + d = patch.get_coord("distance") + span = d.max() - d.min() + d.step + values = d.data + span + if units == "ft": + values = values / 0.3048 + out = patch.update_coords(distance=values) + return out.set_units(distance=units) if units else out + + def test_incompatible_dimensionality_splits(self): + """Metre and second patches with contiguous SI magnitudes stay apart.""" + p = dc.get_example_patch() + pm = p.set_units(distance="m") + ps = self._shifted(p, "s") + sp = dc.spool([pm, ps]) + plan = sp.chunk_plan(distance=None) + assert len(plan.outputs) == 2 + out = sp.chunk(distance=None, conflict="drop") + assert {str(x.get_coord("distance").units) for x in out} == {"1 m", "1 s"} + + def test_unitless_and_unitful_split(self): + """A unitless patch never merges with a unitful one.""" + p = dc.get_example_patch() + sp = dc.spool([p.set_units(distance="m"), self._shifted(p)]) + assert len(sp.chunk(distance=None, conflict="drop")) == 2 + + def test_compatible_units_convert_and_merge(self): + """Metres and feet (one dimensionality) merge, converted, unit-true.""" + p = dc.get_example_patch() + pm = p.set_units(distance="m") + pf = self._shifted(p, "ft") + out = dc.spool([pm, pf]).chunk(distance=None, conflict="drop") + assert len(out) == 1 + patch = out[0] + coord = patch.get_coord("distance") + assert str(coord.units) == "1 m" + n = p.shape[p.get_axis("distance")] + assert patch.shape[patch.get_axis("distance")] == 2 * n + assert float(coord.max()) == pytest.approx(2 * n - 1) + + def test_same_units_unchanged(self): + """The ordinary same-unit merge keeps its behavior and units.""" + p = dc.get_example_patch() + sp = dc.spool([p.set_units(distance="m"), self._shifted(p, "m")]) + out = sp.chunk(distance=None, conflict="drop") + assert len(out) == 1 + assert str(out[0].get_coord("distance").units) == "1 m" diff --git a/tests/test_utils/test_coordmanager_utils.py b/tests/test_utils/test_coordmanager_utils.py index ced4baa4c..d00cbf66f 100644 --- a/tests/test_utils/test_coordmanager_utils.py +++ b/tests/test_utils/test_coordmanager_utils.py @@ -165,3 +165,22 @@ def test_conflicting_non_dimensional_coords(self, conflicting_non_dim_coords): with pytest.raises(CoordMergeError, match="cannot be merged"): merge_coord_managers([c1, c2], dim="time", drop_conflicting=False) + + +class TestRawMergeKeepsUnits: + """The raw-concatenation merge fallback keeps common units.""" + + def test_units_survive_value_merge(self): + """Merging value-backed coords with one common unit keeps it.""" + import numpy as np + + import dascore as dc + from dascore.utils.coordmanager import merge_coord_managers + + p1 = dc.get_example_patch().set_units(distance="m") + d = p1.get_coord("distance") + # non-uniform values force the raw concatenation path + values = np.sort(np.random.default_rng(0).uniform(400, 500, len(d.data))) + p2 = p1.update_coords(distance=values).set_units(distance="m") + merged = merge_coord_managers([p1.coords, p2.coords], dim="distance") + assert str(merged.coord_map["distance"].units) == "1 m" From 43b4016a4e79dabf68097eaa736471bde17ee296 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 13:37:28 +0200 Subject: [PATCH 93/97] Fix the fourth-round review findings: chained chunking, time-true directories, whole transactions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chunking a restructured spool along a different dimension now plans over the spool's current output rows (loaded through the plan resolver), so it keeps the boundaries the earlier operation assembled; re-chunking the same dimension still collapses to the trimmed members. Directory catalogs carry a per-patch default presentation order (ORDER BY time with the ordinal/patch-id tiebreak) because source-grain ordinals cannot interleave a multi-patch file that straddles a patch of another file; the default order is a catalog contract, not view state, so roots still update. The membership-restricted resolver keeps the plan routes its rows reference (mixed planned/live views survive pickling and process-backed map), the segmented-coordinate write guard keys on what a spool resolves rather than where its members live (plan- assembled file-backed spools are guarded; purely file-backed spools still skip inspection), the SQLite statement lock now spans whole transactions so shared-connection readers can never observe a half-applied source replacement, and only mark_initial_update_done — after renumbering succeeds — sets the initial-update marker, keeping the reopen recovery path alive when a sync dies mid-way. --- dascore/core/spool.py | 33 +++++---- dascore/io/core.py | 23 +++++- dascore/io/index/backend.py | 30 +++++--- dascore/io/index/catalog.py | 42 +++++++++-- dascore/io/index/planned.py | 10 +-- docs/changelog.qmd | 1 + tests/test_core/test_coord_segmented.py | 41 +++++++++++ tests/test_core/test_patch_chunk.py | 60 ++++++++++++++++ .../test_index/test_index_edge_cases.py | 60 ++++++++++++++++ tests/test_io/test_index/test_ordering.py | 72 +++++++++++++++++++ tests/test_io/test_index/test_planned.py | 10 +++ tests/test_io/test_index/test_union.py | 38 ++++++++++ 12 files changed, 386 insertions(+), 34 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index c95ea8bba..87d16a711 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -644,22 +644,28 @@ def _materialize_lossy(self): # --- restructuring (materializing) operations ----------------------- - def _plan_frames(self) -> tuple[pd.DataFrame, pd.DataFrame]: + def _plan_frames(self, dim: str | None = None) -> tuple[pd.DataFrame, pd.DataFrame]: """ - Return (source_rows, working) frames for planning. - - Plans collapse (never nest): a derived catalog re-plans from its - members — the trimmed source rows — restricted to the outputs - the current view presents. Patch-local samples residuals adjust - the working envelopes so plans reflect the loading truth. + Return (source_rows, working) frames for planning along ``dim``. + + Re-planning the *same* dimension collapses (never nests): a + derived catalog re-plans from its members — the trimmed source + rows — restricted to the outputs the current view presents. + Planning a *different* dimension must keep the already-assembled + boundaries, so it plans over the current output rows themselves + (loaded through the plan resolver). Patch-local samples + residuals adjust the working envelopes so plans reflect the + loading truth. """ - from dascore.io.index.planned import collapse_working_df + from dascore.io.index.planned import PlanResolver, collapse_working_df from dascore.utils.chunk_plan import ( _ensure_patch_id, samples_adjusted_envelopes, ) - base = collapse_working_df(self._catalog) + resolver = self._catalog.resolver + same_dim = isinstance(resolver, PlanResolver) and resolver.dim == dim + base = collapse_working_df(self._catalog) if same_dim else None if base is None: base = self._catalog.to_df().reset_index(drop=True) base = _ensure_patch_id(base) @@ -703,7 +709,7 @@ def chunk_plan( """ from dascore.utils.chunk_plan import build_chunk_plan - _, working = self._plan_frames() + _, working = self._plan_frames(next(iter(kwargs), None)) return build_chunk_plan( working, overlap=overlap, @@ -732,7 +738,7 @@ def chunk( from dascore.io.index.planned import derived_catalog from dascore.utils.chunk_plan import build_chunk_plan - source_rows, working = self._plan_frames() + source_rows, working = self._plan_frames(next(iter(kwargs), None)) plan = build_chunk_plan( working, overlap=overlap, @@ -773,7 +779,7 @@ def concatenate(self, check_behavior: WARN_LEVELS = "warn", **kwargs) -> Self: raise ParameterError(msg) ((dim, value),) = kwargs.items() value = None if value is Ellipsis else value - source_rows, working = self._plan_frames() + source_rows, working = self._plan_frames(dim) # a dim absent from the metadata envelopes is legal: concatenate # can stack patches along a brand-new dimension has_envelope = f"{dim}_min" in working.columns @@ -837,10 +843,13 @@ def from_directory(cls, path, index_path=None) -> Self: out = cls() if isinstance(path, AbstractIndexer): + from dascore.io.index.catalog import _DIRECTORY_ORDER + out._catalog = PatchCatalog( backend=path._backend, resolver=FileResolver(root=path.path), syncer=path, + default_order=_DIRECTORY_ORDER, ) else: out._catalog = PatchCatalog.from_directory(path, index_path=index_path) diff --git a/dascore/io/core.py b/dascore/io/core.py index dcd78be3d..e9d12805b 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -1343,13 +1343,30 @@ def is_directory_format(path) -> bool: return True +def _resolves_assembled_patches(spool) -> bool: + """ + Return True when the spool can produce patches that are not literal + persisted file reads (live patches or plan-assembled outputs). + + Persisted patches are always contiguous, so purely file-backed + spools skip gap inspection; plan resolvers can assemble several + sources across a real gap into a segmented coordinate. + """ + if getattr(spool, "has_live_patches", False): + return True + catalog = getattr(spool, "_catalog", None) + resolver = getattr(catalog, "resolver", None) + return bool(getattr(resolver, "plan_entries", dict)()) + + def _maybe_split_gapped_patches(spool, fiber_io, split): """Handle patches whose dimensional coords contain gaps before writing.""" from dascore.core.coords import CoordSegmented - # Only in-memory patches are inspected; file-backed patches always have - # contiguous coordinates (gapped patches are never persisted). - if not getattr(spool, "has_live_patches", False): + # Gap inspection depends on what the spool resolves, not on where + # its ultimate members live: only literal file reads are always + # contiguous (gapped patches are never persisted). + if not _resolves_assembled_patches(spool): return spool def _has_gaps(patch): diff --git a/dascore/io/index/backend.py b/dascore/io/index/backend.py index 8daeb9748..84b68b756 100644 --- a/dascore/io/index/backend.py +++ b/dascore/io/index/backend.py @@ -11,7 +11,7 @@ import abc import time import warnings -from contextlib import contextmanager, suppress +from contextlib import contextmanager, nullcontext, suppress from pathlib import Path import numpy as np @@ -202,15 +202,21 @@ def _transaction(self): Commits on normal (or early-return) exit; on any error rolls back without letting a failed rollback mask the original exception. + The backend's statement lock (when present) is held for the whole + transaction, so readers sharing the connection can never observe + a half-applied write; the reentrant lock keeps the statement + helpers inside the body working unchanged. """ - self._begin() - try: - yield - self._commit() - except Exception: - with suppress(Exception): - self._rollback() - raise + lock = getattr(self, "_lock", None) + with lock if lock is not None else nullcontext(): + self._begin() + try: + yield + self._commit() + except Exception: + with suppress(Exception): + self._rollback() + raise # --- schema ------------------------------------------------------ @@ -551,7 +557,11 @@ def write_sources(self, records: list[SourceRecord]) -> None: tuple(PATCH_COORDS), [(pid, name, dims, def_ids[key]) for pid, name, dims, key in link_rows], ) - self._execute("UPDATE meta_data SET last_indexed_ns = ?", (now,)) + # meta_data.last_indexed_ns is the initial-update-complete + # marker; only mark_initial_update_done (after renumbering + # succeeds) may set it, or an interruption here would defeat + # the reopen recovery path. Per-source timestamps already + # live on the sources rows. # Batch size for IN (...) parameter lists; SQLite caps bound # variables (32766 by default) so large replacements must chunk. diff --git a/dascore/io/index/catalog.py b/dascore/io/index/catalog.py index 1b480d5eb..7a4d29586 100644 --- a/dascore/io/index/catalog.py +++ b/dascore/io/index/catalog.py @@ -39,6 +39,11 @@ from dascore.utils.paths import is_memory_uri from dascore.utils.pd import adjust_segments, relative_ranges_to_absolute +# Directory archives present in per-patch time order (source ordinals +# alone cannot interleave multi-patch files); ordinal and patch id stay +# the deterministic tiebreak inside the ORDER BY. +_DIRECTORY_ORDER = ("coord", "time", True) + class _CanonicalRange: """ @@ -358,14 +363,28 @@ def _absolutize_record(record, root): return replace(record, source_path=resolved, base_uri=None) -def _membership_resolver(resolver: PatchResolver, keep: dict) -> PatchResolver: - """Return a copy of resolver whose live registry holds only `keep`.""" +def _membership_resolver( + resolver: PatchResolver, keep: dict, paths=() +) -> PatchResolver: + """ + Return a copy of resolver whose live registry holds only `keep`. + + ``paths`` are the synthetic paths the view's rows reference: plan + routes serving any of them must survive the restriction or mixed + planned/live views lose their plan-backed rows on serialization. + """ if isinstance(resolver, LiveResolver): out = LiveResolver() out._registry = dict(keep) return out out = CompositeResolver() out.live._registry = dict(keep) + plans = getattr(resolver, "plan_entries", dict)() + out.plans = { + prefix: plan + for prefix, plan in plans.items() + if any(str(p).startswith(prefix) for p in paths) + } return out @@ -419,6 +438,7 @@ def __init__( revision: _CatalogRevision | None = None, order: tuple | None = None, ids: tuple | None = None, + default_order: tuple | None = None, ): self._backend = backend self.resolver = resolver @@ -428,6 +448,11 @@ def __init__( # presentation specs (D2): an order override ("attr"|"coord", # name, ascending) and/or an ordered patch-id membership self._order = order + # the catalog's own presentation contract when no user order is + # set (directory archives present in per-patch time order — + # source ordinals alone cannot interleave multi-patch files). + # Not view state: a root with a default order still updates. + self._default_order = default_order self._ids = None if ids is None else tuple(int(x) for x in ids) self._revision = revision or _CatalogRevision() self._df_cache: pd.DataFrame | None = None @@ -533,6 +558,7 @@ def from_directory( backend=syncer._backend, resolver=FileResolver(root=syncer.path), syncer=syncer, + default_order=_DIRECTORY_ORDER, ) # --- internals ------------------------------------------------------ @@ -605,7 +631,7 @@ def __getstate__(self) -> dict: paths = list(dict.fromkeys(df["path"].astype(str))) entries = self.resolver.live_entries() keep = {k: entries[k] for k in paths if k in entries} - state["resolver"] = _membership_resolver(self.resolver, keep) + state["resolver"] = _membership_resolver(self.resolver, keep, paths) return state def _view(self, queries, residuals, order=_KEEP, ids=_KEEP) -> PatchCatalog: @@ -618,6 +644,7 @@ def _view(self, queries, residuals, order=_KEEP, ids=_KEEP) -> PatchCatalog: revision=self._revision, order=self._order if order is _KEEP else order, ids=self._ids if ids is _KEEP else ids, + default_order=self._default_order, ) return out @@ -642,6 +669,11 @@ def order_by(self, attribute: str, ascending: bool = True) -> PatchCatalog: raise IndexError(msg) return self._view(self._queries, self._residuals, order=spec) + @property + def _effective_order(self) -> tuple | None: + """The presentation order: a user order spec, else the default.""" + return self._order if self._order is not None else self._default_order + def _ordered_ids(self) -> tuple[int, ...]: """The view's patch ids in presentation order (ids only, cheap).""" if self._ids is not None and self._order is None: @@ -649,7 +681,7 @@ def _ordered_ids(self) -> tuple[int, ...]: return tuple( self.backend.query_ids( list(self._queries) or None, - order_by=self._order, + order_by=self._effective_order, patch_ids=self._ids, ) ) @@ -797,7 +829,7 @@ def to_df(self) -> pd.DataFrame: if self._df_cache is None or self._df_cache_revision != self._revision.value: df = self.backend.query( list(self._queries) or None, - order_by=self._order, + order_by=self._effective_order, patch_ids=self._ids, ) if self._ids is not None and self._order is None: diff --git a/dascore/io/index/planned.py b/dascore/io/index/planned.py index 4d1ddde52..6fb6c27b9 100644 --- a/dascore/io/index/planned.py +++ b/dascore/io/index/planned.py @@ -475,10 +475,12 @@ def collapse_working_df(catalog: PatchCatalog) -> pd.DataFrame | None: """ Return the re-planning frame for a derived catalog, or None. - Plans collapse (never nest): re-chunking a planned spool plans over - the current view's *members* — the trimmed source rows — restricted - to outputs the view still presents, with the view's value residuals - applied to the envelopes. + Re-planning the *same* dimension collapses: it plans over the + current view's *members* — the trimmed source rows — restricted to + outputs the view still presents, with the view's value residuals + applied to the envelopes. (Planning a different dimension must keep + the assembled boundaries, so its caller plans over the output rows + instead and never collapses.) """ resolver = catalog.resolver if not isinstance(resolver, PlanResolver): diff --git a/docs/changelog.qmd b/docs/changelog.qmd index 1eaf8cfc2..d91439895 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -12,6 +12,7 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f - Directory indexes now use the constrained seven-table SQLite schema in `.dascore_index.sqlite3`. Experimental DuckDB and Parquet index backends and the `engine`/`index_engine` selection parameters were removed. Prototype indexes from the earlier schema must be deleted and rebuilt. - Combining spools (`a + b`) now preserves each operand's full current contents: pending coordinate/samples trims and sort order are baked into the union (as new patch identities backed by the same lazy loading) instead of being silently dropped. Membership-style state (attribute selections, slices, patch-id arrays) still unions by rows, so identity-based deduplication keeps working there. - `Spool.sort(...)` accepts any coordinate of the spool (including renamed and auxiliary coordinates), not just `time`/`distance`; chunk/concatenate outputs keep describing non-dimension coordinates so they remain selectable and sortable; contiguous descending-coordinate patches now merge under `chunk(dim=None)`; and `concatenate` on an empty spool returns an empty spool. +- Chunking a restructured spool along a different dimension now plans over the spool's current patches, so `chunk(time=None).chunk(distance=...)` partitions both dimensions instead of silently undoing the first operation (re-chunking the *same* dimension still re-plans from the original members). Directory spools present patches in per-patch time order even when one multi-patch file straddles a patch of another; mixed planned/live views keep their plan-backed rows through serialization and process-backed `map`; and the segmented-coordinate write guard covers plan-assembled (file-backed) spools, not only in-memory ones. - Chunk partitioning is unit-aware along the chunked dimension: patches whose coordinate units have different dimensionality (or unitful vs unitless) can never plan into one output — they were previously grouped by raw SI magnitude and failed only at load. Compatible unit spellings (e.g. metres and feet) now merge correctly, converted to the first member's units. - Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces (as `name -> selector` mappings, or as a name/collection of names tagging bare keyword arguments), and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. - The legacy PyTables directory index has been replaced by SQLite. SQLite supports concurrent readers and one serialized writer on local filesystems; network filesystems with unreliable locking are not supported. diff --git a/tests/test_core/test_coord_segmented.py b/tests/test_core/test_coord_segmented.py index 36afc3bd3..12c57064d 100644 --- a/tests/test_core/test_coord_segmented.py +++ b/tests/test_core/test_coord_segmented.py @@ -1115,3 +1115,44 @@ def test_units(self): values = np.array([0.0, 1, 2, 10, 11, 12]) coord = CoordSegmented.from_array(values, units="m") assert get_quantity(coord.units) == get_quantity("m") + + +class TestPlannedSpoolWriteGuard: + """The gap write guard covers plan-assembled spools (round-4 F3).""" + + @pytest.fixture() + def gapped_planned_spool(self, tmp_path): + """A file-backed planned spool whose output spans a real gap.""" + import warnings + + src = tmp_path / "src" + src.mkdir() + p1 = dc.get_example_patch() + t = p1.get_coord("time") + p2 = p1.update_coords(time_min=t.max() + 4 * t.step) + dc.write(p1, src / "a.h5", "DASDAE") + dc.write(p2, src / "b.h5", "DASDAE") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + planned = ( + dc.spool(src) + .update(progress=None) + .chunk(time=None, tolerance=5, snap_coords=False, conflict="drop") + ) + assert isinstance(planned[0].get_coord("time"), CoordSegmented) + assert not planned.has_live_patches + return planned + + def test_write_raises_without_split(self, gapped_planned_spool, tmp_path): + """Writing a gapped planned spool raises the documented error.""" + with pytest.raises(ParameterError, match="split"): + dc.write(gapped_planned_spool, tmp_path / "out.h5", "DASDAE") + + def test_write_split_true(self, gapped_planned_spool, tmp_path): + """split=True writes each contiguous section as its own patch.""" + path = tmp_path / "out.h5" + dc.write(gapped_planned_spool, path, "DASDAE", split=True) + back = dc.spool(path) + assert len(back) == 2 + for patch in back: + assert not isinstance(patch.get_coord("time"), CoordSegmented) diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index 4c7d3ccee..278e40f04 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -869,3 +869,63 @@ def test_same_units_unchanged(self): out = sp.chunk(distance=None, conflict="drop") assert len(out) == 1 assert str(out[0].get_coord("distance").units) == "1 m" + + +class TestChainedChunk: + """Chunking a derived spool along another dimension (round-4 F1).""" + + def test_other_dim_keeps_prior_boundaries(self): + """Re-chunking distance must not undo a time concatenation.""" + p1 = dc.get_example_patch() + t = p1.get_coord("time") + p2 = p1.update_coords(time_min=t.max() + t.step) + merged = dc.spool([p1, p2]).chunk(time=None, conflict="drop") + current = merged[0] + d = current.get_coord("distance") + size = (d.max() - d.min()) / 2 + actual = merged.chunk(distance=size, keep_partial=True, conflict="drop") + expected = dc.spool([current]).chunk( + distance=size, keep_partial=True, conflict="drop" + ) + assert sorted(x.shape for x in actual) == sorted(x.shape for x in expected) + got = { + (str(x.get_coord("time").min()), str(x.get_coord("time").max())) + for x in actual + } + want = { + (str(x.get_coord("time").min()), str(x.get_coord("time").max())) + for x in expected + } + assert got == want + + def test_segment_then_segment(self): + """chunk(time=...) then chunk(distance=...) partitions both dims.""" + p = dc.get_example_patch() # (300, 2000), 8 s + out = dc.spool([p]).chunk(time=2).chunk(distance=100) + assert len(out) == 12 + assert {x.shape for x in out} == {(100, 500)} + + def test_same_dim_rechunk_still_collapses(self): + """Re-chunking the same dim re-plans from members (no nesting).""" + p1 = dc.get_example_patch() + t = p1.get_coord("time") + p2 = p1.update_coords(time_min=t.max() + t.step) + merged = dc.spool([p1, p2]).chunk(time=None, conflict="drop") + rechunk = merged.chunk(time=2) + assert len(rechunk) == 8 + assert {x.shape for x in rechunk} == {(300, 500)} + + +class TestMatchMergeUnits: + """The member unit normalizer's defensive paths.""" + + def test_incompatible_units_pass_through(self): + """Dimensionality mismatches pass through for the merge to police.""" + from dascore.units import get_quantity + from dascore.utils.patch_assembly import _match_merge_units + + patch = dc.get_example_patch().set_units(distance="m") + target = get_quantity("s").units + out, kept = _match_merge_units(patch, "distance", target) + assert out is patch # unconverted + assert kept == target diff --git a/tests/test_io/test_index/test_index_edge_cases.py b/tests/test_io/test_index/test_index_edge_cases.py index 982444751..9aed8946b 100644 --- a/tests/test_io/test_index/test_index_edge_cases.py +++ b/tests/test_io/test_index/test_index_edge_cases.py @@ -1282,3 +1282,63 @@ def test_data_units_attr_still_indexed(self): patch = dc.get_example_patch().update_attrs(data_units="strain") spool = dc.spool([patch]) assert "data_units" in spool._catalog.backend.attr_names() + + +class TestTransactionIsolation: + """The statement lock covers whole transactions (round-4 F5).""" + + def test_reader_never_sees_half_written_replacement(self, tmp_path): + """A concurrent reader blocks during a source replacement.""" + import threading + + from dascore.io.index.backend import get_backend + from dascore.io.index.ingest import SourceRecord, patch_record + + patch = dc.get_example_patch() + record = SourceRecord( + source_path="mem://one", + source_format="mem", + format_version="", + patches=(patch_record(patch.summary),), + ) + backend = get_backend(str(tmp_path / "idx.sqlite3")) + backend.write_sources([record]) + assert len(backend.query()) == 1 + + in_delete = threading.Event() + release = threading.Event() + original = type(backend)._delete_by_paths + + def paused_delete(self, *args, **kwargs): + out = original(self, *args, **kwargs) + in_delete.set() + release.wait(timeout=10) + return out + + counts = [] + + def read(): + counts.append(len(backend.query())) + + writer = threading.Thread( + target=lambda: backend.write_sources([record]), daemon=True + ) + type(backend)._delete_by_paths = paused_delete + try: + writer.start() + assert in_delete.wait(timeout=10) + # the writer sits mid-transaction with the row deleted; a + # reader must block on the transaction lock, not observe it + reader = threading.Thread(target=read, daemon=True) + reader.start() + reader.join(timeout=0.3) + assert reader.is_alive(), "reader observed a half-written state" + assert counts == [] + release.set() + writer.join(timeout=10) + reader.join(timeout=10) + finally: + type(backend)._delete_by_paths = original + release.set() + assert counts == [1] + backend.close() diff --git a/tests/test_io/test_index/test_ordering.py b/tests/test_io/test_index/test_ordering.py index 6ac8eb92d..67ec8a46f 100644 --- a/tests/test_io/test_index/test_ordering.py +++ b/tests/test_io/test_index/test_ordering.py @@ -196,3 +196,75 @@ def test_hot_coords_still_sort(self): srt = dc.spool([p2, p]).sort("time") mins = [x.get_coord("time").min() for x in srt] assert mins == sorted(mins) + + +class TestInterleavedSourceOrder: + """Directory time order across interleaved multi-patch files (F4).""" + + def test_multi_patch_file_straddles_another(self, tmp_path): + """A patch between two patches of another file presents in order.""" + p0 = dc.get_example_patch() + t = p0.get_coord("time") + span = t.max() - t.min() + t.step + p1 = p0.update_coords(time_min=t.min() + span) + p2 = p0.update_coords(time_min=t.min() + 2 * span) + dc.write(dc.spool([p0, p2]), tmp_path / "a.h5", "DASDAE") + dc.write(p1, tmp_path / "b.h5", "DASDAE") + spool = dc.spool(tmp_path).update(progress=None) + contents = spool.get_contents() + assert contents["time_min"].is_monotonic_increasing + mins = [x.get_coord("time").min() for x in spool] + assert mins == sorted(mins) + # windows and sorting stay consistent with the presentation + assert spool[1:2][0].get_coord("time").min() == mins[1] + assert spool.sort("distance").get_contents().shape[0] == 3 + + def test_default_order_is_not_view_state(self, tmp_path): + """The presentation contract does not make a root a view.""" + dc.write(dc.get_example_patch(), tmp_path / "a.h5", "DASDAE") + spool = dc.spool(tmp_path).update(progress=None) + assert spool.update() is not None # root update allowed + + +class TestInterruptedInitialUpdate: + """The initial-update marker only sets after renumbering (F6).""" + + def test_interruption_before_renumber_recovers(self, tmp_path): + """A crash after write_sources still renumbers on the retry.""" + from dascore.io.index.indexer import DBDirectoryIndexer + + p0 = dc.get_example_patch() + t = p0.get_coord("time") + span = t.max() - t.min() + t.step + late = p0.update_coords(time_min=t.min() + span) + # walk order (file names) disagrees with time order + dc.write(late, tmp_path / "a_late.h5", "DASDAE") + dc.write(p0, tmp_path / "b_early.h5", "DASDAE") + + class _InterruptedError(RuntimeError): + pass + + indexer = DBDirectoryIndexer(tmp_path) + original = type(indexer._backend).renumber_ordinals_by_time + + def _boom(self): + raise _InterruptedError + + type(indexer._backend).renumber_ordinals_by_time = _boom + try: + with pytest.raises(_InterruptedError): + indexer.ensure_updated() + finally: + type(indexer._backend).renumber_ordinals_by_time = original + del indexer # simulate the process dying after write_sources + + # a fresh open must not treat the interrupted update as done + spool = dc.spool(tmp_path).update(progress=None) + mins = list(spool.get_contents()["time_min"]) + assert mins == sorted(mins) + sources = spool._catalog.backend.get_sources() + by_ordinal = sources.sort_values("ordinal")["source_path"].tolist() + assert [p.split("/")[-1] for p in by_ordinal] == [ + "b_early.h5", + "a_late.h5", + ] diff --git a/tests/test_io/test_index/test_planned.py b/tests/test_io/test_index/test_planned.py index 80370ae8f..163017bdc 100644 --- a/tests/test_io/test_index/test_planned.py +++ b/tests/test_io/test_index/test_planned.py @@ -16,6 +16,7 @@ PlanResolver, _coord_record_from_row, _ns, + collapse_working_df, derived_catalog, ) @@ -316,3 +317,12 @@ def test_all_null_group_skipped(self): {"_patch_id": [1], "sensor_min": [np.nan], "sensor_max": [np.nan]} ) assert _aux_coord_info(sources, members, "time", {"sensor": "distance"}) == {} + + +class TestCollapseGuard: + """collapse_working_df only applies to plan-backed catalogs.""" + + def test_non_plan_catalog_returns_none(self): + """A live catalog has no plan to collapse.""" + catalog = dc.spool([dc.get_example_patch()])._catalog + assert collapse_working_df(catalog) is None diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 3a9ae3187..a2d0fdf5f 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -453,3 +453,41 @@ def test_combined_pickles(self): loaded = pickle.loads(pickle.dumps(combined)) assert len(loaded) == 1 assert loaded[0].shape == combined[0].shape + + +def _patch_shape(patch): + """Module-level shape getter (process pools need a picklable callable).""" + return patch.shape + + +class TestMixedViewPickle: + """Serialization keeps plan routes in mixed views (round-4 F2).""" + + def test_sliced_mixed_union_pickles(self): + """A sliced union of planned and live rows loads all rows back.""" + import pickle + + p = dc.get_example_patch() + t = p.get_coord("time") + trimmed = dc.spool([p]).select( + time=(t.min() + 10 * t.step, t.min() + 20 * t.step) + ) + other = p.new().update_attrs(tag="other") + view = (trimmed + dc.spool([other]))[:] + loaded = pickle.loads(pickle.dumps(view)) + shapes = {loaded[i].shape for i in range(len(loaded))} + assert shapes == {(300, 11), (300, 2000)} + + def test_mixed_union_map_processes(self): + """Process-backed map ships plan routes with each task.""" + from concurrent.futures import ProcessPoolExecutor + + p = dc.get_example_patch() + t = p.get_coord("time") + trimmed = dc.spool([p]).select( + time=(t.min() + 10 * t.step, t.min() + 20 * t.step) + ) + combined = trimmed + dc.spool([p.new().update_attrs(tag="other")]) + with ProcessPoolExecutor(2) as executor: + shapes = set(combined.map(_patch_shape, client=executor)) + assert shapes == {(300, 11), (300, 2000)} From 8d0ba2c6c677a4f6b4caf8a8da427ab439d54ad7 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 13:56:21 +0200 Subject: [PATCH 94/97] Define chunking on dimensions only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A patch carrying the chunk name solely as a non-dimensional coordinate cannot be trimmed or merged along it, but envelope presence made the planner treat it as chunkable: merge outputs failed with CoordError only at patch access, and segmenting happened to slice the riding dimension for numeric coordinates while producing wrong plans for datetime ones. Such patches now fall under missing_dim with the rest of the dimension-less rows — the default raise names how many ride the name as a coordinate, and missing_dim='drop' excludes them. --- dascore/utils/chunk_plan.py | 31 ++++++++++--- docs/changelog.qmd | 1 + tests/test_io/test_index/test_plan.py | 63 +++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index 409197a52..e34e779cb 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -490,18 +490,35 @@ def build_chunk_plan( outputs = pd.DataFrame(columns=[min_name, max_name, "output_id"]) return ChunkPlan(outputs, empty_members, name, value, params) df = _ensure_patch_id(df) - # Missing chunk-dim envelopes (spec 7 / D2). + # Missing chunk-dim envelopes, and patches carrying the name only as + # a non-dimensional coordinate (spec 7 / D2): chunking is defined on + # dimensions, so both fall under missing_dim. Envelope presence is + # not enough — auxiliary coordinates index their envelopes too, but + # their patches cannot be trimmed or merged *along* the name. null_rows = pd.isnull(df[min_name]) | pd.isnull(df[max_name]) - if null_rows.any(): + if "dims" in df.columns: + dim_lists = df["dims"].fillna("").astype(str).str.split(",") + not_a_dim = ~dim_lists.map(lambda dims: name in dims) + else: + not_a_dim = pd.Series(False, index=df.index) + unusable = null_rows | not_a_dim + if unusable.any(): if missing_dim == "raise": - bad = df.loc[null_rows, "_patch_id"].tolist() + bad = df.loc[unusable, "_patch_id"].tolist() + rides = int((not_a_dim & ~null_rows).sum()) + detail = ( + f" ({rides} of them carry {name!r} only as a non-dimensional " + "coordinate; chunking is defined on dimensions)" + if rides + else "" + ) msg = ( - f"{int(null_rows.sum())} patch(es) lack the chunk dimension " - f"{name!r} (patch ids {bad[:5]}...). Pass missing_dim='drop' " - "to exclude them." + f"{int(unusable.sum())} patch(es) lack the chunk dimension " + f"{name!r}{detail} (patch ids {bad[:5]}...). Pass " + "missing_dim='drop' to exclude them." ) raise ChunkError(msg) - df = df[~null_rows] + df = df[~unusable] if df.empty: outputs = pd.DataFrame(columns=[min_name, max_name, "output_id"]) return ChunkPlan(outputs, empty_members, name, value, params) diff --git a/docs/changelog.qmd b/docs/changelog.qmd index d91439895..005e4b540 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -13,6 +13,7 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f - Combining spools (`a + b`) now preserves each operand's full current contents: pending coordinate/samples trims and sort order are baked into the union (as new patch identities backed by the same lazy loading) instead of being silently dropped. Membership-style state (attribute selections, slices, patch-id arrays) still unions by rows, so identity-based deduplication keeps working there. - `Spool.sort(...)` accepts any coordinate of the spool (including renamed and auxiliary coordinates), not just `time`/`distance`; chunk/concatenate outputs keep describing non-dimension coordinates so they remain selectable and sortable; contiguous descending-coordinate patches now merge under `chunk(dim=None)`; and `concatenate` on an empty spool returns an empty spool. - Chunking a restructured spool along a different dimension now plans over the spool's current patches, so `chunk(time=None).chunk(distance=...)` partitions both dimensions instead of silently undoing the first operation (re-chunking the *same* dimension still re-plans from the original members). Directory spools present patches in per-patch time order even when one multi-patch file straddles a patch of another; mixed planned/live views keep their plan-backed rows through serialization and process-backed `map`; and the segmented-coordinate write guard covers plan-assembled (file-backed) spools, not only in-memory ones. +- Chunking is defined on dimensions only: a patch carrying the chunk name solely as a non-dimensional coordinate is treated like a patch missing the dimension — `missing_dim="raise"` (the default) fails eagerly with an explanatory message and `missing_dim="drop"` excludes it. Previously such patches produced plan outputs that failed with `CoordError` only when a patch was accessed. - Chunk partitioning is unit-aware along the chunked dimension: patches whose coordinate units have different dimensionality (or unitful vs unitless) can never plan into one output — they were previously grouped by raw SI magnitude and failed only at load. Compatible unit spellings (e.g. metres and feet) now merge correctly, converted to the first member's units. - Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces (as `name -> selector` mappings, or as a name/collection of names tagging bare keyword arguments), and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. - The legacy PyTables directory index has been replaced by SQLite. SQLite supports concurrent readers and one serialized writer on local filesystems; network filesystems with unreliable locking are not supported. diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index b6652ba54..1c16949cb 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -534,3 +534,66 @@ def test_public_same_dim_envelope_matches_patch(self): got = out.get_contents()["time_max"].iloc[0] assert got == want assert patch.shape[patch.get_axis("time")] == 10 + + +class TestChunkOnlyOnDims: + """Chunking is defined on dimensions; non-dim coords are 'missing'.""" + + @pytest.fixture() + def aux_time_patch(self): + """A patch carrying time only as a coord riding distance.""" + p = dc.get_example_patch() + t = p.get_coord("time") + base = p.mean("time").squeeze() + n = base.shape[base.get_axis("distance")] + return base.update_coords(time=("distance", t.data[:n])) + + def test_aux_only_raises_with_detail(self, aux_time_patch): + """The default error explains the name rides as a coordinate.""" + from dascore.exceptions import ChunkError + + with pytest.raises(ChunkError, match="non-dimensional coordinate"): + dc.spool([aux_time_patch]).chunk(time=None) + + def test_mixed_population_raises_by_default(self, aux_time_patch): + """Mixed dim/aux populations fail eagerly, not at patch access.""" + from dascore.exceptions import ChunkError + + sp = dc.spool([dc.get_example_patch(), aux_time_patch]) + with pytest.raises(ChunkError, match="lack the chunk dimension"): + sp.chunk(time=None) + + def test_drop_excludes_aux_patches(self, aux_time_patch): + """missing_dim='drop' keeps only patches with the real dimension.""" + sp = dc.spool([dc.get_example_patch(), aux_time_patch]) + out = sp.chunk(time=None, missing_dim="drop", conflict="drop") + assert len(out) == 1 + assert out[0].dims == ("distance", "time") + + def test_segmenting_aux_coord_raises(self): + """chunk(=value) is rejected, not accidentally served.""" + import numpy as np + + from dascore.exceptions import ChunkError + + p = dc.get_example_patch() + q = p.update_coords(sensor=("distance", np.arange(p.shape[0], dtype=float))) + with pytest.raises(ChunkError, match="non-dimensional coordinate"): + dc.spool([q]).chunk(sensor=100) + + +class TestDimlessFrames: + """Plain planner frames without a dims column still plan.""" + + def test_frame_without_dims_column(self): + """Dimension membership checks are skipped when dims is absent.""" + df = pd.DataFrame( + { + "time_min": [0.0, 10.0], + "time_max": [10.0, 20.0], + "time_step": [1.0, 1.0], + } + ) + plan = build_chunk_plan(df, time=None) + assert len(plan.outputs) == 1 + assert len(plan.members) == 2 From 5ca467dab1e1c85b1d7acce2ebae54795bdad98c Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 14:23:26 +0200 Subject: [PATCH 95/97] Carry directory time order through unions; sort missing values last MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-patch directory order lived only in the catalog's default-order spec, so combining a directory spool fell back to source-record transfer and re-lost the interleaved presentation the order exists to provide — even against an empty spool, breaking order-sensitive equality with itself. The union handoff now bakes the effective order into an identity plan, but only when record-grain transfer would actually present rows differently, so ordinary archives keep record transfer and same-source deduplication. Ordering also treats missing values consistently: rows without a value for the order key sort last under any direction, matching the ordinal renumberer's missing-time-last rule instead of SQLite's NULLs-first default. --- dascore/core/spool.py | 21 +++++++- dascore/io/index/query.py | 7 ++- docs/changelog.qmd | 1 + tests/test_io/test_index/test_ordering.py | 64 +++++++++++++++++++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index 87d16a711..b531a4192 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -593,13 +593,32 @@ def _as_catalog_member(self): table union as-is, but residual trims and order specs live Python-side and would silently vanish; a spool carrying those first bakes them into a derived catalog (tables only — no patch - data is loaded). + data is loaded). A catalog default order (directory time + presentation) bakes only when the source-record transfer would + actually present rows differently — an interleaved multi-patch + file — so ordinary archives keep record-grain transfer and its + same-source deduplication. """ catalog = self._catalog if catalog._residuals or catalog._order is not None: return self._materialize_lossy(), None + if catalog._default_order is not None and not self._transfer_keeps_order(): + return self._materialize_lossy(), None return catalog, None + def _transfer_keeps_order(self) -> bool: + """True when ordinal-grain transfer matches the presented order.""" + catalog = self._catalog + presented = catalog._ordered_ids() + by_ordinal = tuple( + catalog.backend.query_ids( + list(catalog._queries) or None, + order_by=None, + patch_ids=catalog._ids, + ) + ) + return tuple(presented) == by_ordinal + def _materialize_lossy(self): """ Bake residual trims and presentation order into a derived catalog. diff --git a/dascore/io/index/query.py b/dascore/io/index/query.py index d3eadd184..a7f79d8b3 100644 --- a/dascore/io/index/query.py +++ b/dascore/io/index/query.py @@ -420,7 +420,12 @@ def _order_clause( columns = [dialect.quote(c) for c in rows["column_name"]] # an attr observed under several kinds orders by its first column column = f"a.{columns[0]}" - return f"ORDER BY {column} {direction}, s.ordinal, p.patch_id", params + # rows without a value sort last regardless of direction (matching + # the ordinal renumberer's missing-time-last rule); the null key + # repeats the column expression, so its parameters repeat too + params = [*params, *params] + sql = f"ORDER BY {column} IS NULL, {column} {direction}, s.ordinal, p.patch_id" + return sql, params def build_sql( diff --git a/docs/changelog.qmd b/docs/changelog.qmd index 005e4b540..19a38a62d 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -13,6 +13,7 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f - Combining spools (`a + b`) now preserves each operand's full current contents: pending coordinate/samples trims and sort order are baked into the union (as new patch identities backed by the same lazy loading) instead of being silently dropped. Membership-style state (attribute selections, slices, patch-id arrays) still unions by rows, so identity-based deduplication keeps working there. - `Spool.sort(...)` accepts any coordinate of the spool (including renamed and auxiliary coordinates), not just `time`/`distance`; chunk/concatenate outputs keep describing non-dimension coordinates so they remain selectable and sortable; contiguous descending-coordinate patches now merge under `chunk(dim=None)`; and `concatenate` on an empty spool returns an empty spool. - Chunking a restructured spool along a different dimension now plans over the spool's current patches, so `chunk(time=None).chunk(distance=...)` partitions both dimensions instead of silently undoing the first operation (re-chunking the *same* dimension still re-plans from the original members). Directory spools present patches in per-patch time order even when one multi-patch file straddles a patch of another; mixed planned/live views keep their plan-backed rows through serialization and process-backed `map`; and the segmented-coordinate write guard covers plan-assembled (file-backed) spools, not only in-memory ones. +- Directory presentation order survives combining: `+` bakes the per-patch time order into the union when record-grain transfer would present rows differently (interleaved multi-patch files); ordinary archives keep record transfer and same-source deduplication. Rows without a value for the ordering key (e.g. patches without absolute time) sort last under any spool ordering, matching the ordinal renumberer's rule. - Chunking is defined on dimensions only: a patch carrying the chunk name solely as a non-dimensional coordinate is treated like a patch missing the dimension — `missing_dim="raise"` (the default) fails eagerly with an explanatory message and `missing_dim="drop"` excludes it. Previously such patches produced plan outputs that failed with `CoordError` only when a patch was accessed. - Chunk partitioning is unit-aware along the chunked dimension: patches whose coordinate units have different dimensionality (or unitful vs unitless) can never plan into one output — they were previously grouped by raw SI magnitude and failed only at load. Compatible unit spellings (e.g. metres and feet) now merge correctly, converted to the first member's units. - Spool selection now validates unknown names and quantity dimensionality, composes chained regular expressions with AND, supports explicit `_attrs` and `_coords` namespaces (as `name -> selector` mappings, or as a name/collection of names tagging bare keyword arguments), and applies relative offsets only to coordinate selectors. Values indexed without units stay candidates for quantity selectors, and an attribute observed with dimensionally incompatible units across files skips the incompatible values (with a warning) instead of failing the whole index update. diff --git a/tests/test_io/test_index/test_ordering.py b/tests/test_io/test_index/test_ordering.py index 67ec8a46f..cd14e417d 100644 --- a/tests/test_io/test_index/test_ordering.py +++ b/tests/test_io/test_index/test_ordering.py @@ -268,3 +268,67 @@ def _boom(self): "b_early.h5", "a_late.h5", ] + + +class TestDefaultOrderThroughUnion: + """Directory presentation order survives combining (round-5).""" + + @pytest.fixture() + def interleaved_dir_spool(self, tmp_path): + """A directory whose multi-patch file straddles another file.""" + p0 = dc.get_example_patch() + t = p0.get_coord("time") + span = t.max() - t.min() + t.step + p1 = p0.update_coords(time_min=t.min() + span) + p2 = p0.update_coords(time_min=t.min() + 2 * span) + dc.write(dc.spool([p0, p2]), tmp_path / "a.h5", "DASDAE") + dc.write(p1, tmp_path / "b.h5", "DASDAE") + return dc.spool(tmp_path).update(progress=None) + + def test_empty_union_keeps_order_and_equality(self, interleaved_dir_spool): + """Adding an empty spool preserves contents, order, and equality.""" + source = interleaved_dir_spool + combined = source + dc.spool([]) + want = [x.get_coord("time").min() for x in source] + got = [x.get_coord("time").min() for x in combined] + assert got == want + assert combined == source + + def test_live_append_keeps_directory_prefix(self, interleaved_dir_spool): + """A live operand appends after the directory's presented rows.""" + source = interleaved_dir_spool + later = dc.get_example_patch(time_min="2030-01-01") + combined = source + dc.spool([later]) + mins = [x.get_coord("time").min() for x in combined] + assert mins[:3] == [x.get_coord("time").min() for x in source] + assert len(mins) == 4 + + def test_non_interleaved_union_still_dedups(self, tmp_path): + """Ordinary archives keep record-grain transfer and dedup.""" + p0 = dc.get_example_patch() + t = p0.get_coord("time") + span = t.max() - t.min() + t.step + dc.write(p0, tmp_path / "a.h5", "DASDAE") + dc.write(p0.update_coords(time_min=t.min() + span), tmp_path / "b.h5", "DASDAE") + spool = dc.spool(tmp_path).update(progress=None) + assert len(spool + spool) == len(spool) + + +class TestMissingTimeSortsLast: + """Rows without a value sort last under any order (round-5).""" + + def test_directory_no_time_patch_presents_last(self, tmp_path): + """A distance-only patch follows every time-bearing patch.""" + timed = dc.get_example_patch().update_attrs(tag="time") + no_time = timed.mean("time").squeeze().update_attrs(tag="no_time") + dc.write(timed, tmp_path / "a_time.h5", "DASDAE") + dc.write(no_time, tmp_path / "b_no_time.h5", "DASDAE") + spool = dc.spool(tmp_path).update(progress=None) + assert list(spool.get_contents()["tag"]) == ["time", "no_time"] + + def test_sort_puts_missing_values_last(self): + """Explicit sort also presents value-less rows last.""" + timed = dc.get_example_patch().update_attrs(tag="a_time") + no_time = timed.mean("time").squeeze().update_attrs(tag="b_no_time") + spool = dc.spool([no_time, timed]).sort("time") + assert [x.attrs.tag for x in spool] == ["a_time", "b_no_time"] From 9707f17d9ed4b3fbc1eddf56eed5b0fc7d5e3db8 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 14:44:17 +0200 Subject: [PATCH 96/97] Compare spools by effective contents; resolve negative samples indices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spool equality compared raw rows plus residual state, so a trimmed view never equaled its union-materialized twin despite identical contents, and the view's rows still carried the untrimmed coordinate def keys — an identity a view cannot restate honestly without loading. Equality now folds samples residuals into the compared envelopes (value residuals already present in the realized rows), keeps presented-but-empty rows, and drops def keys and private bookkeeping from the comparison; data values were never compared here anyway. The planner's samples adjustment also resolves negative indices per patch from the envelope-derived sample count (unknown counts keep the candidacy envelope), so chunking a tail selection reports the envelopes the loaded patches actually have. --- dascore/core/spool.py | 47 ++++++++++++------ dascore/utils/chunk_plan.py | 44 +++++++++++++---- docs/changelog.qmd | 1 + tests/test_core/test_spool_contracts.py | 30 ++++++++++++ tests/test_io/test_index/test_plan.py | 65 +++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 25 deletions(-) diff --git a/dascore/core/spool.py b/dascore/core/spool.py index b531a4192..90f5665b3 100644 --- a/dascore/core/spool.py +++ b/dascore/core/spool.py @@ -1001,36 +1001,51 @@ def _eq_state(self) -> dict: """ The spool's semantic state, explicitly enumerated for equality. - Equality is over rows, never backends: same length and order of - patch rows, row-wise equal semantic columns (source identity - like paths and live-vs-file backing stripped), plus equal - pending residual selections. Whether rows come from a live - registry, an index file, or a plan is invisible; data arrays - are never compared (metadata-level, like everything here). - Because the state is enumerated — never ``__dict__`` — new - instance attributes cannot silently join equality. + Equality is over *effective* rows, never backends or + representation: same length and order of patch rows, row-wise + equal semantic columns (source identity like paths and + live-vs-file backing stripped), with pending residual + selections folded into the envelopes — a trimmed view equals + its materialized twin, and spools differing only by a samples + trim differ in their adjusted envelopes. Whether rows come from + a live registry, an index file, or a plan is invisible; data + arrays are never compared (metadata-level, like everything + here). Because the state is enumerated — never ``__dict__`` — + new instance attributes cannot silently join equality. """ def _strip_identity(df): # synthetic per-catalog identities (memory:// paths, ids) and # backend provenance (format/version) are not content; equal # spools must compare equal without them, and column order - # (a construction artifact) must not matter. - drop = ( + # (a construction artifact) must not matter. Coordinate def + # keys are representation artifacts too: a residual-trimmed + # view cannot know its trimmed fingerprint without loading, + # and data values are never compared here anyway. + drop = [ "path", "_patch_id", "source_patch_id", "file_format", "file_version", - ) - out = df.drop(columns=list(drop), errors="ignore") + "_modified", + *[c for c in df.columns if str(c).endswith("_def_key")], + ] + out = df.drop(columns=drop, errors="ignore") return out[sorted(out.columns)] + from dascore.utils.chunk_plan import samples_adjusted_envelopes + catalog = self._catalog - return { - "rows": _strip_identity(self._df), - "residuals": None if catalog is None else catalog._residuals, - } + rows = self._df + # value residuals already trim the presented envelopes (to_df); + # samples residuals fold in here. Presented-but-empty rows stay: + # a spool exposing an emptied patch is not equal to one without. + if catalog is not None and catalog._residuals: + rows = samples_adjusted_envelopes( + rows, catalog._residuals, drop_empty=False + ) + return {"rows": _strip_identity(rows)} def __rich__(self): base = super().__rich__() diff --git a/dascore/utils/chunk_plan.py b/dascore/utils/chunk_plan.py index e34e779cb..57952f975 100644 --- a/dascore/utils/chunk_plan.py +++ b/dascore/utils/chunk_plan.py @@ -95,21 +95,25 @@ def _resolve_group_attrs(group, columns) -> tuple[str, ...]: return tuple(x for x in dc.get_config().groupby_attrs if x in columns) -def samples_adjusted_envelopes(df: pd.DataFrame, residuals) -> pd.DataFrame: +def samples_adjusted_envelopes( + df: pd.DataFrame, residuals, drop_empty: bool = True +) -> pd.DataFrame: """ Adjust envelope columns for patch-local samples residuals. A ``samples=True`` index window trims each patch at load, so the planner must consume the trimmed envelopes or it publishes outputs that lie entirely outside the selected samples (phantom empties). - Only non-negative index windows adjust (Python-slice clamping per - patch); anything else leaves the envelope as a candidacy superset — - exactness is always re-applied at load, and the adjustment only - exists so plans reflect the truth. + Negative indices resolve per patch against the envelope-derived + sample count (rows whose count is unknown keep their envelope as a + candidacy superset — exactness is always re-applied at load). + ``drop_empty`` removes rows whose window selects nothing (planning + truth); equality comparison keeps them, since a presented-but-empty + row is still a presented row. """ def _usable_index(value) -> bool: - return value is None or (isinstance(value, int | np.integer) and value >= 0) + return value is None or isinstance(value, int | np.integer) df = df.copy(deep=False) for coords, samples in residuals: @@ -129,21 +133,43 @@ def _usable_index(value) -> bool: # descending ones. abs_steps = steps.abs() descending = to_float(steps.values) < 0 - lo_off = None if lo_idx is None else lo_idx * abs_steps - hi_off = None if hi_idx is None else (hi_idx - 1) * abs_steps + with np.errstate(invalid="ignore", divide="ignore"): + ratio = to_float((maxs - mins).values) / to_float(abs_steps.values) + counts = pd.Series(np.round(ratio) + 1, index=df.index) + + def _positions(idx, counts=counts, index=df.index): + """Per-row absolute positions (Python-slice clamping).""" + if idx is None: + return None + if idx >= 0: + return pd.Series(float(idx), index=index) + return (counts + idx).clip(lower=0) + + lo_pos, hi_pos = _positions(lo_idx), _positions(hi_idx) + unresolved = pd.Series(False, index=df.index) + for pos in (lo_pos, hi_pos): + if pos is not None: + unresolved |= pos.isna() + lo_off = None if lo_pos is None else lo_pos * abs_steps + hi_off = None if hi_pos is None else (hi_pos - 1) * abs_steps new_min = mins if lo_off is None else mins + lo_off new_max = maxs if hi_off is None else mins + hi_off desc_min = maxs if hi_off is None else maxs - hi_off desc_max = maxs if lo_off is None else maxs - lo_off new_min = new_min.where(~descending, other=desc_min) new_max = new_max.where(~descending, other=desc_max) + # unresolvable rows keep their envelope (candidacy superset) + new_min = new_min.mask(unresolved, mins) + new_max = new_max.mask(unresolved, maxs) # rows whose window is empty or lies entirely outside the # patch contribute nothing; test before clipping so such # windows are not resurrected as one-sample envelopes keep = (new_min <= new_max) & (new_min <= maxs) & (new_max >= mins) + keep |= unresolved df[cols[0]] = new_min.clip(lower=mins, upper=maxs) df[cols[1]] = new_max.clip(lower=mins, upper=maxs) - df = df[keep] + if drop_empty: + df = df[keep] return df diff --git a/docs/changelog.qmd b/docs/changelog.qmd index 19a38a62d..16900ca05 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -13,6 +13,7 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f - Combining spools (`a + b`) now preserves each operand's full current contents: pending coordinate/samples trims and sort order are baked into the union (as new patch identities backed by the same lazy loading) instead of being silently dropped. Membership-style state (attribute selections, slices, patch-id arrays) still unions by rows, so identity-based deduplication keeps working there. - `Spool.sort(...)` accepts any coordinate of the spool (including renamed and auxiliary coordinates), not just `time`/`distance`; chunk/concatenate outputs keep describing non-dimension coordinates so they remain selectable and sortable; contiguous descending-coordinate patches now merge under `chunk(dim=None)`; and `concatenate` on an empty spool returns an empty spool. - Chunking a restructured spool along a different dimension now plans over the spool's current patches, so `chunk(time=None).chunk(distance=...)` partitions both dimensions instead of silently undoing the first operation (re-chunking the *same* dimension still re-plans from the original members). Directory spools present patches in per-patch time order even when one multi-patch file straddles a patch of another; mixed planned/live views keep their plan-backed rows through serialization and process-backed `map`; and the segmented-coordinate write guard covers plan-assembled (file-backed) spools, not only in-memory ones. +- Spool equality compares effective contents rather than representation: pending residual selections fold into the compared envelopes (a trimmed view equals its union-materialized twin; spools differing only by a trim still differ), and internal coordinate identity keys no longer participate. Negative `samples=True` indices now resolve per patch in the planner, so chunking a tail selection reports honest envelopes. - Directory presentation order survives combining: `+` bakes the per-patch time order into the union when record-grain transfer would present rows differently (interleaved multi-patch files); ordinary archives keep record transfer and same-source deduplication. Rows without a value for the ordering key (e.g. patches without absolute time) sort last under any spool ordering, matching the ordinal renumberer's rule. - Chunking is defined on dimensions only: a patch carrying the chunk name solely as a non-dimensional coordinate is treated like a patch missing the dimension — `missing_dim="raise"` (the default) fails eagerly with an explanatory message and `missing_dim="drop"` excludes it. Previously such patches produced plan outputs that failed with `CoordError` only when a patch was accessed. - Chunk partitioning is unit-aware along the chunked dimension: patches whose coordinate units have different dimensionality (or unitful vs unitless) can never plan into one output — they were previously grouped by raw SI magnitude and failed only at load. Compatible unit spellings (e.g. metres and feet) now merge correctly, converted to the first member's units. diff --git a/tests/test_core/test_spool_contracts.py b/tests/test_core/test_spool_contracts.py index 3cb56fc02..88c407418 100644 --- a/tests/test_core/test_spool_contracts.py +++ b/tests/test_core/test_spool_contracts.py @@ -174,3 +174,33 @@ def test_live_patch_predicate(self, patches, tmp_path): assert not dc.spool(file_path).has_live_patches mixed = dc.spool(file_path) + dc.spool(patches[:1]) assert mixed.has_live_patches + + +class TestEqualityOverEffectiveRows: + """Equality compares contents, not representation (2026-07-18).""" + + @pytest.fixture() + def patch(self): + """The example patch.""" + return dc.get_example_patch() + + def test_value_trimmed_view_equals_materialized(self, patch): + """A coordinate-trimmed view equals its union-materialized twin.""" + t = patch.get_coord("time") + sel = dc.spool([patch]).select( + time=(t.min() + 10 * t.step, t.min() + 20 * t.step) + ) + assert sel == sel + dc.spool([]) + + @pytest.mark.parametrize("window", [(0, 10), (-10, None)]) + def test_samples_trimmed_view_equals_materialized(self, patch, window): + """Samples-trimmed views (negative included) equal their twins.""" + sel = dc.spool([patch]).select(time=window, samples=True) + assert sel == sel + dc.spool([]) + + def test_differing_trims_stay_unequal(self, patch): + """Different windows fold to different envelopes and stay unequal.""" + a = dc.spool([patch]).select(time=(0, 10), samples=True) + b = dc.spool([patch]).select(time=(0, 11), samples=True) + assert a != b + assert a != dc.spool([patch]) diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index 1c16949cb..a8db642a3 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -597,3 +597,68 @@ def test_frame_without_dims_column(self): plan = build_chunk_plan(df, time=None) assert len(plan.outputs) == 1 assert len(plan.members) == 2 + + +class TestNegativeSamplesEnvelopes: + """Negative samples indices resolve per patch (2026-07-18).""" + + @staticmethod + def _frame(): + """One ascending row: 10 samples at step 1 spanning [0, 9].""" + return pd.DataFrame({"time_min": [0.0], "time_max": [9.0], "time_step": [1.0]}) + + def test_negative_start_resolves(self): + """(-3, None) selects the last three samples.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + out = samples_adjusted_envelopes(self._frame(), (({"time": (-3, None)}, True),)) + assert out["time_min"].iloc[0] == 7.0 + assert out["time_max"].iloc[0] == 9.0 + + def test_negative_stop_resolves(self): + """(None, -2) drops the last two samples.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + out = samples_adjusted_envelopes(self._frame(), (({"time": (None, -2)}, True),)) + assert out["time_max"].iloc[0] == 7.0 + + def test_unknown_step_keeps_envelope(self): + """Rows whose count is unknown keep their candidacy envelope.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + df = pd.DataFrame({"time_min": [0.0], "time_max": [9.0], "time_step": [np.nan]}) + out = samples_adjusted_envelopes(df, (({"time": (-3, None)}, True),)) + assert len(out) == 1 + assert out["time_min"].iloc[0] == 0.0 + assert out["time_max"].iloc[0] == 9.0 + + def test_drop_empty_false_keeps_rows(self): + """Equality's variant keeps presented-but-empty rows.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + out = samples_adjusted_envelopes( + self._frame(), (({"time": (3, 3)}, True),), drop_empty=False + ) + assert len(out) == 1 + + def test_public_negative_window_contents_honest(self): + """Derived contents match the loaded patch for negative windows.""" + p = dc.get_example_patch() + out = dc.spool([p]).select(time=(-10, None), samples=True).chunk(time=None) + got = out.get_contents()["time_min"].iloc[0] + want = pd.Timestamp(out[0].get_coord("time").min()) + assert got == want + assert out[0].shape[out[0].get_axis("time")] == 10 + + +class TestNonIntSamplesIndices: + """Non-integer samples indices leave envelopes untouched.""" + + def test_float_index_skipped(self): + """A float index cannot adjust; the envelope stays candidacy.""" + from dascore.utils.chunk_plan import samples_adjusted_envelopes + + df = pd.DataFrame({"time_min": [0.0], "time_max": [9.0], "time_step": [1.0]}) + out = samples_adjusted_envelopes(df, (({"time": (0.5, None)}, True),)) + assert out["time_min"].iloc[0] == 0.0 + assert out["time_max"].iloc[0] == 9.0 From e9a873cc0dded70a3a3a7c0e877fe3f9e06ce43b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Sat, 18 Jul 2026 14:57:27 +0200 Subject: [PATCH 97/97] Bring the spool notes up to date with the review-round semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ordering section still described source-ordinal renumbering as the whole directory contract; it now documents the per-patch time presentation (missing-time last, ordinal/patch-id tiebreaks) with renumbering demoted to the stability/dedup grain. Federation documents the row-vs-baked transfer split, plan routing, and effective-contents equality; the flat relation lists the canonical-units column; the chunking note covers one-dim-per-call chaining, unit and orientation partitioning, and non-dim coordinates counting as missing — each new claim with an executable cell. The spool tutorial loses the stale warning that some spool types lack concatenate. --- docs/notes/spool_chunking.qmd | 21 +++++++++++++++++---- docs/notes/spool_index.qmd | 28 ++++++++++++++++++++++++---- docs/tutorial/spool.qmd | 6 +----- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/docs/notes/spool_chunking.qmd b/docs/notes/spool_chunking.qmd index 6ea7f820d..403a3816d 100644 --- a/docs/notes/spool_chunking.qmd +++ b/docs/notes/spool_chunking.qmd @@ -6,7 +6,7 @@ title: Spool Chunking ## Plans -The planner consumes the spool's flat metadata relation and produces a `ChunkPlan`: an `outputs` table (one row per patch the chunked spool will contain) and a `members` table (which slice of which source patch feeds each output). `Spool.chunk_plan` exposes the plan `chunk` would execute, with the same arguments. The chunked spool *is* a fresh in-memory catalog whose patch rows are the plan outputs; a plan resolver loads each output's members through the parent's resolver and executes the assembly engine (`dascore.utils.patch_assembly`) row by row. Re-chunking a chunked spool re-plans from the current view's members, so plans never nest, and `concatenate` is the same machinery with order-based grouping instead of continuity. +The planner consumes the spool's flat metadata relation and produces a `ChunkPlan`: an `outputs` table (one row per patch the chunked spool will contain) and a `members` table (which slice of which source patch feeds each output). `Spool.chunk_plan` exposes the plan `chunk` would execute, with the same arguments. The chunked spool *is* a fresh in-memory catalog whose patch rows are the plan outputs; a plan resolver loads each output's members through the parent's resolver and executes the assembly engine (`dascore.utils.patch_assembly`) row by row. Chunking is one dimension per call; multi-dimensional chunking chains. Re-chunking a chunked spool along the *same* dimension re-plans from the current view's members (the collapse rule); chunking a *different* dimension plans over the spool's current output rows, preserving the boundaries the earlier operation assembled — `chunk(time=2).chunk(distance=100)` partitions both dimensions. `concatenate` is the same machinery with order-based grouping instead of continuity. ```{python} import dascore as dc @@ -18,6 +18,11 @@ assert {"output_id", "_patch_id", "_modified"}.issubset(plan.members.columns) # Resolved parameters are recorded on the plan, never left in config. assert isinstance(plan.params["group"], tuple) assert plan.params["sampling_group_tolerance"] == dc.get_config().sampling_group_tolerance + +# Chaining partitions both dimensions: the second chunk plans over the +# first one's outputs, not the original members. +chained = dc.get_example_spool("random_das").chunk(time=None).chunk(distance=100) +assert all(p.shape[p.get_axis("distance")] <= 100 for p in chained) ``` Plans are deterministic: the same spool produces the same plan regardless of metadata row order, and members with `_modified=False` load whole (no per-patch selection cost). @@ -27,8 +32,8 @@ Plans are deterministic: the same spool produces the same plan regardless of met Patches may only combine when they agree on all of: 1. **Group attributes** — the config option `groupby_attrs` by default (conventional categorical identity: network, station, data type/category, tag, instrument and acquisition ids), overridden per call with `group=`. Differing group values are never an error; the patches simply land in separate outputs. Explicitly passed names must exist somewhere in the spool; config names are best-effort. -2. **Structure** — the dimensions tuple and the coordinate identity of every non-chunked dimension. -3. **Sampling** — steps within the relative tolerance `config.sampling_group_tolerance` (default 5%). +2. **Structure** — the dimensions tuple, the coordinate identity of every non-chunked dimension, and the chunked dimension's canonical units (a metre patch can never plan into one output with a seconds patch, or a unitful with a unitless one; compatible spellings such as metres and feet plan together and assembly converts them to the first member's units). +3. **Sampling** — step magnitudes within the relative tolerance `config.sampling_group_tolerance` (default 5%) of the group's smallest member, with matching orientation (ascending never merges with descending; contiguous descending patches merge with each other). 4. **Continuity** — patches within `tolerance` samples of each other, evaluated within each group so unrelated patches can never bridge a gap. ```{python} @@ -47,7 +52,7 @@ assert len(merged) == 2 Remaining (non-group, non-dimensional) attributes must be single-valued within a partition, policed by `conflict`: `"raise"` (default), `"drop"`, or `"keep_first"`. -Patches lacking the chunked dimension raise by default; `missing_dim="drop"` excludes them instead. Losing patches silently would be data loss, so it requires the explicit opt-in. +Chunking is defined on dimensions: a patch that lacks the chunked dimension — including one that carries the name only as a *non-dimensional coordinate*, which cannot be trimmed or merged along it — raises by default, and `missing_dim="drop"` excludes it instead. Losing patches silently would be data loss, so it requires the explicit opt-in. ```{python} import pytest @@ -57,6 +62,14 @@ no_time = [dc.get_example_patch().mean("time") for _ in range(2)] with pytest.raises(ChunkError, match="missing_dim"): dc.spool(no_time).chunk(time=None) assert len(dc.spool(no_time).chunk(time=None, missing_dim="drop")) == 0 + +# A name carried only as a non-dimensional coordinate counts as missing. +base = dc.get_example_patch() +aux = base.update_coords( + sensor=("distance", np.arange(base.shape[base.get_axis("distance")], dtype=float)) +) +with pytest.raises(ChunkError, match="non-dimensional coordinate"): + dc.spool([aux]).chunk(sensor=100) ``` ## Merged coordinates diff --git a/docs/notes/spool_index.qmd b/docs/notes/spool_index.qmd index 2e3e18363..08a91c870 100644 --- a/docs/notes/spool_index.qmd +++ b/docs/notes/spool_index.qmd @@ -30,13 +30,15 @@ The current schema version is validated before any mutation. An unrelated or inc ## Ordering -Patch rows present in `(sources.ordinal, patch_id)` order — the catalog's explicit ordering contract. Ordinals are assigned at ingest: a replaced source keeps its position while new sources append, so merging catalogs concatenates and duplicate sources keep their first-occurrence position with last-occurrence metadata (dict-merge semantics). Spools built from in-memory patches therefore iterate in construction order on every path. The directory indexer renumbers ordinals to time order (earliest patch per source, path as tiebreak) after each sync, so file archives keep their conventional time-ordered presentation, including files added by later updates. +The base ordering contract is `(sources.ordinal, patch_id)`: ordinals are assigned at ingest, a replaced source keeps its position while new sources append, so merging catalogs concatenates and duplicate sources keep their first-occurrence position with last-occurrence metadata (dict-merge semantics). Spools built from in-memory patches therefore iterate in construction order on every path. + +Directory catalogs additionally carry a **per-patch default presentation order**: rows present by time (`time IS NULL` last, then time, with ordinal and patch id as deterministic tiebreaks), because source-grain ordinals alone cannot interleave a multi-patch file whose span straddles a patch of another file. The default order is a catalog contract, not view state — a directory root still updates — and an explicit `sort(...)` replaces it. The directory indexer still renumbers source ordinals to time order (earliest patch per source, path as tiebreak) after each sync, which keeps the ordinal grain stable for replacement and union dedup. SQLite permits concurrent readers and serializes writers. Initialization and updates use an immediate write transaction and a 30-second busy timeout. This relies on correct local-filesystem locking; reliable operation on network filesystems with weak locking is not promised. ## The flat relation -Spool-facing operations consume the tables through one flat relation: a dataframe with one row per patch carrying `{dim}_min/max/step` envelopes, private structural columns (`_patch_id`, `_{dim}_def_key` coordinate identities), the `dims` signature, and one column per attribute. The chunk planner and selection both operate on this relation (see the [Spool Chunking](spool_chunking.qmd) and [Spool Selection](spool_selection.qmd) notes). The cell below runs against the real catalog so this description cannot silently drift. +Spool-facing operations consume the tables through one flat relation: a dataframe with one row per patch carrying `{dim}_min/max/step` envelopes, private structural columns (`_patch_id`, `_{name}_def_key` coordinate identities, and `_{name}_units` canonical units — the chunk planner's unit-partition key), the `dims` signature, and one column per attribute. The chunk planner and selection both operate on this relation (see the [Spool Chunking](spool_chunking.qmd) and [Spool Selection](spool_selection.qmd) notes). The cell below runs against the real catalog so this description cannot silently drift. ```{python} import dascore as dc @@ -44,7 +46,15 @@ from dascore.io.index.catalog import PatchCatalog catalog = PatchCatalog.from_patches(list(dc.get_example_spool("random_das"))) df = catalog.to_df() -required = {"_patch_id", "_time_def_key", "dims", "time_min", "time_max", "time_step"} +required = { + "_patch_id", + "_time_def_key", + "_time_units", + "dims", + "time_min", + "time_max", + "time_step", +} assert required.issubset(df.columns) ``` @@ -82,13 +92,23 @@ File-backed patches are identified by `(base_uri, source_path, source_patch_id)` ## Federation -`spool + spool` merges catalogs table-to-table: source records are reconstructed from the member backends and re-ingested, so coordinate definitions deduplicate by definition key, the same source appearing in several members keeps a single entry, and file paths are absolutized so members with different roots coexist. A composite resolver routes in-memory rows to the shared registry and everything else through file readers. +`spool + spool` merges catalogs table-to-table: source records are reconstructed from the member backends and re-ingested, so coordinate definitions deduplicate by definition key, the same source appearing in several members keeps a single entry, and file paths are absolutized so members with different roots coexist. A composite resolver routes in-memory rows to the shared registry, plan-output rows to their plan resolvers, and everything else through file readers. + +Row-membership state (attribute predicates, slice windows, patch-id arrays) transfers as rows, preserving the set semantics above. State that only lives Python-side — coordinate/samples residual trims, sort specs, and a directory default order that record-grain transfer would actually scramble — is first **baked into an identity-plan derived catalog** (table work only; no patch data loads), so each operand contributes exactly its current contents in its current order. Baking mints new patch identities, so a trimmed operand no longer deduplicates against its source: its contents genuinely differ. Spool equality follows the same philosophy — it compares *effective contents* (residual trims folded into the envelopes, representation artifacts like def keys and backing excluded), so a trimmed view equals its union-materialized twin. ```{python} other = dc.get_example_patch().new() union = dc.spool([patch]) + dc.spool([other]) assert len(union) == 2 assert len(dc.spool([patch]) + dc.spool([patch])) == 1 # same patch dedups + +# A trimmed operand contributes its trimmed contents (baked, not dropped), +# and equality compares those effective contents. +t = patch.get_coord("time") +trimmed = dc.spool([patch]).select(time=(t.min() + 10 * t.step, t.min() + 20 * t.step)) +combined = trimmed + dc.spool([]) +assert combined[0].shape == trimmed[0].shape +assert combined == trimmed ``` ## Scope diff --git a/docs/tutorial/spool.qmd b/docs/tutorial/spool.qmd index ae7597ff3..dd080418f 100644 --- a/docs/tutorial/spool.qmd +++ b/docs/tutorial/spool.qmd @@ -219,11 +219,7 @@ merged_spool = spool.chunk(time=None) ``` # concatenate -Similar to `chunk`, [`Spool.concatenate`](`dascore.BaseSpool.concatenate`) is used to combine patches together. However, `concatenate` doesn't account for coordinate values along the concatenation axis, and can even be used to create new patch dimensions. - -:::{.callout-warning} -However, unlike [`chunk`](`dascore.BaseSpool.chunk`), not all `Spool` types implement [`concatenate`](`dascore.BaseSpool.concatenate`). -::: +Similar to `chunk`, [`Spool.concatenate`](`dascore.BaseSpool.concatenate`) is used to combine patches together. However, `concatenate` doesn't account for coordinate values along the concatenation axis, and can even be used to create new patch dimensions. Like `chunk`, it is available on every spool and produces a lazy, plan-backed result. ```python import dascore as dc