From 6889e625be26df68f7e6a621496dc3cb447703fd Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 15:12:31 +0200 Subject: [PATCH 01/56] Add the xdas.tiles tile-backed virtual array package Port TileArray and the tile engine registry from the 0.3 line (xdas/virtual) nearly verbatim: lazy positive-step slice folding, manifest-level concatenation, streaming reductions, and per-format load engines. Two 0.2 adaptations: a lazy leading-axis expand_dims supporting the legacy concat-along-a-new-dimension path, and signature-bound streaming dispatch compatible with the DataArray reduction wrappers. --- tests/tiles/conftest.py | 111 ++++ tests/tiles/test_tilearray.py | 878 ++++++++++++++++++++++++++++ xdas/__init__.py | 2 + xdas/core/dataarray.py | 2 +- xdas/tiles/__init__.py | 19 + xdas/tiles/registry.py | 64 +++ xdas/tiles/tilearray.py | 1013 +++++++++++++++++++++++++++++++++ 7 files changed, 2088 insertions(+), 1 deletion(-) create mode 100644 tests/tiles/conftest.py create mode 100644 tests/tiles/test_tilearray.py create mode 100644 xdas/tiles/__init__.py create mode 100644 xdas/tiles/registry.py create mode 100644 xdas/tiles/tilearray.py diff --git a/tests/tiles/conftest.py b/tests/tiles/conftest.py new file mode 100644 index 00000000..d0c7e7a3 --- /dev/null +++ b/tests/tiles/conftest.py @@ -0,0 +1,111 @@ +"""Shared fixtures for the tile-backed virtual array tests.""" + +import h5py +import numpy as np +import pytest + +from xdas.tiles import ENGINES, Engine, TileArray + +NX = 5 + +ENGINE = {"name": "h5py", "dataset": "data"} + + +class H5pyEngine(Engine, name="h5py"): + """Read any HDF5 dataset — the engine of the synthetic test files. + + The format engines each read their own layout; test files belong to + no format, so they are described by this generic load-only engine. + Extra leading selection axes (virtually expanded arrays) pad the + output rank, as the production engines do. + """ + + @staticmethod + def load(path, selection, *, dataset): + with h5py.File(path, "r") as file: + source = file[dataset] + extra = len(selection) - source.ndim + data = source[selection[extra:]] + return data.reshape((1,) * extra + data.shape) + + +@pytest.fixture +def stack(tmp_path): + """Three gzip-compressed HDF5 files with junk edge rows to trim. + + Emulates overlap trimming: each file carries one junk row at its start + and end that the tile's start row (plus the row ``size``) cuts out. + Returns the manifest and the expected stacked values. + """ + paths = [] + sizes = [] + parts = [] + row = 0 + for k, raw_nt in enumerate([12, 9, 14]): + path = str(tmp_path / f"src{k}.h5") + useful = raw_nt - 2 + data = np.full((raw_nt, NX), -999.0) + data[1:-1] = (row + np.arange(useful))[:, None] + np.arange(NX) / 10 + with h5py.File(path, "w") as file: + file.create_dataset("data", data=data, chunks=(4, NX), compression="gzip") + paths.append(path) + sizes.append(useful) + parts.append(data[1:-1]) + row += useful + manifest = TileArray( + paths, + (sizes, NX), + ENGINE, + "float64", + starts=([1, 1, 1], None), + attrs={"units": "strain"}, + ) + return manifest, np.concatenate(parts) + + +@pytest.fixture +def windowed(tmp_path): + """Three files whose rows contribute blob-local windows via ``starts_0``. + + Each file holds junk rows around the useful window; the manifest + exposes blob rows ``[start, start + size)``. The middle file has a + zero start (window at the top of the blob). + """ + paths, sizes, starts, parts = [], [], [], [] + row = 0 + for k, raw_nt in enumerate([12, 9, 14]): + path = str(tmp_path / f"win{k}.h5") + useful = raw_nt - 4 + first = 0 if k == 1 else 2 + data = np.full((raw_nt, NX), -999.0) + good = (row + np.arange(useful))[:, None] + np.arange(NX) / 10 + data[first : first + useful] = good + with h5py.File(path, "w") as file: + file.create_dataset("data", data=data) + paths.append(path) + sizes.append(useful) + starts.append(first) + parts.append(good) + row += useful + manifest = TileArray( + paths, + (sizes, NX), + {"name": "h5py", "dataset": "data"}, + "float64", + starts=(starts, None), + ) + return manifest, np.concatenate(parts) + + +@pytest.fixture +def engine_calls(monkeypatch): + """Record the path of every h5py engine read, delegating to the real one.""" + calls = [] + original = ENGINES["h5py"].load + + def counting(path, selection, **params): + calls.append(path) + return original(path, selection, **params) + + monkeypatch.setattr(ENGINES["h5py"], "load", counting) + return calls diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py new file mode 100644 index 00000000..7daf7acc --- /dev/null +++ b/tests/tiles/test_tilearray.py @@ -0,0 +1,878 @@ +import math + +import h5py +import numpy as np +import numpy.testing as npt +import pytest + +from xdas.tiles import ( + ENGINES, + Engine, + TileArray, +) + +NX = 5 + +ENGINE = {"name": "h5py", "dataset": "data"} + + +def _tile_file(path, data, **kwargs): + """Write *data* to an HDF5 file at *path*.""" + with h5py.File(path, "w") as file: + file.create_dataset("data", data=data, **kwargs) + + +def _random_key(rng, shape, max_step=1): + """A random non-empty positive-step slice per axis.""" + key = [] + for extent in shape: + a = int(rng.integers(0, extent)) + b = int(rng.integers(a + 1, extent + 1)) + s = int(rng.integers(1, max_step + 1)) + key.append(slice(a, b, s)) + return tuple(key) + + +def _random_grid(tmp_path, rng, ndim): + """A random rectilinear grid of junk-padded files, one per tile. + + Per-axis margins model source cropping (axis-separable, as the grid + requires): every tile at index ``i`` along axis ``k`` starts at + ``margins[k][i]`` inside its own file. + """ + counts = tuple(int(rng.integers(1, 4)) for _ in range(ndim)) + sizes = [rng.integers(2, 6, count).astype(np.int64) for count in counts] + margins = [rng.integers(0, 3, count).astype(np.int64) for count in counts] + shape = tuple(int(entry.sum()) for entry in sizes) + edges = [np.concatenate(([0], np.cumsum(entry))) for entry in sizes] + reference = np.empty(shape) + paths = np.empty(counts, dtype=object) + for number, index in enumerate(np.ndindex(counts)): + extents = tuple(int(sizes[k][i]) for k, i in enumerate(index)) + raw = tuple( + int(margins[k][i]) + extent + int(rng.integers(0, 2)) + for (k, i), extent in zip(enumerate(index), extents) + ) + data = np.full(raw, -1.0) + block = 1000.0 * number + np.arange(math.prod(extents)).reshape(extents) + inner = tuple( + slice(int(margins[k][i]), int(margins[k][i]) + extent) + for (k, i), extent in zip(enumerate(index), extents) + ) + data[inner] = block + placed = tuple( + slice(int(edges[k][i]), int(edges[k][i + 1])) for k, i in enumerate(index) + ) + reference[placed] = block + path = str(tmp_path / f"grid{number}.h5") + _tile_file(path, data) + paths[index] = path + manifest = TileArray( + paths, + sizes, + {"name": "h5py", "dataset": "data"}, + "float64", + starts=margins, + ) + return manifest, reference + + +class TestManifest: + def test_shape_and_geometry(self, stack): + manifest, reference = stack + assert manifest.shape == reference.shape + assert manifest.ntiles == 3 + npt.assert_array_equal(manifest._edges[0], [0, 10, 17, 29]) + + def test_reads_across_sources(self, stack): + manifest, reference = stack + npt.assert_array_equal(np.asarray(manifest), reference) + npt.assert_array_equal(np.asarray(manifest[9:13]), reference[9:13]) + npt.assert_array_equal(np.asarray(manifest[3:5]), reference[3:5]) + + def test_dataset_model(self, stack): + manifest, _ = stack + dataset = manifest.dataset + assert tuple(dataset["sizes_0"].dims) == ("tile_0",) + assert tuple(dataset["sizes_1"].dims) == ("tile_1",) + # per-file paths vary along tile_0 only: the trailing axis folds + assert tuple(dataset["paths"].dims) == ("tile_0",) + npt.assert_array_equal(dataset["starts_0"].values, [1, 1, 1]) + # all-default geometry columns are not stored + assert "starts_1" not in dataset and "steps_0" not in dataset + + def test_param_folding(self, stack): + manifest, _ = stack + path = str(manifest.dataset["paths"].values[0]) + uniform = TileArray( + path, ([10, 10, 10], NX), ENGINE, "float64", record=0, nbytes=80 + ) + # one path everywhere: 0-d; uniform per-tile params: 0-d + assert uniform.dataset["paths"].ndim == 0 + assert uniform.dataset["record"].ndim == 0 + assert uniform.shape == (30, NX) + varying = TileArray(path, ([10, 10], NX), ENGINE, "float64", record=[[0], [80]]) + assert tuple(varying.dataset["record"].dims) == ("tile_0",) + + def test_validation(self, stack): + manifest, _ = stack + with pytest.raises(ValueError, match="at least one axis"): + TileArray("a", (), ENGINE, "f8") + with pytest.raises(ValueError, match="little-endian"): + TileArray("a", (5, NX), ENGINE, ">f8") + with pytest.raises(ValueError, match="strictly positive"): + TileArray("a", (0, NX), ENGINE, "f8") + with pytest.raises(ValueError, match="does not match the grid"): + TileArray(np.array(["a", "b"], dtype=object), ([1, 2, 3], NX), ENGINE, "f8") + with pytest.raises(ValueError, match="does not match the grid"): + TileArray("a", ([5, 5], NX), ENGINE, "f8", starts=([1, 2, 3], None)) + with pytest.raises(ValueError, match="reserved"): + TileArray("a", (5, NX), ENGINE, "f8", sizes_0=[5]) + dataset = manifest.dataset.copy() + kwargs = dict(dtype=manifest.dtype, params={"engine": manifest.engine}) + with pytest.raises(ValueError, match="`sizes_0`"): + TileArray.from_dataset(dataset.drop_vars(["sizes_0", "sizes_1"]), **kwargs) + with pytest.raises(ValueError, match="`paths`"): + TileArray.from_dataset(dataset.drop_vars("paths"), **kwargs) + + def test_extra_variables_are_params(self, stack): + """Any non-geometry manifest variable is a per-tile engine parameter.""" + manifest, _ = stack + arr = TileArray.from_dataset( + manifest.dataset.assign(record=(("tile_0",), np.arange(3))), + dtype=manifest.dtype, + params={"engine": manifest.engine}, + ) + assert arr._params == ("record",) + + def test_engine_validation(self): + with pytest.raises(KeyError, match="no engine registered"): + TileArray("a", (5, NX), {"name": "bogus"}, "f8") + with pytest.raises(ValueError, match="`name` key"): + TileArray("a", (5, NX), {"dataset": "data"}, "f8") + with pytest.raises(ValueError, match="`name` key"): + TileArray("a", (5, NX), None, "f8") + + def test_engine_registration(self): + class DummyEngine(Engine, name="dummy"): + @staticmethod + def load(path, selection): + return np.zeros((1, 1)) + + try: + assert ENGINES["dummy"] is DummyEngine + assert DummyEngine.name == "dummy" + # the unimplemented half keeps a telling error + with pytest.raises(NotImplementedError, match="'dummy' cannot open"): + DummyEngine.open("some/path") + finally: + del ENGINES["dummy"] + + def test_unregistered_base_subclass(self): + class HalfBaked(Engine): + pass + + assert HalfBaked.name is None + assert HalfBaked not in ENGINES.values() + with pytest.raises(NotImplementedError, match="cannot load"): + HalfBaked.load("some/path", (slice(0, 1),)) + + def test_repr(self, stack): + manifest, _ = stack + assert "3 tiles" in repr(manifest) + assert "'h5py'" in repr(manifest) + assert manifest._repr_inline_(40) == "TileArray (3 tiles)" + assert manifest._repr_inline_(10) == "TileArray" + + def test_attrs(self, stack): + manifest, _ = stack + assert manifest.attrs == {"units": "strain"} + + +class TestSourcePaths: + """Paths are stored verbatim: an array holds exactly what it was given.""" + + def make(self, path): + return TileArray([str(path)], ([4], NX), ENGINE, " 10 + npt.assert_array_equal(manifest[mask], reference[mask]) + from xdas.tiles.tilearray import _bounding_key + + with pytest.raises(NotImplementedError, match="boolean mask"): + _bounding_key( + (np.zeros(reference.shape, dtype=bool), slice(None)), reference.shape + ) + + def test_empty_index_array(self, stack): + manifest, _ = stack + selected = manifest[np.array([], dtype=np.int64)] + assert selected.shape == (0, NX) + + def test_chunks_report_the_tiling(self, stack): + manifest, _ = stack + assert manifest.chunks == ((10, 7, 12), (NX,)) + + def test_transpose_astype_methods(self, stack): + manifest, reference = stack + npt.assert_array_equal(manifest.transpose((1, 0)), reference.T) + assert manifest.astype("float32").dtype == np.float32 + + def test_materialize_descends_sequences(self, stack): + manifest, reference = stack + out = np.concatenate([manifest, np.ones((1, NX))]) + assert isinstance(out, np.ndarray) + npt.assert_array_equal(out, np.concatenate([reference, np.ones((1, NX))])) + + def test_concatenate_fallbacks(self, stack): + manifest, reference = stack + flat = np.concatenate([manifest, manifest], axis=None) + npt.assert_array_equal(flat, np.concatenate([reference, reference], axis=None)) + casted = np.concatenate([manifest, manifest], dtype="float32") + assert casted.dtype == np.float32 + out = np.concatenate([manifest, manifest], 0, None) + npt.assert_array_equal(out, np.concatenate([reference, reference])) + from xdas.tiles.tilearray import _concatenate_virtual + + assert _concatenate_virtual((), {}) is NotImplemented + assert _concatenate_virtual((5,), {}) is NotImplemented + + def test_incompatible_concat_materializes(self, stack): + manifest, reference = stack + a = manifest[:, 0:3] + b = manifest[:, ::2] # same shape but differing column geometry + out = np.concatenate([a, b], axis=0) + assert isinstance(out, np.ndarray) + npt.assert_array_equal( + out, np.concatenate([reference[:, 0:3], reference[:, ::2]], axis=0) + ) + + def test_streaming_dtype_and_keepdims(self, stack): + manifest, reference = stack + casted = np.sum(manifest, axis=0, dtype="float32") + assert casted.dtype == np.float32 + # the stream accumulates per tile row and casts at the end + npt.assert_allclose(casted, reference.sum(0).astype("float32")) + kept = np.sum(manifest, axis=0, keepdims=True) + assert kept.shape == (1, NX) + npt.assert_allclose(np.sum(manifest, axis=(0, 1)), reference.sum()) + + def test_streaming_guards(self, stack): + manifest, _ = stack + # a reduction whose first argument is not this array is not streamed + assert manifest._reduce_streaming(np.sum, (np.zeros(3),), {}) is ( + NotImplemented + ) + # unbindable arguments fall back to materialization + assert manifest._reduce_streaming(np.sum, (manifest,), {"bogus": 1}) is ( + NotImplemented + ) + + +class TestReadScheduling: + def test_one_read_per_tile(self, stack, engine_calls): + """A full read calls the engine once per tile.""" + manifest, reference = stack + npt.assert_array_equal(np.asarray(manifest), reference) + assert len(engine_calls) == manifest.ntiles + + def test_sliced_reads_touch_only_needed_sources(self, stack, engine_calls): + manifest, reference = stack + npt.assert_array_equal(np.asarray(manifest[0:5]), reference[0:5]) + assert len(engine_calls) == 1 # rows 0..5 live in the first file only + engine_calls.clear() + npt.assert_array_equal(np.asarray(manifest[9:13]), reference[9:13]) + assert len(engine_calls) == 2 + + def test_cache_reads_once(self, stack, engine_calls): + manifest, reference = stack + npt.assert_array_equal(np.asarray(manifest), reference) + first = len(engine_calls) + assert first > 0 + npt.assert_array_equal(np.asarray(manifest), reference) + assert len(engine_calls) == first # served from the cache + + def test_deepcopy_and_pickle(self, stack): + import copy + import pickle + + manifest, reference = stack + copied = copy.deepcopy(manifest) + assert isinstance(copied, TileArray) + assert copied.dataset is manifest.dataset + restored = pickle.loads(pickle.dumps(manifest)) + assert isinstance(restored, TileArray) + npt.assert_array_equal(np.asarray(restored), reference) diff --git a/xdas/__init__.py b/xdas/__init__.py index 023abefb..2f3d4ca0 100644 --- a/xdas/__init__.py +++ b/xdas/__init__.py @@ -25,6 +25,7 @@ "signal", "synthetics", "testing", + "tiles", "virtual", # classes "Coordinate", @@ -69,6 +70,7 @@ signal, synthetics, testing, + tiles, virtual, ) from .coordinates import ( diff --git a/xdas/core/dataarray.py b/xdas/core/dataarray.py index c9b1d60b..cec28a20 100644 --- a/xdas/core/dataarray.py +++ b/xdas/core/dataarray.py @@ -188,7 +188,7 @@ def conjugate(self): @property def data(self): - """The underlying array (numpy, dask, or :class:`~xdas.virtual.VirtualArray`).""" + """The underlying array (numpy, dask, :class:`~xdas.virtual.VirtualArray`, or :class:`~xdas.tiles.TileArray`).""" return self._data @data.setter diff --git a/xdas/tiles/__init__.py b/xdas/tiles/__init__.py new file mode 100644 index 00000000..b1507605 --- /dev/null +++ b/xdas/tiles/__init__.py @@ -0,0 +1,19 @@ +""" +Lazy tile-backed virtual arrays (ported from the 0.3 line). + +:class:`TileArray` exposes a rectilinear grid of file-backed tiles as +one numpy-like lazy array; :class:`Engine` is the per-format tile +reader plugin socket. This backend replaces the serialized-dask-graph +fallback used by the formats that HDF5 virtual datasets cannot serve +(Silixa TDMS, MiniSEED). +""" + +from .registry import ENGINES, Engine +from .tilearray import TileArray, extract_array + +__all__ = [ + "ENGINES", + "Engine", + "TileArray", + "extract_array", +] diff --git a/xdas/tiles/registry.py b/xdas/tiles/registry.py new file mode 100644 index 00000000..03a0512d --- /dev/null +++ b/xdas/tiles/registry.py @@ -0,0 +1,64 @@ +"""Tile engine registry — the format plugin socket of :mod:`xdas.tiles`. + +A tile engine is a subclass of :class:`Engine`, one per format, +registered by subclassing with a ``name``. It carries a ``load`` half +that reads one tile of a source file; the :class:`~xdas.tiles.TileArray` +read path looks it up by the ``name`` key of its engine specification. +This registry is distinct from :class:`xdas.io.Engine`, which handles +whole-file opening and saving of labeled arrays. + +Ported from the 0.3 line (``xdas/virtual/registry.py``). +""" + +ENGINES = {} + + +class Engine: + """Base class of the tile format engines; subclassing registers. + + ``class MyEngine(Engine, name="myformat")`` registers the subclass + in :data:`ENGINES` under *name* (omit it for unregistered + intermediate bases). An engine implements one or both halves as + static methods — a format only referenced by stored manifests needs + only ``load``: + + - ``open(path, **kwargs)``: read only the metadata of one file and + return a lazy tile-backed array. Unused by the 0.2 line, where + the :class:`xdas.io.Engine` subclasses do the opening; kept for + forward compatibility with the 0.3 stack. + - ``load(path, selection, **params)``: read one tile — open the + source itself (h5py, obspy, ...) and return exactly the selected + sub-box of the decoded source as a numpy array, *selection* being + one source-local, possibly strided :class:`slice` per source + axis. The keyword parameters are the manifest's engine + specification merged with the per-tile manifest variables (a + per-tile value shadows a same-named spec constant). + """ + + name = None + + def __init_subclass__(cls, /, name=None, **kwargs): + super().__init_subclass__(**kwargs) + if name is not None: + cls.name = name + ENGINES[name] = cls + + @classmethod + def open(cls, path, **kwargs): + """Scan *path* lazily; overridden by engines that open files.""" + raise NotImplementedError( + f"engine {cls.name!r} cannot open files (no `open` method)" + ) + + @classmethod + def load(cls, path, selection, **params): + """Read one tile of *path*; overridden by engines that load data.""" + raise NotImplementedError( + f"engine {cls.name!r} cannot load tile data (no `load` method)" + ) + + +__all__ = [ + "ENGINES", + "Engine", +] diff --git a/xdas/tiles/tilearray.py b/xdas/tiles/tilearray.py new file mode 100644 index 00000000..25d6e2c3 --- /dev/null +++ b/xdas/tiles/tilearray.py @@ -0,0 +1,1013 @@ +""" +Lazy tile-backed virtual arrays over file archives. + +A :class:`TileArray` is a numpy-like duck array whose values live in a +dense rectilinear grid of file-backed *tiles*, described by a plain +:class:`xarray.Dataset` (used as a container only) with one dimension +``tile_k`` per data axis: + +- 1-D *geometry* variables place the grid along each axis ``k``: + ``sizes_k`` (samples contributed by each tile, required), ``starts_k`` + (origin inside the decoded source, default 0) and ``steps_k`` (source + stride, the per-tile decimation, default 1), reading as + ``virtual[pos : pos + size] = source[start : start + size * step : step]`` + with ``pos`` the running sum of the previous sizes along the axis; +- N-D *parameter* variables: ``paths`` (the source file of each tile, + required) plus any format-private per-tile values forwarded to the + engine as keyword arguments. A parameter carries only the tile + dimensions along which it actually varies — a constant folds to a 0-d + variable — and broadcasts over the grid at read time. + +What arrays cannot carry — the ``dtype`` and the ``engine`` +specification — lives on the array itself, by value, and travels +beside the dataset when the array is persisted (see +:func:`xdas.io.xdas.save_dataarray`). Every dataset attribute is a +user attribute. + +Geometry loads eagerly at construction (tiny); parameters stay lazy and +load only when tiles are read. Positive-step slicing folds into the +geometry and returns a new :class:`TileArray` (as lazy and +self-described as its input); any other indexing reads the bounding box +of the selection and resolves the rest in memory. ``np.asarray`` +materializes: tiles are read one by one by the registered *engine*, +whose ``load`` opens each tile's path itself and returns the tile's +*source selection* (one possibly strided slice per axis, see +:class:`~xdas.tiles.registry.Engine`), every part landing +directly in the output array. + +A tile array is used *raw* as the data of a :class:`xdas.DataArray` +(``DataArray(arr, coords)``), so ``da.data`` returns the inspectable +lazy object. The tiling is the only blocking the array knows: +:attr:`~TileArray.chunks` reports it, and whole-array reductions +stream one tile row at a time. + +:meth:`TileArray.concat` fuses arrays along any axis by concatenating +the geometry and the per-tile parameters (O(tiles), the data is never +read). Tile arrays persist inside the native xdas netCDF format: the +wrapped dataset *is* the stored form. + +Ported from the 0.3 line (``xdas/virtual/tilearray.py``); the lazy +:meth:`TileArray.expand_dims` is a 0.2 extension supporting the legacy +concat-along-a-new-dimension path. +""" + +from __future__ import annotations + +import functools +import inspect +import itertools +import json +import math + +import numpy as np +import xarray as xr + +from .registry import ENGINES + +TILE_PREFIX = "tile_" +"""Prefix of the tile-grid dimensions of a manifest dataset.""" + +_MANIFEST_CHUNK = 65536 +"""Tiles per stored chunk along the manifest's growing axis. + +Tiles are appended on axis 0, so that axis is chunked at a fixed count +rather than at its current length — an append rewrites one tail chunk. +The remaining axes are bounded by the tiling and stay single-chunk. +""" + + +class _Unfoldable(Exception): + """A key that no tile grid can express (private to :meth:`TileArray._fold`). + + Raised for non-foldable entries and for empty selections (a grid + needs at least one tile). Its own type so a genuine failure inside + the fold is never mistaken for "fall back to a bounded read". + """ + + +# reductions that stream tile row by tile row instead of materializing: +# numpy function -> (per-block reduce, pairwise combine, mean-style +# finalizer: None, "count" or "nancount") +_STREAMING_REDUCTIONS = { + np.sum: (np.sum, np.add, None), + np.nansum: (np.nansum, np.add, None), + np.max: (np.max, np.maximum, None), + np.nanmax: (np.nanmax, np.fmax, None), + np.min: (np.min, np.minimum, None), + np.nanmin: (np.nanmin, np.fmin, None), + np.all: (np.all, np.logical_and, None), + np.any: (np.any, np.logical_or, None), + np.mean: (np.sum, np.add, "count"), + np.nanmean: (np.nansum, np.add, "nancount"), +} + + +def _fold_param(values, counts, dims): + """Broadcast *values* over the grid and fold its constant axes away. + + Returns ``(dims, values)`` where the kept dimensions are exactly + those along which the values actually vary — the manifest-level + constant folding, expressed as xarray dimensions. + """ + values = values.reshape(values.shape + (1,) * (len(counts) - values.ndim)) + values = np.broadcast_to(values, counts) + keep = [ + axis + for axis in range(values.ndim) + if values.shape[axis] > 1 + and not bool((values == values.take([0], axis=axis)).all()) + ] + index = tuple(slice(None) if axis in keep else 0 for axis in range(values.ndim)) + # re-wrap: plain indexing of a fully-reduced object array yields the + # bare element, which numpy would re-box as a fixed-width string + return tuple(dims[axis] for axis in keep), np.asarray( + values[index], dtype=values.dtype + ) + + +def _normalize_key(key, ndim): + """Return *key* as a full-length tuple with ``Ellipsis`` expanded. + + Accepts the plain keys produced by xarray's indexing adapters and + by dask-style block slicing, plus a defensive unwrap of explicit + indexer objects carrying a ``tuple`` attribute. + """ + key = getattr(key, "tuple", key) + if not isinstance(key, tuple): + key = (key,) + if any(entry is Ellipsis for entry in key): + index = key.index(Ellipsis) + fill = (slice(None),) * (ndim - len(key) + 1) + key = key[:index] + fill + key[index + 1 :] + if len(key) > ndim: + raise IndexError(f"too many indices: got {len(key)} for {ndim} axes") + return key + (slice(None),) * (ndim - len(key)) + + +def _bounding_key(key, shape): + """Split *key* into a positive-step bounding box and a residual key. + + The box is one positive-step slice per axis covering every selected + index; the residual, applied to the values of the box, produces the + exact selection. Returns ``(box, residual, empty)`` where *empty* + flags a selection with no elements on some axis (the caller can + then skip reading entirely). Raises :class:`NotImplementedError` + for entries that have no bounded reduction (new axes or boolean + masks of more than one dimension). + """ + box, residual = [], [] + empty = False + for entry, extent in zip(key, shape): + if isinstance(entry, slice): + start, stop, step = entry.indices(extent) + size = len(range(start, stop, step)) + if size == 0: + box.append(slice(0, 0)) + residual.append(slice(None)) + empty = True + elif step > 0: + box.append(slice(start, stop, step)) + residual.append(slice(None)) + else: + last = start + (size - 1) * step + box.append(slice(last, start + 1, -step)) + residual.append(slice(None, None, -1)) + elif isinstance(entry, (int, np.integer)): + index = int(entry) + if index < 0: + index += extent + if not 0 <= index < extent: + raise IndexError( + f"index {entry} is out of bounds for axis of size {extent}" + ) + box.append(slice(index, index + 1)) + residual.append(0) + elif isinstance(entry, (list, np.ndarray)): + indices = np.asarray(entry) + if indices.dtype == bool: + if indices.ndim != 1: + raise NotImplementedError("multi-dimensional boolean mask") + (indices,) = np.nonzero(indices) + if not np.issubdtype(indices.dtype, np.integer): + raise IndexError(f"invalid index array dtype: {indices.dtype}") + if indices.size == 0: + box.append(slice(0, 0)) + residual.append(indices) + empty = True + continue + indices = np.where(indices < 0, indices + extent, indices) + if indices.min() < 0 or indices.max() >= extent: + raise IndexError(f"index out of bounds for axis of size {extent}") + low = int(indices.min()) + box.append(slice(low, int(indices.max()) + 1)) + residual.append(indices - low) + else: + raise NotImplementedError(f"unsupported index entry: {entry!r}") + return tuple(box), tuple(residual), empty + + +def _materialize(value): + """Read any :class:`TileArray` in *value*, descending one level.""" + if isinstance(value, TileArray): + return np.asarray(value) + if isinstance(value, (list, tuple)) and any( + isinstance(item, TileArray) for item in value + ): + return type(value)( + np.asarray(item) if isinstance(item, TileArray) else item for item in value + ) + return value + + +def _row_ranges(edges, shape): + """Return the streaming blocks: one tile row along axis 0, whole elsewhere. + + The tiling is the only blocking a tile array has, and a whole row + bounds the memory a streaming pass holds at once. + """ + rows = [slice(int(lo), int(hi)) for lo, hi in zip(edges[:-1], edges[1:])] + return [rows] + [[slice(0, extent)] for extent in shape[1:]] + + +class TileArray(np.lib.mixins.NDArrayOperatorsMixin): + """A dense rectilinear grid of file-backed tiles as one virtual array. + + Numpy-like duck array over the *manifest dataset* described in the + module docstring: construction loads the 1-D geometry (tile sizes + and the optional per-tile source origins and strides) and validates + it; the N-D per-tile parameters are left untouched until a read. + The array is immutable by convention: every tiling-changing + operation returns a new instance over a new dataset, and the first + full read is cached. + + The tiled box is *anonymous*: a tile array carries no dimension + names and no variable name (the ``tile_k`` dimensions are internal). + Those are labeled-array identity, supplied when a + :class:`xdas.DataArray` is emitted around it. + + Parameters + ---------- + paths : str or array-like + Source file of each tile. A scalar describes a + one-tile-per-axis grid; an array is padded with trailing + length-1 axes up to the rank. A path may appear in several + tiles. + sizes : sequence of int or 1-D array-like + One entry per axis (this defines the rank): the samples each + tile contributes along that axis. An int is uniform across the + axis' tiles; an array gives the per-tile extents (its length is + the number of tiles along the axis). + engine : dict + The engine specification: the key ``"name"`` selects a + registered :class:`~xdas.virtual.registry.Engine`; the + remaining keys are passed to its ``load`` as keyword + parameters. + dtype : str or numpy.dtype + Element type of the virtual array (little-endian or + single-byte). + starts : sequence of (None, int, or 1-D array-like), optional + Per-axis origin of each tile inside its decoded source. + Default ``None`` (0 everywhere). + attrs : dict, optional + User attributes of the virtual array. + **params : array-like + Per-tile engine parameters, broadcast over the grid: each read + passes the tile's value to the engine as a keyword argument + (shadowing a same-named specification constant). + """ + + def __init__( + self, + paths, + sizes, + engine, + dtype, + *, + starts=None, + attrs=None, + **params, + ): + ndim = len(sizes) + if ndim == 0: + raise ValueError("a tile array needs at least one axis") + dims = tuple(f"{TILE_PREFIX}{k}" for k in range(ndim)) + paths = np.asarray(paths, dtype=object) + if paths.ndim > ndim: + raise ValueError("`paths` has more axes than `sizes` entries") + paths = paths.reshape(paths.shape + (1,) * (ndim - paths.ndim)) + data = {} + counts = [] + for k, entry in enumerate(sizes): + values = np.atleast_1d(np.asarray(entry, dtype=np.int64)) + if values.size == 1 and paths.shape[k] > 1: + values = np.full(paths.shape[k], values[0], dtype=np.int64) + counts.append(len(values)) + data[f"sizes_{k}"] = (dims[k], values) + counts = tuple(counts) + if any(have not in (1, count) for have, count in zip(paths.shape, counts)): + raise ValueError( + f"`paths` shape {paths.shape} does not match the grid {counts}" + ) + for k, entry in enumerate(starts or ()): + if entry is None: + continue + values = np.atleast_1d(np.asarray(entry, dtype=np.int64)) + if values.size == 1 and counts[k] > 1: + values = np.full(counts[k], values[0], dtype=np.int64) + if len(values) != counts[k]: + raise ValueError(f"`starts[{k}]` does not match the grid") + if values.any(): + data[f"starts_{k}"] = (dims[k], values) + data["paths"] = _fold_param(paths, counts, dims) + reserved = set(data) | {f"steps_{k}" for k in range(ndim)} + for name, values in params.items(): + if name in reserved: + raise ValueError(f"parameter name {name!r} is reserved") + data[name] = _fold_param(np.asarray(values), counts, dims) + dataset = xr.Dataset(data, attrs=dict(attrs or {})) + self._setup(dataset, dtype, engine) + + @classmethod + def from_dataset(cls, dataset, *, name=None, dims=None, dtype, params=None): + """Wrap an existing manifest *dataset* (see the module docstring). + + The dataset — as stored inside a native xdas file — must hold + the geometry and per-tile variables; what the description + arrays cannot carry comes by value, in *params*. + + Parameters + ---------- + dataset : xarray.Dataset + The manifest dataset to wrap. + name, dims : optional + Accepted for interface uniformity and ignored: the tiled box + is anonymous, its name and axis labels are xarray-level + identity. + dtype : str or numpy.dtype + Element type of the virtual array. + params : dict + The by-value description, as :meth:`to_dataset` returned it: + ``engine``, the engine specification (``"name"`` plus its + own parameters). Any other key is ignored, so a view stored + with by-value parameters this class no longer takes still + opens. + + Returns + ------- + TileArray + """ + params = dict(params or {}) + self = cls.__new__(cls) + self._setup(dataset, dtype, params["engine"]) + return self + + def to_dataset(self): + """Encode this tile array as its manifest dataset plus its params. + + The stored form — a copy of the wrapped dataset carrying the + user attributes, and the by-value constructor kwarg the arrays + cannot: the ``engine``. Source paths are stored exactly as the + array holds them, absolute. Each variable pins its chunking + (see :data:`_MANIFEST_CHUNK`). + + Returns + ------- + dataset : xarray.Dataset + The manifest dataset. + params : dict + The by-value keyword arguments of :meth:`from_dataset`, the + ones no manifest variable can carry. They travel beside the + dataset, not in it. + """ + dataset = xr.Dataset(self.dataset.data_vars, attrs=self.attrs) + row = f"{TILE_PREFIX}0" + for variable in dataset.values(): + variable.encoding["chunks"] = tuple( + _MANIFEST_CHUNK if dim == row else int(dataset.sizes[dim]) + for dim in variable.dims + ) + return dataset, {"engine": self.engine} + + def _setup(self, dataset, dtype, engine): + self.dataset = dataset + self._cache = None + # the json round trip deep-copies and normalizes (tuples become + # lists), so equality survives a store round trip + engine = json.loads(json.dumps(engine)) + if not isinstance(engine, dict) or "name" not in engine: + raise ValueError("the engine specification must have a `name` key") + if engine["name"] not in ENGINES: + raise KeyError( + f"no engine registered under {engine['name']!r}; " + f"available: {sorted(ENGINES)}" + ) + self._engine = engine + self.dtype = np.dtype(dtype) + if self.dtype.byteorder == ">": + raise ValueError("only little-endian or single-byte dtypes are supported") + ndim = 0 + while f"sizes_{ndim}" in dataset: + ndim += 1 + if ndim == 0: + raise ValueError("a tile array needs a `sizes_0` geometry variable") + self.ndim = ndim + self.dims = dims = tuple(f"{TILE_PREFIX}{k}" for k in range(ndim)) + self._sizes = self._geometry("sizes", None) + self._starts = self._geometry("starts", 0) + self._steps = self._geometry("steps", 1) + for kind, arrays, bound in ( + ("sizes", self._sizes, 1), + ("starts", self._starts, 0), + ("steps", self._steps, 1), + ): + for k, values in enumerate(arrays): + if np.any(values < bound): + kind_bound = "non-negative" if bound == 0 else "strictly positive" + raise ValueError(f"`{kind}_{k}` must be {kind_bound}") + self._edges = tuple( + np.concatenate(([0], np.cumsum(sizes))) for sizes in self._sizes + ) + self.shape = tuple(int(edges[-1]) for edges in self._edges) + if "paths" not in dataset: + raise ValueError("a tile array needs a `paths` variable") + geometry = { + f"{kind}_{k}" for kind in ("sizes", "starts", "steps") for k in range(ndim) + } + self._params = tuple( + sorted( + name + for name in map(str, dataset.data_vars) + if name not in geometry and name != "paths" + ) + ) + for name in ("paths", *self._params): + vdims = tuple(map(str, dataset[name].dims)) + if tuple(dim for dim in dims if dim in vdims) != vdims: + raise ValueError( + f"`{name}` dimensions must be an ordered subset of {dims}" + ) + + def _geometry(self, kind, default): + """Load the eager 1-D ``{kind}_k`` arrays (*default* where absent).""" + arrays = [] + for k, dim in enumerate(self.dims): + name = f"{kind}_{k}" + if name in self.dataset: + if tuple(self.dataset[name].dims) != (dim,): + raise ValueError(f"`{name}` must have dimensions ({dim!r},)") + arrays.append(np.asarray(self.dataset[name].values, dtype=np.int64)) + else: + count = int(self.dataset.sizes[dim]) + arrays.append(np.full(count, default, dtype=np.int64)) + return tuple(arrays) + + @property + def engine(self): + """dict: the engine specification (``"name"`` plus its parameters).""" + return self._engine + + @property + def attrs(self): + """dict: the user attributes of the virtual array.""" + return dict(self.dataset.attrs) + + @property + def chunks(self): + """Tuple of tuple of int: the tiling, as per-axis tile extents. + + Not a hint — the tiling *is* the only blocking the array has. + """ + return tuple(tuple(int(size) for size in sizes) for sizes in self._sizes) + + @property + def ntiles(self): + """int: total number of tiles in the grid.""" + return math.prod(len(sizes) for sizes in self._sizes) + + @property + def size(self): + """int: total number of elements.""" + return math.prod(self.shape) + + def __getitem__(self, key): + """Index the array, staying virtual whenever possible. + + Positive-step slices fold into the geometry and return a new + :class:`TileArray` without touching the sources: + ``np.asarray(arr[key])`` equals ``np.asarray(arr)[key]``. Per + axis, the overlapping tiles are located by binary search on the + running tile sizes and their geometry is trimmed — and, for + stepped slices, decimated — to the selection (steps multiply, + origins compose, one tile stays one tile); tiles the selection + strides over entirely are dropped. The parameters are sliced + through the wrapped dataset, so a lazy array stays lazy. + + Every other key (integers, index arrays, boolean masks, + reversed slices, empty selections) reads the bounding box of + the selection and applies the remainder in memory, returning a + numpy array. + """ + key = _normalize_key(key, self.ndim) + try: + return self._fold(key) + except _Unfoldable: + pass + try: + box, residual, empty = _bounding_key(key, self.shape) + except NotImplementedError: + return np.asarray(self)[key] + if empty: + # zero-strided: the result is empty, so no value is ever read + # and the full shape is never allocated + return np.broadcast_to(np.zeros((), self.dtype), self.shape)[key].copy() + return np.asarray(self._fold(box))[residual] + + def _fold(self, key): + """Fold a full-length tuple of positive-step slices into a new array. + + Raises :class:`_Unfoldable` for non-foldable entries and for + empty selections (a grid needs at least one tile); + :meth:`__getitem__` then falls back to a bounded read. + """ + indexers = {} + assign = {} + for axis, (entry, extent) in enumerate(zip(key, self.shape)): + if not isinstance(entry, slice) or (entry.step or 1) < 1: + raise _Unfoldable( + "only positive-step slices can be folded into the tile grid" + ) + lo, hi, s = entry.indices(extent) + if len(range(lo, hi, s)) == 0: + raise _Unfoldable(f"empty selection along axis {axis}") + if (lo, hi, s) == (0, extent, 1): + continue + edges = self._edges[axis] + i0 = int(np.searchsorted(edges, lo, "right")) - 1 + i1 = int(np.searchsorted(edges, hi, "left")) + pos = edges[i0:i1] + size = self._sizes[axis][i0:i1] + start = self._starts[axis][i0:i1] + step = self._steps[axis][i0:i1] + # selected positions are lo, lo + s, ...; j0/j1 index the first + # and last of them falling inside each tile + j0 = np.maximum(0, -((lo - pos) // s)) + j1 = (np.minimum(pos + size, hi) - 1 - lo) // s + keep = j1 >= j0 + dim = self.dims[axis] + indexers[dim] = slice(i0, i1) if keep.all() else i0 + np.flatnonzero(keep) + assign[f"sizes_{axis}"] = (dim, (j1 - j0 + 1)[keep]) + assign[f"starts_{axis}"] = (dim, (start + (lo + j0 * s - pos) * step)[keep]) + assign[f"steps_{axis}"] = (dim, (step * s)[keep]) + # all-default starts/steps columns fold away (they stay derivable) + drop = [ + name + for name, (_, values) in assign.items() + if (name.startswith("starts_") and not values.any()) + or (name.startswith("steps_") and not (values != 1).any()) + ] + assign = {name: entry for name, entry in assign.items() if name not in drop} + dataset = self.dataset.isel(indexers).assign(assign) + dataset = dataset.drop_vars([name for name in drop if name in dataset]) + return type(self).from_dataset( + dataset, dtype=self.dtype, params={"engine": self.engine} + ) + + @classmethod + def concat(cls, arrays, dim=0): + """Concatenate tile arrays along axis *dim* into a new array. + + Requires equal engines and dtype, and equal geometry on every + *other* axis — nothing else: differently trimmed or decimated + subviews of the same sources concatenate, and repeated sources + are legitimate. The geometry is chained; parameters stay folded + when every input agrees and are broadcast out and concatenated + otherwise (the tile tables load, the data does not). + + Parameters + ---------- + arrays : list of TileArray + The tile arrays to concatenate, in order along *dim*. + dim : int, optional + The axis along which to concatenate. Default 0. (Dimension + *names* are mapped to axes by the callers — the tile array + mirrors the positional numpy API.) + + Returns + ------- + TileArray + """ + first = arrays[0] + ndim = first.ndim + axis = int(dim) + if not 0 <= axis < ndim: + raise ValueError(f"no axis {dim} in a {ndim}-dimensional tile array") + dims = first.dims + for other in arrays[1:]: + if ( + other.ndim != ndim + or other.dtype != first.dtype + or other.engine != first.engine + or other._params != first._params + or any( + k != axis + and not ( + np.array_equal(other._sizes[k], first._sizes[k]) + and np.array_equal(other._starts[k], first._starts[k]) + and np.array_equal(other._steps[k], first._steps[k]) + ) + for k in range(ndim) + ) + ): + raise ValueError("can only concatenate compatible tile arrays") + data = {} + for kind, per_axis, default in ( + ("sizes", [array._sizes for array in arrays], None), + ("starts", [array._starts for array in arrays], 0), + ("steps", [array._steps for array in arrays], 1), + ): + for k in range(ndim): + if k == axis: + values = np.concatenate([entries[k] for entries in per_axis]) + else: + values = per_axis[0][k] + if default is not None and bool((values == default).all()): + continue + data[f"{kind}_{k}"] = (dims[k], values) + for name in ("paths", *first._params): + variables = [array.dataset[name].variable for array in arrays] + vdims = variables[0].dims + if dims[axis] not in vdims and all(v.dims == vdims for v in variables): + values = variables[0].values + if all(np.array_equal(v.values, values) for v in variables[1:]): + data[name] = (vdims, values) + continue + union = tuple( + d + for d in dims + if d == dims[axis] or any(d in v.dims for v in variables) + ) + parts = [ + _expand(variable, union, array) + for variable, array in zip(variables, arrays) + ] + axis_pos = union.index(dims[axis]) + values = np.concatenate([part.values for part in parts], axis=axis_pos) + data[name] = xr.Variable(union, values) + dataset = xr.Dataset(data, attrs=first.attrs) + return cls.from_dataset( + dataset, dtype=first.dtype, params={"engine": first.engine} + ) + + def _grid_values(self, name): + """Load parameter *name* and broadcast it over the full tile grid.""" + variable = self.dataset[name].variable + values = np.asarray(variable.values) + counts = tuple(len(sizes) for sizes in self._sizes) + shape = tuple( + count if dim in variable.dims else 1 + for dim, count in zip(self.dims, counts) + ) + return np.broadcast_to(values.reshape(shape), counts) + + @functools.cached_property + def _engine_impl(self): + """The ``(load, spec)`` of the engine specification.""" + spec = dict(self.engine) + name = spec.pop("name") + return ENGINES[name].load, spec + + def __array__(self, dtype=None, copy=None): + """Read every tile and return the values as a numpy array. + + The value-materialization primitive: tiles are read one by one, + each exactly once, its part landing directly in the output + array. Read a subset by slicing first: ``np.asarray(arr[key])``. + The first full read is cached. + """ + if self._cache is None: + self._cache = self._read() + values = self._cache + if dtype is not None and np.dtype(dtype) != values.dtype: + return values.astype(dtype) + if copy: + return values.copy() + return values + + def _read(self): + """Read every tile into a fresh output array, one engine call each.""" + out = np.empty(self.shape, dtype=self.dtype) + read, spec = self._engine_impl + counts = tuple(len(sizes) for sizes in self._sizes) + paths = self._grid_values("paths") + params = {name: self._grid_values(name) for name in self._params} + for index in np.ndindex(counts): + selection, dest = [], [] + for k, i in enumerate(index): + first = int(self._starts[k][i]) + size = int(self._sizes[k][i]) + step = int(self._steps[k][i]) + selection.append(slice(first, first + (size - 1) * step + 1, step)) + dest.append(slice(int(self._edges[k][i]), int(self._edges[k][i + 1]))) + selection, dest = tuple(selection), tuple(dest) + kwargs = dict(spec) + for name, values in params.items(): + value = values[index] + kwargs[name] = value.item() if isinstance(value, np.generic) else value + part = np.asarray(read(str(paths[index]), selection, **kwargs)) + widths = tuple(entry.stop - entry.start for entry in dest) + if part.shape != widths: + raise ValueError( + f"engine {self.engine['name']!r} produced a part of shape " + f"{part.shape} where the selection has shape {widths}" + ) + out[dest] = part + return out + + def equals(self, other): + """Whether *other* describes the same tiling (not elementwise). + + Compares the engine, dtype, geometry, parameters and user + attributes; ``==`` stays elementwise, as on any numpy-like + array. + """ + if not isinstance(other, TileArray): + return False + if ( + self.engine != other.engine + or self.dtype != other.dtype + or self.shape != other.shape + or self.attrs != other.attrs + or self._params != other._params + ): + return False + for k in range(self.ndim): + if not ( + np.array_equal(self._sizes[k], other._sizes[k]) + and np.array_equal(self._starts[k], other._starts[k]) + and np.array_equal(self._steps[k], other._steps[k]) + ): + return False + for name in ("paths", *self._params): + mine, theirs = self._grid_values(name), other._grid_values(name) + if not np.array_equal(mine, theirs): + return False + return True + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + """Materialize tile-backed inputs and apply the ufunc.""" + if any(isinstance(value, TileArray) for value in kwargs.get("out", ())): + return NotImplemented + inputs = tuple( + np.asarray(value) if isinstance(value, TileArray) else value + for value in inputs + ) + return getattr(ufunc, method)(*inputs, **kwargs) + + def __array_function__(self, func, types, args, kwargs): + """Dispatch numpy functions, keeping select operations lazy. + + ``numpy.concatenate`` of compatible tile-backed arrays fuses + the tilings and stays virtual; the streaming reductions + (``sum``, ``mean``, ``min``, ``max``, their nan variants, + ``any`` and ``all``) accumulate one tile row at a time with + bounded memory. Everything else materializes and delegates to + numpy. + """ + if func is np.result_type: + args = tuple( + value.dtype if isinstance(value, TileArray) else value for value in args + ) + return np.result_type(*args) + if func is np.concatenate: + result = _concatenate_virtual(args, kwargs) + if result is not NotImplemented: + return result + if func is np.expand_dims: + result = self._expand_virtual(args, kwargs) + if result is not NotImplemented: + return result + if func in _STREAMING_REDUCTIONS: + result = self._reduce_streaming(func, args, kwargs) + if result is not NotImplemented: + return result + args = tuple(_materialize(value) for value in args) + kwargs = {name: _materialize(value) for name, value in kwargs.items()} + return func(*args, **kwargs) + + def _reduce_streaming(self, func, args, kwargs): + """Accumulate a reduction one tile row at a time, bounded memory. + + Arguments are rebound against the reduction's own signature: the + DataArray wrapper calls with every parameter bound positionally + (no-value sentinels included), where plain numpy calls pass + keywords. + """ + try: + bound = inspect.signature(func).bind(*args, **kwargs) + except TypeError: + return NotImplemented + arguments = { + name: value + for name, value in bound.arguments.items() + if value is not np._NoValue + } + if arguments.pop("a", None) is not self: + return NotImplemented + axis = arguments.pop("axis", None) + keepdims = arguments.pop("keepdims", False) + dtype = arguments.pop("dtype", None) + if arguments.pop("out", None) is not None: + return NotImplemented + if any(value is not None for value in arguments.values()): + return NotImplemented + block_reduce, combine, counting = _STREAMING_REDUCTIONS[func] + if axis is None: + axes = tuple(range(self.ndim)) + else: + axis = axis if isinstance(axis, tuple) else (axis,) + axes = tuple(a + self.ndim if a < 0 else a for a in axis) + kept = tuple(a for a in range(self.ndim) if a not in axes) + out_shape = tuple(self.shape[a] for a in kept) + acc = None + filled = np.zeros(out_shape, dtype=bool) + counts = np.zeros(out_shape) if counting else None + for box in itertools.product(*_row_ranges(self._edges[0], self.shape)): + block = np.asarray(self[box]) + partial = np.asarray(block_reduce(block, axis=axes, keepdims=True)) + partial = partial.reshape(tuple(block.shape[a] for a in kept)) + target = tuple(box[a] for a in kept) + if acc is None: + acc = np.zeros(out_shape, dtype=partial.dtype) + acc[target] = np.where( + filled[target], combine(acc[target], partial), partial + ) + filled[target] = True + if counting == "count": + counts[target] += np.prod([block.shape[a] for a in axes]) + elif counting == "nancount": + counts[target] += np.sum(~np.isnan(block), axis=axes).reshape( + partial.shape + ) + result = acc / counts if counting else acc + if dtype is None and counting: + dtype = func(np.zeros(1, self.dtype)).dtype + if dtype is not None: + result = np.asarray(result).astype(dtype) + if keepdims: + full = tuple(1 if a in axes else self.shape[a] for a in range(self.ndim)) + result = np.asarray(result).reshape(full) + elif not kept: + result = np.asarray(result).reshape(())[()] + return result + + def expand_dims(self, axis=0): + """Insert a unit leading axis, staying virtual (0.2 extension). + + The legacy concat-along-a-new-dimension path + (:meth:`xdas.DataArray.expand_dims` then :func:`xdas.concat`) + expands the data with :func:`numpy.expand_dims`; this keeps + that path lazy instead of materializing. Only the leading + position is supported: the new axis holds one tile of size + one, and the engine ``load`` receives one extra leading + ``slice(0, 1)`` per expanded axis, padding its output rank + accordingly (see the silixa and miniseed engines). + + Parameters + ---------- + axis : int, optional + Position of the new axis; only ``0`` (equivalently + ``-ndim - 1``) is supported. + + Returns + ------- + TileArray + """ + axis = int(axis) + if axis == -self.ndim - 1: + axis = 0 + if axis != 0: + raise ValueError("only a leading axis can be virtually expanded") + rename = {} + for k in range(self.ndim - 1, -1, -1): + rename[f"{TILE_PREFIX}{k}"] = f"{TILE_PREFIX}{k + 1}" + for kind in ("sizes", "starts", "steps"): + if f"{kind}_{k}" in self.dataset: + rename[f"{kind}_{k}"] = f"{kind}_{k + 1}" + dataset = self.dataset.rename(rename) + dataset = dataset.assign(sizes_0=(f"{TILE_PREFIX}0", np.ones(1, np.int64))) + return type(self).from_dataset( + dataset, dtype=self.dtype, params={"engine": self.engine} + ) + + def _expand_virtual(self, args, kwargs): + """Dispatch ``numpy.expand_dims``, delegating to :meth:`expand_dims`.""" + kwargs = dict(kwargs) + axis = kwargs.pop("axis", args[1] if len(args) > 1 else None) + if kwargs or len(args) > 2 or args[0] is not self: + return NotImplemented + if not isinstance(axis, (int, np.integer)): + return NotImplemented + if int(axis) not in (0, -self.ndim - 1): + return NotImplemented + return self.expand_dims(0) + + def transpose(self, order): + """Materialize and transpose to the given axis *order*.""" + return np.transpose(np.asarray(self), order) + + def astype(self, dtype, **kwargs): + """Materialize and cast the values to *dtype*.""" + return np.asarray(self).astype(dtype, **kwargs) + + def __deepcopy__(self, memo): + """Copy without the read cache; the dataset is immutable.""" + return type(self).from_dataset( + self.dataset, dtype=self.dtype, params={"engine": self.engine} + ) + + def __repr__(self): + return ( + f"" + ) + + def _repr_inline_(self, max_width): + """Return the one-line summary used by xarray inline reprs.""" + summary = f"TileArray ({self.ntiles} tiles)" + return summary if len(summary) <= max_width else "TileArray" + + +def _expand(variable, union, array): + """Broadcast *variable* over the *union* tile dims of *array*.""" + if variable.dims == union: + return variable + counts = {dim: len(sizes) for dim, sizes in zip(array.dims, array._sizes)} + values = np.asarray(variable.values) + shape = tuple(counts[dim] if dim in variable.dims else 1 for dim in union) + full = tuple(counts[dim] for dim in union) + return xr.Variable(union, np.broadcast_to(values.reshape(shape), full)) + + +def _concatenate_virtual(args, kwargs): + """Fuse tile arrays for ``numpy.concatenate`` when possible.""" + if not args: + return NotImplemented + arrays, *rest = args + if len(rest) > 1 or set(kwargs) - {"axis"}: + return NotImplemented + axis = kwargs.get("axis", rest[0] if rest else 0) + try: + arrays = list(arrays) + except TypeError: + return NotImplemented + if axis is None or not all(isinstance(array, TileArray) for array in arrays): + return NotImplemented + try: + return TileArray.concat(arrays, dim=axis) + except (ValueError, IndexError): + return NotImplemented + + +def extract_array(da): + """Return the :class:`TileArray` backing *da*. + + Slicing folds into the tile grid at indexing time, so the array of + a sliced view describes exactly that view — only the overlapping + sources remain. ``extract_array(xr.DataArray(arr, dims=dims))`` + returns ``arr`` itself. + + Parameters + ---------- + da : DataArray + A tile-backed array, as built by wrapping a :class:`TileArray` + or as returned by the :mod:`xdas.io` openers, possibly sliced + with positive-step slices. + + Returns + ------- + TileArray + + Raises + ------ + TypeError + If *da* holds an in-memory numpy array — because it was + loaded, built eagerly, or indexed in a way no tile grid can + represent (integer, reversed or fancy indexing) — or is + otherwise not backed by a :class:`TileArray`. + """ + data = getattr(da, "data", da) + if isinstance(data, TileArray): + return data + if isinstance(data, np.ndarray): + raise TypeError( + "`da` holds an in-memory numpy array and is no longer backed by " + "a TileArray (it was loaded, built eagerly, or indexed in a way " + "no tile grid can represent)" + ) + raise TypeError("`da` is not backed by a TileArray") + + +__all__ = [ + "TileArray", + "extract_array", +] From 96505983a1d07eafdf535360453ffd09e210728f Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 15:12:31 +0200 Subject: [PATCH 02/56] Rewire the silixa and miniseed engines onto TileArray Replace the serialized-dask-graph fallback: both engines now emit tile-backed lazy DataArrays. Silixa gains time-axis push-down through TdmsReader row bounds; miniseed keeps its header semantics, with the stream method and ignore_last_sample travelling in the manifest's engine specification. --- tests/io/test_miniseed.py | 38 +++++++++++++++++++++++ tests/io/test_silixa.py | 65 +++++++++++++++++++++++++++++++++++++++ xdas/io/miniseed.py | 40 ++++++++++++++++++------ xdas/io/silixa.py | 34 +++++++++++++++----- 4 files changed, 161 insertions(+), 16 deletions(-) create mode 100644 tests/io/test_silixa.py diff --git a/tests/io/test_miniseed.py b/tests/io/test_miniseed.py index 34321d98..81311294 100644 --- a/tests/io/test_miniseed.py +++ b/tests/io/test_miniseed.py @@ -1,9 +1,11 @@ import numpy as np +import numpy.testing as npt import obspy import pytest import xdas as xd from xdas.io.miniseed import MiniSEEDEngine, get_band_code, to_stream +from xdas.tiles import TileArray def make_network(dirpath, gap=False, samples=100): @@ -176,6 +178,42 @@ def test_miniseed(tmp_path): assert values_gap_trimmed.shape == (3, 89) +def test_miniseed_tile_backend(tmp_path): + make_network(tmp_path, samples=100) + + # single files open lazily, tile-backed + paths = sorted(tmp_path.glob("*00.mseed")) + da = xd.open(paths[0], engine="miniseed") + assert isinstance(da.data, TileArray) + assert da.data.engine == { + "name": "miniseed", + "method": "synchronized", + "ignore_last_sample": False, + } + + # concatenation along a new dimension stays lazy and reads correctly + objs = [xd.open(path, engine="miniseed") for path in paths] + stacked = xd.concat(objs, "station") + assert isinstance(stacked.data, TileArray) + npt.assert_array_equal(stacked.values, np.stack([obj.values for obj in objs])) + + # single-trace files fold the channel axis to a scalar coordinate + single = tmp_path / "single.mseed" + st = obspy.Stream([obspy.Trace(np.random.rand(50), make_header(1, "Z", 0))]) + st.write(str(single)) + da = xd.open(single, engine="miniseed") + assert da.dims == ("time",) + assert da.shape == (50,) + npt.assert_allclose(da.values, st[0].data, rtol=1e-6) + + # round trip through the native format + da = xd.open(paths[0], engine="miniseed") + da.to_netcdf(tmp_path / "view.nc") + reopened = xd.open_dataarray(tmp_path / "view.nc") + assert isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, da.values) + + def test_miniseed_helpers(tmp_path): # get_band_code with out-of-range sampling rate assert get_band_code(0.0) == "X" diff --git a/tests/io/test_silixa.py b/tests/io/test_silixa.py new file mode 100644 index 00000000..0456197a --- /dev/null +++ b/tests/io/test_silixa.py @@ -0,0 +1,65 @@ +import numpy as np +import numpy.testing as npt + +from xdas.io import silixa +from xdas.tiles import TileArray + + +class FakeTdms: + """In-memory stand-in for :class:`~xdas.io.tdms.TdmsReader`.""" + + data = np.arange(20.0 * 4).reshape(20, 4) + + channel_length = 20 + fileinfo = {"n_channels": 4} + _data_type = np.dtype("float64") + + def __init__(self, path): + self.path = path + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def get_properties(self): + return { + "GPSTimeStamp": "2020-01-01T00:00:00", + "SamplingFrequency[Hz]": 1000.0, + "Start Distance (m)": 0.0, + "Fibre Length Multiplier": 1.0, + "SpatialResolution[m]": 4.0, + } + + def get_data(self, first_s=None, last_s=None): + first_s = 0 if first_s is None else first_s + last_s = len(self.data) - 1 if last_s is None else last_s + return self.data[first_s : last_s + 1] + + +def test_tile_load(monkeypatch): + monkeypatch.setattr(silixa, "TdmsReader", FakeTdms) + expected = FakeTdms.data + manifest = TileArray("fake.tdms", (20, 4), {"name": "silixa"}, "float64") + npt.assert_array_equal(np.asarray(manifest), expected) + npt.assert_array_equal(np.asarray(manifest[3:15:2, 1:3]), expected[3:15:2, 1:3]) + expanded = np.expand_dims(manifest, 0) + assert isinstance(expanded, TileArray) + npt.assert_array_equal(np.asarray(expanded), expected[None]) + + +def test_read_data(monkeypatch): + monkeypatch.setattr(silixa, "TdmsReader", FakeTdms) + npt.assert_array_equal(silixa.SilixaEngine().read_data("fake.tdms"), FakeTdms.data) + + +def test_open_dataarray(monkeypatch): + monkeypatch.setattr(silixa, "TdmsReader", FakeTdms) + da = silixa.SilixaEngine().open_dataarray("fake.tdms") + assert isinstance(da.data, TileArray) + assert da.dims == ("time", "distance") + assert da.shape == (20, 4) + assert da.coords["time"][0].values == np.datetime64("2020-01-01T00:00:00") + npt.assert_allclose(da.coords["distance"].values, 4.0 * np.arange(4)) + npt.assert_array_equal(da.values, FakeTdms.data) diff --git a/xdas/io/miniseed.py b/xdas/io/miniseed.py index ebdb203f..100ad0dc 100644 --- a/xdas/io/miniseed.py +++ b/xdas/io/miniseed.py @@ -1,6 +1,5 @@ """I/O engine for MiniSEED files via ObsPy (:class:`MiniSEEDEngine`).""" -import dask import numpy as np import obspy @@ -11,27 +10,30 @@ get_sampling_interval, ) from ..core import DataArray, concat_coords +from ..tiles import Engine as TileEngine +from ..tiles import TileArray from .core import Engine class MiniSEEDEngine(Engine, name="miniseed"): - """Engine for reading MiniSEED files via ObsPy as lazy dask-backed DataArrays.""" + """Engine for reading MiniSEED files via ObsPy as lazy tile-backed DataArrays.""" - _supported_vtypes = ["dask"] + _supported_vtypes = ["tiles"] _supported_ctypes = { "time": ["interpolated", "sampled", "dense"], } def open_dataarray(self, fname, ignore_last_sample=False, ctype="interpolated"): - """Return a lazy dask-backed :class:`DataArray` for the MiniSEED file *fname*.""" + """Return a lazy tile-backed :class:`DataArray` for the MiniSEED file *fname*.""" shape, dtype, coords, method = self.read_header( fname, ignore_last_sample, ctype ) - data = dask.array.from_delayed( - dask.delayed(self.read_data)(fname, method, ignore_last_sample), - shape, - dtype, - ) + engine = { + "name": "miniseed", + "method": method, + "ignore_last_sample": bool(ignore_last_sample), + } + data = TileArray(str(fname), shape, engine, np.dtype(dtype)) return DataArray(data, coords) def read_header(self, path, ignore_last_sample, ctype): @@ -112,6 +114,26 @@ def read_data(self, path, method, ignore_last_sample): return np.array(data) +class MiniSEEDTileEngine(TileEngine, name="miniseed"): + """Tile reader for MiniSEED sources, decoding with ObsPy.""" + + @staticmethod + def load(path, selection, *, method, ignore_last_sample): + """Read a source selection of a MiniSEED file. + + Decodes the whole file with ObsPy (as the legacy dask path did) + and crops to *selection*. The decoded rank is padded with unit + leading axes for virtually expanded arrays, or squeezed when a + scalar channel folded an axis away. + """ + data = MiniSEEDEngine().read_data(path, method, ignore_last_sample) + if data.ndim < len(selection): + data = data.reshape((1,) * (len(selection) - data.ndim) + data.shape) + elif data.ndim > len(selection): + data = data.reshape(data.shape[data.ndim - len(selection) :]) + return data[selection] + + def to_stream( da, network="NET", diff --git a/xdas/io/silixa.py b/xdas/io/silixa.py index 5f832056..2ad95e0a 100644 --- a/xdas/io/silixa.py +++ b/xdas/io/silixa.py @@ -1,29 +1,28 @@ """I/O engine for Silixa TDMS files (:class:`SilixaEngine`).""" -import dask import numpy as np from ..coordinates import Coordinate from ..core import DataArray +from ..tiles import Engine as TileEngine +from ..tiles import TileArray from .core import Engine from .tdms import TdmsReader class SilixaEngine(Engine, name="silixa"): - """Engine for reading Silixa iDAS TDMS files as lazy dask-backed DataArrays.""" + """Engine for reading Silixa iDAS TDMS files as lazy tile-backed DataArrays.""" - _supported_vtypes = ["dask"] + _supported_vtypes = ["tiles"] _supported_ctypes = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated", "sampled", "dense"], } def open_dataarray(self, fname): - """Return a lazy dask-backed :class:`DataArray` for the TDMS file *fname*.""" + """Return a lazy tile-backed :class:`DataArray` for the TDMS file *fname*.""" shape, dtype, coords = self.read_header(fname) - data = dask.array.from_delayed( - dask.delayed(self.read_data)(fname), shape, dtype - ) + data = TileArray(str(fname), shape, {"name": "silixa"}, np.dtype(dtype)) return DataArray(data, coords) def read_header(self, fname): @@ -54,3 +53,24 @@ def read_data(self, fname): with TdmsReader(fname) as tdms: data = tdms.get_data() return data + + +class SilixaTileEngine(TileEngine, name="silixa"): + """Tile reader for Silixa TDMS sources (rows are time samples).""" + + @staticmethod + def load(path, selection): + """Read a source selection of a Silixa TDMS file. + + :class:`~xdas.io.tdms.TdmsReader` performs the decoding + (``get_data`` bounds are inclusive, hence the ``stop - 1``); the + residual crop applies as numpy views. Leading extra selection + axes come from virtually expanded arrays and pad the output + rank. + """ + extra = len(selection) - 2 + rows = selection[extra] + with TdmsReader(path) as tdms: + data = tdms.get_data(first_s=rows.start, last_s=rows.stop - 1) + data = data[(slice(None, None, rows.step), *selection[extra + 1 :])] + return data.reshape((1,) * extra + data.shape) From d653d3e839f07fbaa8ae390ae409d8c704a043b6 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 15:12:31 +0200 Subject: [PATCH 03/56] Persist tile manifests in the xdas format, deprecate dask writes A tile-backed variable stores its manifest as a __tiles__ sibling group plus a JSON attribute holding what the arrays cannot (engine specification and dtype); open_dataarray reconstructs the TileArray. Writing dask-backed virtual arrays now emits a FutureWarning; the reader stays. --- docs/api/index.md | 1 + docs/api/tiles.md | 54 ++++++++++++ docs/release-notes.md | 9 ++ tests/tiles/test_integration.py | 144 ++++++++++++++++++++++++++++++++ xdas/io/xdas.py | 38 ++++++++- 5 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 docs/api/tiles.md create mode 100644 tests/tiles/test_integration.py diff --git a/docs/api/index.md b/docs/api/index.md index 2f8a3e99..e2179dc9 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -14,5 +14,6 @@ processing signal synthetics testing +tiles virtual ``` \ No newline at end of file diff --git a/docs/api/tiles.md b/docs/api/tiles.md new file mode 100644 index 00000000..45cf11e1 --- /dev/null +++ b/docs/api/tiles.md @@ -0,0 +1,54 @@ +```{eval-rst} +.. currentmodule:: xdas.tiles +``` + +# xdas.tiles + +Lazy tile-backed virtual arrays, the backend of the formats that HDF5 +virtual datasets cannot serve (Silixa TDMS, MiniSEED). + +## TileArray + +A dense rectilinear grid of file-backed tiles exposed as one lazy +numpy-like array. + +Attributes + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary + + TileArray.shape + TileArray.dtype + TileArray.ndim + TileArray.size + TileArray.chunks + TileArray.ntiles + TileArray.engine + TileArray.attrs +``` + +Methods + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary + + TileArray.from_dataset + TileArray.to_dataset + TileArray.concat + TileArray.expand_dims + TileArray.equals +``` + +## Engine registry + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary + + Engine + Engine.open + Engine.load + extract_array +``` diff --git a/docs/release-notes.md b/docs/release-notes.md index 3c6fdfe0..7e000fdb 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,5 +1,14 @@ # Release notes +## 0.2.9 (unreleased) + +### New Features +- **Tile-backed virtual arrays.** The new `xdas.tiles` package (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). +- Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). + +### Deprecations +- Writing dask-backed virtual arrays (`__dask_array__` attribute) is deprecated and emits a `FutureWarning`; existing files still open. The tile-backed engines replace this mechanism (@atrabattoni). + ## 0.2.8 ### New Features diff --git a/tests/tiles/test_integration.py b/tests/tiles/test_integration.py new file mode 100644 index 00000000..3bc2d75f --- /dev/null +++ b/tests/tiles/test_integration.py @@ -0,0 +1,144 @@ +"""Tile-backed data inside the 0.2 DataArray and the native file format.""" + +import dask.array as da_ +import numpy as np +import numpy.testing as npt +import pytest + +import xdas as xd +from xdas.tiles import TileArray, extract_array + +NX = 5 + +DIMS = ("time", "distance") + + +def wrap(manifest): + """Wrap *manifest* in a DataArray with regular time/distance coordinates.""" + nt, nx = manifest.shape + # ns resolution: the netCDF round trip casts datetimes to M8[ns] + time = xd.Coordinate["interpolated"].from_block( + np.datetime64("2020-01-01T00:00:00", "ns"), + nt, + np.timedelta64(10_000_000, "ns"), + dim="time", + ) + distance = xd.Coordinate["interpolated"].from_block(0.0, nx, 4.0, dim="distance") + return xd.DataArray(manifest, {"time": time, "distance": distance}) + + +class TestDataArray: + def test_data_and_extract(self, stack): + manifest, _ = stack + da = wrap(manifest) + assert da.data is manifest + assert extract_array(da) is manifest + assert "TileArray" in repr(da) + + def test_extract_rejections(self, stack): + manifest, _ = stack + with pytest.raises(TypeError, match="in-memory numpy array"): + extract_array(wrap(manifest).load()) + with pytest.raises(TypeError, match="not backed by"): + extract_array("something else") + + def test_isel_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + view = da.isel(time=slice(9, 13), distance=slice(1, 4)) + assert isinstance(view.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(view.values, reference[9:13, 1:4]) + + def test_sel_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + t0 = da["time"][2].values + t1 = da["time"][20].values + view = da.sel(time=slice(t0, t1)) + assert isinstance(view.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(view.values, reference[2:21]) + + def test_load_materializes(self, stack): + manifest, reference = stack + loaded = wrap(manifest).load() + assert isinstance(loaded.data, np.ndarray) + npt.assert_array_equal(loaded.values, reference) + + def test_concat_along_existing_dim_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + head = da.isel(time=slice(0, 10)) + tail = da.isel(time=slice(10, None)) + out = xd.concat([head, tail], "time") + assert isinstance(out.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(out.values, reference) + + def test_concat_along_new_dim_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + objs = [wrap(manifest), wrap(manifest)] + out = xd.concat(objs, "station") + assert out.dims == ("station", "time", "distance") + assert isinstance(out.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(out.values, np.stack([reference, reference])) + + def test_mean_streams(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + npt.assert_allclose(da.mean("time").values, reference.mean(0)) + assert len(engine_calls) > 0 + assert manifest._cache is None + + +class TestPersistence: + def test_round_trip(self, stack, tmp_path): + manifest, reference = stack + da = wrap(manifest) + path = str(tmp_path / "view.nc") + da.to_netcdf(path) + reopened = xd.open_dataarray(path) + assert isinstance(reopened.data, TileArray) + assert reopened.data.equals(manifest) + assert reopened.coords["time"].equals(da.coords["time"]) + npt.assert_array_equal(reopened.values, reference) + + def test_sliced_view_round_trip(self, stack, tmp_path): + manifest, reference = stack + view = wrap(manifest).isel(time=slice(9, 13)) + path = str(tmp_path / "sliced.nc") + view.to_netcdf(path) + reopened = xd.open_dataarray(path) + assert isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, reference[9:13]) + + def test_grouped_round_trip(self, stack, tmp_path): + manifest, reference = stack + da = wrap(manifest) + path = str(tmp_path / "grouped.nc") + da.to_netcdf(path, group="acquisition") + reopened = xd.open_dataarray(path, group="acquisition") + assert isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, reference) + + def test_eager_save_writes_values(self, stack, tmp_path): + manifest, reference = stack + da = wrap(manifest) + path = str(tmp_path / "eager.nc") + da.to_netcdf(path, virtual=False) + reopened = xd.open_dataarray(path) + assert not isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, reference) + + def test_dask_write_deprecated(self, tmp_path): + import dask + + data = da_.from_delayed(dask.delayed(np.zeros)((4, NX)), (4, NX), np.float64) + da = xd.DataArray(data, dims=DIMS) + path = str(tmp_path / "dask.nc") + with pytest.warns(FutureWarning, match="dask-backed"): + da.to_netcdf(path, virtual=True) + reopened = xd.open_dataarray(path) + npt.assert_array_equal(reopened.values, np.zeros((4, NX))) diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 4461e3fd..6c18458a 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -4,7 +4,9 @@ Supports :class:`DataArray`, :class:`DataSequence`, and :class:`DataMapping`. """ +import json import os +import warnings from pathlib import Path import h5netcdf @@ -16,9 +18,13 @@ from ..coordinates import Coordinates from ..core import DataArray, DataCollection, DataMapping, DataSequence from ..dask import create_variable, loads +from ..tiles import TileArray from ..virtual import VirtualArray, VirtualSource from .core import Engine +TILES_GROUP = "__tiles__" +"""Name of the sibling group holding a virtual variable's tile manifest.""" + class XdasEngine(Engine, name="xdas"): """Engine for the native xdas HDF5/NetCDF4 format.""" @@ -89,7 +95,15 @@ def open_dataarray(fname, group=None): coords = Coordinates._from_dataset(dataset, name) # read data - if "__dask_array__" in dataset[name].attrs: + if "__tile_array__" in dataset[name].attrs: + spec = json.loads(dataset[name].attrs.pop("__tile_array__")) + location = TILES_GROUP if group is None else f"{group}/{TILES_GROUP}" + with xr.open_dataset(fname, group=location, engine="h5netcdf") as manifest: + manifest = manifest.load() + data = TileArray.from_dataset( + manifest, dtype=spec["dtype"], params={"engine": spec["engine"]} + ) + elif "__dask_array__" in dataset[name].attrs: data = loads(dataset[name].attrs.pop("__dask_array__")) else: with h5py.File(fname) as file: @@ -135,7 +149,7 @@ def save_dataarray( fname = str(fname) if virtual is None: - virtual = isinstance(da.data, (VirtualArray, DaskArray)) + virtual = isinstance(da.data, (VirtualArray, DaskArray, TileArray)) # initialize dataset = xr.Dataset(attrs={"Conventions": "CF-1.9"}) @@ -175,7 +189,17 @@ def save_dataarray( variable = da.data.create_variable( file, variable_name, da.dims, da.dtype ) + elif isinstance(da.data, TileArray): + variable = file.create_variable(variable_name, da.dims, da.dtype) + variable.attrs["__tile_array__"] = json.dumps( + {"engine": da.data.engine, "dtype": str(da.data.dtype)} + ) elif isinstance(da.data, DaskArray): + warnings.warn( + "writing dask-backed virtual arrays is deprecated; the " + "tile-backed engines (xdas.tiles) replace them", + FutureWarning, + ) variable = create_variable( da.data, file, variable_name, da.dims, da.dtype ) @@ -191,6 +215,16 @@ def save_dataarray( # write metadata dataset.to_netcdf(fname, mode="a", group=group, engine="h5netcdf") + # write the tile manifest as a sibling group + if virtual and isinstance(da.data, TileArray): + manifest, _ = da.data.to_dataset() + for name in list(manifest.variables): + manifest[name].encoding.clear() + if manifest[name].dtype == object: + manifest[name] = manifest[name].astype(str) + location = TILES_GROUP if group is None else f"{group}/{TILES_GROUP}" + manifest.to_netcdf(fname, mode="a", group=location, engine="h5netcdf") + def open_datacollection(fname, group=None): """Read a :class:`DataCollection` from *fname*, auto-detecting sequence vs. mapping.""" From e275d17aa5f351fbb17ac54cbdd7854a2c509c21 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 22:45:57 +0200 Subject: [PATCH 04/56] Merge the tile engine registry into io.Engine as load_tile Tile decoding becomes the load_tile half of the single per-format plugin socket xdas.io.Engine: the base class gains an abstract load_tile(path, selection, **kwargs), the silixa and miniseed tile engine classes dissolve into their io engines, and xdas.tiles.registry shrinks to a get_engine adapter resolving manifest engine names against the io registry (the 0.3 line hosts the same lookup over its own registry, keeping tilearray.py identical in both lines). --- docs/api/tiles.md | 11 +++-- tests/tiles/conftest.py | 16 ++++--- tests/tiles/test_tilearray.py | 41 ++++++++---------- xdas/io/core.py | 15 +++++++ xdas/io/miniseed.py | 14 +++--- xdas/io/silixa.py | 9 +--- xdas/tiles/__init__.py | 14 +++--- xdas/tiles/registry.py | 81 +++++++++++++---------------------- xdas/tiles/tilearray.py | 30 ++++++------- 9 files changed, 107 insertions(+), 124 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index 45cf11e1..446453ba 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -41,14 +41,17 @@ Methods TileArray.equals ``` -## Engine registry +## Engine lookup + +Tiles are decoded by the ``load_tile`` half of the +{class}`xdas.io.Engine` format plugins; {func}`~xdas.tiles.get_engine` +resolves the engine names stored in tile manifests. ```{eval-rst} .. autosummary:: :toctree: ../_autosummary - Engine - Engine.open - Engine.load + get_engine + xdas.io.Engine.load_tile extract_array ``` diff --git a/tests/tiles/conftest.py b/tests/tiles/conftest.py index d0c7e7a3..d2633d3c 100644 --- a/tests/tiles/conftest.py +++ b/tests/tiles/conftest.py @@ -4,7 +4,8 @@ import numpy as np import pytest -from xdas.tiles import ENGINES, Engine, TileArray +from xdas.io import Engine +from xdas.tiles import TileArray NX = 5 @@ -15,13 +16,14 @@ class H5pyEngine(Engine, name="h5py"): """Read any HDF5 dataset — the engine of the synthetic test files. The format engines each read their own layout; test files belong to - no format, so they are described by this generic load-only engine. - Extra leading selection axes (virtually expanded arrays) pad the - output rank, as the production engines do. + no format, so they are described by this generic load-only engine + (its opening half stays abstract). Extra leading selection axes + (virtually expanded arrays) pad the output rank, as the production + engines do. """ @staticmethod - def load(path, selection, *, dataset): + def load_tile(path, selection, *, dataset): with h5py.File(path, "r") as file: source = file[dataset] extra = len(selection) - source.ndim @@ -101,11 +103,11 @@ def windowed(tmp_path): def engine_calls(monkeypatch): """Record the path of every h5py engine read, delegating to the real one.""" calls = [] - original = ENGINES["h5py"].load + original = Engine["h5py"].load_tile def counting(path, selection, **params): calls.append(path) return original(path, selection, **params) - monkeypatch.setattr(ENGINES["h5py"], "load", counting) + monkeypatch.setattr(Engine["h5py"], "load_tile", counting) return calls diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py index 29671ca8..5180c3a5 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/tiles/test_tilearray.py @@ -5,11 +5,8 @@ import numpy.testing as npt import pytest -from xdas.tiles import ( - ENGINES, - Engine, - TileArray, -) +from xdas.io import Engine +from xdas.tiles import TileArray, get_engine NX = 5 @@ -156,26 +153,26 @@ def test_engine_validation(self): def test_engine_registration(self): class DummyEngine(Engine, name="dummy"): @staticmethod - def load(path, selection): + def load_tile(path, selection): return np.zeros((1, 1)) try: - assert ENGINES["dummy"] is DummyEngine - assert DummyEngine.name == "dummy" - # the unimplemented half keeps a telling error - with pytest.raises(NotImplementedError, match="'dummy' cannot open"): - DummyEngine.open("some/path") + assert get_engine("dummy") is DummyEngine finally: - del ENGINES["dummy"] + del Engine._registry["dummy"] - def test_unregistered_base_subclass(self): - class HalfBaked(Engine): + def test_engine_without_tile_loader(self): + # a registered engine that predates the tiles machinery resolves + # but fails loudly when a manifest asks it to decode + class NoTilesEngine(Engine, name="notiles"): pass - assert HalfBaked.name is None - assert HalfBaked not in ENGINES.values() - with pytest.raises(NotImplementedError, match="cannot load"): - HalfBaked.load("some/path", (slice(0, 1),)) + try: + arr = TileArray("a", (5, NX), {"name": "notiles"}, "f8") + with pytest.raises(NotImplementedError): + np.asarray(arr) + finally: + del Engine._registry["notiles"] def test_repr(self, stack): manifest, _ = stack @@ -474,7 +471,7 @@ def test_per_tile_params_reach_the_engine(self, stack): class ProbeEngine(Engine, name="probe"): @staticmethod - def load(path, selection, *, record, flavor): + def load_tile(path, selection, *, record, flavor): seen.append((path, record, flavor)) widths = tuple( len(range(entry.start, entry.stop, entry.step or 1)) @@ -493,14 +490,14 @@ def load(path, selection, *, record, flavor): assert [entry[1] for entry in seen] == [0, 1, 2] assert {entry[2] for entry in seen} == {"spec"} finally: - del ENGINES["probe"] + del Engine._registry["probe"] def test_wrong_shape_fails_loudly(self, stack): manifest, _ = stack class BadShapeEngine(Engine, name="badshape"): @staticmethod - def load(path, selection): + def load_tile(path, selection): return np.zeros((1, 1)) try: @@ -512,7 +509,7 @@ def load(path, selection): with pytest.raises(ValueError, match="shape"): np.asarray(bad[0:5]) finally: - del ENGINES["badshape"] + del Engine._registry["badshape"] class TestEdgeCases: diff --git a/xdas/io/core.py b/xdas/io/core.py index f4dffb55..9348eb35 100644 --- a/xdas/io/core.py +++ b/xdas/io/core.py @@ -109,6 +109,21 @@ def save_datacollection(self, dc, fname, **kwargs): """Write *dc* to *fname* (abstract).""" raise NotImplementedError + @staticmethod + def load_tile(path, selection, **kwargs): + """Read the selected sub-box of one tile of *path* (abstract). + + The decode half of the tiles machinery: called on the class by + :class:`~xdas.tiles.TileArray` once per tile touched, with one + source-local, possibly strided :class:`slice` per source axis and + the manifest's engine specification (merged with the per-tile + variables) as keyword arguments. It must return exactly the + selected sub-box of the decoded source as a numpy array, and must + depend only on its arguments — never on engine instance state — + so that stored manifests decode identically everywhere. + """ + raise NotImplementedError + def _parse_vtype(self, vtype): if self._supported_vtypes is None: return vtype diff --git a/xdas/io/miniseed.py b/xdas/io/miniseed.py index 85cc2a37..7ed518a3 100644 --- a/xdas/io/miniseed.py +++ b/xdas/io/miniseed.py @@ -12,7 +12,6 @@ get_sampling_interval, ) from ..core import DataArray, concat_coords -from ..tiles import Engine as TileEngine from ..tiles import TileArray from .core import Engine @@ -94,7 +93,8 @@ def read_header(self, path, ignore_last_sample, ctype): ) return shape, dtype, coords, method - def read_data(self, path, method, ignore_last_sample): + @staticmethod + def read_data(path, method, ignore_last_sample): """Load and return the raw data array from *path* using *method*.""" st = obspy.read(path) if method == "synchronized": @@ -115,20 +115,16 @@ def read_data(self, path, method, ignore_last_sample): data.append(np.concatenate(channel_data)) return np.array(data) - -class MiniSEEDTileEngine(TileEngine, name="miniseed"): - """Tile reader for MiniSEED sources, decoding with ObsPy.""" - @staticmethod - def load(path, selection, *, method, ignore_last_sample): - """Read a source selection of a MiniSEED file. + def load_tile(path, selection, *, method="synchronized", ignore_last_sample=False): + """Read a source selection of a MiniSEED file, decoding with ObsPy. Decodes the whole file with ObsPy (as the legacy dask path did) and crops to *selection*. The decoded rank is padded with unit leading axes for virtually expanded arrays, or squeezed when a scalar channel folded an axis away. """ - data = MiniSEEDEngine().read_data(path, method, ignore_last_sample) + data = MiniSEEDEngine.read_data(path, method, ignore_last_sample) if data.ndim < len(selection): data = data.reshape((1,) * (len(selection) - data.ndim) + data.shape) elif data.ndim > len(selection): diff --git a/xdas/io/silixa.py b/xdas/io/silixa.py index db828ab1..6847f67a 100644 --- a/xdas/io/silixa.py +++ b/xdas/io/silixa.py @@ -6,7 +6,6 @@ from ..coordinates import Coordinate from ..core import DataArray -from ..tiles import Engine as TileEngine from ..tiles import TileArray from .core import Engine from .tdms import TdmsReader @@ -56,13 +55,9 @@ def read_data(self, fname): data = tdms.get_data() return data - -class SilixaTileEngine(TileEngine, name="silixa"): - """Tile reader for Silixa TDMS sources (rows are time samples).""" - @staticmethod - def load(path, selection): - """Read a source selection of a Silixa TDMS file. + def load_tile(path, selection): + """Read a source selection of a Silixa TDMS file (rows are time samples). :class:`~xdas.io.tdms.TdmsReader` performs the decoding (``get_data`` bounds are inclusive, hence the ``stop - 1``); the diff --git a/xdas/tiles/__init__.py b/xdas/tiles/__init__.py index b1507605..d43fd4de 100644 --- a/xdas/tiles/__init__.py +++ b/xdas/tiles/__init__.py @@ -2,18 +2,18 @@ Lazy tile-backed virtual arrays (ported from the 0.3 line). :class:`TileArray` exposes a rectilinear grid of file-backed tiles as -one numpy-like lazy array; :class:`Engine` is the per-format tile -reader plugin socket. This backend replaces the serialized-dask-graph -fallback used by the formats that HDF5 virtual datasets cannot serve -(Silixa TDMS, MiniSEED). +one numpy-like lazy array. Tiles are decoded by the ``load_tile`` half +of the :class:`xdas.io.Engine` format plugins, resolved by manifest +engine name with :func:`get_engine`. This backend replaces the +serialized-dask-graph fallback used by the formats that HDF5 virtual +datasets cannot serve (Silixa TDMS, MiniSEED). """ -from .registry import ENGINES, Engine +from .registry import get_engine from .tilearray import TileArray, extract_array __all__ = [ - "ENGINES", - "Engine", "TileArray", "extract_array", + "get_engine", ] diff --git a/xdas/tiles/registry.py b/xdas/tiles/registry.py index 03a0512d..877d4951 100644 --- a/xdas/tiles/registry.py +++ b/xdas/tiles/registry.py @@ -1,64 +1,43 @@ -"""Tile engine registry — the format plugin socket of :mod:`xdas.tiles`. +"""Engine lookup of the tiles machinery. -A tile engine is a subclass of :class:`Engine`, one per format, -registered by subclassing with a ``name``. It carries a ``load`` half -that reads one tile of a source file; the :class:`~xdas.tiles.TileArray` -read path looks it up by the ``name`` key of its engine specification. -This registry is distinct from :class:`xdas.io.Engine`, which handles -whole-file opening and saving of labeled arrays. - -Ported from the 0.3 line (``xdas/virtual/registry.py``). +The 0.2 line keeps a single per-format plugin socket: :class:`xdas.io.Engine`. +Tile decoding is its ``load_tile`` half; :func:`get_engine` resolves the +``name`` key of the engine specifications stored in tile manifests against +that registry. The 0.3 line hosts the same lookup over its own registry, +keeping :mod:`xdas.tiles.tilearray` identical in both lines. """ -ENGINES = {} +def get_engine(name): + """Return the engine class registered under *name*. -class Engine: - """Base class of the tile format engines; subclassing registers. + Parameters + ---------- + name : str + The ``name`` key of a tile manifest's engine specification. - ``class MyEngine(Engine, name="myformat")`` registers the subclass - in :data:`ENGINES` under *name* (omit it for unregistered - intermediate bases). An engine implements one or both halves as - static methods — a format only referenced by stored manifests needs - only ``load``: + Returns + ------- + type + The :class:`xdas.io.Engine` subclass registered under *name*; its + ``load_tile`` static method decodes tiles of that format. - - ``open(path, **kwargs)``: read only the metadata of one file and - return a lazy tile-backed array. Unused by the 0.2 line, where - the :class:`xdas.io.Engine` subclasses do the opening; kept for - forward compatibility with the 0.3 stack. - - ``load(path, selection, **params)``: read one tile — open the - source itself (h5py, obspy, ...) and return exactly the selected - sub-box of the decoded source as a numpy array, *selection* being - one source-local, possibly strided :class:`slice` per source - axis. The keyword parameters are the manifest's engine - specification merged with the per-tile manifest variables (a - per-tile value shadows a same-named spec constant). + Raises + ------ + KeyError + If no engine is registered under *name*. """ + from ..io.core import Engine - name = None - - def __init_subclass__(cls, /, name=None, **kwargs): - super().__init_subclass__(**kwargs) - if name is not None: - cls.name = name - ENGINES[name] = cls - - @classmethod - def open(cls, path, **kwargs): - """Scan *path* lazily; overridden by engines that open files.""" - raise NotImplementedError( - f"engine {cls.name!r} cannot open files (no `open` method)" - ) - - @classmethod - def load(cls, path, selection, **params): - """Read one tile of *path*; overridden by engines that load data.""" - raise NotImplementedError( - f"engine {cls.name!r} cannot load tile data (no `load` method)" - ) + try: + return Engine[name] + except KeyError: + raise KeyError( + f"no engine registered under {name!r}; " + f"available: {sorted(Engine._registry)}" + ) from None __all__ = [ - "ENGINES", - "Engine", + "get_engine", ] diff --git a/xdas/tiles/tilearray.py b/xdas/tiles/tilearray.py index 6a09c6e6..af39b116 100644 --- a/xdas/tiles/tilearray.py +++ b/xdas/tiles/tilearray.py @@ -29,11 +29,11 @@ geometry and returns a new :class:`TileArray` (as lazy and self-described as its input); any other indexing reads the bounding box of the selection and resolves the rest in memory. ``np.asarray`` -materializes: tiles are read one by one by the registered *engine*, -whose ``load`` opens each tile's path itself and returns the tile's -*source selection* (one possibly strided slice per axis, see -:class:`~xdas.tiles.registry.Engine`), every part landing -directly in the output array. +materializes: tiles are read one by one by the registered *engine* +(resolved with :func:`~xdas.tiles.registry.get_engine`), whose +``load_tile`` opens each tile's path itself and returns the tile's +*source selection* (one possibly strided slice per axis), every part +landing directly in the output array. A tile array is used *raw* as the data of a :class:`xdas.DataArray` (``DataArray(arr, coords)``), so ``da.data`` returns the inspectable @@ -62,7 +62,7 @@ import numpy as np import xarray as xr -from .registry import ENGINES +from .registry import get_engine TILE_PREFIX = "tile_" """Prefix of the tile-grid dimensions of a manifest dataset.""" @@ -259,9 +259,9 @@ class TileArray(np.lib.mixins.NDArrayOperatorsMixin): the number of tiles along the axis). engine : dict The engine specification: the key ``"name"`` selects a - registered :class:`~xdas.virtual.registry.Engine`; the - remaining keys are passed to its ``load`` as keyword - parameters. + registered engine (resolved with + :func:`~xdas.tiles.registry.get_engine`); the remaining keys + are passed to its ``load_tile`` as keyword parameters. dtype : str or numpy.dtype Element type of the virtual array (little-endian or single-byte). @@ -396,11 +396,7 @@ def _setup(self, dataset, dtype, engine): engine = json.loads(json.dumps(engine)) if not isinstance(engine, dict) or "name" not in engine: raise ValueError("the engine specification must have a `name` key") - if engine["name"] not in ENGINES: - raise KeyError( - f"no engine registered under {engine['name']!r}; " - f"available: {sorted(ENGINES)}" - ) + get_engine(engine["name"]) self._engine = engine self.dtype = np.dtype(dtype) if self.dtype.byteorder == ">": @@ -671,10 +667,10 @@ def _grid_values(self, name): @functools.cached_property def _engine_impl(self): - """The ``(load, spec)`` of the engine specification.""" + """The ``(load_tile, spec)`` of the engine specification.""" spec = dict(self.engine) name = spec.pop("name") - return ENGINES[name].load, spec + return get_engine(name).load_tile, spec def __array__(self, dtype=None, copy=None): """Read every tile and return the values as a numpy array. @@ -868,7 +864,7 @@ def expand_dims(self, axis=0): expands the data with :func:`numpy.expand_dims`; this keeps that path lazy instead of materializing. Only the leading position is supported: the new axis holds one tile of size - one, and the engine ``load`` receives one extra leading + one, and the engine ``load_tile`` receives one extra leading ``slice(0, 1)`` per expanded axis, padding its output rank accordingly (see the silixa and miniseed engines). From edb605162a4f3247818a4817907640037cee190c Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 22:55:21 +0200 Subject: [PATCH 05/56] Add an optional tiles vtype to every HDF5 io engine asn, febus, terra15, apsensing, prodml and the native xdas engine gain vtype="tiles": open_dataarray then backs the DataArray with a lazy TileArray instead of an HDF5 virtual source, and each engine gains the load_tile decode half (bodies shared verbatim with the 0.3 line, so saved tile views open identically there). The hdf5 vtype stays the default. Febus models a whole file as a single tile: the overlap trimming and 3-D block fusing live in load_tile, instead of one VDS mapping entry per block. --- tests/io/test_tiles_vtype.py | 178 +++++++++++++++++++++++++++++++++++ xdas/io/apsensing.py | 19 +++- xdas/io/asn.py | 16 +++- xdas/io/febus.py | 75 ++++++++++++++- xdas/io/prodml.py | 28 +++++- xdas/io/terra15.py | 17 +++- xdas/io/xdas.py | 42 ++++++++- 7 files changed, 360 insertions(+), 15 deletions(-) create mode 100644 tests/io/test_tiles_vtype.py diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py new file mode 100644 index 00000000..f55c3452 --- /dev/null +++ b/tests/io/test_tiles_vtype.py @@ -0,0 +1,178 @@ +"""Cross-format tests of the optional ``tiles`` vtype of the io engines.""" + +import h5py +import numpy as np +import numpy.testing as npt +import pytest + +import xdas as xd +from xdas.tiles import TileArray + + +def ramp(shape, dtype="float32"): + """Distinct values everywhere, so misplaced reads cannot cancel out.""" + return np.arange(np.prod(shape), dtype=dtype).reshape(shape) + + +def make_asn_file(path, nt=20, nd=4, t0=0.0): + data = ramp((nt, nd)) + with h5py.File(path, "w") as file: + header = file.create_group("header") + header["time"] = t0 + header["dt"] = 0.1 + header["dx"] = 10.0 + file.create_dataset("data", data=data) + cable_spec = file.create_group("cableSpec") + cable_spec["sensorDistances"] = 10.0 * np.arange(nd) + demod_spec = file.create_group("demodSpec") + demod_spec["roiStart"] = np.array([0]) + demod_spec["roiEnd"] = np.array([nd]) + return data + + +def make_febus_file(path, nchunks=3, nt=12, nx=5): + data = ramp((nchunks, nt, nx)) + times = np.arange(nchunks, dtype=np.float64) * 0.01 + with h5py.File(path, "w") as file: + source = file.create_group("DeviceName").create_group("Source1") + source.create_dataset("time", data=times) + zone = source.create_group("Zone1") + zone.attrs["BlockRate"] = np.array([100.0]) + zone.attrs["Spacing"] = np.array([5.0, 1.0]) + zone.attrs["Extent"] = np.array([0.0, (nx - 1) * 5.0]) + zone.attrs["Origin"] = np.array([0.0, 0.0]) + zone.create_dataset("StrainRate", data=data) + return data + + +def make_terra15_file(path, nt=15, nd=6): + data = ramp((nt, nd)) + with h5py.File(path, "w") as file: + product = file.create_group("data_product") + # small epoch offsets stay exact in float64 down to the nanosecond + product.create_dataset("gps_time", data=0.001 * np.arange(nt)) + product.create_dataset("data", data=data) + file.attrs["sensing_range_start"] = 12.0 + file.attrs["dx"] = 2.0 + return data + + +def make_apsensing_file(path, nt=10, nd=8): + data = ramp((nt, nd)) + with h5py.File(path, "w") as file: + file.create_dataset("DAS", data=data) + meta = file.create_group("Metadata") + meta.create_dataset("Timestamp", data=np.bytes_(b"2020-01-01T00:00:00.000Z")) + proc = file.create_group("ProcessingServer") + proc["DataRate"] = 1000.0 + proc["SpatialSampling"] = 2.0 + file.create_group("DAQ")["PositionStart"] = 0.0 + return data + + +def make_prodml_file(path, nt=10, nd=5, swapped=False): + data = ramp((nd, nt) if swapped else (nt, nd)) + with h5py.File(path, "w") as file: + acquisition = file.create_group("Acquisition") + acquisition.attrs["SpatialSamplingInterval"] = 2.0 + acquisition.attrs["StartLocusIndex"] = 0 + rawdata = acquisition.create_group("Raw[0]").create_dataset( + "RawData", data=data + ) + rawdata.attrs["PartStartTime"] = np.bytes_(b"2020-01-01T00:00:00.000+00:00") + rawdata.attrs["PartEndTime"] = np.bytes_(b"2020-01-01T00:00:00.900+00:00") + return data + + +MAKERS = { + "asn": (make_asn_file, {}), + "febus": (make_febus_file, {"overlaps": (1, 1), "offset": 0}), + "terra15": (make_terra15_file, {}), + "apsensing": (make_apsensing_file, {}), + "prodml": (make_prodml_file, {}), +} + + +@pytest.mark.parametrize("fmt", sorted(MAKERS)) +def test_tiles_vtype_matches_hdf5(tmp_path, fmt): + """The opt-in tiles backing yields the exact same array as the VDS one.""" + maker, kwargs = MAKERS[fmt] + path = str(tmp_path / f"{fmt}.h5") + maker(path) + expected = xd.open_dataarray(path, engine=fmt, **kwargs) + result = xd.open_dataarray(path, engine=fmt, vtype="tiles", **kwargs) + assert isinstance(result.data, TileArray) + assert result.data.engine["name"] == fmt + assert result.equals(expected) + sliced = result[3:9:2, 1:3] + assert isinstance(sliced.data, TileArray) + npt.assert_array_equal(sliced.values, expected.values[3:9:2, 1:3]) + + +def test_febus_block_crossing_reads(tmp_path): + """Row ranges spanning block boundaries fuse the right trimmed parts.""" + path = str(tmp_path / "febus.h5") + make_febus_file(path) + kwargs = {"overlaps": (1, 1), "offset": 0} + expected = xd.open_dataarray(path, engine="febus", **kwargs).values + result = xd.open_dataarray(path, engine="febus", vtype="tiles", **kwargs) + assert result.data.engine["block_size"] == 12 + assert result.data.engine["overlaps"] == [1, 1] + npt.assert_array_equal(result[8:12].values, expected[8:12]) + npt.assert_array_equal(result[::3].values, expected[::3]) + npt.assert_array_equal(result[25:].values, expected[25:]) + + +def test_prodml_transpose_param(tmp_path): + """The shared contract reads distance-major files time-major on request. + + The 0.2 engine never writes ``transpose`` (its manifests keep the + on-disk layout), but manifests written by the 0.3 line use it. + """ + path = str(tmp_path / "prodml_swapped.h5") + data = make_prodml_file(path, swapped=True) + manifest = TileArray( + path, data.T.shape, {"name": "prodml", "transpose": True}, data.dtype + ) + npt.assert_array_equal(np.asarray(manifest), data.T) + npt.assert_array_equal(np.asarray(manifest[2:7:2, 1:4]), data.T[2:7:2, 1:4]) + + +def test_tiles_view_roundtrip(tmp_path): + """A tile-backed view persists (spec params included) and reopens lazily.""" + path = str(tmp_path / "febus.h5") + make_febus_file(path) + da = xd.open_dataarray( + path, engine="febus", vtype="tiles", overlaps=(1, 1), offset=0 + ) + out = str(tmp_path / "view.nc") + da.to_netcdf(out) + reopened = xd.open_dataarray(out) + assert isinstance(reopened.data, TileArray) + assert reopened.data.engine == da.data.engine + assert reopened.equals(da) + + +def test_open_mfdataarray_fuses_tiles(tmp_path): + """Multi-file opening fuses at the manifest level and stays lazy.""" + paths, parts = [], [] + for k in range(2): + path = str(tmp_path / f"asn{k}.h5") + parts.append(make_asn_file(path, t0=k * 2.0)) + paths.append(path) + da = xd.open_mfdataarray(paths, engine="asn", vtype="tiles", parallel=False) + assert isinstance(da, xd.DataArray) + assert isinstance(da.data, TileArray) + assert da.data.ntiles == 2 + npt.assert_array_equal(da.values, np.concatenate(parts)) + + +def test_xdas_engine_tiles_vtype(tmp_path): + """Materialized native files reopen lazily as tile arrays on request.""" + da = xd.testing.dummy(shape=(10, 5), step=(1.0, 10.0), dtype=np.float32) + path = str(tmp_path / "da.nc") + da.to_netcdf(path) + result = xd.open_dataarray(path, engine="xdas", vtype="tiles") + assert isinstance(result.data, TileArray) + assert result.data.engine == {"name": "xdas", "dataset": "/__values__"} + assert result.equals(da) diff --git a/xdas/io/apsensing.py b/xdas/io/apsensing.py index b6187713..275290fe 100644 --- a/xdas/io/apsensing.py +++ b/xdas/io/apsensing.py @@ -7,6 +7,7 @@ from ..coordinates import Coordinate from ..core import DataArray +from ..tiles import TileArray from ..virtual import VirtualSource from .core import Engine @@ -14,7 +15,7 @@ class APSensingEngine(Engine, name="apsensing"): """Engine for reading APSensing HDF5 files.""" - _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated", "sampled", "dense"], @@ -27,7 +28,15 @@ def open_dataarray(self, fname): fs = file["ProcessingServer"]["DataRate"][()].item() dx = file["ProcessingServer"]["SpatialSampling"][()].item() x0 = file["DAQ"]["PositionStart"][()].item() - data = VirtualSource(file["DAS"]) + if self.vtype == "tiles": + data = TileArray( + str(fname), + file["DAS"].shape, + {"name": "apsensing"}, + file["DAS"].dtype, + ) + else: + data = VirtualSource(file["DAS"]) nt, nd = data.shape @@ -46,6 +55,12 @@ def open_dataarray(self, fname): ) return DataArray(data, {"time": time, "distance": distance}) + @staticmethod + def load_tile(path, selection): + """Read a source selection of the ``/DAS`` dataset of an APSensing file.""" + with h5py.File(path, "r") as file: + return file["/DAS"][selection] + # NOTE: Distance sample are left aligned. The original number of samples is # `round((xend - xstart) / dx) + 1` with xstart / xend located in # "DAQ/PositionStart" / "DAQ/PositionEnd" and dx located in "DAQ/SamplingInterval". diff --git a/xdas/io/asn.py b/xdas/io/asn.py index 414e28f8..2cf37826 100644 --- a/xdas/io/asn.py +++ b/xdas/io/asn.py @@ -15,6 +15,7 @@ from ..coordinates import Coordinate, get_sampling_interval from ..core import DataArray, concat_coords +from ..tiles import TileArray from ..virtual import VirtualSource from .core import Engine @@ -22,7 +23,7 @@ class ASNEngine(Engine, name="asn"): """Engine for reading ASN HDF5 files.""" - _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated"], @@ -37,7 +38,12 @@ def open_dataarray(self, fname): t0 = np.datetime64(round(header["time"][()] * 1e9), "ns") dt = np.timedelta64(round(1e9 * header["dt"][()]), "ns") dx = float(header["dx"][()]) # Note: dx before (internal) downsampling! - data = VirtualSource(file["data"]) + if self.vtype == "tiles": + data = TileArray( + str(fname), file["data"].shape, {"name": "asn"}, file["data"].dtype + ) + else: + data = VirtualSource(file["data"]) # Get the optical distance for all the recorded channels (after downsampling) # Note that this vector is not continuous for more than one ROI @@ -90,6 +96,12 @@ def open_dataarray(self, fname): distance = concat_coords(roi_blocks, reduce=False, regularize=True) return DataArray(data, {"time": time, "distance": distance}) + @staticmethod + def load_tile(path, selection): + """Read a source selection of the ``/data`` dataset of an ASN file.""" + with h5py.File(path, "r") as file: + return file["/data"][selection] + def _get_roi_bound_indices(self, all_dists, n_start, n_end, dx): start_index = bisect_left(all_dists, n_start * dx) if start_index >= len(all_dists): diff --git a/xdas/io/febus.py b/xdas/io/febus.py index 342f672b..95afdf76 100644 --- a/xdas/io/febus.py +++ b/xdas/io/febus.py @@ -7,7 +7,8 @@ import numpy as np from ..coordinates import Coordinate -from ..core import DataArray, concat +from ..core import DataArray, concat, concat_coords +from ..tiles import TileArray from ..virtual import VirtualSource from .core import Engine @@ -15,7 +16,7 @@ class FebusEngine(Engine, name="febus"): """Engine for reading Febus HDF5 files.""" - _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated", "sampled", "dense"], @@ -67,6 +68,7 @@ def open_dataarray(self, fname, overlaps=None, offset=None): "Could not find the block size, please check file header" ) (name,) = list(zone.keys()) + dataset_path = zone[name].name chunks = VirtualSource(zone[name]) delta = (zone.attrs["Spacing"][1] / 1000.0, zone.attrs["Spacing"][0]) x0 = zone.attrs["Extent"][0] * delta[1] + zone.attrs["Origin"][0] @@ -101,14 +103,42 @@ def open_dataarray(self, fname, overlaps=None, offset=None): case _: raise ValueError("offset must be an integer") - chunks = chunks[:, overlaps[0] : -overlaps[-1], :] times = times + (overlaps[0] - offset) * delta[0] dt, dx = delta - _, nt, nx = chunks.shape + nblocks, block_size, nx = chunks.shape + nt = block_size - overlaps[0] - overlaps[1] dt = np.rint(1e6 * dt).astype("m8[us]").astype("m8[ns]") + if self.vtype == "tiles": + # one tile spans the whole file: the block arithmetic (trimming + # and 3-D to 2-D fusing) lives in `load_tile`, not the manifest + time = concat_coords( + [ + Coordinate[self.ctype["time"]].from_block( + np.rint(1e6 * t0).astype("M8[us]").astype("M8[ns]"), + nt, + dt, + dim="time", + ) + for t0 in times + ] + ) + distance = Coordinate[self.ctype["distance"]].from_block( + x0, nx, dx, dim="distance" + ) + engine = { + "name": "febus", + "dataset": dataset_path, + "block_size": int(block_size), + "overlaps": [int(overlaps[0]), int(overlaps[1])], + } + data = TileArray(str(fname), (nblocks * nt, nx), engine, chunks.dtype) + return DataArray(data, {"time": time, "distance": distance}, name=name) + + chunks = chunks[:, overlaps[0] : -overlaps[-1], :] + dc = [] for t0, chunk in zip(times, chunks): t0 = np.rint(1e6 * t0).astype("M8[us]").astype("M8[ns]") @@ -120,3 +150,40 @@ def open_dataarray(self, fname, overlaps=None, offset=None): dc.append(da) return concat(dc, "time") + + @staticmethod + def load_tile(path, selection, *, dataset, block_size, overlaps): + """Read a post-trim source selection of a Febus file. + + Febus files store a 3-D stack of overlapping ``(time, distance)`` + blocks. Rows are counted post-trim: each block contributes + ``block_size - sum(overlaps)`` rows. Only the blocks overlapping the + requested rows are read; overlap rows are sliced away without being + copied. + + Parameters + ---------- + path : str + Path of the Febus HDF5 file. + selection : tuple of slice + The source selection to read, one possibly strided slice per + axis, post-trim rows along axis 0. + dataset : str + Location of the block stack within the file. + block_size : int + Rows of one untrimmed block. + overlaps : tuple of int + Rows trimmed at the start and end of each block. + """ + rows = selection[0] + start, stop = rows.start, rows.stop + keep = block_size - overlaps[0] - overlaps[1] + with h5py.File(path, "r") as file: + blocks = file[dataset] + parts = [] + for block in range(start // keep, (stop - 1) // keep + 1): + lo = max(start - block * keep, 0) + overlaps[0] + hi = min(stop - block * keep, keep) + overlaps[0] + parts.append(blocks[block, lo:hi]) + rows = np.concatenate(parts) if len(parts) > 1 else parts[0] + return rows[(slice(None, None, selection[0].step), *selection[1:])] diff --git a/xdas/io/prodml.py b/xdas/io/prodml.py index 349d30c6..164e0520 100644 --- a/xdas/io/prodml.py +++ b/xdas/io/prodml.py @@ -11,14 +11,17 @@ from ..coordinates import Coordinate from ..core import DataArray +from ..tiles import TileArray from ..virtual import VirtualSource from .core import Engine +_RAWDATA = "/Acquisition/Raw[0]/RawData" + class ProdML(Engine, name="prodml", aliases=["optasense", "sintela"]): """Engine for reading ProdML / OptaSense / Sintela HDF5 files.""" - _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] _supported_ctypes: ClassVar[dict] = { "time": ["interpolated"], "distance": ["interpolated", "sampled", "dense"], @@ -43,7 +46,14 @@ def open_dataarray(self, fname, swapped_dims=False): .tz_localize(None) .to_numpy() ) - data = VirtualSource(rawdata) + if self.vtype == "tiles": + # the manifest keeps the on-disk layout, whichever way the + # dims are labeled, so the spec needs no `transpose` + data = TileArray( + str(fname), rawdata.shape, {"name": "prodml"}, rawdata.dtype + ) + else: + data = VirtualSource(rawdata) if swapped_dims: nd, nt = data.shape @@ -68,3 +78,17 @@ def open_dataarray(self, fname, swapped_dims=False): else {"time": time, "distance": distance} ) return DataArray(data, coords) + + @staticmethod + def load_tile(path, selection, *, transpose=False): + """Read a source selection of the raw data of a ProdML file. + + With ``transpose`` the on-disk layout is distance-major + ``(distance, time)``; rows are then columns on disk and are + transposed on the way out. + """ + with h5py.File(path, "r") as file: + data = file[_RAWDATA] + if transpose: + return data[selection[1], selection[0]].T + return data[selection] diff --git a/xdas/io/terra15.py b/xdas/io/terra15.py index 7f201b99..8d2b0efe 100644 --- a/xdas/io/terra15.py +++ b/xdas/io/terra15.py @@ -7,6 +7,7 @@ from ..coordinates import Coordinate from ..core import DataArray +from ..tiles import TileArray from ..virtual import VirtualSource from .core import Engine @@ -14,7 +15,7 @@ class Terra15Engine(Engine, name="terra15"): """Engine for reading Terra15 HDF5 files.""" - _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] _supported_ctypes: ClassVar[dict] = { "time": ["interpolated"], "distance": ["interpolated", "sampled", "dense"], @@ -37,7 +38,13 @@ def open_dataarray(self, fname, tz="UTC"): ) d0 = file.attrs["sensing_range_start"] dx = file.attrs["dx"] - data = VirtualSource(file["data_product"]["data"]) + source = file["data_product"]["data"] + if self.vtype == "tiles": + data = TileArray( + str(fname), source.shape, {"name": "terra15"}, source.dtype + ) + else: + data = VirtualSource(source) nt, nd = data.shape # time (regular by declaration, rate derived from the file's own stamps) time = { @@ -49,3 +56,9 @@ def open_dataarray(self, fname, tz="UTC"): d0, nd, dx, dim="distance" ) return DataArray(data, {"time": time, "distance": distance}) + + @staticmethod + def load_tile(path, selection): + """Read a source selection of the data product of a Terra15 file.""" + with h5py.File(path, "r") as file: + return file["/data_product/data"][selection] diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 7a86fd8c..94c40bab 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -8,6 +8,7 @@ import os import warnings from pathlib import Path +from typing import ClassVar import h5netcdf import h5py @@ -29,9 +30,11 @@ class XdasEngine(Engine, name="xdas"): """Engine for the native xdas HDF5/NetCDF4 format.""" + _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] + def open_dataarray(self, fname, **kwargs): """Delegate to module-level :func:`open_dataarray`.""" - return open_dataarray(fname, **kwargs) + return open_dataarray(fname, vtype=self.vtype, **kwargs) def save_dataarray(self, da, fname, **kwargs): """Delegate to module-level :func:`save_dataarray`.""" @@ -45,8 +48,28 @@ def save_datacollection(self, dc, fname, **kwargs): """Delegate to module-level :func:`save_datacollection`.""" return save_datacollection(dc, fname, **kwargs) + @staticmethod + def load_tile(path, selection, *, dataset): + """Read a source selection of a native xdas file. + + The variable is read with h5py, which resolves any HDF5 virtual + dataset the file may store transparently. + + Parameters + ---------- + path : str + Path of the NetCDF4/HDF5 file. + selection : tuple of slice + The source selection to read, one possibly strided slice per + axis. + dataset : str + Location of the data variable within the file. + """ + with h5py.File(path, "r") as file: + return file[dataset][selection] -def open_dataarray(fname, group=None): + +def open_dataarray(fname, group=None, vtype=None): """ Read a :class:`DataArray` from a native xdas NetCDF4/HDF5 file. @@ -56,6 +79,11 @@ def open_dataarray(fname, group=None): Path to the file. group : str, optional HDF5 group path inside the file. + vtype : str, optional + Virtualization backing of the returned data: ``"hdf5"`` (default, + an HDF5 virtual source) or ``"tiles"`` (a lazy + :class:`~xdas.tiles.TileArray` over the stored variable). Files + that store a tile manifest reopen as tile arrays regardless. Returns ------- @@ -110,7 +138,15 @@ def open_dataarray(fname, group=None): if group: file = file[group] variable = file["__values__" if name is None else name] - data = VirtualSource(variable) + if vtype == "tiles": + data = TileArray( + str(fname), + variable.shape, + {"name": "xdas", "dataset": variable.name}, + variable.dtype, + ) + else: + data = VirtualSource(variable) # pack everything return DataArray( From f3d3001fb25c64c2a5fb3a088272bc7d8b4a611d Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 22:56:16 +0200 Subject: [PATCH 06/56] Document the tiles vtype and the load_tile plugin half --- docs/release-notes.md | 4 +++ docs/user-guide/io/data-formats.md | 50 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/docs/release-notes.md b/docs/release-notes.md index b61c2220..04d45f40 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -5,6 +5,10 @@ ### New Features - **Tile-backed virtual arrays.** The new `xdas.tiles` package (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). - Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). +- **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines (the `hdf5` vtype stays the default). Tile views describe a Febus file as a single tile — the overlap trimming lives in the reader, not in one mapping entry per block — and saved tile views are directly readable by the 0.3 line (@atrabattoni). + +### Refactoring +- Tile decoding merged into the io engine plugin socket: `xdas.io.Engine` gained the `load_tile(path, selection, **params)` static half, and the separate `xdas.tiles.Engine` registry was removed — manifest engine names now resolve against the io registry (`xdas.tiles.get_engine`) (@atrabattoni). ### Deprecations - Writing dask-backed virtual arrays (`__dask_array__` attribute) is deprecated and emits a `FutureWarning`; existing files still open. The tile-backed engines replace this mechanism (@atrabattoni). diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index e32b327c..86fbf57e 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -117,3 +117,53 @@ Once the class is created and instanciated you can then use it : da = xd.open("other_format.hdf5", engine="my_engine", ctype="sampled") da ``` + +### Tile-backed engines + +Beside the `hdf5` vtype shown above (an HDF5 virtual source), an engine can +offer the `tiles` vtype: `open_dataarray` then backs the data array with a lazy +{py:class}`xdas.tiles.TileArray` describing the file, and the engine implements +the decoding half as a `load_tile` static method — called once per tile +touched, with one source-local slice per axis and the manifest's engine +specification as keyword arguments, returning exactly the selected sub-box: + +```{code-cell} +from xdas.tiles import TileArray + +class MyTileEngine(Engine, name="my_tile_engine"): + _supported_vtypes = ["hdf5", "tiles"] + _supported_ctypes = { + "distance": ["interpolated", "sampled", "dense"], + "time": ["interpolated", "sampled", "dense"], + } + + def open_dataarray(self, fname): + with h5py.File(fname, "r") as file: + t0 = np.datetime64(file["dataset"].attrs["t0"]).astype("datetime64[ms]") + dt = np.timedelta64(int(file["dataset"].attrs["dt"]*1e3), "ms") + x0 = file["dataset"].attrs["x0"][()] + dx = file["dataset"].attrs["dx"][()] + if self.vtype == "tiles": + data = TileArray( + str(fname), + file["dataset"].shape, + {"name": "my_tile_engine"}, + file["dataset"].dtype, + ) + else: + data = VirtualSource(file["dataset"]) + nt, nx = data.shape + t = Coordinate[self.ctype["time"]].from_block(t0, nt, dt, dim="time") + x = Coordinate[self.ctype["distance"]].from_block(x0, nx, dx, dim="distance") + return DataArray(data, {"time": t, "distance": x}) + + @staticmethod + def load_tile(path, selection): + with h5py.File(path, "r") as file: + return file["dataset"][selection] +``` + +`load_tile` must depend only on its arguments — never on engine instance +state — so that saved tile views decode identically everywhere. This is the +backing used by default for the formats that HDF5 virtual datasets cannot +serve (Silixa TDMS, MiniSEED), and optionally by every built-in HDF5 engine. From 4e6cf315d5a210e4575b40781d9efab08f0cc69e Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 23:00:22 +0200 Subject: [PATCH 07/56] Read febus tiles as one hyperslab instead of per-block reads The touched blocks' trimmed windows form one rectangular hyperslab (every block keeps the same post-overlap window), so a single h5py read replaces the per-block loop and concatenation; the partial first and last blocks crop away as plain numpy slices. --- xdas/io/febus.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/xdas/io/febus.py b/xdas/io/febus.py index 95afdf76..715cd16b 100644 --- a/xdas/io/febus.py +++ b/xdas/io/febus.py @@ -157,9 +157,10 @@ def load_tile(path, selection, *, dataset, block_size, overlaps): Febus files store a 3-D stack of overlapping ``(time, distance)`` blocks. Rows are counted post-trim: each block contributes - ``block_size - sum(overlaps)`` rows. Only the blocks overlapping the - requested rows are read; overlap rows are sliced away without being - copied. + ``block_size - sum(overlaps)`` rows. The touched blocks' trimmed + windows are read as a single hyperslab (overlap rows are never + read) and fused; the partial first and last blocks crop away in + memory. Parameters ---------- @@ -176,14 +177,13 @@ def load_tile(path, selection, *, dataset, block_size, overlaps): Rows trimmed at the start and end of each block. """ rows = selection[0] - start, stop = rows.start, rows.stop keep = block_size - overlaps[0] - overlaps[1] + first = rows.start // keep + last = (rows.stop - 1) // keep + # the touched blocks' trimmed windows form one rectangular + # hyperslab; the partial first/last blocks crop away afterwards + key = (slice(first, last + 1), slice(overlaps[0], overlaps[0] + keep)) with h5py.File(path, "r") as file: - blocks = file[dataset] - parts = [] - for block in range(start // keep, (stop - 1) // keep + 1): - lo = max(start - block * keep, 0) + overlaps[0] - hi = min(stop - block * keep, keep) + overlaps[0] - parts.append(blocks[block, lo:hi]) - rows = np.concatenate(parts) if len(parts) > 1 else parts[0] - return rows[(slice(None, None, selection[0].step), *selection[1:])] + data = file[dataset][key + selection[1:]] + data = data.reshape(-1, *data.shape[2:]) + return data[rows.start - first * keep : rows.stop - first * keep : rows.step] From 23b4b845e6086f31fff4ae23a364b6bdfb493691 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 23:09:06 +0200 Subject: [PATCH 08/56] Drop the tiles registry adapter, resolve engines on Engine[name] The get_engine indirection existed to keep tilearray.py line-identical with 0.3; with the mirror abandoned it lost its purpose. The friendly unknown-name error moves into Engine.__class_getitem__ (where every lookup benefits) and tilearray resolves Engine[name] directly. Also reword the 0.2.9 release notes to describe deltas from 0.2.8 only, not internal churn of the 0.2.9 development. --- docs/api/tiles.md | 5 ++-- docs/release-notes.md | 5 +--- tests/io/test_generic.py | 2 +- tests/tiles/test_tilearray.py | 4 ++-- xdas/io/core.py | 5 +++- xdas/tiles/__init__.py | 8 +++---- xdas/tiles/registry.py | 43 ----------------------------------- xdas/tiles/tilearray.py | 23 ++++++++++--------- 8 files changed, 25 insertions(+), 70 deletions(-) delete mode 100644 xdas/tiles/registry.py diff --git a/docs/api/tiles.md b/docs/api/tiles.md index 446453ba..69adc5fa 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -44,14 +44,13 @@ Methods ## Engine lookup Tiles are decoded by the ``load_tile`` half of the -{class}`xdas.io.Engine` format plugins; {func}`~xdas.tiles.get_engine` -resolves the engine names stored in tile manifests. +{class}`xdas.io.Engine` format plugins; the engine names stored in tile +manifests resolve on that registry (``Engine[name]``). ```{eval-rst} .. autosummary:: :toctree: ../_autosummary - get_engine xdas.io.Engine.load_tile extract_array ``` diff --git a/docs/release-notes.md b/docs/release-notes.md index 04d45f40..0b42fab0 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -5,10 +5,7 @@ ### New Features - **Tile-backed virtual arrays.** The new `xdas.tiles` package (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). - Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). -- **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines (the `hdf5` vtype stays the default). Tile views describe a Febus file as a single tile — the overlap trimming lives in the reader, not in one mapping entry per block — and saved tile views are directly readable by the 0.3 line (@atrabattoni). - -### Refactoring -- Tile decoding merged into the io engine plugin socket: `xdas.io.Engine` gained the `load_tile(path, selection, **params)` static half, and the separate `xdas.tiles.Engine` registry was removed — manifest engine names now resolve against the io registry (`xdas.tiles.get_engine`) (@atrabattoni). +- **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines (the `hdf5` vtype stays the default). Tile views describe a Febus file as a single tile — the overlap trimming lives in the reader, not in one mapping entry per block — and saved tile views are directly readable by the 0.3 line. Custom engines add tile support by implementing the `load_tile(path, selection, **params)` static half of `xdas.io.Engine` (@atrabattoni). ### Deprecations - Writing dask-backed virtual arrays (`__dask_array__` attribute) is deprecated and emits a `FutureWarning`; existing files still open. The tile-backed engines replace this mechanism (@atrabattoni). diff --git a/tests/io/test_generic.py b/tests/io/test_generic.py index 0861e3d7..839ae364 100644 --- a/tests/io/test_generic.py +++ b/tests/io/test_generic.py @@ -9,7 +9,7 @@ class TestEngineRegistry: def test_unknown_engine_raises_key_error(self): - with pytest.raises(KeyError, match="not found"): + with pytest.raises(KeyError, match="no engine registered"): Engine["nonexistent_engine_xyz"] def test_invalid_vtype_raises_value_error(self): diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py index 5180c3a5..ee0619ba 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/tiles/test_tilearray.py @@ -6,7 +6,7 @@ import pytest from xdas.io import Engine -from xdas.tiles import TileArray, get_engine +from xdas.tiles import TileArray NX = 5 @@ -157,7 +157,7 @@ def load_tile(path, selection): return np.zeros((1, 1)) try: - assert get_engine("dummy") is DummyEngine + assert Engine["dummy"] is DummyEngine finally: del Engine._registry["dummy"] diff --git a/xdas/io/core.py b/xdas/io/core.py index 9348eb35..65a05f45 100644 --- a/xdas/io/core.py +++ b/xdas/io/core.py @@ -91,7 +91,10 @@ def __class_getitem__(cls, item): elif item in cls._aliases: return cls._registry[cls._aliases[item]] else: - raise KeyError(f"Item '{item}' not found in registry or aliases") + raise KeyError( + f"no engine registered under {item!r}; " + f"available: {sorted([*cls._registry, *cls._aliases])}" + ) def open_dataarray(self, fname, **kwargs): """Open *fname* and return a :class:`DataArray` (abstract).""" diff --git a/xdas/tiles/__init__.py b/xdas/tiles/__init__.py index d43fd4de..d39fa039 100644 --- a/xdas/tiles/__init__.py +++ b/xdas/tiles/__init__.py @@ -4,16 +4,14 @@ :class:`TileArray` exposes a rectilinear grid of file-backed tiles as one numpy-like lazy array. Tiles are decoded by the ``load_tile`` half of the :class:`xdas.io.Engine` format plugins, resolved by manifest -engine name with :func:`get_engine`. This backend replaces the -serialized-dask-graph fallback used by the formats that HDF5 virtual -datasets cannot serve (Silixa TDMS, MiniSEED). +engine name on that registry (``Engine[name]``). This backend replaces +the serialized-dask-graph fallback used by the formats that HDF5 +virtual datasets cannot serve (Silixa TDMS, MiniSEED). """ -from .registry import get_engine from .tilearray import TileArray, extract_array __all__ = [ "TileArray", "extract_array", - "get_engine", ] diff --git a/xdas/tiles/registry.py b/xdas/tiles/registry.py deleted file mode 100644 index 877d4951..00000000 --- a/xdas/tiles/registry.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Engine lookup of the tiles machinery. - -The 0.2 line keeps a single per-format plugin socket: :class:`xdas.io.Engine`. -Tile decoding is its ``load_tile`` half; :func:`get_engine` resolves the -``name`` key of the engine specifications stored in tile manifests against -that registry. The 0.3 line hosts the same lookup over its own registry, -keeping :mod:`xdas.tiles.tilearray` identical in both lines. -""" - - -def get_engine(name): - """Return the engine class registered under *name*. - - Parameters - ---------- - name : str - The ``name`` key of a tile manifest's engine specification. - - Returns - ------- - type - The :class:`xdas.io.Engine` subclass registered under *name*; its - ``load_tile`` static method decodes tiles of that format. - - Raises - ------ - KeyError - If no engine is registered under *name*. - """ - from ..io.core import Engine - - try: - return Engine[name] - except KeyError: - raise KeyError( - f"no engine registered under {name!r}; " - f"available: {sorted(Engine._registry)}" - ) from None - - -__all__ = [ - "get_engine", -] diff --git a/xdas/tiles/tilearray.py b/xdas/tiles/tilearray.py index af39b116..dbcd52d0 100644 --- a/xdas/tiles/tilearray.py +++ b/xdas/tiles/tilearray.py @@ -30,10 +30,9 @@ self-described as its input); any other indexing reads the bounding box of the selection and resolves the rest in memory. ``np.asarray`` materializes: tiles are read one by one by the registered *engine* -(resolved with :func:`~xdas.tiles.registry.get_engine`), whose -``load_tile`` opens each tile's path itself and returns the tile's -*source selection* (one possibly strided slice per axis), every part -landing directly in the output array. +(``xdas.io.Engine[name]``), whose ``load_tile`` opens each tile's path +itself and returns the tile's *source selection* (one possibly strided +slice per axis), every part landing directly in the output array. A tile array is used *raw* as the data of a :class:`xdas.DataArray` (``DataArray(arr, coords)``), so ``da.data`` returns the inspectable @@ -62,8 +61,6 @@ import numpy as np import xarray as xr -from .registry import get_engine - TILE_PREFIX = "tile_" """Prefix of the tile-grid dimensions of a manifest dataset.""" @@ -259,9 +256,8 @@ class TileArray(np.lib.mixins.NDArrayOperatorsMixin): the number of tiles along the axis). engine : dict The engine specification: the key ``"name"`` selects a - registered engine (resolved with - :func:`~xdas.tiles.registry.get_engine`); the remaining keys - are passed to its ``load_tile`` as keyword parameters. + registered engine (``xdas.io.Engine[name]``); the remaining + keys are passed to its ``load_tile`` as keyword parameters. dtype : str or numpy.dtype Element type of the virtual array (little-endian or single-byte). @@ -396,7 +392,10 @@ def _setup(self, dataset, dtype, engine): engine = json.loads(json.dumps(engine)) if not isinstance(engine, dict) or "name" not in engine: raise ValueError("the engine specification must have a `name` key") - get_engine(engine["name"]) + # imported here: xdas.io imports this module at package init + from ..io.core import Engine + + Engine[engine["name"]] # fail fast on unregistered engines self._engine = engine self.dtype = np.dtype(dtype) if self.dtype.byteorder == ">": @@ -668,9 +667,11 @@ def _grid_values(self, name): @functools.cached_property def _engine_impl(self): """The ``(load_tile, spec)`` of the engine specification.""" + from ..io.core import Engine + spec = dict(self.engine) name = spec.pop("name") - return get_engine(name).load_tile, spec + return Engine[name].load_tile, spec def __array__(self, dtype=None, copy=None): """Read every tile and return the values as a numpy array. From 45493ed1ed64c2d9ebf8296c4bcc4aa0eca38bc4 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 23:12:46 +0200 Subject: [PATCH 09/56] Anchor tile paths to absolute at TileArray construction Reads are lazy and stored views outlive the session, so a relative source path would resolve against whatever the working directory is at read time. Construction is the only moment the relative path is still trustworthy (the scan just used it), so the constructor absolutizes every entry of `paths`; manifests reloaded from disk are trusted as stored. --- tests/tiles/test_tilearray.py | 11 +++++++++++ xdas/tiles/tilearray.py | 8 +++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py index ee0619ba..d51ce21e 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/tiles/test_tilearray.py @@ -1,4 +1,5 @@ import math +import os import h5py import numpy as np @@ -181,6 +182,16 @@ def test_repr(self, stack): assert manifest._repr_inline_(40) == "TileArray (3 tiles)" assert manifest._repr_inline_(10) == "TileArray" + def test_relative_paths_are_anchored(self, tmp_path, monkeypatch): + """Relative paths absolutize at construction and survive a chdir.""" + data = np.arange(4.0 * NX).reshape(4, NX) + _tile_file(tmp_path / "rel.h5", data) + monkeypatch.chdir(tmp_path) + manifest = TileArray("rel.h5", (4, NX), ENGINE, "f8") + assert os.path.isabs(manifest._grid_values("paths").item(0)) + monkeypatch.chdir(tmp_path.parent) + npt.assert_array_equal(np.asarray(manifest), data) + def test_attrs(self, stack): manifest, _ = stack assert manifest.attrs == {"units": "strain"} diff --git a/xdas/tiles/tilearray.py b/xdas/tiles/tilearray.py index dbcd52d0..8e992ec3 100644 --- a/xdas/tiles/tilearray.py +++ b/xdas/tiles/tilearray.py @@ -57,6 +57,7 @@ import itertools import json import math +import os import numpy as np import xarray as xr @@ -248,7 +249,9 @@ class TileArray(np.lib.mixins.NDArrayOperatorsMixin): Source file of each tile. A scalar describes a one-tile-per-axis grid; an array is padded with trailing length-1 axes up to the rank. A path may appear in several - tiles. + tiles. Relative paths are made absolute at construction — the + working directory cannot be trusted later, as reads are lazy + and stored views outlive the session. sizes : sequence of int or 1-D array-like One entry per axis (this defines the rank): the samples each tile contributes along that axis. An int is uniform across the @@ -291,6 +294,9 @@ def __init__( if paths.ndim > ndim: raise ValueError("`paths` has more axes than `sizes` entries") paths = paths.reshape(paths.shape + (1,) * (ndim - paths.ndim)) + # reads are lazy and stored views outlive the session: anchor the + # paths now, while the scan's working directory still applies + paths = np.frompyfunc(os.path.abspath, 1, 1)(paths) data = {} counts = [] for k, entry in enumerate(sizes): From b3000b3a9126c3fb8e72f0870e25486ce2e526c4 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 23:37:15 +0200 Subject: [PATCH 10/56] Remove the regular-coordinates plan from docs --- docs/plan_regular_coordinates.md | 202 ------------------------------- 1 file changed, 202 deletions(-) delete mode 100644 docs/plan_regular_coordinates.md diff --git a/docs/plan_regular_coordinates.md b/docs/plan_regular_coordinates.md deleted file mode 100644 index cd746293..00000000 --- a/docs/plan_regular_coordinates.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -orphan: true ---- - -# Design: regular coordinates — settling the open questions - -Status: implemented on this branch (2026-07-29). -Branch: `feature/fixed-interp-coords`, targeting a PR to `dev` (0.2.8, untagged). - -This document settles the four design questions left open by the regular- -coordinate work, so the remaining implementation (engine emission, docs pass, -failing tests) has a fixed contract to build against. It supersedes the -never-written `docs/plan_propagate_simplify_kwargs.md` referenced by -`concat`'s docstring. - -## Background: the model as implemented - -An `InterpCoordinate` may carry two optional metadata entries: - -- `sampling_interval` — the nominal sample spacing. Its presence is what makes - the coordinate *regular* (`isregular()`). -- `tolerance` — the allowed jitter around that spacing. The validity invariant, - checked at construction (`_is_valid_sampling_interval`), is per continuous - segment: `|num - si * den| <= 2 * tolerance`, evaluated at the dtype - resolution (integer division for datetime64, so sub-resolution drift is - always absorbed). - -`from_block` produces regular coordinates; `_to_regular` enforces or infers a -spacing (raising when it cannot); `simplify(tolerance, reduce, regularize)` -spends an accuracy budget on tie-point reduction and optional promotion to -regular; `_concat` is strict (keeps the spacing only when both sides agree -exactly, takes `max` of tolerances, otherwise drops to irregular). - -## D1. Public API surface: `to_regular` public, `infer_regular` private - -**Decision.** Promote `_to_regular` to public `to_regular`, defined on -`AxisCoordinate` (not just `InterpCoordinate`), honouring the rule that a -public coordinate method exists on the whole axis hierarchy or not at all: - -- `InterpCoordinate.to_regular(sampling_interval=None, tolerance=None)` — - current `_to_regular` behaviour: enforce the given spacing, inferring it when - omitted, raising `ValueError` when the tie points cannot be described by a - single spacing within `tolerance`. -- `SampledCoordinate.to_regular(...)` — regular by construction: with no - arguments return a copy; with explicit arguments validate them against the - stored interval and raise on mismatch. -- `DenseCoordinate.to_regular(...)` — *conversion*: return a regular - `InterpCoordinate` built from the dense values (reduce within `tolerance`, - then enforce the spacing), raising when the values are genuinely irregular. - Returning a different subclass is acceptable: the `to_` prefix already - signals a conversion, and this is the natural "make this axis usable by - signal processing" entry point. - -`_infer_regular` stays private. It is an implementation detail of -`to_regular`/`simplify` (the Chebyshev-center fit); exposing it publicly on -only one subclass would recreate the partial-interface problem, and its -diagnostic value is available through `to_regular`'s behaviour and error -message. `docs/api/coordinates.md` must drop the `infer_regular` entry and the -release notes keep advertising `to_regular` (now truthfully). - -Consequence: `get_sampling_interval` (module level, `core.py:1244`) loses its -`hasattr(coord, "_to_regular")` duck-typing — see D3. - -## D2. What "regular" means per subclass (the Dense question) - -**Decision.** *Regular* means "carries an explicit nominal sampling interval", -uniformly: - -- `InterpCoordinate`: regular iff `sampling_interval` metadata is present. -- `SampledCoordinate`: always regular (the interval is part of its data). -- `DenseCoordinate`: **never regular**. `get_sampling_interval` returns `None` - unconditionally, dropping the current end-to-end average. The average makes - `isregular()` vacuously true for any dense axis and silently hands a - meaningless rate to signal routines on jittery data — the exact failure mode - this branch exists to eliminate. A dense axis that really is evenly sampled - becomes regular explicitly, via `to_regular` (D1) or - `simplify(regularize=True)`. -- `ScalarCoordinate`: `isregular()` moves to the `Coordinate` base and returns - `False` there; `AxisCoordinate` overrides it with the current - `get_sampling_interval() is not None`. This makes the release-notes claim - ("on the base ABC") true and removes the `AttributeError` on scalar coords. - -## D3. The `get_sampling_interval` contract: strict, one choke point - -Three layers, each with a single behaviour: - -1. **Primitive** — `coord.get_sampling_interval(cast=True)`: return the - nominal interval, or `None` when the coordinate is not regular. Never - raises, never infers, O(1). -2. **Conversion** — `coord.to_regular(...)`: the only place inference and - enforcement happen. Raises with an actionable message on genuinely - irregular axes. -3. **Convenience** — `xdas.get_sampling_interval(da, dim)`: return the nominal - interval when the coordinate is regular, otherwise **raise** `ValueError` - telling the user how to fix it (open the files with a `tolerance`, or - `da[dim] = da[dim].to_regular(tolerance=...)`). The current silent - `_to_regular()` fallback is removed: it hides an O(n log n) inference in - every FFT/filter call and only ever succeeds on exactly-uniform axes anyway - (the implicit epsilon tolerance rejects any real jitter), so its benefit is - marginal and its implicitness is not. - - *Amendment (2026-07-30):* data saved by earlier versions carries no - `sampling_interval` metadata, so raising immediately would break every - signal-processing call on existing archives. For one deprecation cycle the - helper therefore falls back to inference on irregular coordinates: it infers - the spacing (and, for `InterpCoordinate`, the minimal tolerance that - validates it via the Chebyshev fit), emits a `FutureWarning` stating both - values and the migration path, and returns the inferred spacing. Dense - coordinates go through the strict `to_regular()` (uniform axes work, jittery - ones still raise — the old end-to-end average was a silent wrong answer not - worth preserving). Raising remains only where no spacing can be inferred at - all. The strict behaviour described above becomes the default when the - deprecation completes. - -**Migration.** All signal-consuming code goes through layer 3 — including -`xdas/signal.py`, which currently open-codes the strict check six times -(`d = coords[dim].get_sampling_interval(); if d is None: raise ...`). Revert -those to the module-level helper so the error message and the policy live in -one place, and keep `fft.py`, `spectral.py`, `atoms/`, `picking.py`, -`miniseed.py` on the helper. Net user-visible behaviour: every signal routine -raises the *same* error on irregular axes, and none of them raise on data -opened through the engines once D5 lands. - -Also fix `DataArrayList`-style compatibility checking -(`routines.py:919-922`): `get_sampling_interval` returning `None` for the -incoming chunk must produce a `CompatibilityError`, not a `TypeError` inside -`np.isclose`. - -## D4. Tolerance semantics and propagation - -**Meaning.** `tolerance` is a *declared jitter bound carried by the -coordinate*: the promise that every continuous segment satisfies -`|num - si * den| <= 2 * tolerance` at the dtype resolution. It is data, not a -processing parameter — processing functions take a *budget* argument that may -default to it. - -**Propagation rules** (R1–R2 already implemented, kept as-is): - -- **R1 — slicing/striding** (`_slice`): spacing scales by the step, tolerance - is preserved. -- **R2 — raw concatenation** (`_concat`): strict; equal spacings are kept with - `max` of tolerances, anything else drops to irregular. Reconciliation is the - job of user-facing routines via `simplify`. -- **R3 — derived rates must carry their quantization error.** Any operation - that synthesizes a new nominal spacing that is not exactly representable in - the coordinate dtype must record the representation error in `tolerance` - instead of claiming `0`. Concretely for `Upsample(factor)` on datetime axes: - `new_delta = delta // factor` truncates, so the coordinate must carry - `tolerance >= (delta - factor * new_delta)` (2 ns in the failing test) on - top of the inherited tolerance. This is what makes chunk seams land within - tolerance of the nominal grid. -- **R4 — `simplify(tolerance=None)` defaults to the coordinate's own stored - tolerance** (falling back to the current zero-like default when the - coordinate has none). Rationale: the coordinate has already declared "my - values are only meaningful to within `tolerance`"; a canonicalisation pass - that refuses to spend that declared slack is pointless strictness. This - applies to `concat(tolerance=None)` too, per-coordinate. `tolerance=False` - keeps its "no simplification" meaning; an explicit scalar overrides. -- **R5 — no unconditional widening.** `InterpCoordinate.simplify` on a regular - coordinate currently stores `self.tolerance + tolerance` whenever `reduce` - runs. Replace with: after reduction, keep the original tolerance if it still - validates, and only widen (to the smallest valid value, bounded by - `self.tolerance + budget`) when it does not. Without this, chunked and - unchunked pipelines can never produce `equals()` coordinates because the - chunked path concatenates and re-simplifies. - -**Why this fixes `test_upsample`.** Each upsampled chunk carries -`sampling_interval = 6_666_666 ns, tolerance = 2 ns` (R3). `_concat` keeps the -spacing (R2). `concat`'s simplify defaults its budget to the stored 2 ns (R4), -Douglas-Peucker drops the seam tie points (they deviate ≤ 2 ns from the global -line), and R5 keeps `tolerance = 2 ns` — identical to the unchunked result. - -**Defaults alignment.** `concat` and `concat_coords` currently disagree -(`regularize=False, tolerance=None` vs `regularize=True, tolerance=False`). -Align `concat_coords` to `concat`: `reduce=True, regularize=False, -tolerance=None` (with R4's meaning). `regularize` stays opt-in for this PR — -with engines emitting regular coordinates (D5) and R2 preserving them, -multi-file opens stay regular without promotion, so the conservative default -costs nothing; flipping it can be revisited once propagation has soaked. - -## D5. IO emission (scope confirmed, design only sketched here) - -Engines construct per-file time/space coordinates with -`InterpCoordinate.from_block(start, size, step)` (the existing `# TODO: use -from_block` sites in `prodml`, `terra15`, `asn`, plus `miniseed.read_stream` -and ObsPy `from_stream`, which must also build at ns resolution to round-trip -`to_stream`). Per-file tolerance is `0`: within one file the grid is exact by -construction. Cross-file jitter is reconciled where it appears — at -`concat`/`open_mfdataarray` time via the user-supplied `tolerance` (R4/R2). -`from_stream` uses `stats.delta`; engines use the file's metadata rate. - -## Acceptance criteria - -- `tests/test_atoms.py::TestFilters::test_upsample` and - `tests/test_dataarray.py::TestIO::test_stream` pass without weakening the - assertions. -- `xd.signal.*`, `xd.fft.*`, `xd.spectral.*`, and the atoms raise one uniform, - actionable error on irregular axes, and raise nothing on engine-opened data. -- Release notes, `docs/api/coordinates.md`, and the user guide describe only - APIs that exist (`to_regular` public, `infer_regular` gone from docs). -- `concat`'s docstring no longer references this document's missing - predecessor. From 76e6644e3f2b1dd294a512289d0204094df12f9a Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 23:52:56 +0200 Subject: [PATCH 11/56] Read the package version from a single source The version was duplicated in pyproject.toml and xdas/__init__.py, and docs/conf.py carried a third copy that had already drifted to 0.2.7. Declare it dynamic and let setuptools read xdas.__version__, which is now the only place to edit; conf.py derives its release from it too. --- docs/conf.py | 4 +++- pyproject.toml | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3a4188cc..8f9dffa1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,6 +4,8 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +from xdas import __version__ + # -- Project information ----------------------------------------------------- project = "xdas" @@ -11,7 +13,7 @@ author = "Alister Trabattoni" # The full version, including alpha/beta/rc tags -release = "0.2.7" +release = __version__ # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 380ee32a..d908894d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "xdas" -version = "0.2.8" +dynamic = ["version"] requires-python = ">= 3.10" authors = [ { name = "Alister Trabattoni", email = "alister.trabattoni@gmail.com" }, @@ -44,6 +44,10 @@ docs = [ ] tests = ["dascore", "psutil", "seisbench", "torch"] +# Single source of truth for the version: xdas/__init__.py +[tool.setuptools.dynamic] +version = { attr = "xdas.__version__" } + [tool.ruff.lint] extend-select = ["I", "D"] extend-ignore = [ From 385cc2d81676c09f9940830a3cfa7b64a79e2cc7 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 30 Jul 2026 23:54:12 +0200 Subject: [PATCH 12/56] Open 0.2.9 development Set the version to 0.2.9.dev0, the canonical PEP 440 spelling: setuptools normalises devN to .devN, so the shorter 0.2.9dev0 would leave __version__ and the distribution metadata spelled differently. test_version only accepted plain digits between dots, so extend it to the pre/post/dev markers. --- tests/test_xdas.py | 9 ++++++--- xdas/__init__.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_xdas.py b/tests/test_xdas.py index 3af18b0b..775815f8 100644 --- a/tests/test_xdas.py +++ b/tests/test_xdas.py @@ -1,9 +1,12 @@ +import re + import xdas as xd +# Release segment, plus the optional PEP 440 pre/post/dev markers (e.g. 0.2.9.dev0). +VERSION_PATTERN = re.compile(r"^\d+(\.\d+)*((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?$") + def test_version(): version = xd.__version__ assert isinstance(version, str) - version_parts = version.split(".") - for part in version_parts: - assert part.isdigit() + assert VERSION_PATTERN.match(version) diff --git a/xdas/__init__.py b/xdas/__init__.py index 9f5251a2..05536145 100644 --- a/xdas/__init__.py +++ b/xdas/__init__.py @@ -6,7 +6,7 @@ for common DAS instrument formats. """ -__version__ = "0.2.8" +__version__ = "0.2.9.dev0" __all__ = [ # noqa: RUF022 - grouped by kind, not alphabetically # submodules From bc100bb51f28bc87a9fb6aef5b9e9512c5cdae50 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Fri, 31 Jul 2026 00:01:03 +0200 Subject: [PATCH 13/56] Note the single version source in the release notes --- docs/release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes.md b/docs/release-notes.md index 0b42fab0..e26b4cf9 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -31,6 +31,7 @@ - Added `xdas.testing.dummy`, a configurable fixture generator replacing `xdas.synthetics.dummy` (@atrabattoni). - Comply with ruff 0.16, whose default rule set is considerably broader (`B`, `C4`, `SIM`, `RUF`, `PERF`, `TRY`, `BLE`, `S`, `DTZ`, `FLY`, `PL`…). Mutable argument defaults (the `dim={...}` mappings of `fft`, `rfft`, `ifft`, `irfft`, `stft`, `to_stream`) became `None` sentinels documenting the same defaults; class-level registries and engine specs are annotated `ClassVar`; deliberate patterns (engine-fallback blind excepts, the long-lived TDMS handle, the grouped `__all__`) carry targeted `noqa`. `TRY004` is disabled project-wide, since xdas raises `ValueError` for all argument validation, including type checks (@atrabattoni). - The abstract `VirtualArray` stubs (`__getitem__`, `__array__`, `shape`, `dtype`, `to_dataset`) now raise `NotImplementedError` instead of silently returning `None` (@atrabattoni). +- The package version is declared in a single place, `xdas/__init__.py`: `pyproject.toml` marks it dynamic and setuptools reads it from there, and `docs/conf.py` derives its `release` from it. The three copies had already drifted — the documentation still advertised 0.2.7 (@atrabattoni). ## 0.2.7 From 4dc69cc77957ffcad4b536195f417e9838e3771a Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Fri, 31 Jul 2026 15:47:25 +0200 Subject: [PATCH 14/56] Raise the multi-file open limit for the tiles vtype The 100 000 file ceiling in `open_mfdataarray` was introduced together with the HDF5 virtual layout linking loop: building that mapping costs one libhdf5 call per source file, so both time and memory grow with the file count. The tiles vtype builds no such mapping, its manifest is a plain array write, so the ceiling does not apply to it. Resolve the engine's effective vtype up front and pick the limit from it. The message now names the real constraint, the per-file data arrays the scan holds until they are combined, and points at the way out. Document the two backends side by side in the virtual datasets guide so the trade-off is written down rather than folklore. --- docs/user-guide/io/virtual-datasets.md | 98 +++++++++++++++++++++++++- tests/test_core.py | 31 ++++++++ xdas/core/routines.py | 37 +++++++++- 3 files changed, 160 insertions(+), 6 deletions(-) diff --git a/docs/user-guide/io/virtual-datasets.md b/docs/user-guide/io/virtual-datasets.md index 6830a05c..74ff9ef8 100644 --- a/docs/user-guide/io/virtual-datasets.md +++ b/docs/user-guide/io/virtual-datasets.md @@ -16,10 +16,13 @@ os.chdir("../../_data") To deal with large multi-file dataset, *Xdas* uses the concept of virtual datasets. A virtual dataset is a file that contains pointers towards an arbitrary number of files that can then be accessed seamlessly as a single, contiguous dataset. -*Xdas* uses two types of virtualization: +*Xdas* uses several types of virtualization, selected with the `vtype` argument: -- For HDF5 based format, it leverages the performance offered by the [virtual datasets](https://docs.h5py.org/en/stable/vds.html) native capabilities of netCDF4/HDF5 which comes with almost no overhead (C compiled). -- For other type of files, it leverage the flexibility of [Dask arrays](https://docs.Dask.org/en/stable/array.html). +- `hdf5`: for HDF5 based formats, it leverages the performance offered by the [virtual datasets](https://docs.h5py.org/en/stable/vds.html) native capabilities of netCDF4/HDF5 which comes with almost no overhead (C compiled). +- `tiles`: a manifest of file-backed tiles stored as a plain array, decoded by the engine itself. It works with any format and keeps the file mapping inspectable. +- For other type of files, it can also leverage the flexibility of [Dask arrays](https://docs.Dask.org/en/stable/array.html). + +Which types an engine offers is declared by its `_supported_vtypes` attribute; the first one listed is the default. See [](#choosing-a-virtualization-backend) for how to pick. ## HDF5 Virtualization @@ -83,6 +86,95 @@ A virtual dataset can point to another virtual dataset. This can be beneficial f When loading large part of a virtual dataset, you might end up with nan values. This normally happens when linked files are missing. But due to a [known limitation](https://forum.hdfgroup.org/t/virtual-datasets-and-open-file-limit/6757) of the HDF5 C library it can be due to the opening of too many files. Try increasing the number of possible file to open with the `ulimit` command. Or load smaller chunk of data. ``` +## Tile Virtualization + +With the `tiles` vtype, the mapping is not delegated to HDF5. *Xdas* stores it as a +{py:class}`xdas.tiles.TileArray`: a plain array manifest that records, for each tile, which +file it comes from and which part of that file it contributes. Reading a region resolves +which tiles it touches and asks the engine to decode each of them through its `load_tile` +method. The manifest is ordinary data, so it can be inspected, sliced and concatenated +like any other array, and it is stored as such inside the *Xdas* netCDF format. + +(choosing-a-virtualization-backend)= +## Choosing a virtualization backend + +For formats that HDF5 virtual datasets cannot serve, the choice is made for you. When an +engine supports both, the trade-off is essentially *who resolves the mapping*: the HDF5 C +library, or *Xdas* itself. + +### HDF5 virtualization + +**Advantages** + +- Resolution happens inside the HDF5 C library, so reading involves no per-file Python + call. This is most visible on reads that touch many files at once. +- Any HDF5-aware tool can read the result, not only *Xdas*. +- Virtual datasets can point at other virtual datasets, so a growing archive can be + linked in batches without relinking everything. +- A subset saved from a virtual dataset is very compact, because it refers to the dataset + it was cut from rather than restating the underlying file list. + +**Limitations** + +- Building the mapping costs one HDF5 call per source file, so both the time and the + memory needed to write a manifest grow in proportion to the number of files. Beyond + some point, writing a single flat manifest stops being practical. +- Reopening a virtual dataset reads its whole mapping table, so opening cost also grows + with the number of linked files. Deep archives therefore tend to require a pyramid of + virtual datasets, which shifts that cost to read time and multiplies the number of + manifest files to keep track of. +- Once written, the mapping is opaque: HDF5 presents a virtual dataset as a regular + dataset, so the list of linked files can no longer be inspected or edited. +- Because a saved subset refers to its parent, extracts are not self-contained. Moving or + deleting the parent breaks them, and each extract adds one more level of indirection. +- Strided (decimating) selection along the concatenation axis is not supported. +- Missing files are read as NaN rather than raising, and exceeding the C library's + open-file limit produces the same symptom, which makes such problems easy to miss. + +### Tile virtualization + +**Advantages** + +- Writing a manifest is an array write rather than a per-file operation, so it stays fast + and light as the number of files grows. A single flat manifest remains workable at + scales where an HDF5 one does not. +- Opening loads only the tile geometry, so open time stays low even for very large + manifests. +- The file list is data: it can be inspected, modified and saved again. +- Concatenation fuses manifests without reading any values. +- A saved subset names the data files it needs directly, so extracts are self-contained + and never gain an extra level of indirection. +- Decimating along the stacking axis is supported: the stride is folded into the tile + geometry and stays lazy. +- The per-file footprint of the manifest is smaller. + +**Limitations** + +- Decoding goes through Python, one call per tile touched. Reads spread over a great many + tiles therefore carry an overhead that the C library avoids. +- For very small manifests the result can be larger than the HDF5 equivalent, since paths + and geometry are written out explicitly instead of referring to a parent dataset. +- The manifest is only meaningful to *Xdas*. +- The engine must implement `load_tile`. + +### Considerations that apply to both + +- Building either manifest starts with reading the metadata of every file. That scan is + dominated by disk access and is usually the bulk of the total build time, so it is not a + criterion for choosing between the two. +- Both keep one data array per file in memory while scanning, so very large file sets must + be opened in batches and combined afterwards, whichever backend is used. +- Neither backend helps when a coordinate is not monotonic — for instance when files + overlap in time. Label-based selection then falls back to a slow path in both cases, and + is better addressed in the data itself. + +```{hint} +As a rule of thumb, prefer `hdf5` when the dataset is modest in file count or when other +HDF5-based tooling has to read it, and `tiles` when the file count is large, when the +mapping needs to remain inspectable, when extracts must stand on their own, or when +decimated reads matter. +``` + ## Dask Virtualization Other type of formats will be loaded as Dask arrays. Those latter are a N-dimensional stack of chunks. At each chunk is associated a task to complete to get the values of that chunk. It results in a computation graph that Xdas is capable to serialize and store within its native NetCDF format. To be able to serialize the graph, it must only contain xdas or Dask functions. diff --git a/tests/test_core.py b/tests/test_core.py index b2ea9ba3..000f845e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -25,6 +25,37 @@ def test_open_mfdataarray(self, tmp_path): with pytest.raises(FileNotFoundError): xd.open_mfdataarray(["not_existing_file.nc"]) + def test_open_mfdataarray_file_limit(self, tmp_path, monkeypatch): + from xdas.core import routines + + for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): + da.to_netcdf(tmp_path / f"{idx:03}.nc") + monkeypatch.setattr(routines, "MAX_OPEN_FILES_DEFAULT", 2) + with pytest.raises(NotImplementedError, match="the limit is 2"): + xd.open_mfdataarray(tmp_path / "00*.nc") + + def test_open_mfdataarray_file_limit_is_higher_for_tiles( + self, tmp_path, monkeypatch + ): + from xdas.core import routines + + for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): + da.to_netcdf(tmp_path / f"{idx:03}.nc") + monkeypatch.setattr(routines, "MAX_OPEN_FILES_DEFAULT", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", {"tiles": 3}) + da = xd.open_mfdataarray(tmp_path / "00*.nc", engine="xdas", vtype="tiles") + assert da.shape == wavelet_wavefronts().shape + + def test_effective_vtype(self): + from xdas.core.routines import _effective_vtype + + assert _effective_vtype("asn", None) == "hdf5" # engine default + assert _effective_vtype("asn", "tiles") == "tiles" + assert _effective_vtype("miniseed", None) == "tiles" # only vtype + assert _effective_vtype(lambda fname: None, "tiles") is None # read function + assert _effective_vtype("not_an_engine", None) is None + assert _effective_vtype("asn", "not_a_vtype") is None + def test_open_mfdataarray_grouping(self, tmp_path): acqs = [ { diff --git a/xdas/core/routines.py b/xdas/core/routines.py index cda37707..364de02c 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -473,6 +473,27 @@ def defaulttree(depth): return defaultdict(lambda: defaulttree(depth - 1)) +def _effective_vtype(engine, vtype): + """Return the vtype *engine* will use, or None if it has no say.""" + if callable(engine): + return None + from ..io.core import Engine + + try: + return Engine[engine](vtype=vtype).vtype + except (KeyError, NotImplementedError, ValueError): + # a bad engine name or vtype: let opening the first file report it + return None + + +# How many files one call may scan, per virtualization type. Every vtype +# holds one data array per file in memory until they are combined; the hdf5 +# one additionally builds an HDF5 virtual mapping per file, which dominates +# both the memory and the time. Tiles pays neither, so it gets far more room. +MAX_OPEN_FILES = {"tiles": 2_000_000} +MAX_OPEN_FILES_DEFAULT = 100_000 + + def open_mfdataarray( paths, dim="first", @@ -526,6 +547,12 @@ def open_mfdataarray( ------ FileNotFound If no file can be found. + NotImplementedError + If more files are given than the vtype allows in one call. Scanning + keeps one data array per file in memory until they are combined, so the + ceiling is `MAX_OPEN_FILES_DEFAULT` and `MAX_OPEN_FILES["tiles"]` for + the much lighter tiles manifests. Larger sets must be opened in batches + and combined with `combine_by_coords`. """ paths = _ensure_str_paths(paths) if isinstance(paths, str): @@ -540,10 +567,14 @@ def open_mfdataarray( ) if len(paths) == 0: raise FileNotFoundError("no file to open") - if len(paths) > 100_000: + vtype = _effective_vtype(engine, kwargs.get("vtype")) + limit = MAX_OPEN_FILES.get(vtype, MAX_OPEN_FILES_DEFAULT) + if len(paths) > limit: raise NotImplementedError( - "The maximum number of file that can be opened at once is for now limited " - "to 100 000." + f"cannot open {len(paths)} files at once: the limit is {limit} for " + f"vtype {vtype!r} because the scan holds one data array per file in " + "memory until they are combined. Open the files in batches and pass " + "the results to `combine_by_coords`." ) max_workers = get_workers_count(parallel) objs = [] From ecc723a1b605e848d68c15647de338045d17cf47 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Fri, 31 Jul 2026 17:23:01 +0200 Subject: [PATCH 15/56] Reopen data collections of tile-backed arrays A tile-backed data array stores its manifest in a `__tiles__` sibling group of its variables. `_get_depth` counted that group, so every such data array looked one level deeper than it is and the collection reader took it for a nested collection, then failed on the datasets it found where it expected a group. Any collection holding more than one tile-backed array was therefore impossible to reopen. Skip the manifest group when measuring depth. --- tests/io/test_tiles_vtype.py | 35 +++++++++++++++++++++++++++++++++++ xdas/io/xdas.py | 15 +++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index f55c3452..fdae9322 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -176,3 +176,38 @@ def test_xdas_engine_tiles_vtype(tmp_path): assert isinstance(result.data, TileArray) assert result.data.engine == {"name": "xdas", "dataset": "/__values__"} assert result.equals(da) + + +def test_tiles_datacollection_roundtrip(tmp_path): + """Collections of tile-backed arrays reopen as collections, not as errors. + + The manifest lives in a sibling group of the data array's variables, which + used to make the array look like a nested collection to the reader. + """ + das = [ + xd.testing.dummy(shape=(10, 5), step=(1.0, 10.0), dtype=np.float32) + for _ in range(2) + ] + paths = [] + for k, da in enumerate(das): + path = str(tmp_path / f"da{k}.nc") + da.to_netcdf(path) + paths.append(path) + tiled = [xd.open_dataarray(path, engine="xdas", vtype="tiles") for path in paths] + + sequence = xd.DataCollection(tiled, name="acquisition") + fname = str(tmp_path / "sequence.nc") + sequence.to_netcdf(fname, virtual=True) + result = xd.open_datacollection(fname) + assert len(result) == 2 + for expected, actual in zip(das, result): + assert isinstance(actual.data, TileArray) + npt.assert_array_equal(actual.values, expected.values) + + mapping = xd.DataCollection({"a": sequence, "b": sequence}, name="node") + fname = str(tmp_path / "mapping.nc") + mapping.to_netcdf(fname, virtual=True) + result = xd.open_datacollection(fname) + assert sorted(result) == ["a", "b"] + assert isinstance(result["a"][1].data, TileArray) + npt.assert_array_equal(result["b"][0].values, das[0].values) diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 94c40bab..5629f034 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -354,8 +354,19 @@ def save_datasequence( def _get_depth(group): + """Nesting depth of *group*, ignoring any tile manifest it contains. + + A tile-backed data array keeps its manifest in a `TILES_GROUP` + sibling of its variables. That group must not count, or the data + array would look one level deeper than it is and be mistaken for a + nested collection. + """ if not isinstance(group, h5py.Group): raise ValueError("not a group") - depths = [] - group.visit(lambda name: depths.append(name.count("/"))) + depths = [0] + group.visit( + lambda name: ( + None if TILES_GROUP in name.split("/") else depths.append(name.count("/")) + ) + ) return max(depths) From 9360838650240e35c04ff1997074cabcd5cdce1f Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sat, 1 Aug 2026 13:10:08 +0200 Subject: [PATCH 16/56] Correct how the two virtualization backends compare on reads Benchmarking the two backends over a multi-million file archive contradicted what the guide claimed. Resolving a region in the HDF5 C library does avoid all per-file Python, but its cost grows with the number of mappings the dataset holds rather than with the size of the request, so the same read gets slower as the archive grows. A tile manifest is searched, so its cost follows how many tiles the read touches and not how many the manifest contains. HDF5 is therefore the quicker reader only while the file count stays modest. Also note that the top of a virtual dataset pyramid opens fast because it defers the work, and charges the first read of each region for it. Fix the tiles API page while here: its `load_tile` entry resolved against `xdas.tiles` and so looked for `xdas.tiles.xdas.io`, which failed the whole documentation build. --- docs/api/tiles.md | 10 +++++++- docs/user-guide/io/virtual-datasets.md | 34 ++++++++++++++++++++------ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index 69adc5fa..483a6520 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -51,6 +51,14 @@ manifests resolve on that registry (``Engine[name]``). .. autosummary:: :toctree: ../_autosummary - xdas.io.Engine.load_tile extract_array ``` + +```{eval-rst} +.. currentmodule:: xdas.io + +.. autosummary:: + :toctree: ../_autosummary + + Engine.load_tile +``` diff --git a/docs/user-guide/io/virtual-datasets.md b/docs/user-guide/io/virtual-datasets.md index 74ff9ef8..76775119 100644 --- a/docs/user-guide/io/virtual-datasets.md +++ b/docs/user-guide/io/virtual-datasets.md @@ -18,7 +18,7 @@ To deal with large multi-file dataset, *Xdas* uses the concept of virtual datase *Xdas* uses several types of virtualization, selected with the `vtype` argument: -- `hdf5`: for HDF5 based formats, it leverages the performance offered by the [virtual datasets](https://docs.h5py.org/en/stable/vds.html) native capabilities of netCDF4/HDF5 which comes with almost no overhead (C compiled). +- `hdf5`: for HDF5 based formats, it leverages the [virtual datasets](https://docs.h5py.org/en/stable/vds.html) native capabilities of netCDF4/HDF5, which resolve the mapping in compiled C with no per-file Python overhead. The cost of that mapping does however grow with the number of linked files. - `tiles`: a manifest of file-backed tiles stored as a plain array, decoded by the engine itself. It works with any format and keeps the file mapping inspectable. - For other type of files, it can also leverage the flexibility of [Dask arrays](https://docs.Dask.org/en/stable/array.html). @@ -102,12 +102,21 @@ For formats that HDF5 virtual datasets cannot serve, the choice is made for you. engine supports both, the trade-off is essentially *who resolves the mapping*: the HDF5 C library, or *Xdas* itself. +That choice decides how each cost scales with the size of the archive. Writing and +reopening an HDF5 virtual dataset both cost one operation per linked file, so both grow +with the file count; a tile manifest is written and read back as an array, so neither +does. Reading inverts the expectation one might have: resolving a region inside the C +library involves no Python at all, but its cost grows with how many mappings the dataset +*contains*, while a tile manifest is searched, so its cost grows only with how many tiles +the read *touches*. Modest file counts therefore favour HDF5, and the advantage moves to +tiles as the archive grows. + ### HDF5 virtualization **Advantages** - Resolution happens inside the HDF5 C library, so reading involves no per-file Python - call. This is most visible on reads that touch many files at once. + call. On modest file counts this makes it the faster of the two to read. - Any HDF5-aware tool can read the result, not only *Xdas*. - Virtual datasets can point at other virtual datasets, so a growing archive can be linked in batches without relinking everything. @@ -122,7 +131,13 @@ library, or *Xdas* itself. - Reopening a virtual dataset reads its whole mapping table, so opening cost also grows with the number of linked files. Deep archives therefore tend to require a pyramid of virtual datasets, which shifts that cost to read time and multiplies the number of - manifest files to keep track of. + manifest files to keep track of. The top of such a pyramid opens quickly precisely + because it defers the work: the first read of a region then has to open the level below + it, a toll that a short-lived process pays on every run. +- Read latency grows with the number of mappings the dataset holds, not only with the + amount of data asked for, so the same request gets slower as the archive it lives in + gets bigger. Past a large enough file count this outweighs the advantage of resolving + in C, and reads become slower than the tile equivalent. - Once written, the mapping is opaque: HDF5 presents a virtual dataset as a regular dataset, so the list of linked files can no longer be inspected or edited. - Because a saved subset refers to its parent, extracts are not self-contained. Moving or @@ -139,7 +154,9 @@ library, or *Xdas* itself. and light as the number of files grows. A single flat manifest remains workable at scales where an HDF5 one does not. - Opening loads only the tile geometry, so open time stays low even for very large - manifests. + manifests, and no cost is deferred to the first read. +- Read latency depends on how much of the manifest a request touches, not on how large + the manifest is, so reads do not get slower as the archive grows. - The file list is data: it can be inspected, modified and saved again. - Concatenation fuses manifests without reading any values. - A saved subset names the data files it needs directly, so extracts are self-contained @@ -150,8 +167,9 @@ library, or *Xdas* itself. **Limitations** -- Decoding goes through Python, one call per tile touched. Reads spread over a great many - tiles therefore carry an overhead that the C library avoids. +- Decoding goes through Python, one call per tile touched. A request spread over a great + many tiles therefore carries a per-tile overhead that the C library avoids, which is + what makes HDF5 the quicker reader while file counts stay modest. - For very small manifests the result can be larger than the HDF5 equivalent, since paths and geometry are written out explicitly instead of referring to a parent dataset. - The manifest is only meaningful to *Xdas*. @@ -172,7 +190,9 @@ library, or *Xdas* itself. As a rule of thumb, prefer `hdf5` when the dataset is modest in file count or when other HDF5-based tooling has to read it, and `tiles` when the file count is large, when the mapping needs to remain inspectable, when extracts must stand on their own, or when -decimated reads matter. +decimated reads matter. The larger the archive, the stronger the case for `tiles`: it is +the only one of the two whose write, open and read costs do not all grow with the number +of files. ``` ## Dask Virtualization From f08de4f0268b6cb186b4873c0113b5eb0fc900de Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sat, 1 Aug 2026 13:27:20 +0200 Subject: [PATCH 17/56] Default Febus to tiles and refresh the I/O documentation A Febus file stores a stack of overlapping blocks rather than one contiguous array. The hdf5 backing has to describe every block with its own mapping, so a Febus manifest grew with the block count on top of the file count; a tile array describes the whole file as one tile and keeps the overlap trimming in the reader. Listing tiles first makes it the default for that engine. The documentation had not kept up. No engine emits dask graphs any more, yet the format table still credited Silixa and MiniSEED to dask and the guide presented it as a live backend; it is now marked deprecated, with the tables stating what each engine supports and which backing it picks by default. A test pins those defaults so the tables cannot drift. --- docs/release-notes.md | 9 +++-- docs/user-guide/io/data-formats.md | 47 ++++++++++++++++---------- docs/user-guide/io/virtual-datasets.md | 26 ++++++++++---- tests/io/test_tiles_vtype.py | 22 ++++++++++++ xdas/io/febus.py | 5 ++- 5 files changed, 81 insertions(+), 28 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 91e982cc..8ab080b4 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -5,10 +5,15 @@ ### New Features - **Tile-backed virtual arrays.** The new `xdas.tiles` package (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). - Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). -- **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines (the `hdf5` vtype stays the default). Tile views describe a Febus file as a single tile — the overlap trimming lives in the reader, not in one mapping entry per block — and saved tile views are directly readable by the 0.3 line. Custom engines add tile support by implementing the `load_tile(path, selection, **params)` static half of `xdas.io.Engine` (@atrabattoni). +- **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines. Saved tile views are directly readable by the 0.3 line. Custom engines add tile support by implementing the `load_tile(path, selection, **params)` static half of `xdas.io.Engine` (@atrabattoni). +- **Febus now defaults to `tiles`.** A tile view describes a Febus file as a single tile — the overlap trimming lives in the reader — whereas the HDF5 backing needs one mapping per block, so its manifest grew with the block count as well as the file count. Every other HDF5 engine still defaults to `hdf5` (@atrabattoni). +- `open_mfdataarray` no longer refuses more than 100 000 paths regardless of backing: the ceiling is now taken from the engine's resolved vtype and is far higher for `tiles`, which does not build one HDF5 mapping per file. The error explains the remaining limit — the scan holds one data array per file in memory until they are combined (@atrabattoni). ### Deprecations -- Writing dask-backed virtual arrays (`__dask_array__` attribute) is deprecated and emits a `FutureWarning`; existing files still open. The tile-backed engines replace this mechanism (@atrabattoni). +- Writing dask-backed virtual arrays (`__dask_array__` attribute) is deprecated and emits a `FutureWarning`; existing files still open. No engine emits them any more: the tile-backed engines replace this mechanism (@atrabattoni). + +### Bug Fixes +- Fix data collections holding more than one tile-backed data array being impossible to reopen. The tile manifest lives in a `__tiles__` sibling group, which the reader counted when deciding whether a group held a data array or a nested collection, so every tile-backed array looked one level too deep (@atrabattoni). ## 0.2.8 diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index 86fbf57e..3c05aec9 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -20,27 +20,34 @@ os.chdir("../../_data") ## Implemented file formats -Here below the list of formats that are currently implemented. All HDF5 based formats support native virtualization. Other formats support Dask virtualization. Please refer to the [](virtual-datasets) section. Xdas should automatically detect the correct file format. You can still specify which one you want in the `engine` argument in {py:func}`xdas.open`. +Here below the list of formats that are currently implemented. Every format supports tile virtualization; HDF5 based formats also support native HDF5 virtualization, and the tables give the backing each engine uses when you do not ask for one. Pass `vtype` to {py:func}`xdas.open` to choose the other, and see [](virtual-datasets) for how to pick. Xdas should automatically detect the correct file format. You can still specify which one you want in the `engine` argument. Xdas support the following DAS formats: -| Constructor | Instrument | `engine` argument | Virtualization | -|:-----------------:|:-----------------:|:-----------------:|:-----------------:| -| AP Sensing | DAS N5* | `"apsensing"` | HDF5 | -| ASN | OptoDAS | `"asn"` | HDF5 | -| FEBUS | A1 | `"febus"` | HDF5 | -| OptaSense | OLA, ODH*, ... | `"optasense"` | HDF5 | -| Silixa | iDAS | `"silixa"` | Dask | -| SINTELA | ONYX | `"sintela"` | HDF5 | -| Terra15 | Treble | `"terra15"` | HDF5 | +| Constructor | Instrument | `engine` argument | Virtualization | Default | +|:-----------------:|:-----------------:|:-----------------:|:-----------------:|:---------:| +| AP Sensing | DAS N5* | `"apsensing"` | HDF5, tiles | `hdf5` | +| ASN | OptoDAS | `"asn"` | HDF5, tiles | `hdf5` | +| FEBUS | A1 | `"febus"` | HDF5, tiles | `tiles` | +| OptaSense | OLA, ODH*, ... | `"optasense"` | HDF5, tiles | `hdf5` | +| Silixa | iDAS | `"silixa"` | tiles | `tiles` | +| SINTELA | ONYX | `"sintela"` | HDF5, tiles | `hdf5` | +| Terra15 | Treble | `"terra15"` | HDF5, tiles | `hdf5` | It also implements its own format and support ProdML and miniSEED: -| Format | `engine` argument | Virtualization | -|:-----------------:|:-----------------:|:-----------------:| -| Xdas | `None` | HDF5 | -| ProdML | `"prodml"` | HDF5 | -| miniSEED | `"miniseed"` | Dask | +| Format | `engine` argument | Virtualization | Default | +|:-----------------:|:-----------------:|:-----------------:|:---------:| +| Xdas | `None` | HDF5, tiles | `hdf5` | +| ProdML | `"prodml"` | HDF5, tiles | `hdf5` | +| miniSEED | `"miniseed"` | tiles | `tiles` | + +```{note} +A Febus file stores a stack of overlapping blocks rather than one contiguous array. The +HDF5 backing needs a separate mapping for every block, so the manifest of a Febus dataset +grows with the number of blocks as well as the number of files; a tile array needs one +tile per file whatever the block count. That is why `febus` defaults to `tiles`. +``` ```{warning} Due to poor documentation of the various version of the Febus format, it is recommended to manually provide the required trimming and the position of the timestamps within each block. For example to trim 100 samples on both side of each block and to set the timestamp location at the center of the block for a block of 2000 samples: @@ -164,6 +171,10 @@ class MyTileEngine(Engine, name="my_tile_engine"): ``` `load_tile` must depend only on its arguments — never on engine instance -state — so that saved tile views decode identically everywhere. This is the -backing used by default for the formats that HDF5 virtual datasets cannot -serve (Silixa TDMS, MiniSEED), and optionally by every built-in HDF5 engine. +state — so that saved tile views decode identically everywhere. + +This is the backing used by default for the formats HDF5 virtual datasets cannot +serve (Silixa TDMS, MiniSEED) and for Febus, whose files hold many blocks each; it +is available on request from every other built-in engine. The order of +`_supported_vtypes` decides the default, so listing `"tiles"` first is all it +takes for a new engine to prefer it. diff --git a/docs/user-guide/io/virtual-datasets.md b/docs/user-guide/io/virtual-datasets.md index 76775119..862ea6cd 100644 --- a/docs/user-guide/io/virtual-datasets.md +++ b/docs/user-guide/io/virtual-datasets.md @@ -20,9 +20,10 @@ To deal with large multi-file dataset, *Xdas* uses the concept of virtual datase - `hdf5`: for HDF5 based formats, it leverages the [virtual datasets](https://docs.h5py.org/en/stable/vds.html) native capabilities of netCDF4/HDF5, which resolve the mapping in compiled C with no per-file Python overhead. The cost of that mapping does however grow with the number of linked files. - `tiles`: a manifest of file-backed tiles stored as a plain array, decoded by the engine itself. It works with any format and keeps the file mapping inspectable. -- For other type of files, it can also leverage the flexibility of [Dask arrays](https://docs.Dask.org/en/stable/array.html). -Which types an engine offers is declared by its `_supported_vtypes` attribute; the first one listed is the default. See [](#choosing-a-virtualization-backend) for how to pick. +Which types an engine offers is declared by its `_supported_vtypes` attribute; the first one listed is the default. See [](#choosing-a-virtualization-backend) for how to pick, and [](data-formats.md) for what each engine supports and defaults to. + +A third backing, [Dask arrays](https://docs.Dask.org/en/stable/array.html), is deprecated and no longer used by any engine — see [](#dask-virtualization). ## HDF5 Virtualization @@ -86,6 +87,7 @@ A virtual dataset can point to another virtual dataset. This can be beneficial f When loading large part of a virtual dataset, you might end up with nan values. This normally happens when linked files are missing. But due to a [known limitation](https://forum.hdfgroup.org/t/virtual-datasets-and-open-file-limit/6757) of the HDF5 C library it can be due to the opening of too many files. Try increasing the number of possible file to open with the `ulimit` command. Or load smaller chunk of data. ``` +(tile-virtualization)= ## Tile Virtualization With the `tiles` vtype, the mapping is not delegated to HDF5. *Xdas* stores it as a @@ -195,10 +197,20 @@ the only one of the two whose write, open and read costs do not all grow with th of files. ``` -## Dask Virtualization - -Other type of formats will be loaded as Dask arrays. Those latter are a N-dimensional stack of chunks. At each chunk is associated a task to complete to get the values of that chunk. It results in a computation graph that Xdas is capable to serialize and store within its native NetCDF format. To be able to serialize the graph, it must only contain xdas or Dask functions. +(dask-virtualization)= +## Dask Virtualization (deprecated) -From an user point of view the use of this type of virtualization is very similar to HDF5 one. +```{deprecated} 0.2.9 +Dask virtualization is no longer used by any engine and writing it is deprecated. The +formats that once relied on it — those HDF5 virtual datasets cannot serve — now use +[tile virtualization](#tile-virtualization) instead. Existing files that store a Dask +graph can still be read, so nothing on disk is lost, but new datasets should not be +written this way. +``` -The main difference is that when opening a dataset with Dask virtualization, the entire graph of pointers to the files is loaded, can be modified and saved again. In the HDF5 case, opening a virtual dataset is handled the same way as if it is a regular file meaning that the underlying mapping of pointers is hidden and cannot be modified. Dask graph can be slow when they start to become very big (more than one million tasks). +Formats that HDF5 could not virtualize used to be loaded as Dask arrays: an +N-dimensional stack of chunks, each with a task attached that produces its values, +serialized into the native *Xdas* netCDF format as a computation graph. Tiles replace it +with a manifest that describes the same mapping as plain array data, which is both more +compact and far quicker to build, and which does not go sluggish once the graph reaches +millions of tasks. diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index fdae9322..f13b3025 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -211,3 +211,25 @@ def test_tiles_datacollection_roundtrip(tmp_path): assert sorted(result) == ["a", "b"] assert isinstance(result["a"][1].data, TileArray) npt.assert_array_equal(result["b"][0].values, das[0].values) + + +def test_default_vtypes(): + """The backing each engine picks when the caller does not say. + + Pinned because it is what the I/O guide documents: formats that store + several blocks per file, or that HDF5 virtual datasets cannot serve at + all, default to tiles. + """ + from xdas.io import Engine + + expected = { + "apsensing": "hdf5", + "asn": "hdf5", + "febus": "tiles", + "miniseed": "tiles", + "prodml": "hdf5", + "silixa": "tiles", + "terra15": "hdf5", + "xdas": "hdf5", + } + assert {name: Engine[name]().vtype for name in expected} == expected diff --git a/xdas/io/febus.py b/xdas/io/febus.py index 715cd16b..6fc1fc02 100644 --- a/xdas/io/febus.py +++ b/xdas/io/febus.py @@ -16,7 +16,10 @@ class FebusEngine(Engine, name="febus"): """Engine for reading Febus HDF5 files.""" - _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] + # tiles first: a Febus file holds a stack of blocks, and the hdf5 + # backing needs one mapping per block while a tile array needs one + # per file, so the manifest stops growing with the block count + _supported_vtypes: ClassVar[list] = ["tiles", "hdf5"] _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated", "sampled", "dense"], From 2969aeb17c57f5bc5f9443b7daf25d08e7ea268b Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sat, 1 Aug 2026 15:06:10 +0200 Subject: [PATCH 18/56] Fold the open-file limits into one vtype-keyed constant The two module constants and the _effective_vtype helper collapse into a single MAX_OPEN_FILES dict (with a None fallback entry) and a _check_file_count helper that resolves the vtype and enforces the limit. --- tests/test_core.py | 25 ++++++++++++--------- xdas/core/routines.py | 52 +++++++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 000f845e..eeb5817a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -30,7 +30,7 @@ def test_open_mfdataarray_file_limit(self, tmp_path, monkeypatch): for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): da.to_netcdf(tmp_path / f"{idx:03}.nc") - monkeypatch.setattr(routines, "MAX_OPEN_FILES_DEFAULT", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", {None: 2, "hdf5": 2}) with pytest.raises(NotImplementedError, match="the limit is 2"): xd.open_mfdataarray(tmp_path / "00*.nc") @@ -41,20 +41,23 @@ def test_open_mfdataarray_file_limit_is_higher_for_tiles( for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): da.to_netcdf(tmp_path / f"{idx:03}.nc") - monkeypatch.setattr(routines, "MAX_OPEN_FILES_DEFAULT", 2) - monkeypatch.setattr(routines, "MAX_OPEN_FILES", {"tiles": 3}) + monkeypatch.setattr( + routines, "MAX_OPEN_FILES", {None: 2, "hdf5": 2, "tiles": 3} + ) da = xd.open_mfdataarray(tmp_path / "00*.nc", engine="xdas", vtype="tiles") assert da.shape == wavelet_wavefronts().shape - def test_effective_vtype(self): - from xdas.core.routines import _effective_vtype + def test_open_mfdataarray_file_limit_unknown_engine(self, tmp_path, monkeypatch): + from xdas.core import routines - assert _effective_vtype("asn", None) == "hdf5" # engine default - assert _effective_vtype("asn", "tiles") == "tiles" - assert _effective_vtype("miniseed", None) == "tiles" # only vtype - assert _effective_vtype(lambda fname: None, "tiles") is None # read function - assert _effective_vtype("not_an_engine", None) is None - assert _effective_vtype("asn", "not_a_vtype") is None + for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): + da.to_netcdf(tmp_path / f"{idx:03}.nc") + monkeypatch.setattr( + routines, "MAX_OPEN_FILES", {None: 2, "hdf5": 2, "tiles": 3} + ) + # a read function has no vtype to ask for: fall back to the strict limit + with pytest.raises(NotImplementedError, match="the limit is 2"): + xd.open_mfdataarray(tmp_path / "00*.nc", engine=xd.open_dataarray) def test_open_mfdataarray_grouping(self, tmp_path): acqs = [ diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 364de02c..c5e3e05a 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -473,25 +473,31 @@ def defaulttree(depth): return defaultdict(lambda: defaulttree(depth - 1)) -def _effective_vtype(engine, vtype): - """Return the vtype *engine* will use, or None if it has no say.""" - if callable(engine): - return None - from ..io.core import Engine - - try: - return Engine[engine](vtype=vtype).vtype - except (KeyError, NotImplementedError, ValueError): - # a bad engine name or vtype: let opening the first file report it - return None - - # How many files one call may scan, per virtualization type. Every vtype # holds one data array per file in memory until they are combined; the hdf5 # one additionally builds an HDF5 virtual mapping per file, which dominates # both the memory and the time. Tiles pays neither, so it gets far more room. -MAX_OPEN_FILES = {"tiles": 2_000_000} -MAX_OPEN_FILES_DEFAULT = 100_000 +# The None entry is the fallback for engines whose vtype cannot be known here. +MAX_OPEN_FILES = {None: 100_000, "hdf5": 100_000, "tiles": 2_000_000} + + +def _check_file_count(nfiles, engine, vtype): + """Refuse file sets too large for the vtype *engine* will end up using.""" + from ..io.core import Engine + + try: + vtype = Engine[engine](vtype=vtype).vtype + except (KeyError, NotImplementedError, ValueError): + # a bad engine or vtype: opening the first file reports it + vtype = None + limit = MAX_OPEN_FILES.get(vtype, MAX_OPEN_FILES[None]) + if nfiles > limit: + raise NotImplementedError( + f"cannot open {nfiles} files at once: the limit is {limit} for " + f"vtype {vtype!r} because the scan holds one data array per file in " + "memory until they are combined. Open the files in batches and pass " + "the results to `combine_by_coords`." + ) def open_mfdataarray( @@ -550,9 +556,9 @@ def open_mfdataarray( NotImplementedError If more files are given than the vtype allows in one call. Scanning keeps one data array per file in memory until they are combined, so the - ceiling is `MAX_OPEN_FILES_DEFAULT` and `MAX_OPEN_FILES["tiles"]` for - the much lighter tiles manifests. Larger sets must be opened in batches - and combined with `combine_by_coords`. + ceiling is given by `MAX_OPEN_FILES`, which leaves far more room to the + much lighter tiles manifests. Larger sets must be opened in batches and + combined with `combine_by_coords`. """ paths = _ensure_str_paths(paths) if isinstance(paths, str): @@ -567,15 +573,7 @@ def open_mfdataarray( ) if len(paths) == 0: raise FileNotFoundError("no file to open") - vtype = _effective_vtype(engine, kwargs.get("vtype")) - limit = MAX_OPEN_FILES.get(vtype, MAX_OPEN_FILES_DEFAULT) - if len(paths) > limit: - raise NotImplementedError( - f"cannot open {len(paths)} files at once: the limit is {limit} for " - f"vtype {vtype!r} because the scan holds one data array per file in " - "memory until they are combined. Open the files in batches and pass " - "the results to `combine_by_coords`." - ) + _check_file_count(len(paths), engine, kwargs.get("vtype")) max_workers = get_workers_count(parallel) objs = [] failures = [] From 57414a96d947f493b0b88edbed3e6b26deafb431 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sat, 1 Aug 2026 20:28:44 +0200 Subject: [PATCH 19/56] Make engine configuration explicit across the open functions The open functions (open, open_dataarray, open_mfdataarray, open_mfdatatree) now declare engine, vtype and ctype explicitly, and engine accepts a configured Engine instance as well as a name. Remaining keyword arguments (**engine_kwargs) are forwarded to the engine constructor only, so format-specific parameters keep working next to the engine name but misspelled ones raise instead of being silently swallowed. Per-engine parameters (febus overlaps/offset, miniseed ignore_last_sample, prodml swapped_dims, terra15 tz, native group) move to the engine constructors, validated before any file is scanned; the callable-engine escape hatch is removed in favor of subclassing Engine. open_mfdataarray resolves the engine once up front: the file-count limit reads the resolved vtype directly and the per-file opens receive the configured instance. Along the way this fixes miniseed silently ignoring its ctype argument and RealTimeLoader defaulting to an engine name that only a doctest side effect ever registered. --- docs/release-notes.md | 5 + docs/user-guide/io/data-formats.md | 67 +++++----- tests/io/test_febus.py | 14 +- tests/io/test_miniseed.py | 7 +- tests/test_core.py | 7 +- tests/test_routines.py | 32 ++++- tests/tiles/test_integration.py | 2 +- xdas/core/routines.py | 206 ++++++++++++++++++++--------- xdas/io/core.py | 27 ++-- xdas/io/febus.py | 94 +++++++------ xdas/io/miniseed.py | 32 ++++- xdas/io/prodml.py | 22 ++- xdas/io/terra15.py | 21 ++- xdas/io/xdas.py | 26 +++- xdas/processing/core.py | 7 +- 15 files changed, 391 insertions(+), 178 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 8ab080b4..fa6fa79e 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -3,6 +3,11 @@ ## 0.2.9 (unreleased) ### New Features +- **Explicit engine configuration.** The open functions (`open`, `open_dataarray`, `open_mfdataarray`, `open_mfdatatree`) now declare `engine`, `vtype` and `ctype` explicitly, and `engine` accepts a configured `xdas.io.Engine` instance as well as a name. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front; passing them next to the engine name keeps working and now raises a `TypeError` on misspelled or unsupported keywords instead of silently ignoring them (@atrabattoni). + +### Breaking Changes +- Passing a bare read function as `engine` is no longer supported: subclass `xdas.io.Engine` instead (see the data-formats documentation). Combining a configured engine instance with `vtype`, `ctype` or extra engine keywords raises a `ValueError` (@atrabattoni). +- The miniseed `ctype` argument is now honored: it previously routed to an unused attribute and the reader always built interpolated time coordinates. The default is unchanged (@atrabattoni). - **Tile-backed virtual arrays.** The new `xdas.tiles` package (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). - Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). - **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines. Saved tile views are directly readable by the 0.3 line. Custom engines add tile support by implementing the `load_tile(path, selection, **params)` static half of `xdas.io.Engine` (@atrabattoni). diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index 3c05aec9..49dbdf8c 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -54,48 +54,53 @@ Due to poor documentation of the various version of the Febus format, it is reco `xdas.open("path.h5", engine="febus", overlaps=(100, 100), offset=1000)` ``` -## Extending *xdas* with your file format +### Engine parameters + +Every open function ({py:func}`xdas.open`, {py:func}`xdas.open_dataarray`, +{py:func}`xdas.open_mfdataarray`, {py:func}`xdas.open_mfdatatree`) takes the same +engine-related arguments. `engine` selects the file format; `vtype` and `ctype` +select the virtualization backend and the coordinate types, and exist for every +engine. Some formats take additional parameters (the trimming of Febus blocks +shown above, the timezone of Terra15 timestamps, ...). Those are engine +constructor parameters: when `engine` is given by name, any extra keyword +argument is forwarded to the engine constructor; alternatively you can configure +an engine instance yourself and pass it as `engine`. The three calls below are +equivalent: + +```python +import xdas as xd +from xdas.io.febus import FebusEngine + +da = xd.open("path.h5", engine="febus", vtype="tiles", overlaps=(100, 100), offset=1000) +da = xd.open( + "path.h5", engine=FebusEngine(vtype="tiles", overlaps=(100, 100), offset=1000) +) +engine = FebusEngine(vtype="tiles", overlaps=(100, 100), offset=1000) # reusable +da = xd.open("path.h5", engine=engine) +``` -*xdas* insists on its extensibility, the power is in the hands of the users. Extending *xdas* usually consists of writing few-line-of-code-long functions. The process consists in dealing with the two main aspects of a {py:class}`xarray.DataArray`: unpacking the data and coordinates objects, eventually processing them and packing them back into a Database object. +Misspelled or unsupported parameters raise a `TypeError` from the engine +constructor. A configured instance is a complete specification: combining it +with `vtype`, `ctype` or extra keyword arguments raises an error. Format +auto-detection (`engine=None`) accepts `vtype` and `ctype` but no +format-specific parameters, since those require knowing the format. + +## Extending *xdas* with your file format -### Function-based solution +*xdas* insists on its extensibility, the power is in the hands of the users. Extending *xdas* usually consists of writing a few-line-of-code-long engine class. The process consists in dealing with the two main aspects of a {py:class}`xarray.DataArray`: unpacking the data and coordinates objects, eventually processing them and packing them back into a Database object. -To add a new file format the user can specify a function that read one file and outputs a {py:class}`xarray.DataArray`. This function can then be passed as an engine keyword argument to the {py:func}`xdas.open` function. The reading function must fetch and parse the data and coordinates information. +### Writing an engine -Adding the support for a new file format generally consists in providing the path to the data array and parsing the start time and spatial and temporal spacing as in the example below. +To add a new file format, create your own engine by inheriting from the `xdas.io.Engine` abstract class. Note that when the class is defined, the `name` keyword argument allows to register the new engine along with the `aliases` one that is useful when several instruments share the same data format. This allows to add your engine to the `Engine._registry` and to retrieve it by doing `Engine[name]`. The `_supported_vtypes` and `_supported_ctypes` class attributes allow to determine which kind of virtualization backend and type of coordinates can be used with this file format. When you open any file, you can additionally provide the `vtype` and `ctype` keyword arguments to specify which backends to use. The `Engine` class defines the `__init__` method that checks those passed kwargs and stores in `self.vtype` and `self.ctype` the chosen backends. If your format needs parameters of its own, define an `__init__` taking them after `vtype` and `ctype` and calling `super().__init__(vtype, ctype)`: they then become available next to the engine name in the open functions, like the built-in ones described above. ```{code-cell} import h5py import numpy as np import xdas as xd from xdas import DataArray -from xdas.virtual import VirtualSource - -def open_dataarray(fname): - with h5py.File(fname, "r") as file: - t0 = np.datetime64(file["dataset"].attrs["t0"]).astype("datetime64[ms]") - dt = np.timedelta64(int(file["dataset"].attrs["dt"]*1e3), "ms") - dx = file["dataset"].attrs["dx"][()] - data = VirtualSource(file["dataset"]) - nt, nx = data.shape - t = {"tie_indices": [0, nt - 1], "tie_values": [t0, t0 + (nt - 1) * dt]} - x = {"tie_indices": [0, nx - 1], "tie_values": [0.0, (nx - 1) * dx]} - return DataArray(data, {"time": t, "distance": x}) - -# Replace "other_format.hdf5" by the path of your file -da = xd.open("other_format.hdf5", engine=open_dataarray) -da -``` - -This example is for one file. For multi-file datasets please indicate the path of your files with a '*' before the file format if all your files are in the same folder or pass a list of paths. - -### Class-based solution - -To add support in a more complete way, you can also create your own engine by inheriting from the `xdas.io.Engine` abstract class. Note that when the class is defined, the `name` keyword argument allows to register the new engine along with the `aliases` one that is useful when several instruments share the same data format. This allows to add your engine to the `Engine._registry` and to retrieve it by doing `Engine[name]`. The `_supported_vtypes` and `_supported_ctypes` class attributes allow to determine which kind of virtualization backend and type of coordinates can be used with this file format. When you open any file, you can additionally provide the `vtype` and `ctype` keyword arguments to specify which backends to use. The `Engine` class defines the `__init__` method that checks those passed kwargs and stores in `self.vtype` and `self.ctype` the chosen backends. - -```{code-cell} -from xdas.io import Engine from xdas.coordinates import Coordinate +from xdas.io import Engine +from xdas.virtual import VirtualSource class MyEngine(Engine, name="my_engine", aliases=["other_engine"]): _supported_vtypes = ["hdf5"] diff --git a/tests/io/test_febus.py b/tests/io/test_febus.py index 5defdd01..85502e68 100644 --- a/tests/io/test_febus.py +++ b/tests/io/test_febus.py @@ -45,17 +45,13 @@ def test_open_with_freqres_attr(self, tmp_path): da = xd.open(str(path), engine="febus", overlaps=(1, 1), offset=0) assert isinstance(da, xd.DataArray) - def test_invalid_overlaps_raises(self, tmp_path): - path = tmp_path / "febus.h5" - make_febus_file(path) + def test_invalid_overlaps_raises(self): with pytest.raises(ValueError, match="overlaps must be"): - FebusEngine().open_dataarray(str(path), overlaps="bad") + FebusEngine(overlaps="bad") - def test_invalid_offset_raises(self, tmp_path): - path = tmp_path / "febus.h5" - make_febus_file(path) + def test_invalid_offset_raises(self): with pytest.raises(ValueError, match="offset must be an integer"): - FebusEngine().open_dataarray(str(path), overlaps=(1, 1), offset="bad") + FebusEngine(overlaps=(1, 1), offset="bad") def test_missing_block_rate_raises(self, tmp_path): path = tmp_path / "febus_no_blockrate.h5" @@ -70,4 +66,4 @@ def test_missing_block_rate_raises(self, tmp_path): zone.attrs["Origin"] = np.array([0.0, 0.0]) zone.create_dataset("Data", data=np.zeros((nchunks, nt, nx))) with pytest.raises(KeyError, match="Could not find the block size"): - FebusEngine().open_dataarray(str(path), overlaps=(0, 0), offset=0) + FebusEngine(overlaps=(0, 0), offset=0).open_dataarray(str(path)) diff --git a/tests/io/test_miniseed.py b/tests/io/test_miniseed.py index 81311294..9974dfd3 100644 --- a/tests/io/test_miniseed.py +++ b/tests/io/test_miniseed.py @@ -4,6 +4,7 @@ import pytest import xdas as xd +from xdas.coordinates import Coordinate from xdas.io.miniseed import MiniSEEDEngine, get_band_code, to_stream from xdas.tiles import TileArray @@ -81,6 +82,10 @@ def test_miniseed(tmp_path): assert da.coords["location"].values == "00" assert da.coords["channel"].values.tolist() == ["HHZ", "HHN", "HHE"] + # the ctype engine parameter drives the time coordinate flavor + da = xd.open(paths[0], engine="miniseed", ctype="dense") + assert isinstance(da.coords["time"], Coordinate["dense"]) + # read one file with gaps make_network(tmp_path, gap=True, samples=100) paths = sorted(tmp_path.glob("*_gap.mseed")) @@ -242,4 +247,4 @@ def test_miniseed_unsynchronized_traces(tmp_path): ) st.write(str(path), format="MSEED") with pytest.raises(ValueError, match="synchronized"): - MiniSEEDEngine().read_header(str(path), False, "interpolated") + MiniSEEDEngine().read_header(str(path)) diff --git a/tests/test_core.py b/tests/test_core.py index eeb5817a..a92c5f4f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -47,17 +47,18 @@ def test_open_mfdataarray_file_limit_is_higher_for_tiles( da = xd.open_mfdataarray(tmp_path / "00*.nc", engine="xdas", vtype="tiles") assert da.shape == wavelet_wavefronts().shape - def test_open_mfdataarray_file_limit_unknown_engine(self, tmp_path, monkeypatch): + def test_open_mfdataarray_file_limit_engine_instance(self, tmp_path, monkeypatch): from xdas.core import routines + from xdas.io.xdas import XdasEngine for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): da.to_netcdf(tmp_path / f"{idx:03}.nc") monkeypatch.setattr( routines, "MAX_OPEN_FILES", {None: 2, "hdf5": 2, "tiles": 3} ) - # a read function has no vtype to ask for: fall back to the strict limit + # the configured instance carries the vtype the limit is keyed on with pytest.raises(NotImplementedError, match="the limit is 2"): - xd.open_mfdataarray(tmp_path / "00*.nc", engine=xd.open_dataarray) + xd.open_mfdataarray(tmp_path / "00*.nc", engine=XdasEngine(vtype="hdf5")) def test_open_mfdataarray_grouping(self, tmp_path): acqs = [ diff --git a/tests/test_routines.py b/tests/test_routines.py index 8fafc7b8..2733b1cf 100644 --- a/tests/test_routines.py +++ b/tests/test_routines.py @@ -468,22 +468,42 @@ def test_invalid_paths_type_raises(self): with pytest.raises(Exception, match="paths"): xd.open(123) - def test_callable_engine(self, tmp_path): + def test_engine_instance(self, tmp_path): + from xdas.io import Engine + da = xd.testing.dummy(shape=(10, 5)) path = str(tmp_path / "test.nc") da.to_netcdf(path) + result = xd.open_dataarray(path, engine=Engine["xdas"]()) + assert result.equals(da) - def my_engine(fname, **kwargs): - return xd.open_dataarray(fname) + def test_engine_instance_rejects_extra_config(self, tmp_path): + from xdas.io import Engine - result = xd.open_dataarray(path, engine=my_engine) - assert result.equals(da) + da = xd.testing.dummy(shape=(10, 5)) + path = str(tmp_path / "test.nc") + da.to_netcdf(path) + engine = Engine["xdas"]() + with pytest.raises(ValueError, match="configured engine instance"): + xd.open_dataarray(path, engine=engine, vtype="tiles") + with pytest.raises(ValueError, match="configured engine instance"): + xd.open_dataarray(path, engine=engine, group="somegroup") + + def test_unknown_engine_kwarg_raises(self, tmp_path): + da = xd.testing.dummy(shape=(10, 5)) + path = str(tmp_path / "test.nc") + da.to_netcdf(path) + with pytest.raises(TypeError, match="overlpas"): + xd.open_dataarray(path, engine="febus", overlpas=(1, 1)) + # auto-detection accepts no format-specific parameters + with pytest.raises(TypeError, match="overlaps"): + xd.open_dataarray(path, overlaps=(1, 1)) def test_invalid_engine_type_raises(self, tmp_path): da = xd.testing.dummy(shape=(10, 5)) path = str(tmp_path / "test.nc") da.to_netcdf(path) - with pytest.raises(ValueError, match="engine"): + with pytest.raises(TypeError, match="engine must be"): xd.open_dataarray(path, engine=42) diff --git a/tests/tiles/test_integration.py b/tests/tiles/test_integration.py index 3bc2d75f..41b16852 100644 --- a/tests/tiles/test_integration.py +++ b/tests/tiles/test_integration.py @@ -119,7 +119,7 @@ def test_grouped_round_trip(self, stack, tmp_path): da = wrap(manifest) path = str(tmp_path / "grouped.nc") da.to_netcdf(path, group="acquisition") - reopened = xd.open_dataarray(path, group="acquisition") + reopened = xd.open_dataarray(path, engine="xdas", group="acquisition") assert isinstance(reopened.data, TileArray) npt.assert_array_equal(reopened.values, reference) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index c5e3e05a..7dfcdce3 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -34,9 +34,11 @@ def open( tolerance=None, squeeze=None, engine=None, + vtype=None, + ctype=None, parallel=None, verbose=False, - **kwargs, + **engine_kwargs, ): """ Open one or several files as a data array or collection. @@ -75,10 +77,17 @@ def open( contains only one data array. When ``None`` (default), the behaviour depends on the dispatch path: ``True`` for multi-file data arrays, ``False`` otherwise. Ignored when opening a single file. - engine : str or callable, optional - The file format engine to use, or a custom read callable. When ``None`` - (default), the xdas NetCDF format is assumed. Providing an engine skips the - automatic DataCollection detection. + engine : str or Engine, optional + The file format engine to use, given by name or as a configured + :class:`~xdas.io.Engine` instance. When ``None`` (default), the format is + auto-detected. Providing an engine skips the automatic DataCollection + detection. + vtype : str, optional + The virtualization type to use. If None, the engine default is used. + Only valid when `engine` is given by name (or None). + ctype : str or dict, optional + The coordinate type(s) to use. If None, the engine defaults are used. + Only valid when `engine` is given by name (or None). parallel: bool or int, optional Whether to use multiprocessing to fetch file metadata. If False or 1, runs in single-process mode. If an integer, use that many processes. @@ -87,9 +96,9 @@ def open( verbose : bool, optional Whether to display a progress bar while reading metadata. Ignored when opening a single file. Default is ``False``. - **kwargs - Additional keyword arguments forwarded to the underlying engine read - function. Only used when `engine` is not ``None``. + **engine_kwargs + Format-specific engine parameters forwarded to the engine constructor + (e.g. ``overlaps`` for "febus"). Only valid when `engine` is given by name. Returns ------- @@ -153,7 +162,9 @@ def open( return open_datacollection(paths) except Exception: # noqa: BLE001, S110 - fall back to dataarray pass - return open_dataarray(paths, engine=engine, **kwargs) + return open_dataarray( + paths, engine=engine, vtype=vtype, ctype=ctype, **engine_kwargs + ) case "multi-file": if engine is None: try: @@ -173,9 +184,11 @@ def open( tolerance, squeeze=True if squeeze is None else squeeze, engine=engine, + vtype=vtype, + ctype=ctype, parallel=parallel, verbose=verbose, - **kwargs, + **engine_kwargs, ) case "tree-like": # pragma: no branch return open_mfdatatree( @@ -184,9 +197,11 @@ def open( tolerance, squeeze=False if squeeze is None else squeeze, engine=engine, + vtype=vtype, + ctype=ctype, parallel=parallel, verbose=verbose, - **kwargs, + **engine_kwargs, ) @@ -278,9 +293,11 @@ def open_mfdatatree( tolerance=None, squeeze=False, engine=None, + vtype=None, + ctype=None, verbose=False, parallel=None, - **kwargs, + **engine_kwargs, ): """ Open a directory tree structure as a data collection. @@ -312,8 +329,15 @@ def open_mfdatatree( squeeze : bool, optional Whether to return a DataArray instead of a DataCollection if the combination results in a data collection containing a unique data array. - engine: str or callable, optional - The type of file to open or a read function. Default to xdas netcdf format. + engine: str or Engine, optional + The file format engine to use, given by name or as a configured + :class:`~xdas.io.Engine` instance. Default to format auto-detection. + vtype : str, optional + The virtualization type to use. If None, the engine default is used. + Only valid when `engine` is given by name (or None). + ctype : str or dict, optional + The coordinate type(s) to use. If None, the engine defaults are used. + Only valid when `engine` is given by name (or None). parallel: bool or int, optional Whether to use multiprocessing to fetch file metadata. If False or 1, runs in single-process mode. If an integer, use that many processes. @@ -321,8 +345,9 @@ def open_mfdatatree( global xdas configuration. Default to None. verbose: bool Whether to display a progress bar. Default to False. - **kwargs - Additional keyword arguments to be passed to the read function. + **engine_kwargs + Format-specific engine parameters forwarded to the engine constructor + (e.g. ``overlaps`` for "febus"). Only valid when `engine` is given by name. Returns ------- @@ -389,7 +414,17 @@ def open_mfdatatree( bag.append(fname) return collect( - tree, fields, dim, tolerance, squeeze, engine, parallel, verbose, **kwargs + tree, + fields, + dim, + tolerance, + squeeze, + engine, + vtype, + ctype, + parallel, + verbose, + **engine_kwargs, ) @@ -400,9 +435,11 @@ def collect( tolerance=None, squeeze=False, engine=None, + vtype=None, + ctype=None, parallel=None, verbose=False, - **kwargs, + **engine_kwargs, ): """ Collect the data from a tree of paths using `fields` as level names. @@ -422,8 +459,15 @@ def collect( squeeze : bool, optional Whether to return a DataArray instead of a DataCollection if the combination results in a data collection containing a unique data array. - engine: str or callable, optional - The type of file to open or a read function. Default to xdas netcdf format. + engine: str or Engine, optional + The file format engine to use, given by name or as a configured + :class:`~xdas.io.Engine` instance. Default to format auto-detection. + vtype : str, optional + The virtualization type to use. If None, the engine default is used. + Only valid when `engine` is given by name (or None). + ctype : str or dict, optional + The coordinate type(s) to use. If None, the engine defaults are used. + Only valid when `engine` is given by name (or None). parallel: bool or int, optional Whether to use multiprocessing to fetch file metadata. If False or 1, runs in single-process mode. If an integer, use that many processes. @@ -431,8 +475,9 @@ def collect( global xdas configuration. Default to None. verbose: bool Whether to display a progress bar. Default to False. - **kwargs - Additional keyword arguments to be passed to the read function. + **engine_kwargs + Format-specific engine parameters forwarded to the engine constructor + (e.g. ``overlaps`` for "febus"). Only valid when `engine` is given by name. Returns @@ -446,7 +491,16 @@ def collect( for key, value in tree.items(): if isinstance(value, list): dc = open_mfdataarray( - value, dim, tolerance, squeeze, engine, parallel, verbose, **kwargs + value, + dim, + tolerance, + squeeze, + engine, + vtype, + ctype, + parallel, + verbose, + **engine_kwargs, ) dc.name = fields[0] collection[key] = dc @@ -458,9 +512,11 @@ def collect( tolerance, squeeze, engine, + vtype, + ctype, parallel, verbose, - **kwargs, + **engine_kwargs, ) return collection @@ -481,15 +537,29 @@ def defaulttree(depth): MAX_OPEN_FILES = {None: 100_000, "hdf5": 100_000, "tiles": 2_000_000} -def _check_file_count(nfiles, engine, vtype): - """Refuse file sets too large for the vtype *engine* will end up using.""" +def _resolve_engine(engine, vtype, ctype, engine_kwargs): + """Turn the `engine` argument of the open functions into an Engine instance.""" from ..io.core import Engine - try: - vtype = Engine[engine](vtype=vtype).vtype - except (KeyError, NotImplementedError, ValueError): - # a bad engine or vtype: opening the first file reports it - vtype = None + if isinstance(engine, Engine): + if vtype is not None or ctype is not None or engine_kwargs: + raise ValueError( + "`vtype`, `ctype` and engine keyword arguments cannot be combined " + "with an already configured engine instance; configure the instance " + "instead" + ) + return engine + elif engine is None or isinstance(engine, str): + return Engine[engine](vtype=vtype, ctype=ctype, **engine_kwargs) + else: + raise TypeError( + "engine must be None, a registered engine name or an Engine instance, " + f"found {type(engine)}" + ) + + +def _check_file_count(nfiles, vtype): + """Refuse file sets too large for the vtype the engine will end up using.""" limit = MAX_OPEN_FILES.get(vtype, MAX_OPEN_FILES[None]) if nfiles > limit: raise NotImplementedError( @@ -506,9 +576,11 @@ def open_mfdataarray( tolerance=None, squeeze=True, engine=None, + vtype=None, + ctype=None, parallel=None, verbose=False, - **kwargs, + **engine_kwargs, ): """ Open a multiple file dataset. @@ -531,8 +603,15 @@ def open_mfdataarray( squeeze : bool, optional Whether to return a DataArray instead of a DataCollection if the combination results in a data collection containing a unique data array. - engine: str or callable, optional - The type of file to open or a read function. Default to xdas netcdf format. + engine: str or Engine, optional + The file format engine to use, given by name or as a configured + :class:`~xdas.io.Engine` instance. Default to format auto-detection. + vtype : str, optional + The virtualization type to use. If None, the engine default is used. + Only valid when `engine` is given by name (or None). + ctype : str or dict, optional + The coordinate type(s) to use. If None, the engine defaults are used. + Only valid when `engine` is given by name (or None). parallel: bool or int, optional Whether to use multiprocessing to fetch file metadata. If False or 1, runs in single-process mode. If an integer, use that many processes. @@ -540,8 +619,9 @@ def open_mfdataarray( global xdas configuration. Default to None. verbose: bool Whether to display a progress bar. Default to False. - **kwargs - Additional keyword arguments to be passed to the read function. + **engine_kwargs + Format-specific engine parameters forwarded to the engine constructor + (e.g. ``overlaps`` for "febus"). Only valid when `engine` is given by name. Returns ------- @@ -573,25 +653,25 @@ def open_mfdataarray( ) if len(paths) == 0: raise FileNotFoundError("no file to open") - _check_file_count(len(paths), engine, kwargs.get("vtype")) + engine = _resolve_engine(engine, vtype, ctype, engine_kwargs) + _check_file_count(len(paths), engine.vtype) max_workers = get_workers_count(parallel) objs = [] failures = [] - if (max_workers == 1) or (engine == "miniseed"): # TODO: dirty miniseed fix + if (max_workers == 1) or (engine.name == "miniseed"): # TODO: dirty miniseed fix iterator = ( tqdm(paths, desc="Fetching metadata from files") if verbose else paths ) for path in iterator: try: - objs.append(open_dataarray(path, engine=engine, **kwargs)) + objs.append(open_dataarray(path, engine=engine)) except Exception as error: # noqa: BLE001 - collected and warned below failures.append((path, error)) warnings.warn(f"could not open {path}: {error}", RuntimeWarning) else: executor = get_reusable_executor(max_workers) futures_to_paths = { - executor.submit(open_dataarray, path, engine=engine, **kwargs): path - for path in paths + executor.submit(open_dataarray, path, engine=engine): path for path in paths } if verbose: iterator = tqdm( @@ -613,12 +693,14 @@ def open_mfdataarray( if len(objs) == 0: # there must be failures path, error = failures[0] raise RuntimeError( - f"could not open any file with engine: {engine}; first failure was {path}: {error}" + f"could not open any file with engine: " + f"{engine.name or type(engine).__name__}; " + f"first failure was {path}: {error}" ) from error return combine_by_coords(objs, dim, tolerance, squeeze, None, verbose) -def open_dataarray(fname, engine=None, vtype=None, ctype=None, **kwargs): +def open_dataarray(fname, engine=None, vtype=None, ctype=None, **engine_kwargs): """ Open a dataarray. @@ -626,10 +708,18 @@ def open_dataarray(fname, engine=None, vtype=None, ctype=None, **kwargs): ---------- fname : str The path of the dataarray. - engine: str or callable, optional - The type of file to open or a read function. Default to xdas netcdf format. - **kwargs - Additional keyword arguments to be passed to the read function. + engine: str or Engine, optional + The file format engine to use, given by name or as a configured + :class:`~xdas.io.Engine` instance. Default to format auto-detection. + vtype : str, optional + The virtualization type to use. If None, the engine default is used. + Only valid when `engine` is given by name (or None). + ctype : str or dict, optional + The coordinate type(s) to use. If None, the engine defaults are used. + Only valid when `engine` is given by name (or None). + **engine_kwargs + Format-specific engine parameters forwarded to the engine constructor + (e.g. ``overlaps`` for "febus"). Only valid when `engine` is given by name. Returns ------- @@ -638,12 +728,13 @@ def open_dataarray(fname, engine=None, vtype=None, ctype=None, **kwargs): Raises ------ + TypeError + If `engine` is neither None, an engine name nor an Engine instance, or + if an engine keyword argument is unknown to the engine. ValueError - If the engine is not recognized. - - Raises - ------ - FileNotFound + If `vtype`, `ctype` or engine keyword arguments are combined with an + already configured engine instance. + FileNotFoundError If no file can be found. """ # parse & checks @@ -652,15 +743,8 @@ def open_dataarray(fname, engine=None, vtype=None, ctype=None, **kwargs): raise FileNotFoundError("no file to open") # dispatch & open - if engine is None or isinstance(engine, str): - from ..io.core import Engine - - engine = Engine[engine](vtype=vtype, ctype=ctype) - return engine.open_dataarray(fname, **kwargs) - elif callable(engine): - return engine(fname, **kwargs) - else: - raise ValueError("engine not recognized") + engine = _resolve_engine(engine, vtype, ctype, engine_kwargs) + return engine.open_dataarray(fname) def open_datacollection(fname, group=None): diff --git a/xdas/io/core.py b/xdas/io/core.py index 65a05f45..cb3b122b 100644 --- a/xdas/io/core.py +++ b/xdas/io/core.py @@ -48,28 +48,34 @@ class Engine: - `_supported_ctypes` (dict): Maps component names to lists of supported coordinate types + Engines with format-specific parameters define their own `__init__` taking those + parameters after `vtype` and `ctype` and calling `super().__init__(vtype, ctype)`. + They are then reachable from the open functions either by configuring an instance + or as extra keyword arguments next to the engine name. + Examples -------- Subclass registration (automatic via `__init_subclass__`): - >>> class NetCDFEngine(Engine, name="netcdf", aliases=["nc"]): + >>> class MyFormatEngine(Engine, name="myformat", aliases=["my"]): ... _supported_vtypes = ["hdf5"] ... _supported_ctypes = { ... "time": ["sampled", "dense"], "distance": ["sampled", "dense"] ... } - ... def open_dataarray(self, fname, **kwargs): - ... ... + ... def open_dataarray(self, fname): + ... raise NotImplementedError Access registered engines: - >>> engine = Engine["netcdf"](vtype="hdf5") - >>> engine = Engine["nc"](ctype="dense") # Using alias + >>> engine = Engine["myformat"](vtype="hdf5") + >>> engine = Engine["my"](ctype="dense") # Using alias """ _registry: ClassVar[dict] = {} _aliases: ClassVar[dict] = {} _supported_vtypes = None _supported_ctypes = None + name = None def __init__(self, vtype=None, ctype=None): self.vtype = self._parse_vtype(vtype) @@ -78,6 +84,7 @@ def __init__(self, vtype=None, ctype=None): def __init_subclass__(cls, *, name=None, aliases=None, **kwargs): super().__init_subclass__(**kwargs) if name is not None: + cls.name = name Engine._registry[name] = cls if aliases is not None: for alias in aliases: @@ -96,7 +103,7 @@ def __class_getitem__(cls, item): f"available: {sorted([*cls._registry, *cls._aliases])}" ) - def open_dataarray(self, fname, **kwargs): + def open_dataarray(self, fname): """Open *fname* and return a :class:`DataArray` (abstract).""" raise NotImplementedError @@ -104,7 +111,7 @@ def save_dataarray(self, da, fname, **kwargs): """Write *da* to *fname* (abstract).""" raise NotImplementedError - def open_datacollection(self, fname, **kwargs): + def open_datacollection(self, fname): """Open *fname* and return a :class:`DataCollection` (abstract).""" raise NotImplementedError @@ -193,6 +200,8 @@ class AutoEngine(Engine): ctype : str or dict, optional The coordinate type(s) to use. Passed to all engines during auto-detection. Can be a string, dict, or None (each engine uses its default). + Format-specific engine parameters cannot be used with auto-detection: + they require naming a concrete engine. Attributes ---------- @@ -216,12 +225,12 @@ class AutoEngine(Engine): _last_successful_engine = "xdas" - def open_dataarray(self, fname, **kwargs): + def open_dataarray(self, fname): """Try each registered engine in order and return the first successful result.""" for engine in self._ordered_engines(): try: out = Engine[engine](vtype=self.vtype, ctype=self.ctype).open_dataarray( - fname, **kwargs + fname ) AutoEngine._last_successful_engine = engine return out diff --git a/xdas/io/febus.py b/xdas/io/febus.py index 6fc1fc02..541484c7 100644 --- a/xdas/io/febus.py +++ b/xdas/io/febus.py @@ -14,7 +14,25 @@ class FebusEngine(Engine, name="febus"): - """Engine for reading Febus HDF5 files.""" + """ + Engine for reading Febus HDF5 files. + + Parameters + ---------- + vtype : str, optional + The virtualization type to use. Default to "tiles". + ctype : str or dict, optional + The coordinate type(s) to use. Default to "interpolated". + overlaps : tuple of int, optional + A tuple specifying the overlap in number of sample to trim on both side of each + chunk of the data. If not provided, the engine will attempt to determine the + correct overlap at your own risk. + offset : int, optional + The location of the timestamp within each block given as the number of samples + from the beginning. If not provided, the engine will attempt to determine the + correct offset at you own risk. + + """ # tiles first: a Febus file holds a stack of blocks, and the hdf5 # backing needs one mapping per block while a tile array needs one @@ -25,7 +43,24 @@ class FebusEngine(Engine, name="febus"): "distance": ["interpolated", "sampled", "dense"], } - def open_dataarray(self, fname, overlaps=None, offset=None): + def __init__(self, vtype=None, ctype=None, overlaps=None, offset=None): + super().__init__(vtype, ctype) + match overlaps: + case None | (int(), int()): + pass + case _: + raise ValueError( + "overlaps must be a integer or a tuple of two integers" + ) + match offset: + case None | int(): + pass + case _: + raise ValueError("offset must be an integer") + self.overlaps = overlaps + self.offset = offset + + def open_dataarray(self, fname): """ Open a Febus file into a xdas DataArray object. @@ -35,21 +70,14 @@ def open_dataarray(self, fname, overlaps=None, offset=None): timestamp that is located at a fixed offset from the beginning of the chunk. Because of poor documentation of the evolution of the Febus file format, it is - recommended to manually specify the overlap and offset parameters. If not provided, - the function will attempt to determine the correct values at your own risk. + recommended to manually specify the `overlaps` and `offset` engine parameters. + If not provided, the function will attempt to determine the correct values at + your own risk. Parameters ---------- fname : str The filename of the Febus file to read. - overlaps : tuple of int, optional - A tuple specifying the overlap in number of sample to trim on both side of each - chunk of the data. If not provided, the function will attempt to determine the - correct overlap at your own risk. - offset : int, optional - The location of the timestamp within each block given as the number of samples - from the beginning. If not provided, the function will attempt to determine the - correct offset at you own risk. Returns ------- @@ -57,6 +85,8 @@ def open_dataarray(self, fname, overlaps=None, offset=None): A data array containing the data from the Febus file. """ + overlaps = self.overlaps + offset = self.offset with h5py.File(fname, "r") as file: (device_name,) = list(file.keys()) source = file[device_name]["Source1"] @@ -79,32 +109,20 @@ def open_dataarray(self, fname, overlaps=None, offset=None): "_" ) - match overlaps: - case None: - warnings.warn( - "No overlap specified, Xdas will try its best to find the correct trimming" - ) - noverlap = chunks.shape[1] - round((1 / blockrate) / delta[0]) - before = noverlap // 2 - after = noverlap - before - overlaps = (before, after) - case (int(), int()): - pass - case _: - raise ValueError( - "overlaps must be a integer or a tuple of two integers" - ) - - match offset: - case None: - warnings.warn( - "No offset specified, Xdas will try its best to place the timestamps" - ) - offset = chunks.shape[1] // 2 - case int(): - pass - case _: - raise ValueError("offset must be an integer") + if overlaps is None: + warnings.warn( + "No overlap specified, Xdas will try its best to find the correct trimming" + ) + noverlap = chunks.shape[1] - round((1 / blockrate) / delta[0]) + before = noverlap // 2 + after = noverlap - before + overlaps = (before, after) + + if offset is None: + warnings.warn( + "No offset specified, Xdas will try its best to place the timestamps" + ) + offset = chunks.shape[1] // 2 times = times + (overlaps[0] - offset) * delta[0] diff --git a/xdas/io/miniseed.py b/xdas/io/miniseed.py index 7ed518a3..a2e2606b 100644 --- a/xdas/io/miniseed.py +++ b/xdas/io/miniseed.py @@ -17,28 +17,46 @@ class MiniSEEDEngine(Engine, name="miniseed"): - """Engine for reading MiniSEED files via ObsPy as lazy tile-backed DataArrays.""" + """ + Engine for reading MiniSEED files via ObsPy as lazy tile-backed DataArrays. + + Parameters + ---------- + vtype : str, optional + The virtualization type to use. Default to "tiles". + ctype : str or dict, optional + The coordinate type to use for the time axis. Default to "interpolated". + ignore_last_sample : bool, optional + Whether to drop the last sample of each contiguous segment. Useful for + files whose last sample overlaps the first one of the next file. + Default to False. + + """ _supported_vtypes: ClassVar[list] = ["tiles"] _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], } - def open_dataarray(self, fname, ignore_last_sample=False, ctype="interpolated"): + def __init__(self, vtype=None, ctype=None, ignore_last_sample=False): + super().__init__(vtype, ctype) + self.ignore_last_sample = bool(ignore_last_sample) + + def open_dataarray(self, fname): """Return a lazy tile-backed :class:`DataArray` for the MiniSEED file *fname*.""" - shape, dtype, coords, method = self.read_header( - fname, ignore_last_sample, ctype - ) + shape, dtype, coords, method = self.read_header(fname) engine = { "name": "miniseed", "method": method, - "ignore_last_sample": bool(ignore_last_sample), + "ignore_last_sample": self.ignore_last_sample, } data = TileArray(str(fname), shape, engine, np.dtype(dtype)) return DataArray(data, coords) - def read_header(self, path, ignore_last_sample, ctype): + def read_header(self, path): """Read metadata from *path* and return ``(shape, dtype, coords, method)``.""" + ignore_last_sample = self.ignore_last_sample + ctype = self.ctype["time"] st = obspy.read(path, headonly=True) dtype = uniquifiy(tr.data.dtype for tr in st) diff --git a/xdas/io/prodml.py b/xdas/io/prodml.py index 164e0520..a2a1ba67 100644 --- a/xdas/io/prodml.py +++ b/xdas/io/prodml.py @@ -19,7 +19,20 @@ class ProdML(Engine, name="prodml", aliases=["optasense", "sintela"]): - """Engine for reading ProdML / OptaSense / Sintela HDF5 files.""" + """ + Engine for reading ProdML / OptaSense / Sintela HDF5 files. + + Parameters + ---------- + vtype : str, optional + The virtualization type to use. Default to "hdf5". + ctype : str or dict, optional + The coordinate type(s) to use. Default to "interpolated". + swapped_dims : bool, optional + Whether the on-disk array is (distance, time) instead of the usual + (time, distance). Default to False. + + """ _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] _supported_ctypes: ClassVar[dict] = { @@ -27,8 +40,13 @@ class ProdML(Engine, name="prodml", aliases=["optasense", "sintela"]): "distance": ["interpolated", "sampled", "dense"], } - def open_dataarray(self, fname, swapped_dims=False): + def __init__(self, vtype=None, ctype=None, swapped_dims=False): + super().__init__(vtype, ctype) + self.swapped_dims = bool(swapped_dims) + + def open_dataarray(self, fname): """Read a ProdML HDF5 file *fname* and return a virtual :class:`DataArray`.""" + swapped_dims = self.swapped_dims with h5py.File(fname, "r") as file: acquisition = file["Acquisition"] dx = acquisition.attrs["SpatialSamplingInterval"] diff --git a/xdas/io/terra15.py b/xdas/io/terra15.py index 8d2b0efe..c5d7c3c2 100644 --- a/xdas/io/terra15.py +++ b/xdas/io/terra15.py @@ -13,7 +13,19 @@ class Terra15Engine(Engine, name="terra15"): - """Engine for reading Terra15 HDF5 files.""" + """ + Engine for reading Terra15 HDF5 files. + + Parameters + ---------- + vtype : str, optional + The virtualization type to use. Default to "hdf5". + ctype : str or dict, optional + The coordinate type(s) to use. Default to "interpolated". + tz : str, optional + The timezone of the GPS timestamps stored in the file. Default to "UTC". + + """ _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] _supported_ctypes: ClassVar[dict] = { @@ -21,8 +33,13 @@ class Terra15Engine(Engine, name="terra15"): "distance": ["interpolated", "sampled", "dense"], } - def open_dataarray(self, fname, tz="UTC"): + def __init__(self, vtype=None, ctype=None, tz="UTC"): + super().__init__(vtype, ctype) + self.tz = tz + + def open_dataarray(self, fname): """Read a Terra15 HDF5 file *fname* and return a virtual :class:`DataArray`.""" + tz = self.tz with h5py.File(fname, "r") as file: ti = ( pd.Timestamp(file["data_product"]["gps_time"][0], unit="s", tz=tz) diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 5629f034..3a4b25fa 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -28,21 +28,37 @@ class XdasEngine(Engine, name="xdas"): - """Engine for the native xdas HDF5/NetCDF4 format.""" + """ + Engine for the native xdas HDF5/NetCDF4 format. + + Parameters + ---------- + vtype : str, optional + The virtualization type to use. Default to "hdf5". + ctype : str or dict, optional + Ignored: the native format stores coordinates as they were written. + group : str, optional + The location of the data array within the file. Default to the root group. + + """ _supported_vtypes: ClassVar[list] = ["hdf5", "tiles"] - def open_dataarray(self, fname, **kwargs): + def __init__(self, vtype=None, ctype=None, group=None): + super().__init__(vtype, ctype) + self.group = group + + def open_dataarray(self, fname): """Delegate to module-level :func:`open_dataarray`.""" - return open_dataarray(fname, vtype=self.vtype, **kwargs) + return open_dataarray(fname, group=self.group, vtype=self.vtype) def save_dataarray(self, da, fname, **kwargs): """Delegate to module-level :func:`save_dataarray`.""" return save_dataarray(da, fname, **kwargs) - def open_datacollection(self, fname, **kwargs): + def open_datacollection(self, fname): """Delegate to module-level :func:`open_datacollection`.""" - return open_datacollection(fname, **kwargs) + return open_datacollection(fname, group=self.group) def save_datacollection(self, dc, fname, **kwargs): """Delegate to module-level :func:`save_datacollection`.""" diff --git a/xdas/processing/core.py b/xdas/processing/core.py index 2c12eb81..5ff15832 100644 --- a/xdas/processing/core.py +++ b/xdas/processing/core.py @@ -184,11 +184,12 @@ class RealTimeLoader(Observer): ---------- path : str or Path Directory to watch. - engine : str, optional - Engine used to open arriving files. Defaults to ``"netcdf"``. + engine : str or Engine, optional + Engine used to open arriving files, given by name or as a configured + :class:`~xdas.io.Engine` instance. Defaults to ``"xdas"``. """ - def __init__(self, path, engine="netcdf"): + def __init__(self, path, engine="xdas"): super().__init__() self.path = str(path) if isinstance(path, Path) else path self.queue = Queue() From 46b88d65f0d42a4ef70c3bfa5077491148eba039 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 02:06:53 +0200 Subject: [PATCH 20/56] Split the TileArray constructor from the scan-time encoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The constructor now wraps a manifest dataset directly — hand-built or reopened from a stored file — while the new from_tiles classmethod builds the manifest from per-tile descriptions at scan time; the from_dataset/_setup indirection is gone. The starts kwarg goes with it: trimmed and decimated geometry is view state, arising from slicing or entering through a stored manifest. --- docs/api/tiles.md | 2 +- docs/user-guide/io/data-formats.md | 2 +- tests/io/test_silixa.py | 2 +- tests/io/test_tiles_vtype.py | 2 +- tests/tiles/conftest.py | 24 +-- tests/tiles/test_tilearray.py | 171 +++++++++--------- xdas/io/apsensing.py | 2 +- xdas/io/asn.py | 2 +- xdas/io/febus.py | 4 +- xdas/io/miniseed.py | 2 +- xdas/io/prodml.py | 2 +- xdas/io/silixa.py | 4 +- xdas/io/terra15.py | 2 +- xdas/io/xdas.py | 9 +- xdas/tiles/tilearray.py | 276 ++++++++++++----------------- 15 files changed, 235 insertions(+), 271 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index 483a6520..e52a1af1 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -34,7 +34,7 @@ Methods .. autosummary:: :toctree: ../_autosummary - TileArray.from_dataset + TileArray.from_tiles TileArray.to_dataset TileArray.concat TileArray.expand_dims diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index 49dbdf8c..2b30c859 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -156,7 +156,7 @@ class MyTileEngine(Engine, name="my_tile_engine"): x0 = file["dataset"].attrs["x0"][()] dx = file["dataset"].attrs["dx"][()] if self.vtype == "tiles": - data = TileArray( + data = TileArray.from_tiles( str(fname), file["dataset"].shape, {"name": "my_tile_engine"}, diff --git a/tests/io/test_silixa.py b/tests/io/test_silixa.py index 0456197a..bea0572b 100644 --- a/tests/io/test_silixa.py +++ b/tests/io/test_silixa.py @@ -41,7 +41,7 @@ def get_data(self, first_s=None, last_s=None): def test_tile_load(monkeypatch): monkeypatch.setattr(silixa, "TdmsReader", FakeTdms) expected = FakeTdms.data - manifest = TileArray("fake.tdms", (20, 4), {"name": "silixa"}, "float64") + manifest = TileArray.from_tiles("fake.tdms", (20, 4), {"name": "silixa"}, "float64") npt.assert_array_equal(np.asarray(manifest), expected) npt.assert_array_equal(np.asarray(manifest[3:15:2, 1:3]), expected[3:15:2, 1:3]) expanded = np.expand_dims(manifest, 0) diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index f13b3025..f620657d 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -131,7 +131,7 @@ def test_prodml_transpose_param(tmp_path): """ path = str(tmp_path / "prodml_swapped.h5") data = make_prodml_file(path, swapped=True) - manifest = TileArray( + manifest = TileArray.from_tiles( path, data.T.shape, {"name": "prodml", "transpose": True}, data.dtype ) npt.assert_array_equal(np.asarray(manifest), data.T) diff --git a/tests/tiles/conftest.py b/tests/tiles/conftest.py index d2633d3c..5d27985b 100644 --- a/tests/tiles/conftest.py +++ b/tests/tiles/conftest.py @@ -54,13 +54,14 @@ def stack(tmp_path): sizes.append(useful) parts.append(data[1:-1]) row += useful + manifest = TileArray.from_tiles( + paths, (sizes, NX), ENGINE, "float64", attrs={"units": "strain"} + ) + # per-tile source origins are view state: assigned through the manifest manifest = TileArray( - paths, - (sizes, NX), - ENGINE, - "float64", - starts=([1, 1, 1], None), - attrs={"units": "strain"}, + manifest.dataset.assign(starts_0=("tile_0", np.array([1, 1, 1]))), + manifest.dtype, + manifest.engine, ) return manifest, np.concatenate(parts) @@ -89,12 +90,13 @@ def windowed(tmp_path): starts.append(first) parts.append(good) row += useful + manifest = TileArray.from_tiles( + paths, (sizes, NX), {"name": "h5py", "dataset": "data"}, "float64" + ) manifest = TileArray( - paths, - (sizes, NX), - {"name": "h5py", "dataset": "data"}, - "float64", - starts=(starts, None), + manifest.dataset.assign(starts_0=("tile_0", np.array(starts))), + manifest.dtype, + manifest.engine, ) return manifest, np.concatenate(parts) diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py index d51ce21e..3b41c3a3 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/tiles/test_tilearray.py @@ -20,6 +20,21 @@ def _tile_file(path, data, **kwargs): file.create_dataset("data", data=data, **kwargs) +def _with_starts(manifest, *starts): + """Rebuild *manifest* with per-axis tile origins inside their sources. + + ``starts_k`` is view state the :meth:`TileArray.from_tiles` encoder + does not take: windowed manifests assign it through the dataset and + the canonical constructor. + """ + assign = { + f"starts_{k}": (f"tile_{k}", np.asarray(entry, dtype=np.int64)) + for k, entry in enumerate(starts) + if entry is not None + } + return TileArray(manifest.dataset.assign(assign), manifest.dtype, manifest.engine) + + def _random_key(rng, shape, max_step=1): """A random non-empty positive-step slice per axis.""" key = [] @@ -65,14 +80,10 @@ def _random_grid(tmp_path, rng, ndim): path = str(tmp_path / f"grid{number}.h5") _tile_file(path, data) paths[index] = path - manifest = TileArray( - paths, - sizes, - {"name": "h5py", "dataset": "data"}, - "float64", - starts=margins, + manifest = TileArray.from_tiles( + paths, sizes, {"name": "h5py", "dataset": "data"}, "float64" ) - return manifest, reference + return _with_starts(manifest, *margins), reference class TestManifest: @@ -102,54 +113,61 @@ def test_dataset_model(self, stack): def test_param_folding(self, stack): manifest, _ = stack path = str(manifest.dataset["paths"].values[0]) - uniform = TileArray( + uniform = TileArray.from_tiles( path, ([10, 10, 10], NX), ENGINE, "float64", record=0, nbytes=80 ) # one path everywhere: 0-d; uniform per-tile params: 0-d assert uniform.dataset["paths"].ndim == 0 assert uniform.dataset["record"].ndim == 0 assert uniform.shape == (30, NX) - varying = TileArray(path, ([10, 10], NX), ENGINE, "float64", record=[[0], [80]]) + varying = TileArray.from_tiles( + path, ([10, 10], NX), ENGINE, "float64", record=[[0], [80]] + ) assert tuple(varying.dataset["record"].dims) == ("tile_0",) def test_validation(self, stack): manifest, _ = stack with pytest.raises(ValueError, match="at least one axis"): - TileArray("a", (), ENGINE, "f8") + TileArray.from_tiles("a", (), ENGINE, "f8") with pytest.raises(ValueError, match="little-endian"): - TileArray("a", (5, NX), ENGINE, ">f8") + TileArray.from_tiles("a", (5, NX), ENGINE, ">f8") with pytest.raises(ValueError, match="strictly positive"): - TileArray("a", (0, NX), ENGINE, "f8") + TileArray.from_tiles("a", (0, NX), ENGINE, "f8") with pytest.raises(ValueError, match="does not match the grid"): - TileArray(np.array(["a", "b"], dtype=object), ([1, 2, 3], NX), ENGINE, "f8") - with pytest.raises(ValueError, match="does not match the grid"): - TileArray("a", ([5, 5], NX), ENGINE, "f8", starts=([1, 2, 3], None)) + TileArray.from_tiles( + np.array(["a", "b"], dtype=object), ([1, 2, 3], NX), ENGINE, "f8" + ) + with pytest.raises(ValueError, match="reserved"): + TileArray.from_tiles("a", (5, NX), ENGINE, "f8", sizes_0=[5]) with pytest.raises(ValueError, match="reserved"): - TileArray("a", (5, NX), ENGINE, "f8", sizes_0=[5]) + TileArray.from_tiles("a", (5, NX), ENGINE, "f8", starts_0=[0]) dataset = manifest.dataset.copy() - kwargs = {"dtype": manifest.dtype, "params": {"engine": manifest.engine}} with pytest.raises(ValueError, match="`sizes_0`"): - TileArray.from_dataset(dataset.drop_vars(["sizes_0", "sizes_1"]), **kwargs) + TileArray( + dataset.drop_vars(["sizes_0", "sizes_1"]), + manifest.dtype, + manifest.engine, + ) with pytest.raises(ValueError, match="`paths`"): - TileArray.from_dataset(dataset.drop_vars("paths"), **kwargs) + TileArray(dataset.drop_vars("paths"), manifest.dtype, manifest.engine) def test_extra_variables_are_params(self, stack): """Any non-geometry manifest variable is a per-tile engine parameter.""" manifest, _ = stack - arr = TileArray.from_dataset( + arr = TileArray( manifest.dataset.assign(record=(("tile_0",), np.arange(3))), - dtype=manifest.dtype, - params={"engine": manifest.engine}, + manifest.dtype, + manifest.engine, ) assert arr._params == ("record",) def test_engine_validation(self): with pytest.raises(KeyError, match="no engine registered"): - TileArray("a", (5, NX), {"name": "bogus"}, "f8") + TileArray.from_tiles("a", (5, NX), {"name": "bogus"}, "f8") with pytest.raises(ValueError, match="`name` key"): - TileArray("a", (5, NX), {"dataset": "data"}, "f8") + TileArray.from_tiles("a", (5, NX), {"dataset": "data"}, "f8") with pytest.raises(ValueError, match="`name` key"): - TileArray("a", (5, NX), None, "f8") + TileArray.from_tiles("a", (5, NX), None, "f8") def test_engine_registration(self): class DummyEngine(Engine, name="dummy"): @@ -169,7 +187,7 @@ class NoTilesEngine(Engine, name="notiles"): pass try: - arr = TileArray("a", (5, NX), {"name": "notiles"}, "f8") + arr = TileArray.from_tiles("a", (5, NX), {"name": "notiles"}, "f8") with pytest.raises(NotImplementedError): np.asarray(arr) finally: @@ -187,7 +205,7 @@ def test_relative_paths_are_anchored(self, tmp_path, monkeypatch): data = np.arange(4.0 * NX).reshape(4, NX) _tile_file(tmp_path / "rel.h5", data) monkeypatch.chdir(tmp_path) - manifest = TileArray("rel.h5", (4, NX), ENGINE, "f8") + manifest = TileArray.from_tiles("rel.h5", (4, NX), ENGINE, "f8") assert os.path.isabs(manifest._grid_values("paths").item(0)) monkeypatch.chdir(tmp_path.parent) npt.assert_array_equal(np.asarray(manifest), data) @@ -201,16 +219,15 @@ class TestSourcePaths: """Paths are stored verbatim: an array holds exactly what it was given.""" def make(self, path): - return TileArray([str(path)], ([4], NX), ENGINE, " ndim: - raise ValueError("`paths` has more axes than `sizes` entries") - paths = paths.reshape(paths.shape + (1,) * (ndim - paths.ndim)) - # reads are lazy and stored views outlive the session: anchor the - # paths now, while the scan's working directory still applies - paths = np.frompyfunc(os.path.abspath, 1, 1)(paths) - data = {} - counts = [] - for k, entry in enumerate(sizes): - values = np.atleast_1d(np.asarray(entry, dtype=np.int64)) - if values.size == 1 and paths.shape[k] > 1: - values = np.full(paths.shape[k], values[0], dtype=np.int64) - counts.append(len(values)) - data[f"sizes_{k}"] = (dims[k], values) - counts = tuple(counts) - if any(have not in (1, count) for have, count in zip(paths.shape, counts)): - raise ValueError( - f"`paths` shape {paths.shape} does not match the grid {counts}" - ) - for k, entry in enumerate(starts or ()): - if entry is None: - continue - values = np.atleast_1d(np.asarray(entry, dtype=np.int64)) - if values.size == 1 and counts[k] > 1: - values = np.full(counts[k], values[0], dtype=np.int64) - if len(values) != counts[k]: - raise ValueError(f"`starts[{k}]` does not match the grid") - if values.any(): - data[f"starts_{k}"] = (dims[k], values) - data["paths"] = _fold_param(paths, counts, dims) - reserved = set(data) | {f"steps_{k}" for k in range(ndim)} - for name, values in params.items(): - if name in reserved: - raise ValueError(f"parameter name {name!r} is reserved") - data[name] = _fold_param(np.asarray(values), counts, dims) - dataset = xr.Dataset(data, attrs=dict(attrs or {})) - self._setup(dataset, dtype, engine) - - @classmethod - def from_dataset(cls, dataset, *, name=None, dims=None, dtype, params=None): - """Wrap an existing manifest *dataset* (see the module docstring). - - The dataset — as stored inside a native xdas file — must hold - the geometry and per-tile variables; what the description - arrays cannot carry comes by value, in *params*. - - Parameters - ---------- - dataset : xarray.Dataset - The manifest dataset to wrap. - name, dims : optional - Accepted for interface uniformity and ignored: the tiled box - is anonymous, its name and axis labels are xarray-level - identity. - dtype : str or numpy.dtype - Element type of the virtual array. - params : dict - The by-value description, as :meth:`to_dataset` returned it: - ``engine``, the engine specification (``"name"`` plus its - own parameters). Any other key is ignored, so a view stored - with by-value parameters this class no longer takes still - opens. - - Returns - ------- - TileArray - """ - params = dict(params or {}) - self = cls.__new__(cls) - self._setup(dataset, dtype, params["engine"]) - return self - - def to_dataset(self): - """Encode this tile array as its manifest dataset plus its params. - - The stored form — a copy of the wrapped dataset carrying the - user attributes, and the by-value constructor kwarg the arrays - cannot: the ``engine``. Source paths are stored exactly as the - array holds them, absolute. Each variable pins its chunking - (see :data:`_MANIFEST_CHUNK`). - - Returns - ------- - dataset : xarray.Dataset - The manifest dataset. - params : dict - The by-value keyword arguments of :meth:`from_dataset`, the - ones no manifest variable can carry. They travel beside the - dataset, not in it. - """ - dataset = xr.Dataset(self.dataset.data_vars, attrs=self.attrs) - row = f"{TILE_PREFIX}0" - for variable in dataset.values(): - variable.encoding["chunks"] = tuple( - _MANIFEST_CHUNK if dim == row else int(dataset.sizes[dim]) - for dim in variable.dims - ) - return dataset, {"engine": self.engine} - - def _setup(self, dataset, dtype, engine): + def __init__(self, dataset, dtype, engine): self.dataset = dataset self._cache = None # the json round trip deep-copies and normalizes (tuples become @@ -448,6 +313,99 @@ def _setup(self, dataset, dtype, engine): f"`{name}` dimensions must be an ordered subset of {dims}" ) + @classmethod + def from_tiles(cls, paths, sizes, engine, dtype, *, attrs=None, **params): + """Build a tile array from per-tile descriptions of fresh sources. + + The scan-time encoder: every tile is read from the origin of + its decoded source, without decimation. Trimmed (``starts_k``) + and decimated (``steps_k``) geometry is view state — it arises + by slicing, or comes from a stored manifest through the class + constructor. + + Parameters + ---------- + paths : str or array-like + Source file of each tile. A scalar describes a + one-tile-per-axis grid; an array is padded with trailing + length-1 axes up to the rank. A path may appear in several + tiles. Relative paths are made absolute at construction — + the working directory cannot be trusted later, as reads are + lazy and stored views outlive the session. + sizes : sequence of int or 1-D array-like + One entry per axis (this defines the rank): the samples each + tile contributes along that axis. An int is uniform across + the axis' tiles; an array gives the per-tile extents (its + length is the number of tiles along the axis). + engine : dict + The engine specification: the key ``"name"`` selects a + registered engine (``xdas.io.Engine[name]``); the remaining + keys are passed to its ``load_tile`` as keyword parameters. + dtype : str or numpy.dtype + Element type of the virtual array (little-endian or + single-byte). + attrs : dict, optional + User attributes of the virtual array. + **params : array-like + Per-tile engine parameters, broadcast over the grid: each + read passes the tile's value to the engine as a keyword + argument (shadowing a same-named specification constant). + + Returns + ------- + TileArray + """ + ndim = len(sizes) + if ndim == 0: + raise ValueError("a tile array needs at least one axis") + dims = tuple(f"{TILE_PREFIX}{k}" for k in range(ndim)) + paths = np.asarray(paths, dtype=object) + if paths.ndim > ndim: + raise ValueError("`paths` has more axes than `sizes` entries") + paths = paths.reshape(paths.shape + (1,) * (ndim - paths.ndim)) + # reads are lazy and stored views outlive the session: anchor the + # paths now, while the scan's working directory still applies + paths = np.frompyfunc(os.path.abspath, 1, 1)(paths) + data = {} + counts = [] + for k, entry in enumerate(sizes): + values = np.atleast_1d(np.asarray(entry, dtype=np.int64)) + if values.size == 1 and paths.shape[k] > 1: + values = np.full(paths.shape[k], values[0], dtype=np.int64) + counts.append(len(values)) + data[f"sizes_{k}"] = (dims[k], values) + counts = tuple(counts) + if any(have not in (1, count) for have, count in zip(paths.shape, counts)): + raise ValueError( + f"`paths` shape {paths.shape} does not match the grid {counts}" + ) + data["paths"] = _fold_param(paths, counts, dims) + reserved = set(data) | { + f"{kind}_{k}" for kind in ("starts", "steps") for k in range(ndim) + } + for name, values in params.items(): + if name in reserved: + raise ValueError(f"parameter name {name!r} is reserved") + data[name] = _fold_param(np.asarray(values), counts, dims) + dataset = xr.Dataset(data, attrs=dict(attrs or {})) + return cls(dataset, dtype, engine) + + def to_dataset(self): + """Encode this tile array as its manifest dataset. + + The stored form — a copy of the wrapped dataset carrying the + user attributes. What the dataset cannot hold, the by-value + ``dtype`` and ``engine``, the caller reads off the properties + and stores beside it. Source paths are stored exactly as the + array holds them, absolute. + + Returns + ------- + xarray.Dataset + The manifest dataset. + """ + return xr.Dataset(self.dataset.data_vars, attrs=self.attrs) + def _geometry(self, kind, default): """Load the eager 1-D ``{kind}_k`` arrays (*default* where absent).""" arrays = [] @@ -569,9 +527,7 @@ def _fold(self, key): assign = {name: entry for name, entry in assign.items() if name not in drop} dataset = self.dataset.isel(indexers).assign(assign) dataset = dataset.drop_vars([name for name in drop if name in dataset]) - return type(self).from_dataset( - dataset, dtype=self.dtype, params={"engine": self.engine} - ) + return type(self)(dataset, self.dtype, self.engine) @classmethod def concat(cls, arrays, dim=0): @@ -655,9 +611,7 @@ def concat(cls, arrays, dim=0): values = np.concatenate([part.values for part in parts], axis=axis_pos) data[name] = xr.Variable(union, values) dataset = xr.Dataset(data, attrs=first.attrs) - return cls.from_dataset( - dataset, dtype=first.dtype, params={"engine": first.engine} - ) + return cls(dataset, first.dtype, first.engine) def _grid_values(self, name): """Load parameter *name* and broadcast it over the full tile grid.""" @@ -898,9 +852,7 @@ def expand_dims(self, axis=0): rename[f"{kind}_{k}"] = f"{kind}_{k + 1}" dataset = self.dataset.rename(rename) dataset = dataset.assign(sizes_0=(f"{TILE_PREFIX}0", np.ones(1, np.int64))) - return type(self).from_dataset( - dataset, dtype=self.dtype, params={"engine": self.engine} - ) + return type(self)(dataset, self.dtype, self.engine) def _expand_virtual(self, args, kwargs): """Dispatch ``numpy.expand_dims``, delegating to :meth:`expand_dims`.""" @@ -924,9 +876,7 @@ def astype(self, dtype, **kwargs): def __deepcopy__(self, memo): """Copy without the read cache; the dataset is immutable.""" - return type(self).from_dataset( - self.dataset, dtype=self.dtype, params={"engine": self.engine} - ) + return type(self)(self.dataset, self.dtype, self.engine) def __repr__(self): return ( From 7dcad98ad1402e4a5d18fe4d51be459226e02237 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 01:52:23 +0200 Subject: [PATCH 21/56] Demote the tile dtype to recorded metadata The dtype of a tile array is not a decode target: engines decode into the element type of their sources, and the array records that type at scan time so the lazy array answers dtype without touching files. Make that role explicit: from_tiles takes (paths, sizes, dtype, engine) in the numpy order, reads verify each decoded part against the recorded dtype instead of silently casting, and the stored sidecar spec drops its dtype copy in favor of the placeholder variable's own element type. Casting stays an explicit astype step outside the tiles module. --- docs/user-guide/io/data-formats.md | 2 +- tests/io/test_silixa.py | 2 +- tests/io/test_tiles_vtype.py | 2 +- tests/tiles/conftest.py | 4 +- tests/tiles/test_tilearray.py | 78 +++++++++++++++++++----------- xdas/io/apsensing.py | 2 +- xdas/io/asn.py | 2 +- xdas/io/febus.py | 2 +- xdas/io/miniseed.py | 2 +- xdas/io/prodml.py | 2 +- xdas/io/silixa.py | 2 +- xdas/io/terra15.py | 2 +- xdas/io/xdas.py | 11 +++-- xdas/tiles/tilearray.py | 31 +++++++----- 14 files changed, 89 insertions(+), 55 deletions(-) diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index 2b30c859..e5da5384 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -159,8 +159,8 @@ class MyTileEngine(Engine, name="my_tile_engine"): data = TileArray.from_tiles( str(fname), file["dataset"].shape, - {"name": "my_tile_engine"}, file["dataset"].dtype, + {"name": "my_tile_engine"}, ) else: data = VirtualSource(file["dataset"]) diff --git a/tests/io/test_silixa.py b/tests/io/test_silixa.py index bea0572b..5d14563b 100644 --- a/tests/io/test_silixa.py +++ b/tests/io/test_silixa.py @@ -41,7 +41,7 @@ def get_data(self, first_s=None, last_s=None): def test_tile_load(monkeypatch): monkeypatch.setattr(silixa, "TdmsReader", FakeTdms) expected = FakeTdms.data - manifest = TileArray.from_tiles("fake.tdms", (20, 4), {"name": "silixa"}, "float64") + manifest = TileArray.from_tiles("fake.tdms", (20, 4), "float64", {"name": "silixa"}) npt.assert_array_equal(np.asarray(manifest), expected) npt.assert_array_equal(np.asarray(manifest[3:15:2, 1:3]), expected[3:15:2, 1:3]) expanded = np.expand_dims(manifest, 0) diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index f620657d..2ea88409 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -132,7 +132,7 @@ def test_prodml_transpose_param(tmp_path): path = str(tmp_path / "prodml_swapped.h5") data = make_prodml_file(path, swapped=True) manifest = TileArray.from_tiles( - path, data.T.shape, {"name": "prodml", "transpose": True}, data.dtype + path, data.T.shape, data.dtype, {"name": "prodml", "transpose": True} ) npt.assert_array_equal(np.asarray(manifest), data.T) npt.assert_array_equal(np.asarray(manifest[2:7:2, 1:4]), data.T[2:7:2, 1:4]) diff --git a/tests/tiles/conftest.py b/tests/tiles/conftest.py index 5d27985b..9bd338d8 100644 --- a/tests/tiles/conftest.py +++ b/tests/tiles/conftest.py @@ -55,7 +55,7 @@ def stack(tmp_path): parts.append(data[1:-1]) row += useful manifest = TileArray.from_tiles( - paths, (sizes, NX), ENGINE, "float64", attrs={"units": "strain"} + paths, (sizes, NX), "float64", ENGINE, attrs={"units": "strain"} ) # per-tile source origins are view state: assigned through the manifest manifest = TileArray( @@ -91,7 +91,7 @@ def windowed(tmp_path): parts.append(good) row += useful manifest = TileArray.from_tiles( - paths, (sizes, NX), {"name": "h5py", "dataset": "data"}, "float64" + paths, (sizes, NX), "float64", {"name": "h5py", "dataset": "data"} ) manifest = TileArray( manifest.dataset.assign(starts_0=("tile_0", np.array(starts))), diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py index 3b41c3a3..87725f0e 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/tiles/test_tilearray.py @@ -81,7 +81,7 @@ def _random_grid(tmp_path, rng, ndim): _tile_file(path, data) paths[index] = path manifest = TileArray.from_tiles( - paths, sizes, {"name": "h5py", "dataset": "data"}, "float64" + paths, sizes, "float64", {"name": "h5py", "dataset": "data"} ) return _with_starts(manifest, *margins), reference @@ -114,33 +114,33 @@ def test_param_folding(self, stack): manifest, _ = stack path = str(manifest.dataset["paths"].values[0]) uniform = TileArray.from_tiles( - path, ([10, 10, 10], NX), ENGINE, "float64", record=0, nbytes=80 + path, ([10, 10, 10], NX), "float64", ENGINE, record=0, nbytes=80 ) # one path everywhere: 0-d; uniform per-tile params: 0-d assert uniform.dataset["paths"].ndim == 0 assert uniform.dataset["record"].ndim == 0 assert uniform.shape == (30, NX) varying = TileArray.from_tiles( - path, ([10, 10], NX), ENGINE, "float64", record=[[0], [80]] + path, ([10, 10], NX), "float64", ENGINE, record=[[0], [80]] ) assert tuple(varying.dataset["record"].dims) == ("tile_0",) def test_validation(self, stack): manifest, _ = stack with pytest.raises(ValueError, match="at least one axis"): - TileArray.from_tiles("a", (), ENGINE, "f8") + TileArray.from_tiles("a", (), "f8", ENGINE) with pytest.raises(ValueError, match="little-endian"): - TileArray.from_tiles("a", (5, NX), ENGINE, ">f8") + TileArray.from_tiles("a", (5, NX), ">f8", ENGINE) with pytest.raises(ValueError, match="strictly positive"): - TileArray.from_tiles("a", (0, NX), ENGINE, "f8") + TileArray.from_tiles("a", (0, NX), "f8", ENGINE) with pytest.raises(ValueError, match="does not match the grid"): TileArray.from_tiles( - np.array(["a", "b"], dtype=object), ([1, 2, 3], NX), ENGINE, "f8" + np.array(["a", "b"], dtype=object), ([1, 2, 3], NX), "f8", ENGINE ) with pytest.raises(ValueError, match="reserved"): - TileArray.from_tiles("a", (5, NX), ENGINE, "f8", sizes_0=[5]) + TileArray.from_tiles("a", (5, NX), "f8", ENGINE, sizes_0=[5]) with pytest.raises(ValueError, match="reserved"): - TileArray.from_tiles("a", (5, NX), ENGINE, "f8", starts_0=[0]) + TileArray.from_tiles("a", (5, NX), "f8", ENGINE, starts_0=[0]) dataset = manifest.dataset.copy() with pytest.raises(ValueError, match="`sizes_0`"): TileArray( @@ -163,11 +163,11 @@ def test_extra_variables_are_params(self, stack): def test_engine_validation(self): with pytest.raises(KeyError, match="no engine registered"): - TileArray.from_tiles("a", (5, NX), {"name": "bogus"}, "f8") + TileArray.from_tiles("a", (5, NX), "f8", {"name": "bogus"}) with pytest.raises(ValueError, match="`name` key"): - TileArray.from_tiles("a", (5, NX), {"dataset": "data"}, "f8") + TileArray.from_tiles("a", (5, NX), "f8", {"dataset": "data"}) with pytest.raises(ValueError, match="`name` key"): - TileArray.from_tiles("a", (5, NX), None, "f8") + TileArray.from_tiles("a", (5, NX), "f8", None) def test_engine_registration(self): class DummyEngine(Engine, name="dummy"): @@ -187,7 +187,7 @@ class NoTilesEngine(Engine, name="notiles"): pass try: - arr = TileArray.from_tiles("a", (5, NX), {"name": "notiles"}, "f8") + arr = TileArray.from_tiles("a", (5, NX), "f8", {"name": "notiles"}) with pytest.raises(NotImplementedError): np.asarray(arr) finally: @@ -205,7 +205,7 @@ def test_relative_paths_are_anchored(self, tmp_path, monkeypatch): data = np.arange(4.0 * NX).reshape(4, NX) _tile_file(tmp_path / "rel.h5", data) monkeypatch.chdir(tmp_path) - manifest = TileArray.from_tiles("rel.h5", (4, NX), ENGINE, "f8") + manifest = TileArray.from_tiles("rel.h5", (4, NX), "f8", ENGINE) assert os.path.isabs(manifest._grid_values("paths").item(0)) monkeypatch.chdir(tmp_path.parent) npt.assert_array_equal(np.asarray(manifest), data) @@ -220,7 +220,7 @@ class TestSourcePaths: def make(self, path): # the file's first row is skipped: sliced away, as views are made - return TileArray.from_tiles([str(path)], ([5], NX), ENGINE, " Date: Sun, 2 Aug 2026 02:04:13 +0200 Subject: [PATCH 22/56] Accept a plain engine name as TileArray specification shorthand --- docs/user-guide/io/data-formats.md | 2 +- tests/tiles/test_tilearray.py | 4 ++++ xdas/io/apsensing.py | 2 +- xdas/io/asn.py | 2 +- xdas/io/prodml.py | 2 +- xdas/io/silixa.py | 4 +--- xdas/io/terra15.py | 2 +- xdas/tiles/tilearray.py | 25 +++++++++++++++++-------- 8 files changed, 27 insertions(+), 16 deletions(-) diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index e5da5384..8cd778bf 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -160,7 +160,7 @@ class MyTileEngine(Engine, name="my_tile_engine"): str(fname), file["dataset"].shape, file["dataset"].dtype, - {"name": "my_tile_engine"}, + "my_tile_engine", ) else: data = VirtualSource(file["dataset"]) diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py index 87725f0e..7479a462 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/tiles/test_tilearray.py @@ -169,6 +169,10 @@ def test_engine_validation(self): with pytest.raises(ValueError, match="`name` key"): TileArray.from_tiles("a", (5, NX), "f8", None) + def test_engine_string_shorthand(self): + arr = TileArray.from_tiles("a", (5, NX), "f8", "h5py") + assert arr.engine == {"name": "h5py"} + def test_engine_registration(self): class DummyEngine(Engine, name="dummy"): @staticmethod diff --git a/xdas/io/apsensing.py b/xdas/io/apsensing.py index 93bae025..c2a52e22 100644 --- a/xdas/io/apsensing.py +++ b/xdas/io/apsensing.py @@ -33,7 +33,7 @@ def open_dataarray(self, fname): str(fname), file["DAS"].shape, file["DAS"].dtype, - {"name": "apsensing"}, + "apsensing", ) else: data = VirtualSource(file["DAS"]) diff --git a/xdas/io/asn.py b/xdas/io/asn.py index 10d1e64c..4d991a0e 100644 --- a/xdas/io/asn.py +++ b/xdas/io/asn.py @@ -40,7 +40,7 @@ def open_dataarray(self, fname): dx = float(header["dx"][()]) # Note: dx before (internal) downsampling! if self.vtype == "tiles": data = TileArray.from_tiles( - str(fname), file["data"].shape, file["data"].dtype, {"name": "asn"} + str(fname), file["data"].shape, file["data"].dtype, "asn" ) else: data = VirtualSource(file["data"]) diff --git a/xdas/io/prodml.py b/xdas/io/prodml.py index 6a88ea3e..1d9b30a2 100644 --- a/xdas/io/prodml.py +++ b/xdas/io/prodml.py @@ -68,7 +68,7 @@ def open_dataarray(self, fname): # the manifest keeps the on-disk layout, whichever way the # dims are labeled, so the spec needs no `transpose` data = TileArray.from_tiles( - str(fname), rawdata.shape, rawdata.dtype, {"name": "prodml"} + str(fname), rawdata.shape, rawdata.dtype, "prodml" ) else: data = VirtualSource(rawdata) diff --git a/xdas/io/silixa.py b/xdas/io/silixa.py index 8f28a412..4b38c109 100644 --- a/xdas/io/silixa.py +++ b/xdas/io/silixa.py @@ -23,9 +23,7 @@ class SilixaEngine(Engine, name="silixa"): def open_dataarray(self, fname): """Return a lazy tile-backed :class:`DataArray` for the TDMS file *fname*.""" shape, dtype, coords = self.read_header(fname) - data = TileArray.from_tiles( - str(fname), shape, np.dtype(dtype), {"name": "silixa"} - ) + data = TileArray.from_tiles(str(fname), shape, np.dtype(dtype), "silixa") return DataArray(data, coords) def read_header(self, fname): diff --git a/xdas/io/terra15.py b/xdas/io/terra15.py index 3073bd73..376be295 100644 --- a/xdas/io/terra15.py +++ b/xdas/io/terra15.py @@ -58,7 +58,7 @@ def open_dataarray(self, fname): source = file["data_product"]["data"] if self.vtype == "tiles": data = TileArray.from_tiles( - str(fname), source.shape, source.dtype, {"name": "terra15"} + str(fname), source.shape, source.dtype, "terra15" ) else: data = VirtualSource(source) diff --git a/xdas/tiles/tilearray.py b/xdas/tiles/tilearray.py index 78b41c43..60760e40 100644 --- a/xdas/tiles/tilearray.py +++ b/xdas/tiles/tilearray.py @@ -256,15 +256,22 @@ class TileArray(np.lib.mixins.NDArrayOperatorsMixin): (little-endian or single-byte), recorded so the lazy array can report it without reading. Verified against every decoded tile, never used to cast. - engine : dict - The engine specification: the key ``"name"`` selects a - registered engine (``xdas.io.Engine[name]``); the remaining - keys are passed to its ``load_tile`` as keyword parameters. + engine : str or dict + The engine specification, stored by value with the array: the + key ``"name"`` selects a registered engine + (``xdas.io.Engine[name]``); the remaining keys are passed to + its ``load_tile`` as keyword parameters. A plain string is + shorthand for ``{"name": engine}``. Never an + :class:`~xdas.io.Engine` instance: the specification must + reproduce the decode with no instance alive, so open-time + settings (``vtype``, ``ctype``) do not belong in it. """ def __init__(self, dataset, dtype, engine): self.dataset = dataset self._cache = None + if isinstance(engine, str): + engine = {"name": engine} # the json round trip deep-copies and normalizes (tuples become # lists), so equality survives a store round trip engine = json.loads(json.dumps(engine)) @@ -348,10 +355,12 @@ def from_tiles(cls, paths, sizes, dtype, engine, *, attrs=None, **params): Element type of the sources as the engine decodes them (little-endian or single-byte) — recorded, not a cast target; the scanner reads it off the file it describes. - engine : dict - The engine specification: the key ``"name"`` selects a - registered engine (``xdas.io.Engine[name]``); the remaining - keys are passed to its ``load_tile`` as keyword parameters. + engine : str or dict + The engine specification, stored by value with the array: + the key ``"name"`` selects a registered engine + (``xdas.io.Engine[name]``); the remaining keys are passed + to its ``load_tile`` as keyword parameters. A plain string + is shorthand for ``{"name": engine}``. attrs : dict, optional User attributes of the virtual array. **params : array-like From dca625f55e18809aa0d6af47b1949242043774fb Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 02:04:52 +0200 Subject: [PATCH 23/56] Summarize a tile array on one line under its data array The old repr led with the shape and the dtype, both of which the labeled array already prints one line above, and reported the engine as a quoted kwarg. Report instead what only the tiling knows: the volume the array stands for and the number of tiles, keyed by engine. Bytes render in decimal units, as xarray renders its Size: header, so the two lines agree rather than differing by the 1024/1000 base. The inline form drops the size and the dtype, which an inline row already carries, as dask's does. --- tests/tiles/test_tilearray.py | 13 +++++++++--- xdas/tiles/tilearray.py | 39 +++++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/tests/tiles/test_tilearray.py b/tests/tiles/test_tilearray.py index 7479a462..29c486fb 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/tiles/test_tilearray.py @@ -199,11 +199,18 @@ class NoTilesEngine(Engine, name="notiles"): def test_repr(self, stack): manifest, _ = stack - assert "3 tiles" in repr(manifest) - assert "'h5py'" in repr(manifest) - assert manifest._repr_inline_(40) == "TileArray (3 tiles)" + assert repr(manifest) == "TileArray[h5py] 1kB (float64) 3 tiles" + assert manifest._repr_inline_(40) == "TileArray[h5py] (3 tiles)" assert manifest._repr_inline_(10) == "TileArray" + def test_repr_of_a_single_tile(self, tmp_path): + """One tile reads as one tile, and the volume scales with the array.""" + path = str(tmp_path / "one.h5") + _tile_file(path, np.zeros((250, NX))) + arr = TileArray.from_tiles(path, (250, NX), "float64", ENGINE) + assert repr(arr) == "TileArray[h5py] 10kB (float64) 1 tile" + assert arr._repr_inline_(40) == "TileArray[h5py] (1 tile)" + def test_relative_paths_are_anchored(self, tmp_path, monkeypatch): """Relative paths absolutize at construction and survive a chdir.""" data = np.arange(4.0 * NX).reshape(4, NX) diff --git a/xdas/tiles/tilearray.py b/xdas/tiles/tilearray.py index 60760e40..b95dfd4f 100644 --- a/xdas/tiles/tilearray.py +++ b/xdas/tiles/tilearray.py @@ -71,6 +71,9 @@ TILE_PREFIX = "tile_" """Prefix of the tile-grid dimensions of a manifest dataset.""" +_UNITS = ("B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") +"""Decimal byte units, as xarray spells them in its ``Size:`` header.""" + class _Unfoldable(Exception): """A key that no tile grid can express (private to :meth:`TileArray._fold`). @@ -215,6 +218,20 @@ def _materialize(value): return value +def _to_si(nbytes): + """Render a byte count the way xarray renders its ``Size:`` header. + + Decimal units, no decimals: the repr sits right under that header, + so a base-1024 count would read as a different number. + """ + dividend = float(nbytes) + index = 0 + while dividend >= 1000.0 and index < len(_UNITS) - 1: + dividend /= 1000.0 + index += 1 + return f"{dividend:.0f}{_UNITS[index]}" + + def _row_ranges(edges, shape): """Return the streaming blocks: one tile row along axis 0, whole elsewhere. @@ -897,14 +914,28 @@ def __deepcopy__(self, memo): return type(self)(self.dataset, self.dtype, self.engine) def __repr__(self): + """Summarize the array on one line, as the data of a data array. + + The shape is left out: the labeled array prints it one line + above. What remains is what only the tiling knows — the volume + it stands for, and how many tiles it took. + """ return ( - f"" + f"TileArray[{self.engine['name']}] " + f"{_to_si(self.size * self.dtype.itemsize)} ({self.dtype}) " + f"{self.ntiles} {'tile' if self.ntiles == 1 else 'tiles'}" ) def _repr_inline_(self, max_width): - """Return the one-line summary used by xarray inline reprs.""" - summary = f"TileArray ({self.ntiles} tiles)" + """Return the one-line summary used by xarray inline reprs. + + Shorter than :meth:`__repr__`: an inline row already prints the + dtype and the size, so only the tiling is left to report. + """ + summary = ( + f"TileArray[{self.engine['name']}] " + f"({self.ntiles} {'tile' if self.ntiles == 1 else 'tiles'})" + ) return summary if len(summary) <= max_width else "TileArray" From daa751fe648f3c35f87e1c0cfd746347a6bface6 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 02:19:48 +0200 Subject: [PATCH 24/56] Drop the extract_array helper in favor of plain .data access --- docs/api/tiles.md | 7 ------ tests/tiles/test_integration.py | 12 ++-------- xdas/tiles/__init__.py | 3 +-- xdas/tiles/tilearray.py | 40 --------------------------------- 4 files changed, 3 insertions(+), 59 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index e52a1af1..09af5682 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -47,13 +47,6 @@ Tiles are decoded by the ``load_tile`` half of the {class}`xdas.io.Engine` format plugins; the engine names stored in tile manifests resolve on that registry (``Engine[name]``). -```{eval-rst} -.. autosummary:: - :toctree: ../_autosummary - - extract_array -``` - ```{eval-rst} .. currentmodule:: xdas.io diff --git a/tests/tiles/test_integration.py b/tests/tiles/test_integration.py index 41b16852..00c799f0 100644 --- a/tests/tiles/test_integration.py +++ b/tests/tiles/test_integration.py @@ -6,7 +6,7 @@ import pytest import xdas as xd -from xdas.tiles import TileArray, extract_array +from xdas.tiles import TileArray NX = 5 @@ -28,20 +28,12 @@ def wrap(manifest): class TestDataArray: - def test_data_and_extract(self, stack): + def test_data(self, stack): manifest, _ = stack da = wrap(manifest) assert da.data is manifest - assert extract_array(da) is manifest assert "TileArray" in repr(da) - def test_extract_rejections(self, stack): - manifest, _ = stack - with pytest.raises(TypeError, match="in-memory numpy array"): - extract_array(wrap(manifest).load()) - with pytest.raises(TypeError, match="not backed by"): - extract_array("something else") - def test_isel_stays_virtual(self, stack, engine_calls): manifest, reference = stack da = wrap(manifest) diff --git a/xdas/tiles/__init__.py b/xdas/tiles/__init__.py index d39fa039..eaee8af3 100644 --- a/xdas/tiles/__init__.py +++ b/xdas/tiles/__init__.py @@ -9,9 +9,8 @@ virtual datasets cannot serve (Silixa TDMS, MiniSEED). """ -from .tilearray import TileArray, extract_array +from .tilearray import TileArray __all__ = [ "TileArray", - "extract_array", ] diff --git a/xdas/tiles/tilearray.py b/xdas/tiles/tilearray.py index b95dfd4f..d4ab06e9 100644 --- a/xdas/tiles/tilearray.py +++ b/xdas/tiles/tilearray.py @@ -970,46 +970,6 @@ def _concatenate_virtual(args, kwargs): return NotImplemented -def extract_array(da): - """Return the :class:`TileArray` backing *da*. - - Slicing folds into the tile grid at indexing time, so the array of - a sliced view describes exactly that view — only the overlapping - sources remain. ``extract_array(xr.DataArray(arr, dims=dims))`` - returns ``arr`` itself. - - Parameters - ---------- - da : DataArray - A tile-backed array, as built by wrapping a :class:`TileArray` - or as returned by the :mod:`xdas.io` openers, possibly sliced - with positive-step slices. - - Returns - ------- - TileArray - - Raises - ------ - TypeError - If *da* holds an in-memory numpy array — because it was - loaded, built eagerly, or indexed in a way no tile grid can - represent (integer, reversed or fancy indexing) — or is - otherwise not backed by a :class:`TileArray`. - """ - data = getattr(da, "data", da) - if isinstance(data, TileArray): - return data - if isinstance(data, np.ndarray): - raise TypeError( - "`da` holds an in-memory numpy array and is no longer backed by " - "a TileArray (it was loaded, built eagerly, or indexed in a way " - "no tile grid can represent)" - ) - raise TypeError("`da` is not backed by a TileArray") - - __all__ = [ "TileArray", - "extract_array", ] From dc7512a96ac17ff6b42478142876ee80569523a7 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 02:25:01 +0200 Subject: [PATCH 25/56] Flatten the single-module tiles package into xdas/tiles.py --- docs/release-notes.md | 2 +- .../test_tilearray.py => test_tiles.py} | 236 +++++++++++++++++- tests/tiles/conftest.py | 115 --------- tests/tiles/test_integration.py | 136 ---------- xdas/{tiles/tilearray.py => tiles.py} | 4 +- xdas/tiles/__init__.py | 16 -- 6 files changed, 237 insertions(+), 272 deletions(-) rename tests/{tiles/test_tilearray.py => test_tiles.py} (81%) delete mode 100644 tests/tiles/conftest.py delete mode 100644 tests/tiles/test_integration.py rename xdas/{tiles/tilearray.py => tiles.py} (99%) delete mode 100644 xdas/tiles/__init__.py diff --git a/docs/release-notes.md b/docs/release-notes.md index fa6fa79e..4f7d5b10 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,7 +8,7 @@ ### Breaking Changes - Passing a bare read function as `engine` is no longer supported: subclass `xdas.io.Engine` instead (see the data-formats documentation). Combining a configured engine instance with `vtype`, `ctype` or extra engine keywords raises a `ValueError` (@atrabattoni). - The miniseed `ctype` argument is now honored: it previously routed to an unused attribute and the reader always built interpolated time coordinates. The default is unchanged (@atrabattoni). -- **Tile-backed virtual arrays.** The new `xdas.tiles` package (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). +- **Tile-backed virtual arrays.** The new `xdas.tiles` module (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). - Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). - **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines. Saved tile views are directly readable by the 0.3 line. Custom engines add tile support by implementing the `load_tile(path, selection, **params)` static half of `xdas.io.Engine` (@atrabattoni). - **Febus now defaults to `tiles`.** A tile view describes a Febus file as a single tile — the overlap trimming lives in the reader — whereas the HDF5 backing needs one mapping per block, so its manifest grew with the block count as well as the file count. Every other HDF5 engine still defaults to `hdf5` (@atrabattoni). diff --git a/tests/tiles/test_tilearray.py b/tests/test_tiles.py similarity index 81% rename from tests/tiles/test_tilearray.py rename to tests/test_tiles.py index 29c486fb..e39ae208 100644 --- a/tests/tiles/test_tilearray.py +++ b/tests/test_tiles.py @@ -1,19 +1,128 @@ +"""The tile-backed virtual array and its integration in the DataArray and native format.""" + import math import os +import dask.array as da_ import h5py import numpy as np import numpy.testing as npt import pytest +import xdas as xd from xdas.io import Engine from xdas.tiles import TileArray NX = 5 +DIMS = ("time", "distance") + ENGINE = {"name": "h5py", "dataset": "data"} +class H5pyEngine(Engine, name="h5py"): + """Read any HDF5 dataset — the engine of the synthetic test files. + + The format engines each read their own layout; test files belong to + no format, so they are described by this generic load-only engine + (its opening half stays abstract). Extra leading selection axes + (virtually expanded arrays) pad the output rank, as the production + engines do. + """ + + @staticmethod + def load_tile(path, selection, *, dataset): + with h5py.File(path, "r") as file: + source = file[dataset] + extra = len(selection) - source.ndim + data = source[selection[extra:]] + return data.reshape((1,) * extra + data.shape) + + +@pytest.fixture +def stack(tmp_path): + """Three gzip-compressed HDF5 files with junk edge rows to trim. + + Emulates overlap trimming: each file carries one junk row at its start + and end that the tile's start row (plus the row ``size``) cuts out. + Returns the manifest and the expected stacked values. + """ + paths = [] + sizes = [] + parts = [] + row = 0 + for k, raw_nt in enumerate([12, 9, 14]): + path = str(tmp_path / f"src{k}.h5") + useful = raw_nt - 2 + data = np.full((raw_nt, NX), -999.0) + data[1:-1] = (row + np.arange(useful))[:, None] + np.arange(NX) / 10 + with h5py.File(path, "w") as file: + file.create_dataset("data", data=data, chunks=(4, NX), compression="gzip") + paths.append(path) + sizes.append(useful) + parts.append(data[1:-1]) + row += useful + manifest = TileArray.from_tiles( + paths, (sizes, NX), "float64", ENGINE, attrs={"units": "strain"} + ) + # per-tile source origins are view state: assigned through the manifest + manifest = TileArray( + manifest.dataset.assign(starts_0=("tile_0", np.array([1, 1, 1]))), + manifest.dtype, + manifest.engine, + ) + return manifest, np.concatenate(parts) + + +@pytest.fixture +def windowed(tmp_path): + """Three files whose rows contribute blob-local windows via ``starts_0``. + + Each file holds junk rows around the useful window; the manifest + exposes blob rows ``[start, start + size)``. The middle file has a + zero start (window at the top of the blob). + """ + paths, sizes, starts, parts = [], [], [], [] + row = 0 + for k, raw_nt in enumerate([12, 9, 14]): + path = str(tmp_path / f"win{k}.h5") + useful = raw_nt - 4 + first = 0 if k == 1 else 2 + data = np.full((raw_nt, NX), -999.0) + good = (row + np.arange(useful))[:, None] + np.arange(NX) / 10 + data[first : first + useful] = good + with h5py.File(path, "w") as file: + file.create_dataset("data", data=data) + paths.append(path) + sizes.append(useful) + starts.append(first) + parts.append(good) + row += useful + manifest = TileArray.from_tiles( + paths, (sizes, NX), "float64", {"name": "h5py", "dataset": "data"} + ) + manifest = TileArray( + manifest.dataset.assign(starts_0=("tile_0", np.array(starts))), + manifest.dtype, + manifest.engine, + ) + return manifest, np.concatenate(parts) + + +@pytest.fixture +def engine_calls(monkeypatch): + """Record the path of every h5py engine read, delegating to the real one.""" + calls = [] + original = Engine["h5py"].load_tile + + def counting(path, selection, **params): + calls.append(path) + return original(path, selection, **params) + + monkeypatch.setattr(Engine["h5py"], "load_tile", counting) + return calls + + def _tile_file(path, data, **kwargs): """Write *data* to an HDF5 file at *path*.""" with h5py.File(path, "w") as file: @@ -820,7 +929,7 @@ def test_boolean_masks(self, stack): manifest, reference = stack mask = reference[:, 0] > 10 npt.assert_array_equal(manifest[mask], reference[mask]) - from xdas.tiles.tilearray import _bounding_key + from xdas.tiles import _bounding_key with pytest.raises(NotImplementedError, match="boolean mask"): _bounding_key( @@ -855,7 +964,7 @@ def test_concatenate_fallbacks(self, stack): assert casted.dtype == np.float32 out = np.concatenate([manifest, manifest], 0, None) npt.assert_array_equal(out, np.concatenate([reference, reference])) - from xdas.tiles.tilearray import _concatenate_virtual + from xdas.tiles import _concatenate_virtual assert _concatenate_virtual((), {}) is NotImplemented assert _concatenate_virtual((5,), {}) is NotImplemented @@ -926,3 +1035,126 @@ def test_deepcopy_and_pickle(self, stack): restored = pickle.loads(pickle.dumps(manifest)) assert isinstance(restored, TileArray) npt.assert_array_equal(np.asarray(restored), reference) + + +def wrap(manifest): + """Wrap *manifest* in a DataArray with regular time/distance coordinates.""" + nt, nx = manifest.shape + # ns resolution: the netCDF round trip casts datetimes to M8[ns] + time = xd.Coordinate["interpolated"].from_block( + np.datetime64("2020-01-01T00:00:00", "ns"), + nt, + np.timedelta64(10_000_000, "ns"), + dim="time", + ) + distance = xd.Coordinate["interpolated"].from_block(0.0, nx, 4.0, dim="distance") + return xd.DataArray(manifest, {"time": time, "distance": distance}) + + +class TestDataArray: + def test_data(self, stack): + manifest, _ = stack + da = wrap(manifest) + assert da.data is manifest + assert "TileArray" in repr(da) + + def test_isel_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + view = da.isel(time=slice(9, 13), distance=slice(1, 4)) + assert isinstance(view.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(view.values, reference[9:13, 1:4]) + + def test_sel_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + t0 = da["time"][2].values + t1 = da["time"][20].values + view = da.sel(time=slice(t0, t1)) + assert isinstance(view.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(view.values, reference[2:21]) + + def test_load_materializes(self, stack): + manifest, reference = stack + loaded = wrap(manifest).load() + assert isinstance(loaded.data, np.ndarray) + npt.assert_array_equal(loaded.values, reference) + + def test_concat_along_existing_dim_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + head = da.isel(time=slice(0, 10)) + tail = da.isel(time=slice(10, None)) + out = xd.concat([head, tail], "time") + assert isinstance(out.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(out.values, reference) + + def test_concat_along_new_dim_stays_virtual(self, stack, engine_calls): + manifest, reference = stack + objs = [wrap(manifest), wrap(manifest)] + out = xd.concat(objs, "station") + assert out.dims == ("station", "time", "distance") + assert isinstance(out.data, TileArray) + assert engine_calls == [] + npt.assert_array_equal(out.values, np.stack([reference, reference])) + + def test_mean_streams(self, stack, engine_calls): + manifest, reference = stack + da = wrap(manifest) + npt.assert_allclose(da.mean("time").values, reference.mean(0)) + assert len(engine_calls) > 0 + assert manifest._cache is None + + +class TestPersistence: + def test_round_trip(self, stack, tmp_path): + manifest, reference = stack + da = wrap(manifest) + path = str(tmp_path / "view.nc") + da.to_netcdf(path) + reopened = xd.open_dataarray(path) + assert isinstance(reopened.data, TileArray) + assert reopened.data.equals(manifest) + assert reopened.coords["time"].equals(da.coords["time"]) + npt.assert_array_equal(reopened.values, reference) + + def test_sliced_view_round_trip(self, stack, tmp_path): + manifest, reference = stack + view = wrap(manifest).isel(time=slice(9, 13)) + path = str(tmp_path / "sliced.nc") + view.to_netcdf(path) + reopened = xd.open_dataarray(path) + assert isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, reference[9:13]) + + def test_grouped_round_trip(self, stack, tmp_path): + manifest, reference = stack + da = wrap(manifest) + path = str(tmp_path / "grouped.nc") + da.to_netcdf(path, group="acquisition") + reopened = xd.open_dataarray(path, engine="xdas", group="acquisition") + assert isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, reference) + + def test_eager_save_writes_values(self, stack, tmp_path): + manifest, reference = stack + da = wrap(manifest) + path = str(tmp_path / "eager.nc") + da.to_netcdf(path, virtual=False) + reopened = xd.open_dataarray(path) + assert not isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, reference) + + def test_dask_write_deprecated(self, tmp_path): + import dask + + data = da_.from_delayed(dask.delayed(np.zeros)((4, NX)), (4, NX), np.float64) + da = xd.DataArray(data, dims=DIMS) + path = str(tmp_path / "dask.nc") + with pytest.warns(FutureWarning, match="dask-backed"): + da.to_netcdf(path, virtual=True) + reopened = xd.open_dataarray(path) + npt.assert_array_equal(reopened.values, np.zeros((4, NX))) diff --git a/tests/tiles/conftest.py b/tests/tiles/conftest.py deleted file mode 100644 index 9bd338d8..00000000 --- a/tests/tiles/conftest.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Shared fixtures for the tile-backed virtual array tests.""" - -import h5py -import numpy as np -import pytest - -from xdas.io import Engine -from xdas.tiles import TileArray - -NX = 5 - -ENGINE = {"name": "h5py", "dataset": "data"} - - -class H5pyEngine(Engine, name="h5py"): - """Read any HDF5 dataset — the engine of the synthetic test files. - - The format engines each read their own layout; test files belong to - no format, so they are described by this generic load-only engine - (its opening half stays abstract). Extra leading selection axes - (virtually expanded arrays) pad the output rank, as the production - engines do. - """ - - @staticmethod - def load_tile(path, selection, *, dataset): - with h5py.File(path, "r") as file: - source = file[dataset] - extra = len(selection) - source.ndim - data = source[selection[extra:]] - return data.reshape((1,) * extra + data.shape) - - -@pytest.fixture -def stack(tmp_path): - """Three gzip-compressed HDF5 files with junk edge rows to trim. - - Emulates overlap trimming: each file carries one junk row at its start - and end that the tile's start row (plus the row ``size``) cuts out. - Returns the manifest and the expected stacked values. - """ - paths = [] - sizes = [] - parts = [] - row = 0 - for k, raw_nt in enumerate([12, 9, 14]): - path = str(tmp_path / f"src{k}.h5") - useful = raw_nt - 2 - data = np.full((raw_nt, NX), -999.0) - data[1:-1] = (row + np.arange(useful))[:, None] + np.arange(NX) / 10 - with h5py.File(path, "w") as file: - file.create_dataset("data", data=data, chunks=(4, NX), compression="gzip") - paths.append(path) - sizes.append(useful) - parts.append(data[1:-1]) - row += useful - manifest = TileArray.from_tiles( - paths, (sizes, NX), "float64", ENGINE, attrs={"units": "strain"} - ) - # per-tile source origins are view state: assigned through the manifest - manifest = TileArray( - manifest.dataset.assign(starts_0=("tile_0", np.array([1, 1, 1]))), - manifest.dtype, - manifest.engine, - ) - return manifest, np.concatenate(parts) - - -@pytest.fixture -def windowed(tmp_path): - """Three files whose rows contribute blob-local windows via ``starts_0``. - - Each file holds junk rows around the useful window; the manifest - exposes blob rows ``[start, start + size)``. The middle file has a - zero start (window at the top of the blob). - """ - paths, sizes, starts, parts = [], [], [], [] - row = 0 - for k, raw_nt in enumerate([12, 9, 14]): - path = str(tmp_path / f"win{k}.h5") - useful = raw_nt - 4 - first = 0 if k == 1 else 2 - data = np.full((raw_nt, NX), -999.0) - good = (row + np.arange(useful))[:, None] + np.arange(NX) / 10 - data[first : first + useful] = good - with h5py.File(path, "w") as file: - file.create_dataset("data", data=data) - paths.append(path) - sizes.append(useful) - starts.append(first) - parts.append(good) - row += useful - manifest = TileArray.from_tiles( - paths, (sizes, NX), "float64", {"name": "h5py", "dataset": "data"} - ) - manifest = TileArray( - manifest.dataset.assign(starts_0=("tile_0", np.array(starts))), - manifest.dtype, - manifest.engine, - ) - return manifest, np.concatenate(parts) - - -@pytest.fixture -def engine_calls(monkeypatch): - """Record the path of every h5py engine read, delegating to the real one.""" - calls = [] - original = Engine["h5py"].load_tile - - def counting(path, selection, **params): - calls.append(path) - return original(path, selection, **params) - - monkeypatch.setattr(Engine["h5py"], "load_tile", counting) - return calls diff --git a/tests/tiles/test_integration.py b/tests/tiles/test_integration.py deleted file mode 100644 index 00c799f0..00000000 --- a/tests/tiles/test_integration.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Tile-backed data inside the 0.2 DataArray and the native file format.""" - -import dask.array as da_ -import numpy as np -import numpy.testing as npt -import pytest - -import xdas as xd -from xdas.tiles import TileArray - -NX = 5 - -DIMS = ("time", "distance") - - -def wrap(manifest): - """Wrap *manifest* in a DataArray with regular time/distance coordinates.""" - nt, nx = manifest.shape - # ns resolution: the netCDF round trip casts datetimes to M8[ns] - time = xd.Coordinate["interpolated"].from_block( - np.datetime64("2020-01-01T00:00:00", "ns"), - nt, - np.timedelta64(10_000_000, "ns"), - dim="time", - ) - distance = xd.Coordinate["interpolated"].from_block(0.0, nx, 4.0, dim="distance") - return xd.DataArray(manifest, {"time": time, "distance": distance}) - - -class TestDataArray: - def test_data(self, stack): - manifest, _ = stack - da = wrap(manifest) - assert da.data is manifest - assert "TileArray" in repr(da) - - def test_isel_stays_virtual(self, stack, engine_calls): - manifest, reference = stack - da = wrap(manifest) - view = da.isel(time=slice(9, 13), distance=slice(1, 4)) - assert isinstance(view.data, TileArray) - assert engine_calls == [] - npt.assert_array_equal(view.values, reference[9:13, 1:4]) - - def test_sel_stays_virtual(self, stack, engine_calls): - manifest, reference = stack - da = wrap(manifest) - t0 = da["time"][2].values - t1 = da["time"][20].values - view = da.sel(time=slice(t0, t1)) - assert isinstance(view.data, TileArray) - assert engine_calls == [] - npt.assert_array_equal(view.values, reference[2:21]) - - def test_load_materializes(self, stack): - manifest, reference = stack - loaded = wrap(manifest).load() - assert isinstance(loaded.data, np.ndarray) - npt.assert_array_equal(loaded.values, reference) - - def test_concat_along_existing_dim_stays_virtual(self, stack, engine_calls): - manifest, reference = stack - da = wrap(manifest) - head = da.isel(time=slice(0, 10)) - tail = da.isel(time=slice(10, None)) - out = xd.concat([head, tail], "time") - assert isinstance(out.data, TileArray) - assert engine_calls == [] - npt.assert_array_equal(out.values, reference) - - def test_concat_along_new_dim_stays_virtual(self, stack, engine_calls): - manifest, reference = stack - objs = [wrap(manifest), wrap(manifest)] - out = xd.concat(objs, "station") - assert out.dims == ("station", "time", "distance") - assert isinstance(out.data, TileArray) - assert engine_calls == [] - npt.assert_array_equal(out.values, np.stack([reference, reference])) - - def test_mean_streams(self, stack, engine_calls): - manifest, reference = stack - da = wrap(manifest) - npt.assert_allclose(da.mean("time").values, reference.mean(0)) - assert len(engine_calls) > 0 - assert manifest._cache is None - - -class TestPersistence: - def test_round_trip(self, stack, tmp_path): - manifest, reference = stack - da = wrap(manifest) - path = str(tmp_path / "view.nc") - da.to_netcdf(path) - reopened = xd.open_dataarray(path) - assert isinstance(reopened.data, TileArray) - assert reopened.data.equals(manifest) - assert reopened.coords["time"].equals(da.coords["time"]) - npt.assert_array_equal(reopened.values, reference) - - def test_sliced_view_round_trip(self, stack, tmp_path): - manifest, reference = stack - view = wrap(manifest).isel(time=slice(9, 13)) - path = str(tmp_path / "sliced.nc") - view.to_netcdf(path) - reopened = xd.open_dataarray(path) - assert isinstance(reopened.data, TileArray) - npt.assert_array_equal(reopened.values, reference[9:13]) - - def test_grouped_round_trip(self, stack, tmp_path): - manifest, reference = stack - da = wrap(manifest) - path = str(tmp_path / "grouped.nc") - da.to_netcdf(path, group="acquisition") - reopened = xd.open_dataarray(path, engine="xdas", group="acquisition") - assert isinstance(reopened.data, TileArray) - npt.assert_array_equal(reopened.values, reference) - - def test_eager_save_writes_values(self, stack, tmp_path): - manifest, reference = stack - da = wrap(manifest) - path = str(tmp_path / "eager.nc") - da.to_netcdf(path, virtual=False) - reopened = xd.open_dataarray(path) - assert not isinstance(reopened.data, TileArray) - npt.assert_array_equal(reopened.values, reference) - - def test_dask_write_deprecated(self, tmp_path): - import dask - - data = da_.from_delayed(dask.delayed(np.zeros)((4, NX)), (4, NX), np.float64) - da = xd.DataArray(data, dims=DIMS) - path = str(tmp_path / "dask.nc") - with pytest.warns(FutureWarning, match="dask-backed"): - da.to_netcdf(path, virtual=True) - reopened = xd.open_dataarray(path) - npt.assert_array_equal(reopened.values, np.zeros((4, NX))) diff --git a/xdas/tiles/tilearray.py b/xdas/tiles.py similarity index 99% rename from xdas/tiles/tilearray.py rename to xdas/tiles.py index d4ab06e9..658fd881 100644 --- a/xdas/tiles/tilearray.py +++ b/xdas/tiles.py @@ -295,7 +295,7 @@ def __init__(self, dataset, dtype, engine): if not isinstance(engine, dict) or "name" not in engine: raise ValueError("the engine specification must have a `name` key") # imported here: xdas.io imports this module at package init - from ..io.core import Engine + from .io.core import Engine Engine[engine["name"]] # fail fast on unregistered engines self._engine = engine @@ -661,7 +661,7 @@ def _grid_values(self, name): @functools.cached_property def _engine_impl(self): """The ``(load_tile, spec)`` of the engine specification.""" - from ..io.core import Engine + from .io.core import Engine spec = dict(self.engine) name = spec.pop("name") diff --git a/xdas/tiles/__init__.py b/xdas/tiles/__init__.py deleted file mode 100644 index eaee8af3..00000000 --- a/xdas/tiles/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Lazy tile-backed virtual arrays (ported from the 0.3 line). - -:class:`TileArray` exposes a rectilinear grid of file-backed tiles as -one numpy-like lazy array. Tiles are decoded by the ``load_tile`` half -of the :class:`xdas.io.Engine` format plugins, resolved by manifest -engine name on that registry (``Engine[name]``). This backend replaces -the serialized-dask-graph fallback used by the formats that HDF5 -virtual datasets cannot serve (Silixa TDMS, MiniSEED). -""" - -from .tilearray import TileArray - -__all__ = [ - "TileArray", -] From d2010d695d572cfa2913812680b891541a237825 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 15:27:21 +0200 Subject: [PATCH 26/56] Store tile manifest strings as fixed-width char arrays Variable-length HDF5 strings pay per-object global-heap overhead on disk and decode eagerly and slowly when the manifest is reopened (xarray materializes vlen columns at open regardless of decode flags). Writing string columns as netCDF-standard fixed-width char arrays (encoding dtype S1) instead makes a 1M-tile manifest open 3x faster (629 -> 206 ms), halves its memory (289 -> 144 MB) and shrinks it 36% on disk (96 -> 62 MB), with identical values on reopen. Legacy vlen manifests still open as before. --- tests/io/test_tiles_vtype.py | 53 ++++++++++++++++++++++++++++++++++++ xdas/io/xdas.py | 7 +++-- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index 2ea88409..715bd75a 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -178,6 +178,59 @@ def test_xdas_engine_tiles_vtype(tmp_path): assert result.equals(da) +def test_manifest_strings_stored_as_char_arrays(tmp_path): + """Manifest strings land on disk as fixed-width char arrays, not vlen.""" + da = xd.testing.dummy(shape=(10, 5), step=(1.0, 10.0), dtype=np.float32) + path = str(tmp_path / "da.nc") + da.to_netcdf(path) + tiled = xd.open_dataarray(path, engine="xdas", vtype="tiles") + out = str(tmp_path / "view.nc") + tiled.to_netcdf(out) + with h5py.File(out, "r") as file: + assert file["__tiles__/paths"].dtype == np.dtype("S1") + reopened = xd.open_dataarray(out) + assert isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, da.values) + + +def test_reopened_tile_file_stays_writable(tmp_path): + """Opening loads the manifest and closes the file: it accepts appends.""" + import h5netcdf + + da = xd.testing.dummy(shape=(10, 5), step=(1.0, 10.0), dtype=np.float32) + path = str(tmp_path / "da.nc") + da.to_netcdf(path) + tiled = xd.open_dataarray(path, engine="xdas", vtype="tiles") + out = str(tmp_path / "view.nc") + tiled.to_netcdf(out) + reopened = xd.open_dataarray(out) + with h5netcdf.File(out, "a") as file: + file.attrs["appended"] = 1 + npt.assert_array_equal(reopened.values, da.values) + + +def test_legacy_vlen_manifest_reopens(tmp_path): + """Manifests stored with variable-length strings (pre char-array) reopen.""" + da = xd.testing.dummy(shape=(10, 5), step=(1.0, 10.0), dtype=np.float32) + path = str(tmp_path / "da.nc") + da.to_netcdf(path) + tiled = xd.open_dataarray(path, engine="xdas", vtype="tiles") + out = str(tmp_path / "legacy.nc") + tiled.to_netcdf(out) + # rewrite the manifest group the way the old writer did: vlen strings + manifest = tiled.data.to_dataset() + for name in list(manifest.variables): + manifest[name].encoding.clear() + if manifest[name].dtype == object: + manifest[name] = manifest[name].astype(str) + with h5py.File(out, "a") as file: + del file["__tiles__"] + manifest.to_netcdf(out, mode="a", group="__tiles__", engine="h5netcdf") + reopened = xd.open_dataarray(out) + assert isinstance(reopened.data, TileArray) + npt.assert_array_equal(reopened.values, da.values) + + def test_tiles_datacollection_roundtrip(tmp_path): """Collections of tile-backed arrays reopen as collections, not as errors. diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 8e7cf186..f61ca51c 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -274,8 +274,11 @@ def save_dataarray( manifest = da.data.to_dataset() for name in list(manifest.variables): manifest[name].encoding.clear() - if manifest[name].dtype == object: - manifest[name] = manifest[name].astype(str) + if manifest[name].dtype.kind in "OU": + # fixed-width char arrays: variable-length strings pay + # heap overhead on disk and decode slowly at open + manifest[name] = manifest[name].astype(object) + manifest[name].encoding["dtype"] = "S1" location = TILES_GROUP if group is None else f"{group}/{TILES_GROUP}" manifest.to_netcdf(fname, mode="a", group=location, engine="h5netcdf") From aab8b3d06af76ef7d458b6112ed1b305122990cc Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 15:48:36 +0200 Subject: [PATCH 27/56] Split the common source directory out of tile manifest paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every tile stored its full absolute path, repeating the archive directory once per tile — O(tiles) redundant bytes in memory and in stored manifests, and relocating an archive meant rewriting an N-D variable. from_tiles now factors the deepest directory containing every path (commonpath of the dirnames, so a basename always remains) into a 0-d `root` manifest variable and keeps the per-tile paths root-relative; reads join the two per tile. Equality compares joined paths, so arrays naming the same files are equal however each splits its root. Concatenation rebases the inputs under the deepest directory containing every root, falling back to absolute per-tile paths when none exists (several drives). Manifests without a `root` variable — the previous stored form — reopen and read unchanged. --- docs/release-notes.md | 1 + tests/test_tiles.py | 138 +++++++++++++++++++++++++++++++++++++++--- xdas/tiles.py | 113 ++++++++++++++++++++++++++++++---- 3 files changed, 231 insertions(+), 21 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 4f7d5b10..7b11b5e1 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -10,6 +10,7 @@ - The miniseed `ctype` argument is now honored: it previously routed to an unused attribute and the reader always built interpolated time coordinates. The default is unchanged (@atrabattoni). - **Tile-backed virtual arrays.** The new `xdas.tiles` module (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). - Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). +- **Compact manifest path storage.** Tile manifests split the common directory of their source paths into a single 0-d `root` variable and keep only the root-relative rest per tile, and manifest strings are written as fixed-width char arrays instead of variable-length HDF5 strings: manifests shrink in memory and on disk and open faster, and relocating an archive amounts to editing one stored value. Arrays rooted in different directories still concatenate (the fusion is rebased under the deepest directory containing every root), and manifests written by earlier versions reopen unchanged (@atrabattoni). - **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines. Saved tile views are directly readable by the 0.3 line. Custom engines add tile support by implementing the `load_tile(path, selection, **params)` static half of `xdas.io.Engine` (@atrabattoni). - **Febus now defaults to `tiles`.** A tile view describes a Febus file as a single tile — the overlap trimming lives in the reader — whereas the HDF5 backing needs one mapping per block, so its manifest grew with the block count as well as the file count. Every other HDF5 engine still defaults to `hdf5` (@atrabattoni). - `open_mfdataarray` no longer refuses more than 100 000 paths regardless of backing: the ceiling is now taken from the engine's resolved vtype and is far higher for `tiles`, which does not build one HDF5 mapping per file. The error explains the remaining limit — the scan holds one data array per file in memory until they are combined (@atrabattoni). diff --git a/tests/test_tiles.py b/tests/test_tiles.py index e39ae208..3b827816 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -8,6 +8,7 @@ import numpy as np import numpy.testing as npt import pytest +import xarray as xr import xdas as xd from xdas.io import Engine @@ -208,20 +209,24 @@ def test_reads_across_sources(self, stack): npt.assert_array_equal(np.asarray(manifest[9:13]), reference[9:13]) npt.assert_array_equal(np.asarray(manifest[3:5]), reference[3:5]) - def test_dataset_model(self, stack): + def test_dataset_model(self, stack, tmp_path): manifest, _ = stack dataset = manifest.dataset assert tuple(dataset["sizes_0"].dims) == ("tile_0",) assert tuple(dataset["sizes_1"].dims) == ("tile_1",) # per-file paths vary along tile_0 only: the trailing axis folds assert tuple(dataset["paths"].dims) == ("tile_0",) + # the common directory splits off: 0-d root, root-relative paths + assert dataset["root"].ndim == 0 + assert str(dataset["root"].values[()]) == str(tmp_path) + assert dataset["paths"].values.tolist() == ["src0.h5", "src1.h5", "src2.h5"] npt.assert_array_equal(dataset["starts_0"].values, [1, 1, 1]) # all-default geometry columns are not stored assert "starts_1" not in dataset and "steps_0" not in dataset def test_param_folding(self, stack): manifest, _ = stack - path = str(manifest.dataset["paths"].values[0]) + path = manifest._full_paths().item(0) uniform = TileArray.from_tiles( path, ([10, 10, 10], NX), "float64", ENGINE, record=0, nbytes=80 ) @@ -250,6 +255,12 @@ def test_validation(self, stack): TileArray.from_tiles("a", (5, NX), "f8", ENGINE, sizes_0=[5]) with pytest.raises(ValueError, match="reserved"): TileArray.from_tiles("a", (5, NX), "f8", ENGINE, starts_0=[0]) + with pytest.raises(ValueError, match="reserved"): + TileArray.from_tiles("a", (5, NX), "f8", ENGINE, root=["r"]) + bad_root = manifest.dataset.copy() + bad_root["root"] = (("tile_0",), np.array(["a", "b", "c"], dtype=object)) + with pytest.raises(ValueError, match="0-d"): + TileArray(bad_root, manifest.dtype, manifest.engine) dataset = manifest.dataset.copy() with pytest.raises(ValueError, match="`sizes_0`"): TileArray( @@ -326,7 +337,8 @@ def test_relative_paths_are_anchored(self, tmp_path, monkeypatch): _tile_file(tmp_path / "rel.h5", data) monkeypatch.chdir(tmp_path) manifest = TileArray.from_tiles("rel.h5", (4, NX), "f8", ENGINE) - assert os.path.isabs(manifest._grid_values("paths").item(0)) + assert manifest.root == str(tmp_path) + assert os.path.isabs(manifest._full_paths().item(0)) monkeypatch.chdir(tmp_path.parent) npt.assert_array_equal(np.asarray(manifest), data) @@ -336,7 +348,7 @@ def test_attrs(self, stack): class TestSourcePaths: - """Paths are stored verbatim: an array holds exactly what it was given.""" + """Paths are stored split: a common 0-d root and root-relative values.""" def make(self, path): # the file's first row is skipped: sliced away, as views are made @@ -349,9 +361,17 @@ def stored(self, manifest): def round_trip(self, manifest): return TileArray(manifest.to_dataset(), manifest.dtype, manifest.engine) + def test_root_splits_off(self, tmp_path): + manifest = self.make(tmp_path / "sources" / "f.h5") + assert manifest.root == str(tmp_path / "sources") + assert self.stored(manifest) == ["f.h5"] + def test_paths_round_trip(self, tmp_path): path = tmp_path / "sources" / "f.h5" - assert self.stored(self.round_trip(self.make(path))) == [str(path)] + restored = self.round_trip(self.make(path)) + assert restored.root == str(tmp_path / "sources") + assert self.stored(restored) == ["f.h5"] + assert restored._full_paths().item(0) == str(path) def test_stored_paths_read(self, tmp_path): # the tile's start row skips the first row of the file @@ -361,6 +381,41 @@ def test_stored_paths_read(self, tmp_path): restored = self.round_trip(self.make(tmp_path / "sources" / "f.h5")) npt.assert_array_equal(np.asarray(restored), data[1:]) + def test_rootless_manifest_reads(self, tmp_path): + """Manifests without a `root` (the pre-split stored form) still work.""" + data = np.arange(5 * NX, dtype=" Date: Sun, 2 Aug 2026 23:12:47 +0200 Subject: [PATCH 28/56] Return a plain copy of the wrapped dataset from to_dataset --- xdas/tiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xdas/tiles.py b/xdas/tiles.py index 658fd881..5a35b03c 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -438,7 +438,7 @@ def to_dataset(self): xarray.Dataset The manifest dataset. """ - return xr.Dataset(self.dataset.data_vars, attrs=self.attrs) + return self.dataset.copy() def _geometry(self, kind, default): """Load the eager 1-D ``{kind}_k`` arrays (*default* where absent).""" From d969c44040bb7418cf6d9544efd92f8fced1d6ae Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Sun, 2 Aug 2026 23:21:13 +0200 Subject: [PATCH 29/56] Find the manifest root from the lexicographic path extremes --- xdas/tiles.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/xdas/tiles.py b/xdas/tiles.py index bf48743c..f61a7516 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -137,11 +137,18 @@ def _split_root(paths): (paths spread over several drives). """ try: - root = os.path.commonpath([os.path.dirname(path) for path in paths.flat]) + extremes = [min(paths.flat), max(paths.flat)] except ValueError: return "", paths - # commonpath ends on a component boundary: a plain strip is exact - strip = np.frompyfunc(lambda path: path[len(root) :].lstrip(os.sep), 1, 1) + # the paths are normalized, so they sort by component: the shared + # prefix of the lexicographic extremes -- hence of every path -- cut + # at its last separator is the common directory of them all, found + # without one python-level call per path + cut = os.path.commonprefix(extremes).rfind(os.sep) + if cut < 0: + return "", paths + root = extremes[0][:cut] if cut else os.sep + strip = np.frompyfunc(lambda path: path[cut + 1 :], 1, 1) return root, strip(paths) From 6ed73b592ea875b38d735cbfc272c203a8e1a60c Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 08:29:34 +0200 Subject: [PATCH 30/56] Hold manifest strings as fixed-width bytes in memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The char-array disk encoding fixed the stored form but left the in-memory manifest an object array of str: 8-byte pointers plus a heap-allocated str per tile, python-level comparisons, and a str-to-char re-encoding step at save plus a char-to-str decode at open. Making fixed-width bytes (S kind, filesystem encoding) the canonical dtype of every string variable — normalized once in the TileArray constructor, so scan-time, hand-built and legacy stored manifests all converge — removes the save-site special case entirely (S arrays land on disk as netCDF char arrays natively) and reopens without any decode. Bytes reach str only at the engine boundary, one os.fsdecode per tile read. On a 1M-tile manifest: memory 80 -> 31 MB (-61%), save 0.44 -> 0.11 s (4.0x), open 206 -> 56 ms (3.7x, on top of the 3x the char encoding already bought), manifest fusion 46 -> 37 ms, identical 31 MB file. Scan-time build pays one os.fsencode per path (0.52 -> 0.78 s per million tiles), noise against real scan I/O. --- tests/io/test_tiles_vtype.py | 7 +++- tests/test_tiles.py | 39 ++++++++++++++----- xdas/io/xdas.py | 7 +--- xdas/tiles.py | 75 ++++++++++++++++++++++++++---------- 4 files changed, 92 insertions(+), 36 deletions(-) diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index 715bd75a..11fe68a8 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -190,6 +190,8 @@ def test_manifest_strings_stored_as_char_arrays(tmp_path): assert file["__tiles__/paths"].dtype == np.dtype("S1") reopened = xd.open_dataarray(out) assert isinstance(reopened.data, TileArray) + # and reopen as fixed-width bytes: no per-string decode, no heap + assert reopened.data.dataset["paths"].dtype.kind == "S" npt.assert_array_equal(reopened.values, da.values) @@ -221,13 +223,16 @@ def test_legacy_vlen_manifest_reopens(tmp_path): manifest = tiled.data.to_dataset() for name in list(manifest.variables): manifest[name].encoding.clear() - if manifest[name].dtype == object: + if manifest[name].dtype.kind == "S": manifest[name] = manifest[name].astype(str) with h5py.File(out, "a") as file: del file["__tiles__"] manifest.to_netcdf(out, mode="a", group="__tiles__", engine="h5netcdf") + with h5py.File(out, "r") as file: + assert h5py.check_string_dtype(file["__tiles__/paths"].dtype) is not None reopened = xd.open_dataarray(out) assert isinstance(reopened.data, TileArray) + assert reopened.data.dataset["paths"].dtype.kind == "S" npt.assert_array_equal(reopened.values, da.values) diff --git a/tests/test_tiles.py b/tests/test_tiles.py index 3b827816..cfbad84f 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -218,8 +218,10 @@ def test_dataset_model(self, stack, tmp_path): assert tuple(dataset["paths"].dims) == ("tile_0",) # the common directory splits off: 0-d root, root-relative paths assert dataset["root"].ndim == 0 - assert str(dataset["root"].values[()]) == str(tmp_path) - assert dataset["paths"].values.tolist() == ["src0.h5", "src1.h5", "src2.h5"] + assert os.fsdecode(dataset["root"].values[()]) == str(tmp_path) + # strings are held as fixed-width bytes, not str objects + assert dataset["paths"].dtype.kind == "S" + assert dataset["paths"].values.tolist() == [b"src0.h5", b"src1.h5", b"src2.h5"] npt.assert_array_equal(dataset["starts_0"].values, [1, 1, 1]) # all-default geometry columns are not stored assert "starts_1" not in dataset and "steps_0" not in dataset @@ -281,6 +283,22 @@ def test_extra_variables_are_params(self, stack): ) assert arr._params == ("record",) + def test_string_params_decode_to_str(self, tmp_path): + """Per-tile string parameters store as bytes but reach the engine as str.""" + paths, parts = [], [] + for k in range(2): + path = str(tmp_path / f"named{k}.h5") + data = 100.0 * k + np.arange(3.0 * NX).reshape(3, NX) + with h5py.File(path, "w") as file: + file.create_dataset(f"data{k}", data=data) + paths.append(path) + parts.append(data) + manifest = TileArray.from_tiles( + paths, ([3, 3], NX), "float64", "h5py", dataset=["data0", "data1"] + ) + assert manifest.dataset["dataset"].dtype.kind == "S" + npt.assert_array_equal(np.asarray(manifest), np.concatenate(parts)) + def test_engine_validation(self): with pytest.raises(KeyError, match="no engine registered"): TileArray.from_tiles("a", (5, NX), "f8", {"name": "bogus"}) @@ -364,14 +382,14 @@ def round_trip(self, manifest): def test_root_splits_off(self, tmp_path): manifest = self.make(tmp_path / "sources" / "f.h5") assert manifest.root == str(tmp_path / "sources") - assert self.stored(manifest) == ["f.h5"] + assert self.stored(manifest) == [b"f.h5"] def test_paths_round_trip(self, tmp_path): path = tmp_path / "sources" / "f.h5" restored = self.round_trip(self.make(path)) assert restored.root == str(tmp_path / "sources") - assert self.stored(restored) == ["f.h5"] - assert restored._full_paths().item(0) == str(path) + assert self.stored(restored) == [b"f.h5"] + assert os.fsdecode(restored._full_paths().item(0)) == str(path) def test_stored_paths_read(self, tmp_path): # the tile's start row skips the first row of the file @@ -399,9 +417,12 @@ def test_rootless_manifest_reads(self, tmp_path): def test_no_common_directory_keeps_paths_whole(self): from xdas.tiles import _common_root, _split_root - mixed = np.array(["rel/f.h5", "/abs/g.h5"], dtype=object) + mixed = np.array([b"rel/f.h5", b"/abs/g.h5"], dtype=object) root, kept = _split_root(mixed) - assert root == "" and kept is mixed + assert root == b"" and kept is mixed + empty = np.array([], dtype=object) + root, kept = _split_root(empty) + assert root == b"" and kept is empty assert _common_root(["/a/b", ""]) == "" assert _common_root(["/a/b", "relative"]) == "" @@ -642,8 +663,8 @@ def test_concat_rebases_differing_roots(self, tmp_path): fused = TileArray.concat(manifests) assert fused.root == str(tmp_path) assert fused.dataset["paths"].values.tolist() == [ - os.path.join("a", "p0.h5"), - os.path.join("b", "p1.h5"), + os.fsencode(os.path.join("a", "p0.h5")), + os.fsencode(os.path.join("b", "p1.h5")), ] npt.assert_array_equal(np.asarray(fused), np.concatenate(parts)) diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index f61ca51c..ba06e75e 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -272,13 +272,10 @@ def save_dataarray( # write the tile manifest as a sibling group if virtual and isinstance(da.data, TileArray): manifest = da.data.to_dataset() + # strings are fixed-width bytes by construction and land on disk + # as char arrays; only stale open-time encodings need clearing for name in list(manifest.variables): manifest[name].encoding.clear() - if manifest[name].dtype.kind in "OU": - # fixed-width char arrays: variable-length strings pay - # heap overhead on disk and decode slowly at open - manifest[name] = manifest[name].astype(object) - manifest[name].encoding["dtype"] = "S1" location = TILES_GROUP if group is None else f"{group}/{TILES_GROUP}" manifest.to_netcdf(fname, mode="a", group=location, engine="h5netcdf") diff --git a/xdas/tiles.py b/xdas/tiles.py index 35de448c..d235dc71 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -22,6 +22,14 @@ one shared constant instead of a per-tile repeat. Absent (or empty), the paths are used as stored. +String variables (``paths``, ``root`` and any string parameter) are +held as fixed-width bytes (numpy ``S`` kind, filesystem encoding): one +contiguous block instead of a heap of str objects, comparisons that +vectorize, and a stored form that lands on disk as netCDF char arrays +with no encoding step. The constructor recodes str-valued variables +(hand-built manifests, legacy stored forms) on entry; values decode +back to str only when handed to the engine. + What arrays cannot carry — the ``dtype`` and the ``engine`` specification — lives on the array itself, by value, and travels beside the dataset when the array is persisted (see @@ -121,33 +129,44 @@ def _fold_param(values, counts, dims): and not bool((values == values.take([0], axis=axis)).all()) ] index = tuple(slice(None) if axis in keep else 0 for axis in range(values.ndim)) - # re-wrap: plain indexing of a fully-reduced object array yields the - # bare element, which numpy would re-box as a fixed-width string + # re-wrap: plain indexing of a fully-reduced array yields the bare + # element; np.asarray with the input dtype keeps width and kind return tuple(dims[axis] for axis in keep), np.asarray( values[index], dtype=values.dtype ) +def _as_bytes(values): + """Recode string *values* to a fixed-width bytes (``S``) array. + + Filesystem encoding, element by element; the re-wrap keeps 0-d + inputs 0-d (``frompyfunc`` unboxes them to a bare element). + """ + encoded = np.frompyfunc(os.fsencode, 1, 1)(np.asarray(values, dtype=object)) + return np.asarray(encoded, dtype=object).astype("S") + + def _split_root(paths): - """Split absolute *paths* into their common directory and relative rest. + """Split absolute byte *paths* into their common directory and relative rest. The root is the deepest directory containing every path (dirnames only, so at least a basename always remains in the relative part). - Falls back to ``("", paths)`` when no common directory exists + Falls back to ``(b"", paths)`` when no common directory exists (paths spread over several drives). """ + sep = os.fsencode(os.sep) try: extremes = [min(paths.flat), max(paths.flat)] except ValueError: - return "", paths + return b"", paths # the paths are normalized, so they sort by component: the shared # prefix of the lexicographic extremes -- hence of every path -- cut # at its last separator is the common directory of them all, found # without one python-level call per path - cut = os.path.commonprefix(extremes).rfind(os.sep) + cut = os.path.commonprefix(extremes).rfind(sep) if cut < 0: - return "", paths - root = extremes[0][:cut] if cut else os.sep + return b"", paths + root = extremes[0][:cut] if cut else sep strip = np.frompyfunc(lambda path: path[cut + 1 :], 1, 1) return root, strip(paths) @@ -323,6 +342,15 @@ class TileArray(np.lib.mixins.NDArrayOperatorsMixin): """ def __init__(self, dataset, dtype, engine): + # canonical string dtype is fixed-width bytes: str-valued + # variables (hand-built or legacy stored manifests) recode here + recode = { + name: xr.Variable(dataset[name].dims, _as_bytes(dataset[name].values)) + for name in map(str, dataset.data_vars) + if dataset[name].dtype.kind in "OU" + } + if recode: + dataset = dataset.assign(recode) self.dataset = dataset self._cache = None if isinstance(engine, str): @@ -368,7 +396,7 @@ def __init__(self, dataset, dtype, engine): if "root" in dataset: if tuple(dataset["root"].dims) != (): raise ValueError("`root` must be a 0-d variable") - self.root = str(dataset["root"].values[()]) + self.root = os.fsdecode(dataset["root"].values[()]) else: self.root = "" geometry = { @@ -446,7 +474,9 @@ def from_tiles(cls, paths, sizes, dtype, engine, *, attrs=None, **params): paths = paths.reshape(paths.shape + (1,) * (ndim - paths.ndim)) # reads are lazy and stored views outlive the session: anchor the # paths now, while the scan's working directory still applies - paths = np.frompyfunc(os.path.abspath, 1, 1)(paths) + paths = np.frompyfunc(lambda path: os.path.abspath(os.fsencode(path)), 1, 1)( + paths + ) # the common directory is one shared constant, not a per-tile # repeat: split it off, the stored paths stay root-relative root, paths = _split_root(paths) @@ -463,9 +493,9 @@ def from_tiles(cls, paths, sizes, dtype, engine, *, attrs=None, **params): raise ValueError( f"`paths` shape {paths.shape} does not match the grid {counts}" ) - data["paths"] = _fold_param(paths, counts, dims) + data["paths"] = _fold_param(paths.astype("S"), counts, dims) if root: - data["root"] = ((), np.asarray(root, dtype=object)) + data["root"] = ((), np.asarray(root)) reserved = ( set(data) | {"root"} @@ -684,7 +714,7 @@ def concat(cls, arrays, dim=0): data[f"{kind}_{k}"] = (dims[k], values) root = _common_root([array.root for array in arrays]) if root: - data["root"] = ((), np.asarray(root, dtype=object)) + data["root"] = ((), np.asarray(os.fsencode(root))) for name in ("paths", *first._params): if name == "paths": variables = [array._rebased_paths(root) for array in arrays] @@ -712,11 +742,12 @@ def concat(cls, arrays, dim=0): return cls(dataset, first.dtype, first.engine) def _full_paths(self): - """Return the full source path of every tile, root joined, over the grid.""" + """Return the full source byte path of every tile, root joined, over the grid.""" paths = self._grid_values("paths") if not self.root: return paths - return np.frompyfunc(lambda path: os.path.join(self.root, path), 1, 1)(paths) + root = os.fsencode(self.root) + return np.frompyfunc(lambda path: os.path.join(root, path), 1, 1)(paths) def _rebased_paths(self, root): """Return the ``paths`` variable of this array, rebased on directory *root*. @@ -727,13 +758,14 @@ def _rebased_paths(self, root): variable = self.dataset["paths"].variable if root == self.root: return variable + mine, target = os.fsencode(self.root), os.fsencode(root) def rebase(path): - full = os.path.join(self.root, path) - return os.path.relpath(full, root) if root else full + full = os.path.join(mine, path) + return os.path.relpath(full, target) if root else full values = np.frompyfunc(rebase, 1, 1)(np.asarray(variable.values, dtype=object)) - return xr.Variable(variable.dims, values) + return xr.Variable(variable.dims, _as_bytes(values)) def _grid_values(self, name): """Load parameter *name* and broadcast it over the full tile grid.""" @@ -790,9 +822,10 @@ def _read(self): selection, dest = tuple(selection), tuple(dest) kwargs = dict(spec) for name, values in params.items(): - value = values[index] - kwargs[name] = value.item() if isinstance(value, np.generic) else value - path = os.path.join(self.root, str(paths[index])) + value = values[index].item() + # engines take str: bytes decode at this boundary only + kwargs[name] = os.fsdecode(value) if isinstance(value, bytes) else value + path = os.path.join(self.root, os.fsdecode(paths[index])) part = np.asarray(read(path, selection, **kwargs)) widths = tuple(entry.stop - entry.start for entry in dest) if part.shape != widths or part.dtype != self.dtype: From e631db908b12c6f8ab1ba6704b7c06e854430ef9 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 09:01:36 +0200 Subject: [PATCH 31/56] Vectorize the manifest path handling with np.strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every string operation on the manifest went through np.frompyfunc: an object array boxing each element, one python call per tile, then a re-wrap back to fixed-width bytes. Two of the five were pure numpy work — stripping the root prefix and joining it back — and the third, rebasing paths on a new root, only looked per-tile: concat always rebases onto an ancestor directory, so where the old root sits under the new one is one constant prefix shared by every tile. Those become np.strings.slice/add ufuncs. The two that genuinely need python, os.path.abspath at scan time and encoding object arrays that may mix str and bytes, become plain comprehensions writing S arrays directly, which also drops an astype at the call site. The np.strings ufuncs size their output from the input widths, so the results are trimmed back to their longest element — otherwise the root split would stop paying for itself. Requires numpy >= 2.1 for np.strings.slice. --- pyproject.toml | 2 +- xdas/tiles.py | 63 +++++++++++++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d908894d..6d56f87d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "loky", "msgpack", "numba", - "numpy", + "numpy>=2.1", "obspy", "pandas", "plotly", diff --git a/xdas/tiles.py b/xdas/tiles.py index d235dc71..f238730d 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -76,6 +76,7 @@ import json import math import os +import sys import numpy as np import xarray as xr @@ -139,11 +140,32 @@ def _fold_param(values, counts, dims): def _as_bytes(values): """Recode string *values* to a fixed-width bytes (``S``) array. - Filesystem encoding, element by element; the re-wrap keeps 0-d - inputs 0-d (``frompyfunc`` unboxes them to a bare element). + Filesystem encoding, as :func:`os.fsencode` spells it. """ - encoded = np.frompyfunc(os.fsencode, 1, 1)(np.asarray(values, dtype=object)) - return np.asarray(encoded, dtype=object).astype("S") + values = np.asarray(values) + if values.dtype.kind == "U": + return np.strings.encode( + values, sys.getfilesystemencoding(), sys.getfilesystemencodeerrors() + ) + # object arrays may mix str and bytes, so encode element by element + encoded = [os.fsencode(value) for value in values.flat] + return np.asarray(encoded, dtype="S").reshape(values.shape) + + +def _trim(values): + """Shrink a bytes array to the width of its longest element. + + The ``np.strings`` ufuncs size their output from the input widths, + which overshoots once a common part is added or removed. + """ + width = max(int(np.strings.str_len(values).max(initial=0)), 1) + return values.astype(f"S{width}") if width < values.dtype.itemsize else values + + +def _as_prefix(root): + """Return byte directory *root* as a plain concatenation prefix.""" + sep = os.fsencode(os.sep) + return root if root.endswith(sep) else root + sep def _split_root(paths): @@ -167,8 +189,7 @@ def _split_root(paths): if cut < 0: return b"", paths root = extremes[0][:cut] if cut else sep - strip = np.frompyfunc(lambda path: path[cut + 1 :], 1, 1) - return root, strip(paths) + return root, _trim(np.strings.slice(paths, cut + 1, None)) def _common_root(roots): @@ -474,9 +495,8 @@ def from_tiles(cls, paths, sizes, dtype, engine, *, attrs=None, **params): paths = paths.reshape(paths.shape + (1,) * (ndim - paths.ndim)) # reads are lazy and stored views outlive the session: anchor the # paths now, while the scan's working directory still applies - paths = np.frompyfunc(lambda path: os.path.abspath(os.fsencode(path)), 1, 1)( - paths - ) + anchored = [os.path.abspath(os.fsencode(path)) for path in paths.flat] + paths = np.asarray(anchored, dtype="S").reshape(paths.shape) # the common directory is one shared constant, not a per-tile # repeat: split it off, the stored paths stay root-relative root, paths = _split_root(paths) @@ -493,7 +513,7 @@ def from_tiles(cls, paths, sizes, dtype, engine, *, attrs=None, **params): raise ValueError( f"`paths` shape {paths.shape} does not match the grid {counts}" ) - data["paths"] = _fold_param(paths.astype("S"), counts, dims) + data["paths"] = _fold_param(paths, counts, dims) if root: data["root"] = ((), np.asarray(root)) reserved = ( @@ -746,26 +766,27 @@ def _full_paths(self): paths = self._grid_values("paths") if not self.root: return paths - root = os.fsencode(self.root) - return np.frompyfunc(lambda path: os.path.join(root, path), 1, 1)(paths) + # the stored paths are root-relative, so the join is a prefix + return np.strings.add(_as_prefix(os.fsencode(self.root)), paths) def _rebased_paths(self, root): """Return the ``paths`` variable of this array, rebased on directory *root*. Folded dimensions are preserved: the rebase rewrites the stored - values under the new root, it never broadcasts. + values under the new root, it never broadcasts. *root* must be a + parent of (or equal to) the current one, or empty — the only + cases :meth:`concat` produces — so that where the old root sits + under the new one is a plain prefix, shared by every tile. """ variable = self.dataset["paths"].variable if root == self.root: return variable - mine, target = os.fsencode(self.root), os.fsencode(root) - - def rebase(path): - full = os.path.join(mine, path) - return os.path.relpath(full, target) if root else full - - values = np.frompyfunc(rebase, 1, 1)(np.asarray(variable.values, dtype=object)) - return xr.Variable(variable.dims, _as_bytes(values)) + mine = os.fsencode(self.root) + prefix = os.path.relpath(mine, os.fsencode(root)) if root else mine + prefix = b"" if prefix == b"." else _as_prefix(prefix) + return xr.Variable( + variable.dims, _trim(np.strings.add(prefix, variable.values)) + ) def _grid_values(self, name): """Load parameter *name* and broadcast it over the full tile grid.""" From ac3b02014e55e8dbd2848db5d4f261e26885019e Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 09:12:51 +0200 Subject: [PATCH 32/56] Lean on Variable.set_dims and inline the streaming row loop --- xdas/tiles.py | 53 ++++++++++++++------------------------------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/xdas/tiles.py b/xdas/tiles.py index f238730d..c3c9dd98 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -310,16 +310,6 @@ def _to_si(nbytes): return f"{dividend:.0f}{_UNITS[index]}" -def _row_ranges(edges, shape): - """Return the streaming blocks: one tile row along axis 0, whole elsewhere. - - The tiling is the only blocking a tile array has, and a whole row - bounds the memory a streaming pass holds at once. - """ - rows = [slice(int(lo), int(hi)) for lo, hi in itertools.pairwise(edges)] - return [rows] + [[slice(0, extent)] for extent in shape[1:]] - - class TileArray(np.lib.mixins.NDArrayOperatorsMixin): """A dense rectilinear grid of file-backed tiles as one virtual array. @@ -516,11 +506,9 @@ def from_tiles(cls, paths, sizes, dtype, engine, *, attrs=None, **params): data["paths"] = _fold_param(paths, counts, dims) if root: data["root"] = ((), np.asarray(root)) - reserved = ( - set(data) - | {"root"} - | {f"{kind}_{k}" for kind in ("starts", "steps") for k in range(ndim)} - ) + reserved = {"paths", "root"} | { + f"{kind}_{k}" for kind in ("sizes", "starts", "steps") for k in range(ndim) + } for name, values in params.items(): if name in reserved: raise ValueError(f"parameter name {name!r} is reserved") @@ -751,10 +739,10 @@ def concat(cls, arrays, dim=0): for d in dims if d == dims[axis] or any(d in v.dims for v in variables) ) - parts = [ - _expand(variable, union, array) - for variable, array in zip(variables, arrays) - ] + parts = [] + for variable, array in zip(variables, arrays): + counts = dict(zip(dims, (len(sizes) for sizes in array._sizes))) + parts.append(variable.set_dims({dim: counts[dim] for dim in union})) axis_pos = union.index(dims[axis]) values = np.concatenate([part.values for part in parts], axis=axis_pos) data[name] = xr.Variable(union, values) @@ -790,14 +778,8 @@ def _rebased_paths(self, root): def _grid_values(self, name): """Load parameter *name* and broadcast it over the full tile grid.""" - variable = self.dataset[name].variable - values = np.asarray(variable.values) - counts = tuple(len(sizes) for sizes in self._sizes) - shape = tuple( - count if dim in variable.dims else 1 - for dim, count in zip(self.dims, counts) - ) - return np.broadcast_to(values.reshape(shape), counts) + counts = dict(zip(self.dims, (len(sizes) for sizes in self._sizes))) + return self.dataset[name].variable.set_dims(counts).values @functools.cached_property def _engine_impl(self): @@ -971,7 +953,11 @@ def _reduce_streaming(self, func, args, kwargs): acc = None filled = np.zeros(out_shape, dtype=bool) counts = np.zeros(out_shape) if counting else None - for box in itertools.product(*_row_ranges(self._edges[0], self.shape)): + # stream one tile row at a time: the tiling is the only blocking + # the array has, and a whole row bounds the memory held at once + rest = tuple(slice(0, extent) for extent in self.shape[1:]) + for lo, hi in itertools.pairwise(self._edges[0]): + box = (slice(int(lo), int(hi)), *rest) block = np.asarray(self[box]) partial = np.asarray(block_reduce(block, axis=axes, keepdims=True)) partial = partial.reshape(tuple(block.shape[a] for a in kept)) @@ -1087,17 +1073,6 @@ def _repr_inline_(self, max_width): return summary if len(summary) <= max_width else "TileArray" -def _expand(variable, union, array): - """Broadcast *variable* over the *union* tile dims of *array*.""" - if variable.dims == union: - return variable - counts = {dim: len(sizes) for dim, sizes in zip(array.dims, array._sizes)} - values = np.asarray(variable.values) - shape = tuple(counts[dim] if dim in variable.dims else 1 for dim in union) - full = tuple(counts[dim] for dim in union) - return xr.Variable(union, np.broadcast_to(values.reshape(shape), full)) - - def _concatenate_virtual(args, kwargs): """Fuse tile arrays for ``numpy.concatenate`` when possible.""" if not args: From ccacb8bb18f5220d510cd9a21abc6b1f3d290e96 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 09:13:00 +0200 Subject: [PATCH 33/56] Inline the file-count check and name the depth-scan visitor --- xdas/core/routines.py | 21 ++++++++------------- xdas/io/xdas.py | 11 ++++++----- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 7dfcdce3..8ccd74dc 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -558,18 +558,6 @@ def _resolve_engine(engine, vtype, ctype, engine_kwargs): ) -def _check_file_count(nfiles, vtype): - """Refuse file sets too large for the vtype the engine will end up using.""" - limit = MAX_OPEN_FILES.get(vtype, MAX_OPEN_FILES[None]) - if nfiles > limit: - raise NotImplementedError( - f"cannot open {nfiles} files at once: the limit is {limit} for " - f"vtype {vtype!r} because the scan holds one data array per file in " - "memory until they are combined. Open the files in batches and pass " - "the results to `combine_by_coords`." - ) - - def open_mfdataarray( paths, dim="first", @@ -654,7 +642,14 @@ def open_mfdataarray( if len(paths) == 0: raise FileNotFoundError("no file to open") engine = _resolve_engine(engine, vtype, ctype, engine_kwargs) - _check_file_count(len(paths), engine.vtype) + limit = MAX_OPEN_FILES.get(engine.vtype, MAX_OPEN_FILES[None]) + if len(paths) > limit: + raise NotImplementedError( + f"cannot open {len(paths)} files at once: the limit is {limit} for " + f"vtype {engine.vtype!r} because the scan holds one data array per " + "file in memory until they are combined. Open the files in batches " + "and pass the results to `combine_by_coords`." + ) max_workers = get_workers_count(parallel) objs = [] failures = [] diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index ba06e75e..04d67b1e 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -382,9 +382,10 @@ def _get_depth(group): if not isinstance(group, h5py.Group): raise ValueError("not a group") depths = [0] - group.visit( - lambda name: ( - None if TILES_GROUP in name.split("/") else depths.append(name.count("/")) - ) - ) + + def visit(name): + if TILES_GROUP not in name.split("/"): + depths.append(name.count("/")) + + group.visit(visit) return max(depths) From d17402478af3f5e50f197392f7c18613ee7c74ad Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 11:32:25 +0200 Subject: [PATCH 34/56] Dispatch the geometry-rewriting numpy routines lazily on TileArray --- docs/release-notes.md | 1 + tests/test_tiles.py | 343 +++++++++++++++++++++++++++++++++- xdas/tiles.py | 419 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 729 insertions(+), 34 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 7b11b5e1..73c41614 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -3,6 +3,7 @@ ## 0.2.9 (unreleased) ### New Features +- **Lazy numpy manipulation routines on tile arrays.** The `split` family (`split`, `array_split`, `vsplit`, `hsplit`, `dsplit`), the `stack` family (`stack`, `vstack`, `hstack`, `column_stack`) and `atleast_1d`/`atleast_2d`, plus `roll`, `tile`, `delete`, and `append`/`insert` between tile arrays, now dispatch on `TileArray` as rewrites of the tile geometry and stay lazy. Cases the tile grid cannot express — axis fusion, element repetition, trailing-axis promotion, eager operands — keep materializing as before (@atrabattoni). - **Explicit engine configuration.** The open functions (`open`, `open_dataarray`, `open_mfdataarray`, `open_mfdatatree`) now declare `engine`, `vtype` and `ctype` explicitly, and `engine` accepts a configured `xdas.io.Engine` instance as well as a name. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front; passing them next to the engine name keeps working and now raises a `TypeError` on misspelled or unsupported keywords instead of silently ignoring them (@atrabattoni). ### Breaking Changes diff --git a/tests/test_tiles.py b/tests/test_tiles.py index cfbad84f..aba29ebf 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -971,9 +971,18 @@ def test_negative_axis_method(self, stack): npt.assert_array_equal(np.asarray(expanded), reference[np.newaxis]) def test_dispatch_guards(self, stack): + from xdas.tiles import _expand_dims_virtual + manifest, _ = stack - assert manifest._expand_virtual((np.zeros(3), 0), {}) is NotImplemented - assert manifest._expand_virtual((manifest, 0), {"extra": 1}) is NotImplemented + expand = np.expand_dims + assert ( + _expand_dims_virtual(manifest, expand, (np.zeros(3), 0), {}) + is NotImplemented + ) + assert ( + _expand_dims_virtual(manifest, expand, (manifest, 0), {"extra": 1}) + is NotImplemented + ) class TestGetitem: @@ -1049,6 +1058,331 @@ def test_matches_numpy(self, tmp_path, ndim, seed): npt.assert_array_equal(np.asarray(sliced[key2]), reference[key][key2]) +class TestManipulationRoutines: + """Numpy manipulation routines that rewrite the tile geometry lazily.""" + + @pytest.fixture + def line(self, tmp_path): + """A 1-D two-tile manifest and its values.""" + paths, parts = [], [] + for k in range(2): + path = str(tmp_path / f"line{k}.h5") + data = 10.0 * k + np.arange(7.0) + _tile_file(path, data) + paths.append(path) + parts.append(data) + manifest = TileArray.from_tiles( + paths, ([7, 7],), "float64", {"name": "h5py", "dataset": "data"} + ) + return manifest, np.concatenate(parts) + + def test_split_stays_lazy(self, stack, engine_calls): + manifest, reference = stack + pieces = np.split(manifest, [10, 17]) + assert all(isinstance(piece, TileArray) for piece in pieces) + assert engine_calls == [] + for piece, expected in zip(pieces, np.split(reference, [10, 17])): + npt.assert_array_equal(np.asarray(piece), expected) + + def test_split_sections(self, stack): + manifest, reference = stack + with pytest.raises(ValueError, match="equal division"): + np.split(manifest, 2) + pieces = np.array_split(manifest, 4, axis=1) + assert all(isinstance(piece, TileArray) for piece in pieces) + for piece, expected in zip(pieces, np.array_split(reference, 4, axis=1)): + npt.assert_array_equal(np.asarray(piece), expected) + + def test_split_variants(self, stack, line): + manifest, reference = stack + for got, expected in zip(np.vsplit(manifest, [12]), np.vsplit(reference, [12])): + npt.assert_array_equal(np.asarray(got), expected) + for got, expected in zip(np.hsplit(manifest, [2]), np.hsplit(reference, [2])): + npt.assert_array_equal(np.asarray(got), expected) + with pytest.raises(ValueError, match="3 or more"): + np.dsplit(manifest, 1) + line_manifest, line_reference = line + with pytest.raises(ValueError, match="2 or more"): + np.vsplit(line_manifest, 2) + pieces = np.hsplit(line_manifest, 2) # 1-D hsplit works on axis 0 + assert all(isinstance(piece, TileArray) for piece in pieces) + for piece, expected in zip(pieces, np.hsplit(line_reference, 2)): + npt.assert_array_equal(np.asarray(piece), expected) + + def test_split_empty_pieces_are_plain(self, stack): + manifest, reference = stack + pieces = np.split(manifest, [17, 12]) # unsorted: middle piece empty + assert isinstance(pieces[1], np.ndarray) + for piece, expected in zip(pieces, np.split(reference, [17, 12])): + npt.assert_array_equal(np.asarray(piece), expected) + + def test_roll_stays_lazy(self, stack, engine_calls): + manifest, reference = stack + rolled = np.roll(manifest, 12, axis=0) + assert isinstance(rolled, TileArray) + assert engine_calls == [] + npt.assert_array_equal(np.asarray(rolled), np.roll(reference, 12, axis=0)) + + def test_roll_variants(self, stack): + manifest, reference = stack + for shift, axis in [(-4, 0), (100, 0), ((3, 2), (0, 1)), (2, (0, 1)), (0, 1)]: + rolled = np.roll(manifest, shift, axis=axis) + assert isinstance(rolled, TileArray) + npt.assert_array_equal( + np.asarray(rolled), np.roll(reference, shift, axis=axis) + ) + + def test_roll_flat_materializes(self, line): + manifest, reference = line + rolled = np.roll(manifest, 3) # axis=None rolls the flattened array + assert isinstance(rolled, np.ndarray) + npt.assert_array_equal(rolled, np.roll(reference, 3)) + + def test_tile_stays_lazy(self, stack, engine_calls): + manifest, reference = stack + tiled = np.tile(manifest, (2, 3)) + assert isinstance(tiled, TileArray) + assert engine_calls == [] + npt.assert_array_equal(np.asarray(tiled), np.tile(reference, (2, 3))) + + def test_tile_promotes_rank(self, stack, line): + manifest, reference = stack + tiled = np.tile(manifest, (2, 1, 1)) + assert isinstance(tiled, TileArray) + npt.assert_array_equal(np.asarray(tiled), np.tile(reference, (2, 1, 1))) + line_manifest, line_reference = line + tiled = np.tile(line_manifest, 3) + assert isinstance(tiled, TileArray) + npt.assert_array_equal(np.asarray(tiled), np.tile(line_reference, 3)) + + def test_tile_zero_rep_reads_nothing(self, stack, engine_calls): + manifest, reference = stack + tiled = np.tile(manifest, (0, 2)) + assert isinstance(tiled, np.ndarray) + assert engine_calls == [] + npt.assert_array_equal(tiled, np.tile(reference, (0, 2))) + + def test_delete_lazy_cases(self, stack): + manifest, reference = stack + for obj in [4, -1, slice(5, 20), slice(3, 3), slice(20, 5, -1)]: + deleted = np.delete(manifest, obj, axis=0) + assert isinstance(deleted, TileArray) + npt.assert_array_equal( + np.asarray(deleted), np.delete(reference, obj, axis=0) + ) + + def test_delete_everything(self, stack): + manifest, _ = stack + deleted = np.delete(manifest, slice(None), axis=1) + assert isinstance(deleted, np.ndarray) + assert deleted.shape == (29, 0) + + def test_delete_fallbacks_and_errors(self, stack): + manifest, reference = stack + strided = np.delete(manifest, slice(None, None, 2), axis=0) + assert isinstance(strided, np.ndarray) + npt.assert_array_equal( + strided, np.delete(reference, slice(None, None, 2), axis=0) + ) + listed = np.delete(manifest, [1, 4], axis=0) + npt.assert_array_equal(listed, np.delete(reference, [1, 4], axis=0)) + assert isinstance(np.delete(manifest, 3), np.ndarray) # axis=None flattens + with pytest.raises(IndexError, match="out of bounds"): + np.delete(manifest, 40, axis=0) + + def test_append_insert_stay_lazy(self, stack): + manifest, reference = stack + appended = np.append(manifest, manifest[0:4], axis=0) + assert isinstance(appended, TileArray) + npt.assert_array_equal( + np.asarray(appended), np.append(reference, reference[0:4], axis=0) + ) + for pos in [0, 17, 29, -29]: + inserted = np.insert(manifest, pos, manifest[3:5], axis=0) + assert isinstance(inserted, TileArray) + npt.assert_array_equal( + np.asarray(inserted), np.insert(reference, pos, reference[3:5], axis=0) + ) + + def test_append_insert_fallbacks(self, stack, line): + manifest, reference = stack + eager = np.append(manifest, np.ones((1, NX)), axis=0) + assert isinstance(eager, np.ndarray) + npt.assert_array_equal(eager, np.append(reference, np.ones((1, NX)), axis=0)) + flat = np.append(manifest, manifest) # axis=None flattens 2-D inputs + assert isinstance(flat, np.ndarray) + npt.assert_array_equal(flat, np.append(reference, reference)) + line_manifest, line_reference = line + joined = np.append(line_manifest, line_manifest) # 1-D stays lazy + assert isinstance(joined, TileArray) + npt.assert_array_equal( + np.asarray(joined), np.append(line_reference, line_reference) + ) + scalar = np.insert(manifest, 3, 0.0, axis=0) # eager values broadcast + assert isinstance(scalar, np.ndarray) + npt.assert_array_equal(scalar, np.insert(reference, 3, 0.0, axis=0)) + with pytest.raises(IndexError, match="out of bounds"): + np.insert(manifest, 100, manifest[0:1], axis=0) + + def test_stack(self, stack, engine_calls): + manifest, reference = stack + stacked = np.stack([manifest, manifest]) + assert isinstance(stacked, TileArray) + assert engine_calls == [] + npt.assert_array_equal(np.asarray(stacked), np.stack([reference, reference])) + negative = np.stack([manifest, manifest], axis=-3) + assert isinstance(negative, TileArray) + npt.assert_array_equal( + np.asarray(negative), np.stack([reference, reference], axis=-3) + ) + middle = np.stack([manifest, manifest], axis=1) # non-leading: fallback + assert isinstance(middle, np.ndarray) + npt.assert_array_equal(middle, np.stack([reference, reference], axis=1)) + + def test_stack_wrappers(self, stack): + manifest, reference = stack + piled = np.vstack([manifest, manifest]) + assert isinstance(piled, TileArray) + npt.assert_array_equal(np.asarray(piled), np.vstack([reference, reference])) + wide = np.hstack([manifest, manifest]) + assert isinstance(wide, TileArray) + npt.assert_array_equal(np.asarray(wide), np.hstack([reference, reference])) + cols = np.column_stack([manifest, manifest]) + assert isinstance(cols, TileArray) + npt.assert_array_equal( + np.asarray(cols), np.column_stack([reference, reference]) + ) + deep = np.dstack([manifest, manifest]) # 2-D needs a trailing axis + assert isinstance(deep, np.ndarray) + npt.assert_array_equal(deep, np.dstack([reference, reference])) + + def test_stack_wrappers_1d(self, line): + manifest, reference = line + rows = np.vstack([manifest, manifest]) + assert isinstance(rows, TileArray) + npt.assert_array_equal(np.asarray(rows), np.vstack([reference, reference])) + flat = np.hstack([manifest, manifest]) + assert isinstance(flat, TileArray) + npt.assert_array_equal(np.asarray(flat), np.hstack([reference, reference])) + cols = np.column_stack([manifest, manifest]) # trailing axis: fallback + assert isinstance(cols, np.ndarray) + npt.assert_array_equal(cols, np.column_stack([reference, reference])) + + def test_atleast(self, stack, line): + manifest, reference = stack + assert np.atleast_1d(manifest) is manifest + assert np.atleast_2d(manifest) is manifest + line_manifest, line_reference = line + promoted = np.atleast_2d(line_manifest) + assert isinstance(promoted, TileArray) + npt.assert_array_equal(np.asarray(promoted), np.atleast_2d(line_reference)) + deep = np.atleast_3d(manifest) # trailing axis: fallback + assert isinstance(deep, np.ndarray) + npt.assert_array_equal(deep, np.atleast_3d(reference)) + + def test_mixed_operands_materialize(self, stack): + manifest, reference = stack + result = np.vstack([manifest, np.ones((1, NX))]) + assert isinstance(result, np.ndarray) + npt.assert_array_equal(result, np.vstack([reference, np.ones((1, NX))])) + + def test_3d_variants_stay_lazy(self, stack): + manifest, reference = stack + deep = manifest.expand_dims(0) + reference = reference[np.newaxis] + pieces = np.dsplit(deep, [2]) + assert all(isinstance(piece, TileArray) for piece in pieces) + for piece, expected in zip(pieces, np.dsplit(reference, [2])): + npt.assert_array_equal(np.asarray(piece), expected) + stacked = np.dstack([deep, deep]) + assert isinstance(stacked, TileArray) + npt.assert_array_equal(np.asarray(stacked), np.dstack([reference, reference])) + columns = np.array_split(deep, 2, axis=-1) + assert all(isinstance(piece, TileArray) for piece in columns) + for piece, expected in zip(columns, np.array_split(reference, 2, axis=-1)): + npt.assert_array_equal(np.asarray(piece), expected) + + def test_numpy_only_variants_materialize(self, stack): + """Calls the grid cannot express take the numpy path with equal values.""" + manifest, reference = stack + rolled = np.roll(manifest, 1.5, axis=0) # non-integer shift + assert isinstance(rolled, np.ndarray) + npt.assert_array_equal(rolled, np.roll(reference, 1.5, axis=0)) + multi = np.insert(manifest, [1, 2], manifest[0:2], axis=0) + assert isinstance(multi, np.ndarray) + npt.assert_array_equal( + multi, np.insert(reference, [1, 2], reference[0:2], axis=0) + ) + flat = np.insert(manifest, 3, 7.0) # axis=None flattens + assert isinstance(flat, np.ndarray) + npt.assert_array_equal(flat, np.insert(reference, 3, 7.0)) + casted = np.stack([manifest, manifest], dtype="float32") + assert isinstance(casted, np.ndarray) and casted.dtype == np.float32 + mixed = np.stack([manifest, np.asarray(reference)]) + assert isinstance(mixed, np.ndarray) + piled = np.vstack([manifest, manifest], dtype="float32") + assert isinstance(piled, np.ndarray) and piled.dtype == np.float32 + first, _ = np.atleast_2d(manifest, manifest) + assert isinstance(first, np.ndarray) + npt.assert_array_equal(first, reference) + + def test_error_parity(self, stack, line): + """Inexpressible or invalid calls raise the numpy errors.""" + manifest, _ = stack + line_manifest, _ = line + with pytest.raises(ValueError, match="larger than 0"): + np.split(manifest, 0) + with pytest.raises(TypeError): + np.split(manifest, [2.5]) + with pytest.raises(TypeError): + np.split(manifest, 2, axis="bad") + with pytest.raises(ValueError): + np.roll(manifest, (1, 2, 3), axis=(0, 1)) + with pytest.raises(np.exceptions.AxisError): + np.roll(manifest, 1, axis=5) + with pytest.raises(TypeError): + np.tile(manifest, 1.5) + with pytest.raises(ValueError): + np.tile(manifest, -1) + with pytest.raises(np.exceptions.AxisError): + np.delete(manifest, 1, axis=5) + with pytest.raises(np.exceptions.AxisError): + np.append(manifest, manifest, axis=5) + with pytest.raises(np.exceptions.AxisError): + np.insert(manifest, 1, manifest[0:1], axis=5) + with pytest.raises(TypeError): + np.stack([manifest, manifest], axis=None) + with pytest.raises(ValueError, match="same number of dimensions"): + np.hstack([line_manifest, manifest]) + + def test_lazy_rewrite_guards(self, stack): + """Handlers step aside for calls that are not theirs to rewrite.""" + from xdas import tiles + + manifest, _ = stack + other = np.zeros(3) + assert ( + tiles._split_virtual(manifest, np.split, (other, 2), {}) is NotImplemented + ) + assert tiles._roll_virtual(manifest, np.roll, (manifest,), {}) is NotImplemented + assert tiles._tile_virtual(manifest, np.tile, (other, 2), {}) is NotImplemented + assert ( + tiles._append_virtual(manifest, np.append, (manifest,), {}) + is NotImplemented + ) + assert ( + tiles._insert_virtual( + manifest, np.insert, (manifest, 1, manifest.expand_dims(0)), {"axis": 0} + ) + is NotImplemented + ) + assert tiles._stack_virtual(manifest, np.stack, (5,), {}) is NotImplemented + assert tiles._stack_virtual(manifest, np.stack, (), {}) is NotImplemented + assert ( + tiles._stack_like_virtual(manifest, np.vstack, (5,), {}) is NotImplemented + ) + + class TestNumpyProtocols: """Direct duck-array protocol behavior, without a wrapping DataArray.""" @@ -1108,8 +1442,9 @@ def test_concatenate_fallbacks(self, stack): npt.assert_array_equal(out, np.concatenate([reference, reference])) from xdas.tiles import _concatenate_virtual - assert _concatenate_virtual((), {}) is NotImplemented - assert _concatenate_virtual((5,), {}) is NotImplemented + concat = np.concatenate + assert _concatenate_virtual(manifest, concat, (), {}) is NotImplemented + assert _concatenate_virtual(manifest, concat, (5,), {}) is NotImplemented def test_incompatible_concat_materializes(self, stack): manifest, reference = stack diff --git a/xdas/tiles.py b/xdas/tiles.py index c3c9dd98..bbdf955a 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -60,8 +60,15 @@ :meth:`TileArray.concat` fuses arrays along any axis by concatenating the geometry and the per-tile parameters (O(tiles), the data is never -read). Tile arrays persist inside the native xdas netCDF format: the -wrapped dataset *is* the stored form. +read). On top of slicing and concatenation, the numpy manipulation +routines whose effect is a rewrite of the tile geometry dispatch +lazily too (see ``_LAZY_ROUTINES``): the ``split``, ``stack`` and +``atleast`` families, ``roll``, ``tile``, ``delete``, and +``append``/``insert`` between tile arrays. Each falls back to a +materializing read for the cases the grid cannot express (axis +fusion, element repetition, eager operands). Tile arrays persist +inside the native xdas netCDF format: the wrapped dataset *is* the +stored form. Ported from the 0.3 line (``xdas/virtual/tilearray.py``); the lazy :meth:`TileArray.expand_dims` is a 0.2 extension supporting the legacy @@ -75,6 +82,7 @@ import itertools import json import math +import operator import os import sys @@ -888,24 +896,24 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): def __array_function__(self, func, types, args, kwargs): """Dispatch numpy functions, keeping select operations lazy. - ``numpy.concatenate`` of compatible tile-backed arrays fuses - the tilings and stays virtual; the streaming reductions - (``sum``, ``mean``, ``min``, ``max``, their nan variants, - ``any`` and ``all``) accumulate one tile row at a time with - bounded memory. Everything else materializes and delegates to - numpy. + The manipulation routines in ``_LAZY_ROUTINES`` (the + ``concatenate``/``stack``/``split``/``atleast`` families, + ``expand_dims``, ``roll``, ``tile``, ``delete``, ``append`` + and ``insert``) rewrite the tile geometry and stay virtual + whenever their effect is expressible on the grid; the + streaming reductions (``sum``, ``mean``, ``min``, ``max``, + their nan variants, ``any`` and ``all``) accumulate one tile + row at a time with bounded memory. Everything else — including + any case the lazy rewrites cannot express — materializes and + delegates to numpy. """ if func is np.result_type: args = tuple( value.dtype if isinstance(value, TileArray) else value for value in args ) return np.result_type(*args) - if func is np.concatenate: - result = _concatenate_virtual(args, kwargs) - if result is not NotImplemented: - return result - if func is np.expand_dims: - result = self._expand_virtual(args, kwargs) + if func in _LAZY_ROUTINES: + result = _LAZY_ROUTINES[func](self, func, args, kwargs) if result is not NotImplemented: return result if func in _STREAMING_REDUCTIONS: @@ -1023,18 +1031,6 @@ def expand_dims(self, axis=0): dataset = dataset.assign(sizes_0=(f"{TILE_PREFIX}0", np.ones(1, np.int64))) return type(self)(dataset, self.dtype, self.engine) - def _expand_virtual(self, args, kwargs): - """Dispatch ``numpy.expand_dims``, delegating to :meth:`expand_dims`.""" - kwargs = dict(kwargs) - axis = kwargs.pop("axis", args[1] if len(args) > 1 else None) - if kwargs or len(args) > 2 or args[0] is not self: - return NotImplemented - if not isinstance(axis, (int, np.integer)): - return NotImplemented - if int(axis) not in (0, -self.ndim - 1): - return NotImplemented - return self.expand_dims(0) - def transpose(self, order): """Materialize and transpose to the given axis *order*.""" return np.transpose(np.asarray(self), order) @@ -1073,7 +1069,47 @@ def _repr_inline_(self, max_width): return summary if len(summary) <= max_width else "TileArray" -def _concatenate_virtual(args, kwargs): +def _bind(func, args, kwargs): + """Bind a dispatched call against *func*'s own signature. + + Returns the complete arguments dict (defaults applied), or None + when the call does not fit — the caller then falls back. + """ + try: + bound = inspect.signature(func).bind(*args, **kwargs) + except TypeError: + return None + bound.apply_defaults() + return dict(bound.arguments) + + +def _axis_slice(array, axis, start, stop): + """Return the lazy slice ``[start:stop]`` of *array* along *axis*.""" + key = [slice(None)] * array.ndim + key[axis] = slice(start, stop) + return array[tuple(key)] + + +def _normalize_axis(axis, ndim): + """Return *axis* as a valid non-negative int, or None when it is not one.""" + try: + axis = operator.index(axis) + except TypeError: + return None + if axis < 0: + axis += ndim + return axis if 0 <= axis < ndim else None + + +def _try_concat(arrays, axis): + """Concatenate tile *arrays*, falling back on incompatibility.""" + try: + return TileArray.concat(arrays, dim=axis) + except (ValueError, IndexError): + return NotImplemented + + +def _concatenate_virtual(array, func, args, kwargs): """Fuse tile arrays for ``numpy.concatenate`` when possible.""" if not args: return NotImplemented @@ -1085,12 +1121,335 @@ def _concatenate_virtual(args, kwargs): arrays = list(arrays) except TypeError: return NotImplemented - if axis is None or not all(isinstance(array, TileArray) for array in arrays): + if axis is None or not all(isinstance(entry, TileArray) for entry in arrays): + return NotImplemented + return _try_concat(arrays, axis) + + +def _expand_dims_virtual(array, func, args, kwargs): + """Dispatch ``numpy.expand_dims``, delegating to :meth:`TileArray.expand_dims`.""" + kwargs = dict(kwargs) + axis = kwargs.pop("axis", args[1] if len(args) > 1 else None) + if kwargs or len(args) > 2 or args[0] is not array: + return NotImplemented + if not isinstance(axis, (int, np.integer)): return NotImplemented + if int(axis) not in (0, -array.ndim - 1): + return NotImplemented + return array.expand_dims(0) + + +def _split_virtual(array, func, args, kwargs): + """Split into lazy sub-arrays for the ``numpy.split`` family. + + Every piece is a positive-step slice along one axis, so each comes + out a lazy :class:`TileArray` (empty pieces, which no tile grid can + hold, come out as plain empty arrays, as do pieces of unsorted + index sections — numpy parity). + """ + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["ary"] is not array: + return NotImplemented + sections = arguments["indices_or_sections"] + if func is np.vsplit: + if array.ndim < 2: + raise ValueError("vsplit only works on arrays of 2 or more dimensions") + axis = 0 + elif func is np.hsplit: + axis = 1 if array.ndim > 1 else 0 + elif func is np.dsplit: + if array.ndim < 3: + raise ValueError("dsplit only works on arrays of 3 or more dimensions") + axis = 2 + else: + axis = arguments["axis"] + axis = _normalize_axis(axis, array.ndim) + if axis is None: + return NotImplemented + extent = array.shape[axis] + if isinstance(sections, (int, np.integer)): + count = int(sections) + if count <= 0: + raise ValueError("number sections must be larger than 0.") + if func is np.split and extent % count: + raise ValueError("array split does not result in an equal division") + each, extras = divmod(extent, count) + sizes = [each + 1] * extras + [each] * (count - extras) + points = list(itertools.accumulate([0, *sizes])) + else: + try: + points = [0, *(operator.index(index) for index in sections), extent] + except TypeError: + return NotImplemented + return [ + _axis_slice(array, axis, start, stop) + for start, stop in itertools.pairwise(points) + ] + + +def _roll_virtual(array, func, args, kwargs): + """Roll lazily: a circular shift is two slices concatenated.""" + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["a"] is not array or arguments["axis"] is None: + return NotImplemented + shifts, axes = arguments["shift"], arguments["axis"] + shifts = shifts if isinstance(shifts, tuple) else (shifts,) + axes = axes if isinstance(axes, tuple) else (axes,) + if len(shifts) == 1: + shifts = shifts * len(axes) + if len(axes) == 1: + axes = axes * len(shifts) + if len(shifts) != len(axes): + return NotImplemented + result = array[(slice(None),) * array.ndim] + for shift, axis in zip(shifts, axes): + axis = _normalize_axis(axis, array.ndim) + try: + shift = operator.index(shift) + except TypeError: + axis = None + if axis is None: + return NotImplemented + extent = result.shape[axis] + shift = shift % extent if extent else 0 + if shift == 0: + continue + result = TileArray.concat( + [ + _axis_slice(result, axis, extent - shift, extent), + _axis_slice(result, axis, 0, extent - shift), + ], + dim=axis, + ) + return result + + +def _tile_virtual(array, func, args, kwargs): + """Tile lazily: whole-array repetitions are self-concatenations.""" + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["A"] is not array: + return NotImplemented + reps = arguments["reps"] + if not isinstance(reps, (tuple, list)): + reps = (reps,) try: - return TileArray.concat(arrays, dim=axis) - except (ValueError, IndexError): + reps = tuple(operator.index(rep) for rep in reps) + except TypeError: + return NotImplemented + if any(rep < 0 for rep in reps): + return NotImplemented + result = array[(slice(None),) * array.ndim] + while result.ndim < len(reps): + result = result.expand_dims(0) + reps = (1,) * (result.ndim - len(reps)) + reps + if 0 in reps: + shape = tuple(extent * rep for extent, rep in zip(result.shape, reps)) + return np.empty(shape, array.dtype) + for axis, rep in enumerate(reps): + if rep > 1: + result = TileArray.concat([result] * rep, dim=axis) + return result + + +def _delete_virtual(array, func, args, kwargs): + """Delete lazily when the removed run is contiguous: concat what remains.""" + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["arr"] is not array or arguments["axis"] is None: return NotImplemented + axis = _normalize_axis(arguments["axis"], array.ndim) + if axis is None: + return NotImplemented + obj = arguments["obj"] + extent = array.shape[axis] + if isinstance(obj, slice): + removed = range(*obj.indices(extent)) + if len(removed) == 0: + return array[(slice(None),) * array.ndim] + if abs(removed.step) != 1: + return NotImplemented + lo, hi = min(removed[0], removed[-1]), max(removed[0], removed[-1]) + 1 + else: + try: + index = operator.index(obj) + except TypeError: + return NotImplemented + if index < 0: + index += extent + if not 0 <= index < extent: + raise IndexError( + f"index {obj} is out of bounds for axis {axis} with size {extent}" + ) + lo, hi = index, index + 1 + pieces = [ + _axis_slice(array, axis, start, stop) + for start, stop in ((0, lo), (hi, extent)) + if stop > start + ] + if not pieces: + shape = tuple( + 0 if k == axis else extent for k, extent in enumerate(array.shape) + ) + return np.empty(shape, array.dtype) + if len(pieces) == 1: + return pieces[0] + return _try_concat(pieces, axis) + + +def _append_virtual(array, func, args, kwargs): + """Append lazily when both operands are tile arrays (a concatenation).""" + arguments = _bind(func, args, kwargs) + if arguments is None: + return NotImplemented + arr, values, axis = arguments["arr"], arguments["values"], arguments["axis"] + if not (isinstance(arr, TileArray) and isinstance(values, TileArray)): + return NotImplemented + if axis is None: + if arr.ndim != 1 or values.ndim != 1: + return NotImplemented + axis = 0 + axis = _normalize_axis(axis, arr.ndim) + if axis is None: + return NotImplemented + return _try_concat([arr, values], axis) + + +def _insert_virtual(array, func, args, kwargs): + """Insert lazily when the values are a compatible tile array.""" + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["axis"] is None: + return NotImplemented + arr, values = arguments["arr"], arguments["values"] + if not (isinstance(arr, TileArray) and isinstance(values, TileArray)): + return NotImplemented + if arr.ndim != values.ndim: + return NotImplemented + axis = _normalize_axis(arguments["axis"], arr.ndim) + if axis is None: + return NotImplemented + try: + index = operator.index(arguments["obj"]) + except TypeError: + return NotImplemented + extent = arr.shape[axis] + if index < 0: + index += extent + if not 0 <= index <= extent: + raise IndexError( + f"index {arguments['obj']} is out of bounds for axis {axis} " + f"with size {extent}" + ) + # a tile array is never empty (every tile spans at least one sample), + # so there is always the values piece plus at least one arr piece + pieces = [values] + if index > 0: + pieces.insert(0, _axis_slice(arr, axis, 0, index)) + if index < extent: + pieces.append(_axis_slice(arr, axis, index, extent)) + return _try_concat(pieces, axis) + + +def _stack_virtual(array, func, args, kwargs): + """Stack lazily along a new leading axis (expand then concatenate).""" + arguments = _bind(func, args, kwargs) + if arguments is None: + return NotImplemented + if arguments.get("out") is not None or arguments.get("dtype") is not None: + return NotImplemented + try: + arrays = list(arguments["arrays"]) + except TypeError: + return NotImplemented + if not arrays or not all(isinstance(entry, TileArray) for entry in arrays): + return NotImplemented + try: + axis = operator.index(arguments["axis"]) + except TypeError: + return NotImplemented + if axis not in (0, -arrays[0].ndim - 1): + return NotImplemented + return _try_concat([entry.expand_dims(0) for entry in arrays], 0) + + +def _stack_like_virtual(array, func, args, kwargs): + """Dispatch the ``*stack`` wrappers where leading-only expansion suffices. + + ``dstack`` and ``column_stack`` promote low-rank inputs by + *appending* a unit axis, which the tile grid cannot express yet: + those cases fall back to materializing. + """ + arguments = _bind(func, args, kwargs) + if arguments is None or arguments.get("dtype") is not None: + return NotImplemented + try: + arrays = list(arguments["tup"]) + except (KeyError, TypeError): + return NotImplemented + if not arrays or not all(isinstance(entry, TileArray) for entry in arrays): + return NotImplemented + ndims = {entry.ndim for entry in arrays} + if func is np.vstack: + arrays = [ + entry.expand_dims(0) if entry.ndim == 1 else entry for entry in arrays + ] + axis = 0 + elif func is np.hstack: + if ndims == {1}: + axis = 0 + elif 1 in ndims: + return NotImplemented + else: + axis = 1 + elif func is np.dstack: + if min(ndims) < 3: + return NotImplemented + axis = 2 + else: # column_stack + if min(ndims) < 2: + return NotImplemented + axis = 1 + return _try_concat(arrays, axis) + + +def _atleast_virtual(array, func, args, kwargs): + """``atleast_1d``/``2d``/``3d`` on a single tile array, virtually. + + ``atleast_3d`` on a 1-D or 2-D array would append a trailing unit + axis, which the grid cannot express yet: it falls back. + """ + if kwargs or len(args) != 1 or args[0] is not array: + return NotImplemented + if func is np.atleast_3d and array.ndim < 3: + return NotImplemented + if func is np.atleast_2d and array.ndim == 1: + return array.expand_dims(0) + return array + + +# numpy manipulation routines that stay lazy: each handler rewrites the +# tile geometry when the call is expressible on the grid, and returns +# NotImplemented otherwise (the dispatcher then materializes) +_LAZY_ROUTINES = { + np.concatenate: _concatenate_virtual, + np.expand_dims: _expand_dims_virtual, + np.split: _split_virtual, + np.array_split: _split_virtual, + np.vsplit: _split_virtual, + np.hsplit: _split_virtual, + np.dsplit: _split_virtual, + np.roll: _roll_virtual, + np.tile: _tile_virtual, + np.delete: _delete_virtual, + np.append: _append_virtual, + np.insert: _insert_virtual, + np.stack: _stack_virtual, + np.vstack: _stack_like_virtual, + np.hstack: _stack_like_virtual, + np.dstack: _stack_like_virtual, + np.column_stack: _stack_like_virtual, + np.atleast_1d: _atleast_virtual, + np.atleast_2d: _atleast_virtual, + np.atleast_3d: _atleast_virtual, +} __all__ = [ From 7dfeb51539daedaa0c9371fc7e00da34c2f72567 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 14:16:05 +0200 Subject: [PATCH 35/56] Record an axis map in tile manifests, freeing the engine contract --- docs/api/tiles.md | 2 + docs/release-notes.md | 3 +- docs/user-guide/io/data-formats.md | 6 +- tests/test_tiles.py | 404 +++++++++++++++++++--- xdas/io/core.py | 8 +- xdas/io/miniseed.py | 9 +- xdas/io/silixa.py | 10 +- xdas/tiles.py | 531 +++++++++++++++++++++-------- 8 files changed, 770 insertions(+), 203 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index 09af5682..f8e2bff5 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -38,6 +38,8 @@ Methods TileArray.to_dataset TileArray.concat TileArray.expand_dims + TileArray.squeeze + TileArray.transpose TileArray.equals ``` diff --git a/docs/release-notes.md b/docs/release-notes.md index 73c41614..ddebc737 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -3,7 +3,8 @@ ## 0.2.9 (unreleased) ### New Features -- **Lazy numpy manipulation routines on tile arrays.** The `split` family (`split`, `array_split`, `vsplit`, `hsplit`, `dsplit`), the `stack` family (`stack`, `vstack`, `hstack`, `column_stack`) and `atleast_1d`/`atleast_2d`, plus `roll`, `tile`, `delete`, and `append`/`insert` between tile arrays, now dispatch on `TileArray` as rewrites of the tile geometry and stay lazy. Cases the tile grid cannot express — axis fusion, element repetition, trailing-axis promotion, eager operands — keep materializing as before (@atrabattoni). +- **Tile manifests carry an axis map.** Manifests gain two optional entries — a 1-D `axes` variable (which stored geometry axis each virtual axis presents) and a 0-d `source_ndim` — that make transpose-like operations (`transpose`, `permute_dims`, `matrix_transpose`, `swapaxes`, `moveaxis`), `expand_dims`/`stack`/`np.newaxis` at any position, `squeeze`, and integer indexing all lazy rewrites of the map. Engines now always receive exactly one slice per source axis, in source order — custom `load_tile` implementations no longer need any rank-padding logic — and freshly scanned manifests store neither entry (absent means identity), so existing files are unaffected (@atrabattoni). +- **Lazy numpy manipulation routines on tile arrays.** The `split` family (`split`, `array_split`, `vsplit`, `hsplit`, `dsplit`), the `stack` family (`stack`, `vstack`, `hstack`, `dstack`, `column_stack`) and `atleast_1d`/`atleast_2d`/`atleast_3d`, plus `roll`, `tile`, `delete`, and `append`/`insert` between tile arrays, now dispatch on `TileArray` as rewrites of the tile geometry and stay lazy. Cases the tile grid cannot express — axis fusion, element repetition, eager operands — keep materializing as before (@atrabattoni). - **Explicit engine configuration.** The open functions (`open`, `open_dataarray`, `open_mfdataarray`, `open_mfdatatree`) now declare `engine`, `vtype` and `ctype` explicitly, and `engine` accepts a configured `xdas.io.Engine` instance as well as a name. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front; passing them next to the engine name keeps working and now raises a `TypeError` on misspelled or unsupported keywords instead of silently ignoring them (@atrabattoni). ### Breaking Changes diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index 8cd778bf..e422be04 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -136,8 +136,10 @@ Beside the `hdf5` vtype shown above (an HDF5 virtual source), an engine can offer the `tiles` vtype: `open_dataarray` then backs the data array with a lazy {py:class}`xdas.tiles.TileArray` describing the file, and the engine implements the decoding half as a `load_tile` static method — called once per tile -touched, with one source-local slice per axis and the manifest's engine -specification as keyword arguments, returning exactly the selected sub-box: +touched, with exactly one source-local slice per source axis, in source order +(whatever transposes or inserted axes the tile array presents), and the +manifest's engine specification as keyword arguments, returning exactly the +selected sub-box: ```{code-cell} from xdas.tiles import TileArray diff --git a/tests/test_tiles.py b/tests/test_tiles.py index aba29ebf..10be65fd 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -26,18 +26,15 @@ class H5pyEngine(Engine, name="h5py"): The format engines each read their own layout; test files belong to no format, so they are described by this generic load-only engine - (its opening half stays abstract). Extra leading selection axes - (virtually expanded arrays) pad the output rank, as the production - engines do. + (its opening half stays abstract). The selection always has one + slice per source axis, in source order, whatever the virtual + arrangement. """ @staticmethod def load_tile(path, selection, *, dataset): with h5py.File(path, "r") as file: - source = file[dataset] - extra = len(selection) - source.ndim - data = source[selection[extra:]] - return data.reshape((1,) * extra + data.shape) + return file[dataset][selection] @pytest.fixture @@ -863,11 +860,14 @@ def test_index_errors(self, stack): npt.assert_array_equal(manifest[-1], reference[-1]) npt.assert_array_equal(manifest[[-1, -2]], reference[[-1, -2]]) - def test_new_axis_materializes(self, stack): + def test_new_axis_stays_virtual(self, stack): manifest, reference = stack expanded = manifest[np.newaxis] - assert isinstance(expanded, np.ndarray) - npt.assert_array_equal(expanded, reference[np.newaxis]) + assert isinstance(expanded, TileArray) + npt.assert_array_equal(np.asarray(expanded), reference[np.newaxis]) + mixed = manifest[5:20, None, ::2] + assert isinstance(mixed, TileArray) + npt.assert_array_equal(np.asarray(mixed), reference[5:20, None, ::2]) def test_bad_shapes(self, stack): manifest, _ = stack @@ -922,14 +922,18 @@ def test_repeated_expansion(self, stack): npt.assert_array_equal(np.asarray(expanded), reference[None, None]) def test_expanded_geometry_carries_over(self, stack): + """Expansion appends a synthetic axis: the stored geometry never moves.""" manifest, _ = stack expanded = manifest.expand_dims() npt.assert_array_equal( - expanded.dataset["sizes_1"].values, manifest.dataset["sizes_0"].values + expanded.dataset["sizes_0"].values, manifest.dataset["sizes_0"].values ) npt.assert_array_equal( - expanded.dataset["starts_1"].values, manifest.dataset["starts_0"].values + expanded.dataset["starts_0"].values, manifest.dataset["starts_0"].values ) + npt.assert_array_equal(expanded.dataset["sizes_2"].values, [1]) + npt.assert_array_equal(expanded.dataset["axes"].values, [2, 0, 1]) + assert int(expanded.dataset["source_ndim"].values[()]) == 2 def test_expanded_slicing_folds(self, stack, engine_calls): manifest, reference = stack @@ -947,22 +951,28 @@ def test_expanded_concat_stacks(self, stack): assert fused.shape == (2, *manifest.shape) npt.assert_array_equal(np.asarray(fused), np.stack([reference, reference])) - def test_non_leading_axis_materializes(self, stack): + def test_any_axis_stays_virtual(self, stack): manifest, reference = stack - expanded = np.expand_dims(manifest, 1) - assert isinstance(expanded, np.ndarray) - npt.assert_array_equal(expanded, reference[:, np.newaxis]) + for axis in [1, 2, -1, -2]: + expanded = np.expand_dims(manifest, axis) + assert isinstance(expanded, TileArray) + npt.assert_array_equal( + np.asarray(expanded), np.expand_dims(reference, axis) + ) - def test_tuple_axis_materializes(self, stack): + def test_tuple_axis_stays_virtual(self, stack): manifest, reference = stack - expanded = np.expand_dims(manifest, (0, 1)) - assert isinstance(expanded, np.ndarray) - npt.assert_array_equal(expanded, reference[np.newaxis, np.newaxis]) + for axis in [(0, 1), (0, 3), (3, 1), (-1, 0)]: + expanded = np.expand_dims(manifest, axis) + assert isinstance(expanded, TileArray) + npt.assert_array_equal( + np.asarray(expanded), np.expand_dims(reference, axis) + ) - def test_non_leading_method_axis_raises(self, stack): + def test_out_of_range_method_axis_raises(self, stack): manifest, _ = stack - with pytest.raises(ValueError, match="leading"): - manifest.expand_dims(1) + with pytest.raises(ValueError, match="position"): + manifest.expand_dims(4) def test_negative_axis_method(self, stack): manifest, reference = stack @@ -1058,24 +1068,25 @@ def test_matches_numpy(self, tmp_path, ndim, seed): npt.assert_array_equal(np.asarray(sliced[key2]), reference[key][key2]) +@pytest.fixture +def line(tmp_path): + """A 1-D two-tile manifest and its values.""" + paths, parts = [], [] + for k in range(2): + path = str(tmp_path / f"line{k}.h5") + data = 10.0 * k + np.arange(7.0) + _tile_file(path, data) + paths.append(path) + parts.append(data) + manifest = TileArray.from_tiles( + paths, ([7, 7],), "float64", {"name": "h5py", "dataset": "data"} + ) + return manifest, np.concatenate(parts) + + class TestManipulationRoutines: """Numpy manipulation routines that rewrite the tile geometry lazily.""" - @pytest.fixture - def line(self, tmp_path): - """A 1-D two-tile manifest and its values.""" - paths, parts = [], [] - for k in range(2): - path = str(tmp_path / f"line{k}.h5") - data = 10.0 * k + np.arange(7.0) - _tile_file(path, data) - paths.append(path) - parts.append(data) - manifest = TileArray.from_tiles( - paths, ([7, 7],), "float64", {"name": "h5py", "dataset": "data"} - ) - return manifest, np.concatenate(parts) - def test_split_stays_lazy(self, stack, engine_calls): manifest, reference = stack pieces = np.split(manifest, [10, 17]) @@ -1235,9 +1246,12 @@ def test_stack(self, stack, engine_calls): npt.assert_array_equal( np.asarray(negative), np.stack([reference, reference], axis=-3) ) - middle = np.stack([manifest, manifest], axis=1) # non-leading: fallback - assert isinstance(middle, np.ndarray) - npt.assert_array_equal(middle, np.stack([reference, reference], axis=1)) + for axis in [1, 2, -1]: + middle = np.stack([manifest, manifest], axis=axis) + assert isinstance(middle, TileArray) + npt.assert_array_equal( + np.asarray(middle), np.stack([reference, reference], axis=axis) + ) def test_stack_wrappers(self, stack): manifest, reference = stack @@ -1252,9 +1266,9 @@ def test_stack_wrappers(self, stack): npt.assert_array_equal( np.asarray(cols), np.column_stack([reference, reference]) ) - deep = np.dstack([manifest, manifest]) # 2-D needs a trailing axis - assert isinstance(deep, np.ndarray) - npt.assert_array_equal(deep, np.dstack([reference, reference])) + deep = np.dstack([manifest, manifest]) + assert isinstance(deep, TileArray) + npt.assert_array_equal(np.asarray(deep), np.dstack([reference, reference])) def test_stack_wrappers_1d(self, line): manifest, reference = line @@ -1264,9 +1278,11 @@ def test_stack_wrappers_1d(self, line): flat = np.hstack([manifest, manifest]) assert isinstance(flat, TileArray) npt.assert_array_equal(np.asarray(flat), np.hstack([reference, reference])) - cols = np.column_stack([manifest, manifest]) # trailing axis: fallback - assert isinstance(cols, np.ndarray) - npt.assert_array_equal(cols, np.column_stack([reference, reference])) + cols = np.column_stack([manifest, manifest]) + assert isinstance(cols, TileArray) + npt.assert_array_equal( + np.asarray(cols), np.column_stack([reference, reference]) + ) def test_atleast(self, stack, line): manifest, reference = stack @@ -1276,9 +1292,12 @@ def test_atleast(self, stack, line): promoted = np.atleast_2d(line_manifest) assert isinstance(promoted, TileArray) npt.assert_array_equal(np.asarray(promoted), np.atleast_2d(line_reference)) - deep = np.atleast_3d(manifest) # trailing axis: fallback - assert isinstance(deep, np.ndarray) - npt.assert_array_equal(deep, np.atleast_3d(reference)) + deep = np.atleast_3d(manifest) + assert isinstance(deep, TileArray) + npt.assert_array_equal(np.asarray(deep), np.atleast_3d(reference)) + line_deep = np.atleast_3d(line_manifest) + assert isinstance(line_deep, TileArray) + npt.assert_array_equal(np.asarray(line_deep), np.atleast_3d(line_reference)) def test_mixed_operands_materialize(self, stack): manifest, reference = stack @@ -1383,6 +1402,291 @@ def test_lazy_rewrite_guards(self, stack): ) +class TestAxisMap: + """The axis map: transposes, inserted, hidden and pinned axes stay lazy.""" + + def test_transpose_stays_lazy(self, stack, engine_calls): + manifest, reference = stack + flipped = np.transpose(manifest) + assert isinstance(flipped, TileArray) + assert flipped.shape == reference.T.shape + assert engine_calls == [] + npt.assert_array_equal(np.asarray(flipped), reference.T) + + def test_transpose_variants(self, stack): + manifest, reference = stack + npt.assert_array_equal( + np.asarray(np.swapaxes(manifest, 0, 1)), np.swapaxes(reference, 0, 1) + ) + npt.assert_array_equal( + np.asarray(np.moveaxis(manifest, 0, -1)), np.moveaxis(reference, 0, -1) + ) + npt.assert_array_equal( + np.asarray(np.matrix_transpose(manifest)), np.matrix_transpose(reference) + ) + npt.assert_array_equal( + np.asarray(np.permute_dims(manifest, (1, 0))), reference.T + ) + npt.assert_array_equal(np.asarray(manifest.transpose()), reference.T) + + def test_transpose_composes_with_slicing(self, stack, engine_calls): + manifest, reference = stack + view = np.transpose(manifest)[1:4, 9:20:2] + assert isinstance(view, TileArray) + assert engine_calls == [] + npt.assert_array_equal(np.asarray(view), reference.T[1:4, 9:20:2]) + + def test_transpose_composes_with_concat(self, stack): + manifest, reference = stack + flipped = np.transpose(manifest) + fused = np.concatenate([flipped, flipped], axis=1) + assert isinstance(fused, TileArray) + npt.assert_array_equal( + np.asarray(fused), np.concatenate([reference.T, reference.T], axis=1) + ) + + def test_engine_receives_source_order(self, stack): + """The selection reaches the engine in source order, full rank.""" + manifest, _ = stack + seen = [] + + class SelectionProbe(Engine, name="selection-probe"): + @staticmethod + def load_tile(path, selection, **params): + seen.append(selection) + widths = tuple( + len(range(entry.start, entry.stop, entry.step or 1)) + for entry in selection + ) + return np.zeros(widths) + + try: + probe = TileArray( + manifest.dataset.drop_vars("record", errors="ignore"), + manifest.dtype, + "selection-probe", + ) + np.asarray(np.expand_dims(np.transpose(probe), 1)[:, :, 9:13]) + assert all(len(selection) == 2 for selection in seen) + # the time trim reaches source axis 0 whatever the virtual order + rows = [ + len(range(sel[0].start, sel[0].stop, sel[0].step or 1)) for sel in seen + ] + columns = [ + len(range(sel[1].start, sel[1].stop, sel[1].step or 1)) for sel in seen + ] + assert sum(rows) == 4 and set(columns) == {NX} + finally: + del Engine._registry["selection-probe"] + + def test_integer_indexing_stays_lazy(self, stack, engine_calls): + manifest, reference = stack + column = manifest[:, 2] + assert isinstance(column, TileArray) + assert column.shape == (29,) + assert engine_calls == [] + npt.assert_array_equal(np.asarray(column), reference[:, 2]) + row = manifest[11] + assert isinstance(row, TileArray) + npt.assert_array_equal(np.asarray(row), reference[11]) + npt.assert_array_equal(np.asarray(manifest[-1]), reference[-1]) + # a pinned axis composes with later slicing and reads one tile + npt.assert_array_equal(np.asarray(row[1:4]), reference[11, 1:4]) + + def test_scalar_selection_materializes(self, stack): + manifest, reference = stack + value = manifest[11, 2] + assert not isinstance(value, TileArray) + npt.assert_array_equal(value, reference[11, 2]) + + def test_squeeze(self, stack): + manifest, reference = stack + expanded = np.expand_dims(manifest, 1) + squeezed = np.squeeze(expanded, axis=1) + assert isinstance(squeezed, TileArray) + npt.assert_array_equal(np.asarray(squeezed), reference) + # squeezing a real unit axis hides it: the source is still read + thin = manifest[:, 1:2] + squeezed = np.squeeze(thin) + assert isinstance(squeezed, TileArray) + assert squeezed.shape == (29,) + npt.assert_array_equal(np.asarray(squeezed), reference[:, 1]) + assert np.squeeze(manifest) is manifest # nothing to squeeze + with pytest.raises(ValueError, match="not equal to one"): + np.squeeze(manifest, axis=1) + scalar = manifest[0:1, 0:1].squeeze() + assert isinstance(scalar, np.ndarray) and scalar.shape == () + npt.assert_array_equal(scalar, reference[0, 0]) + + def test_expand_middle_and_concat_along_it(self, stack): + manifest, reference = stack + expanded = np.expand_dims(manifest, 1) + fused = np.concatenate([expanded, expanded], axis=1) + assert isinstance(fused, TileArray) + npt.assert_array_equal( + np.asarray(fused), np.stack([reference, reference], axis=1) + ) + + def test_mapped_round_trip(self, stack, tmp_path): + """A transposed, pinned view survives the native format.""" + manifest, reference = stack + view = np.transpose(manifest)[1:4] + da = wrap(view) + path = str(tmp_path / "mapped.nc") + da.to_netcdf(path) + reopened = xd.open_dataarray(path) + assert isinstance(reopened.data, TileArray) + assert reopened.data.equals(view) + npt.assert_array_equal(reopened.values, reference.T[1:4]) + + def test_expanded_round_trip(self, stack, tmp_path): + manifest, reference = stack + expanded = np.expand_dims(manifest, 1) + da = xd.DataArray(expanded, dims=("time", "extra", "distance")) + path = str(tmp_path / "expanded.nc") + da.to_netcdf(path) + reopened = xd.open_dataarray(path) + assert isinstance(reopened.data, TileArray) + assert reopened.data.equals(expanded) + npt.assert_array_equal(reopened.values, reference[:, np.newaxis]) + + def test_equals_distinguishes_maps(self, stack): + manifest, _ = stack + assert not manifest.equals(np.transpose(manifest)) + assert not manifest.equals(np.expand_dims(manifest, 0)) + assert np.transpose(manifest).equals(np.transpose(manifest)) + + def test_map_validation(self, stack): + manifest, _ = stack + with pytest.raises(ValueError, match="distinct geometry axes"): + TileArray( + manifest.dataset.assign(axes=("axis", np.array([0, 0]))), + manifest.dtype, + manifest.engine, + ) + with pytest.raises(ValueError, match="1-D over its own"): + TileArray( + manifest.dataset.assign(axes=("tile_0", np.array([0, 1, 0]))), + manifest.dtype, + manifest.engine, + ) + with pytest.raises(ValueError, match="between 1 and"): + TileArray( + manifest.dataset.assign(source_ndim=((), np.int64(3))), + manifest.dtype, + manifest.engine, + ) + with pytest.raises(ValueError, match="sizes must be 1"): + TileArray( + manifest.dataset.assign(axes=("axis", np.array([1]))), + manifest.dtype, + manifest.engine, + ) + two = manifest[9:11] # two one-row tiles along the time axis + with pytest.raises(ValueError, match="single tile"): + TileArray( + two.dataset.assign(axes=("axis", np.array([1]))), + two.dtype, + two.engine, + ) + with pytest.raises(ValueError, match="sizes must be 1"): + TileArray( + manifest.dataset.assign(source_ndim=((), np.int64(1))), + manifest.dtype, + manifest.engine, + ) + with pytest.raises(ValueError, match="at least one visible"): + TileArray( + manifest.dataset.assign(axes=("axis", np.array([], dtype=np.int64))), + manifest.dtype, + manifest.engine, + ) + + def test_transpose_method_validation(self, stack): + manifest, _ = stack + with pytest.raises(ValueError, match="permute"): + manifest.transpose((0, 0)) + + def test_streaming_reduction_on_transposed(self, stack): + manifest, reference = stack + flipped = np.transpose(manifest) + npt.assert_allclose(np.mean(flipped, axis=1), reference.T.mean(1)) + npt.assert_allclose(np.sum(flipped), reference.sum()) + + def test_chunks_follow_the_map(self, stack): + manifest, _ = stack + assert np.transpose(manifest).chunks == ( + manifest.chunks[1], + manifest.chunks[0], + ) + assert np.expand_dims(manifest, 1).chunks == ( + manifest.chunks[0], + (1,), + manifest.chunks[1], + ) + + def test_map_error_parity(self, stack, line): + """Inexpressible or invalid map calls raise the numpy errors.""" + manifest, _ = stack + line_manifest, _ = line + with pytest.raises(ValueError): + np.transpose(manifest, 0) # too few axes + with pytest.raises(ValueError): + np.matrix_transpose(line_manifest) # ndim < 2 + with pytest.raises(np.exceptions.AxisError): + np.swapaxes(manifest, 0, 5) + with pytest.raises(np.exceptions.AxisError): + np.moveaxis(manifest, 0, 5) + with pytest.raises(ValueError): + np.moveaxis(manifest, (0, 1), (0,)) # length mismatch + with pytest.raises(ValueError): + np.moveaxis(manifest, (0, 0), (0, 1)) # repeated source + with pytest.raises(TypeError): + np.expand_dims(manifest, "bad") + with pytest.raises(ValueError): + np.expand_dims(manifest, (0, 0)) # repeated position + with pytest.raises(np.exceptions.AxisError): + np.expand_dims(manifest, 5) + with pytest.raises(np.exceptions.AxisError): + np.stack([manifest, manifest], axis=5) + with pytest.raises(TypeError): + np.squeeze(manifest, axis="bad") + with pytest.raises(ValueError, match="no axis"): + manifest.squeeze(axis=5) + with pytest.raises(ValueError, match="0-d"): + TileArray( + manifest.dataset.assign(source_ndim=("axis", np.array([2]))), + manifest.dtype, + manifest.engine, + ) + + def test_map_dispatch_guards(self, stack): + """Handlers step aside for calls that are not theirs to rewrite.""" + from xdas import tiles + + manifest, _ = stack + other = np.zeros((3, 4)) + assert ( + tiles._transpose_virtual(manifest, np.transpose, (other,), {}) + is NotImplemented + ) + assert ( + tiles._squeeze_virtual(manifest, np.squeeze, (other,), {}) is NotImplemented + ) + + def test_masked_selection_on_bounded_path(self, stack): + """Advanced keys with integers still take the bounded read.""" + manifest, reference = stack + picked = manifest[[3, 7, 20], -1] + assert isinstance(picked, np.ndarray) + npt.assert_array_equal(picked, reference[[3, 7, 20], -1]) + with pytest.raises(IndexError, match="out of bounds"): + manifest[[3, 7], 99] + mask = np.zeros(manifest.shape, dtype=bool) + mask[3, 2] = True + npt.assert_array_equal(manifest[mask], reference[mask]) + + class TestNumpyProtocols: """Direct duck-array protocol behavior, without a wrapping DataArray.""" diff --git a/xdas/io/core.py b/xdas/io/core.py index cb3b122b..5938e086 100644 --- a/xdas/io/core.py +++ b/xdas/io/core.py @@ -124,9 +124,11 @@ def load_tile(path, selection, **kwargs): """Read the selected sub-box of one tile of *path* (abstract). The decode half of the tiles machinery: called on the class by - :class:`~xdas.tiles.TileArray` once per tile touched, with one - source-local, possibly strided :class:`slice` per source axis and - the manifest's engine specification (merged with the per-tile + :class:`~xdas.tiles.TileArray` once per tile touched, with + exactly one source-local, possibly strided :class:`slice` per + source axis, in source order — whatever virtual arrangement + (transposes, inserted axes) the tile array presents — and the + manifest's engine specification (merged with the per-tile variables) as keyword arguments. It must return exactly the selected sub-box of the decoded source as a numpy array, and must depend only on its arguments — never on engine instance state — diff --git a/xdas/io/miniseed.py b/xdas/io/miniseed.py index 26d19fc8..a70bc6d4 100644 --- a/xdas/io/miniseed.py +++ b/xdas/io/miniseed.py @@ -138,14 +138,11 @@ def load_tile(path, selection, *, method="synchronized", ignore_last_sample=Fals """Read a source selection of a MiniSEED file, decoding with ObsPy. Decodes the whole file with ObsPy (as the legacy dask path did) - and crops to *selection*. The decoded rank is padded with unit - leading axes for virtually expanded arrays, or squeezed when a - scalar channel folded an axis away. + and crops to *selection*. The decoded rank is squeezed when a + scalar channel folded an axis out of the scanned shape. """ data = MiniSEEDEngine.read_data(path, method, ignore_last_sample) - if data.ndim < len(selection): - data = data.reshape((1,) * (len(selection) - data.ndim) + data.shape) - elif data.ndim > len(selection): + if data.ndim > len(selection): data = data.reshape(data.shape[data.ndim - len(selection) :]) return data[selection] diff --git a/xdas/io/silixa.py b/xdas/io/silixa.py index 4b38c109..dd9ff82c 100644 --- a/xdas/io/silixa.py +++ b/xdas/io/silixa.py @@ -61,13 +61,9 @@ def load_tile(path, selection): :class:`~xdas.io.tdms.TdmsReader` performs the decoding (``get_data`` bounds are inclusive, hence the ``stop - 1``); the - residual crop applies as numpy views. Leading extra selection - axes come from virtually expanded arrays and pad the output - rank. + residual crop applies as numpy views. """ - extra = len(selection) - 2 - rows = selection[extra] + rows = selection[0] with TdmsReader(path) as tdms: data = tdms.get_data(first_s=rows.start, last_s=rows.stop - 1) - data = data[(slice(None, None, rows.step), *selection[extra + 1 :])] - return data.reshape((1,) * extra + data.shape) + return data[(slice(None, None, rows.step), *selection[1:])] diff --git a/xdas/tiles.py b/xdas/tiles.py index bbdf955a..145db3d6 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -20,7 +20,17 @@ - an optional 0-d ``root`` variable: the common directory of the sources, split off so the per-tile ``paths`` stay root-relative — one shared constant instead of a per-tile repeat. Absent (or empty), - the paths are used as stored. + the paths are used as stored; +- an optional *axis map*: the geometry axes are stored in *source* + order (``sizes_g`` describes source axis ``g``), and a 1-D ``axes`` + variable lists the geometry axis each virtual axis presents, in + virtual order — how transposes and inserted axes stay lazy. Axes + beyond the 0-d ``source_ndim`` are *synthetic* (inserted, one sample + wide, backed by no source data); geometry axes left out of the map + are *hidden* (pinned to a single sample, read but not presented — + how integer indexing stays lazy). Both variables are absent from + freshly scanned manifests: the default is the identity, every + geometry axis a source axis presented in stored order. String variables (``paths``, ``root`` and any string parameter) are held as fixed-width bytes (numpy ``S`` kind, filesystem encoding): one @@ -43,14 +53,17 @@ Geometry loads eagerly at construction (tiny); parameters stay folded — a constant occupies one element whatever the grid size — and broadcast -over the grid only as tiles are read. Positive-step slicing folds into -the geometry and returns a new :class:`TileArray` (as lazy and -self-described as its input); any other indexing reads the bounding box -of the selection and resolves the rest in memory. ``np.asarray`` -materializes: tiles are read one by one by the registered *engine* -(``xdas.io.Engine[name]``), whose ``load_tile`` opens each tile's path -itself and returns the tile's *source selection* (one possibly strided -slice per axis), every part landing directly in the output array. +over the grid only as tiles are read. Positive-step slicing, integer +indexing and ``np.newaxis`` fold into the geometry and the axis map +and return a new :class:`TileArray` (as lazy and self-described as its +input); any other indexing reads the bounding box of the selection and +resolves the rest in memory. ``np.asarray`` materializes: tiles are +read one by one by the registered *engine* (``xdas.io.Engine[name]``), +whose ``load_tile`` opens each tile's path itself and returns the +tile's *source selection* — always one possibly strided slice per +source axis, in source order, whatever the virtual arrangement — every +part landing (permuted through the axis map) directly in the output +array. A tile array is used *raw* as the data of a :class:`xdas.DataArray` (``DataArray(arr, coords)``), so ``da.data`` returns the inspectable @@ -61,18 +74,19 @@ :meth:`TileArray.concat` fuses arrays along any axis by concatenating the geometry and the per-tile parameters (O(tiles), the data is never read). On top of slicing and concatenation, the numpy manipulation -routines whose effect is a rewrite of the tile geometry dispatch -lazily too (see ``_LAZY_ROUTINES``): the ``split``, ``stack`` and -``atleast`` families, ``roll``, ``tile``, ``delete``, and +routines whose effect is a rewrite of the tile geometry or the axis +map dispatch lazily too (see ``_LAZY_ROUTINES``): the ``split``, +``stack``, ``atleast`` and transpose (``transpose``, ``swapaxes``, +``moveaxis``, ``matrix_transpose``) families, ``expand_dims`` and +``squeeze`` at any position, ``roll``, ``tile``, ``delete``, and ``append``/``insert`` between tile arrays. Each falls back to a materializing read for the cases the grid cannot express (axis fusion, element repetition, eager operands). Tile arrays persist inside the native xdas netCDF format: the wrapped dataset *is* the stored form. -Ported from the 0.3 line (``xdas/virtual/tilearray.py``); the lazy -:meth:`TileArray.expand_dims` is a 0.2 extension supporting the legacy -concat-along-a-new-dimension path. +Ported from the 0.3 line (``xdas/virtual/tilearray.py``); the axis map +and the lazy manipulation routines are 0.2 extensions. """ from __future__ import annotations @@ -210,23 +224,45 @@ def _common_root(roots): return "" +def _consumed(entry): + """How many data axes one key *entry* consumes, as numpy counts them. + + ``None`` (a new axis) consumes none; a multi-dimensional boolean + mask consumes one axis per mask dimension; everything else one. + """ + if entry is None: + return 0 + if isinstance(entry, np.ndarray) and entry.dtype == bool: + return entry.ndim + return 1 + + def _normalize_key(key, ndim): """Return *key* as a full-length tuple with ``Ellipsis`` expanded. Accepts the plain keys produced by xarray's indexing adapters and by dask-style block slicing, plus a defensive unwrap of explicit - indexer objects carrying a ``tuple`` attribute. + indexer objects carrying a ``tuple`` attribute. ``None`` entries + (new axes) consume no data axis. """ key = getattr(key, "tuple", key) if not isinstance(key, tuple): key = (key,) if any(entry is Ellipsis for entry in key): index = key.index(Ellipsis) - fill = (slice(None),) * (ndim - len(key) + 1) + consumed = sum(_consumed(entry) for entry in key) - 1 + fill = (slice(None),) * (ndim - consumed) key = key[:index] + fill + key[index + 1 :] - if len(key) > ndim: - raise IndexError(f"too many indices: got {len(key)} for {ndim} axes") - return key + (slice(None),) * (ndim - len(key)) + consumed = sum(_consumed(entry) for entry in key) + if consumed > ndim: + raise IndexError(f"too many indices: got {consumed} for {ndim} axes") + return key + (slice(None),) * (ndim - consumed) + + +def _reinsert_newaxes(key, residual): + """Weave the ``None`` entries of *key* back into the per-axis *residual*.""" + residual = iter(residual) + return tuple(None if entry is None else next(residual) for entry in key) def _bounding_key(key, shape): @@ -387,13 +423,12 @@ def __init__(self, dataset, dtype, engine): self.dtype = np.dtype(dtype) if self.dtype.byteorder == ">": raise ValueError("only little-endian or single-byte dtypes are supported") - ndim = 0 - while f"sizes_{ndim}" in dataset: - ndim += 1 - if ndim == 0: + ngrid = 0 + while f"sizes_{ngrid}" in dataset: + ngrid += 1 + if ngrid == 0: raise ValueError("a tile array needs a `sizes_0` geometry variable") - self.ndim = ndim - self.dims = dims = tuple(f"{TILE_PREFIX}{k}" for k in range(ndim)) + self.dims = dims = tuple(f"{TILE_PREFIX}{g}" for g in range(ngrid)) self._sizes = self._geometry("sizes", None) self._starts = self._geometry("starts", 0) self._steps = self._geometry("steps", 1) @@ -402,14 +437,49 @@ def __init__(self, dataset, dtype, engine): ("starts", self._starts, 0), ("steps", self._steps, 1), ): - for k, values in enumerate(arrays): + for g, values in enumerate(arrays): if np.any(values < bound): kind_bound = "non-negative" if bound == 0 else "strictly positive" - raise ValueError(f"`{kind}_{k}` must be {kind_bound}") + raise ValueError(f"`{kind}_{g}` must be {kind_bound}") self._edges = tuple( np.concatenate(([0], np.cumsum(sizes))) for sizes in self._sizes ) - self.shape = tuple(int(edges[-1]) for edges in self._edges) + # the axis map: which geometry axis each virtual axis presents. + # absent variables mean the identity — every geometry axis is a + # source axis, presented in stored (= source) order + if "source_ndim" in dataset: + if tuple(dataset["source_ndim"].dims) != (): + raise ValueError("`source_ndim` must be a 0-d variable") + self._source_ndim = int(dataset["source_ndim"].values[()]) + else: + self._source_ndim = ngrid + if not 0 < self._source_ndim <= ngrid: + raise ValueError("`source_ndim` must be between 1 and the geometry rank") + if "axes" in dataset: + if len(dataset["axes"].dims) != 1 or dataset["axes"].dims[0] in dims: + raise ValueError("`axes` must be 1-D over its own dimension") + self._axes = tuple(int(g) for g in dataset["axes"].values) + else: + self._axes = tuple(range(ngrid)) + if not self._axes: + raise ValueError("a tile array needs at least one visible axis") + if len(set(self._axes)) != len(self._axes) or not all( + 0 <= g < ngrid for g in self._axes + ): + raise ValueError(f"`axes` must name distinct geometry axes below {ngrid}") + # synthetic axes (beyond the source rank) and hidden axes (absent + # from the map) have no extent to offer: their tiles are one + # sample wide, and a hidden axis holds a single tile — several + # would write the same destination + for g in range(ngrid): + if (g >= self._source_ndim or g not in self._axes) and not bool( + (self._sizes[g] == 1).all() + ): + raise ValueError(f"axis {g} is synthetic or hidden: sizes must be 1") + if g not in self._axes and len(self._sizes[g]) != 1: + raise ValueError(f"hidden axis {g} must hold a single tile") + self.ndim = len(self._axes) + self.shape = tuple(int(self._edges[g][-1]) for g in self._axes) if "paths" not in dataset: raise ValueError("a tile array needs a `paths` variable") if "root" in dataset: @@ -419,13 +489,14 @@ def __init__(self, dataset, dtype, engine): else: self.root = "" geometry = { - f"{kind}_{k}" for kind in ("sizes", "starts", "steps") for k in range(ndim) + f"{kind}_{g}" for kind in ("sizes", "starts", "steps") for g in range(ngrid) } self._params = tuple( sorted( name for name in map(str, dataset.data_vars) - if name not in geometry and name not in ("paths", "root") + if name not in geometry + and name not in ("paths", "root", "axes", "source_ndim") ) ) for name in ("paths", *self._params): @@ -571,7 +642,24 @@ def chunks(self): Not a hint — the tiling *is* the only blocking the array has. """ - return tuple(tuple(int(size) for size in sizes) for sizes in self._sizes) + return tuple(tuple(int(size) for size in self._sizes[g]) for g in self._axes) + + def _assign_axes(self, dataset, axes, ngrid): + """Set the axis map of *dataset* to *axes*, dropping what is derivable. + + The identity parts stay absent: `axes` is stored only when it + differs from the stored geometry order, `source_ndim` only when + synthetic axes exist. The source rank is this array's. + """ + drop = [name for name in ("axes", "source_ndim") if name in dataset] + if drop: + dataset = dataset.drop_vars(drop) + assign = {} + if axes != tuple(range(ngrid)): + assign["axes"] = ("axis", np.asarray(axes, dtype=np.int64)) + if self._source_ndim != ngrid: + assign["source_ndim"] = ((), np.asarray(self._source_ndim, dtype=np.int64)) + return dataset.assign(assign) if assign else dataset @property def ntiles(self): @@ -596,10 +684,9 @@ def __getitem__(self, key): strides over entirely are dropped. The parameters are sliced through the wrapped dataset, so a lazy array stays lazy. - Every other key (integers, index arrays, boolean masks, - reversed slices, empty selections) reads the bounding box of - the selection and applies the remainder in memory, returning a - numpy array. + Every other key (index arrays, boolean masks, reversed slices, + empty selections) reads the bounding box of the selection and + applies the remainder in memory, returning a numpy array. """ key = _normalize_key(key, self.ndim) try: @@ -607,51 +694,72 @@ def __getitem__(self, key): except _Unfoldable: pass try: - box, residual, empty = _bounding_key(key, self.shape) + box, residual, empty = _bounding_key( + tuple(entry for entry in key if entry is not None), self.shape + ) except NotImplementedError: return np.asarray(self)[key] if empty: # zero-strided: the result is empty, so no value is ever read # and the full shape is never allocated return np.broadcast_to(np.zeros((), self.dtype), self.shape)[key].copy() - return np.asarray(self._fold(box))[residual] + return np.asarray(self._fold(box))[_reinsert_newaxes(key, residual)] def _fold(self, key): - """Fold a full-length tuple of positive-step slices into a new array. + """Fold slices, integers and new axes into a new array, virtually. - Raises :class:`_Unfoldable` for non-foldable entries and for - empty selections (a grid needs at least one tile); + Positive-step slices trim the geometry; an integer pins the + geometry axis at one sample and hides it from the map; ``None`` + inserts a synthetic axis. Raises :class:`_Unfoldable` for other + entries, for empty selections (a grid needs at least one tile) + and for all-integer keys (a scalar is not a tile array); :meth:`__getitem__` then falls back to a bounded read. """ indexers = {} assign = {} - for axis, (entry, extent) in enumerate(zip(key, self.shape)): - if not isinstance(entry, slice) or (entry.step or 1) < 1: + new_axes = [] + entries = (entry for entry in key if entry is not None) + for axis, (entry, extent) in enumerate(zip(entries, self.shape)): + g = self._axes[axis] + if isinstance(entry, (int, np.integer)): + index = int(entry) + if index < 0: + index += extent + if not 0 <= index < extent: + raise IndexError( + f"index {entry} is out of bounds for axis of size {extent}" + ) + lo, hi, s = index, index + 1, 1 + elif isinstance(entry, slice) and (entry.step or 1) >= 1: + lo, hi, s = entry.indices(extent) + if len(range(lo, hi, s)) == 0: + raise _Unfoldable(f"empty selection along axis {axis}") + new_axes.append(g) + else: raise _Unfoldable( - "only positive-step slices can be folded into the tile grid" + "only slices, integers and new axes fold into the tile grid" ) - lo, hi, s = entry.indices(extent) - if len(range(lo, hi, s)) == 0: - raise _Unfoldable(f"empty selection along axis {axis}") if (lo, hi, s) == (0, extent, 1): continue - edges = self._edges[axis] + edges = self._edges[g] i0 = int(np.searchsorted(edges, lo, "right")) - 1 i1 = int(np.searchsorted(edges, hi, "left")) pos = edges[i0:i1] - size = self._sizes[axis][i0:i1] - start = self._starts[axis][i0:i1] - step = self._steps[axis][i0:i1] + size = self._sizes[g][i0:i1] + start = self._starts[g][i0:i1] + step = self._steps[g][i0:i1] # selected positions are lo, lo + s, ...; j0/j1 index the first # and last of them falling inside each tile j0 = np.maximum(0, -((lo - pos) // s)) j1 = (np.minimum(pos + size, hi) - 1 - lo) // s keep = j1 >= j0 - dim = self.dims[axis] + dim = self.dims[g] indexers[dim] = slice(i0, i1) if keep.all() else i0 + np.flatnonzero(keep) - assign[f"sizes_{axis}"] = (dim, (j1 - j0 + 1)[keep]) - assign[f"starts_{axis}"] = (dim, (start + (lo + j0 * s - pos) * step)[keep]) - assign[f"steps_{axis}"] = (dim, (step * s)[keep]) + assign[f"sizes_{g}"] = (dim, (j1 - j0 + 1)[keep]) + assign[f"starts_{g}"] = (dim, (start + (lo + j0 * s - pos) * step)[keep]) + assign[f"steps_{g}"] = (dim, (step * s)[keep]) + if not new_axes: + raise _Unfoldable("an all-integer key selects a scalar, not a grid") # all-default starts/steps columns fold away (they stay derivable) drop = [ name @@ -662,7 +770,19 @@ def _fold(self, key): assign = {name: entry for name, entry in assign.items() if name not in drop} dataset = self.dataset.isel(indexers).assign(assign) dataset = dataset.drop_vars([name for name in drop if name in dataset]) - return type(self)(dataset, self.dtype, self.engine) + new_axes = tuple(new_axes) + if new_axes != self._axes: + dataset = self._assign_axes(dataset, new_axes, len(self.dims)) + result = type(self)(dataset, self.dtype, self.engine) + # np.newaxis entries insert synthetic axes at their output position + position = 0 + for entry in key: + if entry is None: + result = result.expand_dims(position) + position += 1 + elif isinstance(entry, slice): + position += 1 + return result @classmethod def concat(cls, arrays, dim=0): @@ -697,20 +817,24 @@ def concat(cls, arrays, dim=0): if not 0 <= axis < ndim: raise ValueError(f"no axis {dim} in a {ndim}-dimensional tile array") dims = first.dims + ngrid = len(dims) + gaxis = first._axes[axis] for other in arrays[1:]: if ( - other.ndim != ndim + other._axes != first._axes + or other._source_ndim != first._source_ndim + or len(other.dims) != ngrid or other.dtype != first.dtype or other.engine != first.engine or other._params != first._params or any( - k != axis + g != gaxis and not ( - np.array_equal(other._sizes[k], first._sizes[k]) - and np.array_equal(other._starts[k], first._starts[k]) - and np.array_equal(other._steps[k], first._steps[k]) + np.array_equal(other._sizes[g], first._sizes[g]) + and np.array_equal(other._starts[g], first._starts[g]) + and np.array_equal(other._steps[g], first._steps[g]) ) - for k in range(ndim) + for g in range(ngrid) ) ): raise ValueError("can only concatenate compatible tile arrays") @@ -720,14 +844,18 @@ def concat(cls, arrays, dim=0): ("starts", [array._starts for array in arrays], 0), ("steps", [array._steps for array in arrays], 1), ): - for k in range(ndim): - if k == axis: - values = np.concatenate([entries[k] for entries in per_axis]) + for g in range(ngrid): + if g == gaxis: + values = np.concatenate([entries[g] for entries in per_axis]) else: - values = per_axis[0][k] + values = per_axis[0][g] if default is not None and bool((values == default).all()): continue - data[f"{kind}_{k}"] = (dims[k], values) + data[f"{kind}_{g}"] = (dims[g], values) + if first._axes != tuple(range(ngrid)): + data["axes"] = ("axis", np.asarray(first._axes, dtype=np.int64)) + if first._source_ndim != ngrid: + data["source_ndim"] = ((), np.asarray(first._source_ndim, dtype=np.int64)) root = _common_root([array.root for array in arrays]) if root: data["root"] = ((), np.asarray(os.fsencode(root))) @@ -737,7 +865,7 @@ def concat(cls, arrays, dim=0): else: variables = [array.dataset[name].variable for array in arrays] vdims = variables[0].dims - if dims[axis] not in vdims and all(v.dims == vdims for v in variables): + if dims[gaxis] not in vdims and all(v.dims == vdims for v in variables): values = variables[0].values if all(np.array_equal(v.values, values) for v in variables[1:]): data[name] = (vdims, values) @@ -745,13 +873,13 @@ def concat(cls, arrays, dim=0): union = tuple( d for d in dims - if d == dims[axis] or any(d in v.dims for v in variables) + if d == dims[gaxis] or any(d in v.dims for v in variables) ) parts = [] for variable, array in zip(variables, arrays): counts = dict(zip(dims, (len(sizes) for sizes in array._sizes))) parts.append(variable.set_dims({dim: counts[dim] for dim in union})) - axis_pos = union.index(dims[axis]) + axis_pos = union.index(dims[gaxis]) values = np.concatenate([part.values for part in parts], axis=axis_pos) data[name] = xr.Variable(union, values) dataset = xr.Dataset(data, attrs=first.attrs) @@ -816,21 +944,35 @@ def __array__(self, dtype=None, copy=None): return values def _read(self): - """Read every tile into a fresh output array, one engine call each.""" + """Read every tile into a fresh output array, one engine call each. + + The engine always receives one slice per *source* axis, in + source order — whatever the virtual arrangement. Its part comes + back source-ordered: the visible axes transpose into virtual + order, and the one-sample-wide hidden and synthetic axes fold + away in the final reshape. + """ out = np.empty(self.shape, dtype=self.dtype) read, spec = self._engine_impl counts = tuple(len(sizes) for sizes in self._sizes) paths = self._grid_values("paths") params = {name: self._grid_values(name) for name in self._params} + rank = self._source_ndim + order = [g for g in self._axes if g < rank] + order += [g for g in range(rank) if g not in order] for index in np.ndindex(counts): - selection, dest = [], [] - for k, i in enumerate(index): - first = int(self._starts[k][i]) - size = int(self._sizes[k][i]) - step = int(self._steps[k][i]) + selection, widths = [], [] + for g in range(rank): + first = int(self._starts[g][index[g]]) + size = int(self._sizes[g][index[g]]) + step = int(self._steps[g][index[g]]) selection.append(slice(first, first + (size - 1) * step + 1, step)) - dest.append(slice(int(self._edges[k][i]), int(self._edges[k][i + 1]))) - selection, dest = tuple(selection), tuple(dest) + widths.append(size) + selection, widths = tuple(selection), tuple(widths) + dest = tuple( + slice(int(self._edges[g][index[g]]), int(self._edges[g][index[g] + 1])) + for g in self._axes + ) kwargs = dict(spec) for name, values in params.items(): value = values[index].item() @@ -838,14 +980,14 @@ def _read(self): kwargs[name] = os.fsdecode(value) if isinstance(value, bytes) else value path = os.path.join(self.root, os.fsdecode(paths[index])) part = np.asarray(read(path, selection, **kwargs)) - widths = tuple(entry.stop - entry.start for entry in dest) if part.shape != widths or part.dtype != self.dtype: raise ValueError( f"engine {self.engine['name']!r} produced a {part.dtype} " f"part of shape {part.shape} where the array records " f"{self.dtype} parts and the selection has shape {widths}" ) - out[dest] = part + part = np.transpose(part, order) + out[dest] = part.reshape(tuple(entry.stop - entry.start for entry in dest)) return out def equals(self, other): @@ -864,13 +1006,16 @@ def equals(self, other): or self.shape != other.shape or self.attrs != other.attrs or self._params != other._params + or self._axes != other._axes + or self._source_ndim != other._source_ndim + or len(self.dims) != len(other.dims) ): return False - for k in range(self.ndim): + for g in range(len(self.dims)): if not ( - np.array_equal(self._sizes[k], other._sizes[k]) - and np.array_equal(self._starts[k], other._starts[k]) - and np.array_equal(self._steps[k], other._steps[k]) + np.array_equal(self._sizes[g], other._sizes[g]) + and np.array_equal(self._starts[g], other._starts[g]) + and np.array_equal(self._steps[g], other._steps[g]) ): return False for name in self._params: @@ -964,7 +1109,7 @@ def _reduce_streaming(self, func, args, kwargs): # stream one tile row at a time: the tiling is the only blocking # the array has, and a whole row bounds the memory held at once rest = tuple(slice(0, extent) for extent in self.shape[1:]) - for lo, hi in itertools.pairwise(self._edges[0]): + for lo, hi in itertools.pairwise(self._edges[self._axes[0]]): box = (slice(int(lo), int(hi)), *rest) block = np.asarray(self[box]) partial = np.asarray(block_reduce(block, axis=axes, keepdims=True)) @@ -995,45 +1140,93 @@ def _reduce_streaming(self, func, args, kwargs): return result def expand_dims(self, axis=0): - """Insert a unit leading axis, staying virtual (0.2 extension). + """Insert a unit axis at position *axis*, staying virtual. - The legacy concat-along-a-new-dimension path - (:meth:`xdas.DataArray.expand_dims` then :func:`xdas.concat`) - expands the data with :func:`numpy.expand_dims`; this keeps - that path lazy instead of materializing. Only the leading - position is supported: the new axis holds one tile of size - one, and the engine ``load_tile`` receives one extra leading - ``slice(0, 1)`` per expanded axis, padding its output rank - accordingly (see the silixa and miniseed engines). + A new synthetic geometry axis (one tile, one sample wide) is + appended to the stored grid and mapped into the virtual order + at *axis*; the sources — and the engine calls — are untouched. Parameters ---------- axis : int, optional - Position of the new axis; only ``0`` (equivalently - ``-ndim - 1``) is supported. + Position of the new axis in the result. Default 0. Returns ------- TileArray """ axis = int(axis) - if axis == -self.ndim - 1: - axis = 0 - if axis != 0: - raise ValueError("only a leading axis can be virtually expanded") - rename = {} - for k in range(self.ndim - 1, -1, -1): - rename[f"{TILE_PREFIX}{k}"] = f"{TILE_PREFIX}{k + 1}" - for kind in ("sizes", "starts", "steps"): - if f"{kind}_{k}" in self.dataset: - rename[f"{kind}_{k}"] = f"{kind}_{k + 1}" - dataset = self.dataset.rename(rename) - dataset = dataset.assign(sizes_0=(f"{TILE_PREFIX}0", np.ones(1, np.int64))) + if axis < 0: + axis += self.ndim + 1 + if not 0 <= axis <= self.ndim: + raise ValueError(f"no position {axis} in a {self.ndim}-dimensional array") + ngrid = len(self.dims) + dataset = self.dataset.assign( + {f"sizes_{ngrid}": (f"{TILE_PREFIX}{ngrid}", np.ones(1, np.int64))} + ) + axes = self._axes[:axis] + (ngrid,) + self._axes[axis:] + dataset = self._assign_axes(dataset, axes, ngrid + 1) + return type(self)(dataset, self.dtype, self.engine) + + def squeeze(self, axis=None): + """Drop unit axes, staying virtual. + + The dropped axes leave the virtual order (real ones stay in the + grid as hidden, one-sample reads); squeezing every axis away + materializes the value as a 0-d array, as a grid needs at least + one visible axis. + + Parameters + ---------- + axis : int or tuple of int, optional + The unit axes to drop; all of them when None (default). + + Returns + ------- + TileArray or numpy.ndarray + """ + if axis is None: + drop = tuple(k for k in range(self.ndim) if self.shape[k] == 1) + else: + axis = axis if isinstance(axis, tuple) else (axis,) + axis = tuple(operator.index(a) for a in axis) + drop = tuple(a + self.ndim if a < 0 else a for a in axis) + for k in drop: + if not 0 <= k < self.ndim: + raise ValueError(f"no axis {k} in a {self.ndim}-dimensional array") + if self.shape[k] != 1: + raise ValueError( + "cannot select an axis to squeeze out which has size " + "not equal to one" + ) + if not drop: + return self + if len(drop) == self.ndim: + return np.asarray(self).reshape(()) + axes = tuple(g for k, g in enumerate(self._axes) if k not in drop) + dataset = self._assign_axes(self.dataset, axes, len(self.dims)) return type(self)(dataset, self.dtype, self.engine) - def transpose(self, order): - """Materialize and transpose to the given axis *order*.""" - return np.transpose(np.asarray(self), order) + def transpose(self, order=None): + """Permute the axes, staying virtual (reversed order by default). + + Parameters + ---------- + order : sequence of int, optional + The new order of the current axes; all of them, each once. + + Returns + ------- + TileArray + """ + if order is None: + order = range(self.ndim - 1, -1, -1) + order = tuple(int(a) + self.ndim if int(a) < 0 else int(a) for a in order) + if sorted(order) != list(range(self.ndim)): + raise ValueError(f"axes {order} do not permute a {self.ndim}-d array") + axes = tuple(self._axes[a] for a in order) + dataset = self._assign_axes(self.dataset, axes, len(self.dims)) + return type(self)(dataset, self.dtype, self.engine) def astype(self, dtype, **kwargs): """Materialize and cast the values to *dtype*.""" @@ -1132,11 +1325,74 @@ def _expand_dims_virtual(array, func, args, kwargs): axis = kwargs.pop("axis", args[1] if len(args) > 1 else None) if kwargs or len(args) > 2 or args[0] is not array: return NotImplemented - if not isinstance(axis, (int, np.integer)): + axis = axis if isinstance(axis, tuple) else (axis,) + if not all(isinstance(entry, (int, np.integer)) for entry in axis): + return NotImplemented + final = array.ndim + len(axis) + positions = tuple(int(entry) + final if entry < 0 else int(entry) for entry in axis) + if len(set(positions)) != len(positions) or not all( + 0 <= position < final for position in positions + ): + return NotImplemented + result = array + # ascending insertion keeps every later position valid in the + # grown array, whatever order the caller listed them in + for position in sorted(positions): + result = result.expand_dims(position) + return result + + +def _transpose_virtual(array, func, args, kwargs): + """Dispatch the transpose-like functions as axis-map permutations.""" + arguments = _bind(func, args, kwargs) + if arguments is None or next(iter(arguments.values())) is not array: + return NotImplemented + ndim = array.ndim + if func is np.transpose: + order = arguments["axes"] + if order is None: + order = range(ndim - 1, -1, -1) + elif isinstance(order, (int, np.integer)): + order = (order,) + elif func is np.matrix_transpose: + if ndim < 2: + return NotImplemented + order = (*range(ndim - 2), ndim - 1, ndim - 2) + elif func is np.swapaxes: + one, two = arguments["axis1"], arguments["axis2"] + one, two = _normalize_axis(one, ndim), _normalize_axis(two, ndim) + if one is None or two is None: + return NotImplemented + order = list(range(ndim)) + order[one], order[two] = order[two], order[one] + else: # moveaxis + source, destination = arguments["source"], arguments["destination"] + source = source if isinstance(source, (tuple, list)) else (source,) + destination = ( + destination if isinstance(destination, (tuple, list)) else (destination,) + ) + source = tuple(_normalize_axis(entry, ndim) for entry in source) + destination = tuple(_normalize_axis(entry, ndim) for entry in destination) + if None in source or None in destination or len(source) != len(destination): + return NotImplemented + order = [a for a in range(ndim) if a not in source] + for position, a in sorted(zip(destination, source)): + order.insert(position, a) + try: + return array.transpose(order) + except (TypeError, ValueError): + return NotImplemented + + +def _squeeze_virtual(array, func, args, kwargs): + """Dispatch ``numpy.squeeze``, delegating to :meth:`TileArray.squeeze`.""" + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["a"] is not array: return NotImplemented - if int(axis) not in (0, -array.ndim - 1): + try: + return array.squeeze(arguments["axis"]) + except TypeError: return NotImplemented - return array.expand_dims(0) def _split_virtual(array, func, args, kwargs): @@ -1349,7 +1605,7 @@ def _insert_virtual(array, func, args, kwargs): def _stack_virtual(array, func, args, kwargs): - """Stack lazily along a new leading axis (expand then concatenate).""" + """Stack lazily along a new axis (expand then concatenate).""" arguments = _bind(func, args, kwargs) if arguments is None: return NotImplemented @@ -1365,18 +1621,15 @@ def _stack_virtual(array, func, args, kwargs): axis = operator.index(arguments["axis"]) except TypeError: return NotImplemented - if axis not in (0, -arrays[0].ndim - 1): + if axis < 0: + axis += arrays[0].ndim + 1 + if not 0 <= axis <= arrays[0].ndim: return NotImplemented - return _try_concat([entry.expand_dims(0) for entry in arrays], 0) + return _try_concat([entry.expand_dims(axis) for entry in arrays], axis) def _stack_like_virtual(array, func, args, kwargs): - """Dispatch the ``*stack`` wrappers where leading-only expansion suffices. - - ``dstack`` and ``column_stack`` promote low-rank inputs by - *appending* a unit axis, which the tile grid cannot express yet: - those cases fall back to materializing. - """ + """Dispatch the ``*stack`` wrappers: promote the inputs, concatenate.""" arguments = _bind(func, args, kwargs) if arguments is None or arguments.get("dtype") is not None: return NotImplemented @@ -1400,26 +1653,31 @@ def _stack_like_virtual(array, func, args, kwargs): else: axis = 1 elif func is np.dstack: - if min(ndims) < 3: - return NotImplemented + arrays = [_at_least_3d(entry) for entry in arrays] axis = 2 else: # column_stack - if min(ndims) < 2: - return NotImplemented + arrays = [ + entry.expand_dims(1) if entry.ndim == 1 else entry for entry in arrays + ] axis = 1 return _try_concat(arrays, axis) -def _atleast_virtual(array, func, args, kwargs): - """``atleast_1d``/``2d``/``3d`` on a single tile array, virtually. +def _at_least_3d(array): + """Promote *array* to 3-D the way ``numpy.atleast_3d`` does, virtually.""" + if array.ndim == 1: + return array.expand_dims(0).expand_dims(2) + if array.ndim == 2: + return array.expand_dims(2) + return array - ``atleast_3d`` on a 1-D or 2-D array would append a trailing unit - axis, which the grid cannot express yet: it falls back. - """ + +def _atleast_virtual(array, func, args, kwargs): + """``atleast_1d``/``2d``/``3d`` on a single tile array, virtually.""" if kwargs or len(args) != 1 or args[0] is not array: return NotImplemented - if func is np.atleast_3d and array.ndim < 3: - return NotImplemented + if func is np.atleast_3d: + return _at_least_3d(array) if func is np.atleast_2d and array.ndim == 1: return array.expand_dims(0) return array @@ -1449,6 +1707,11 @@ def _atleast_virtual(array, func, args, kwargs): np.atleast_1d: _atleast_virtual, np.atleast_2d: _atleast_virtual, np.atleast_3d: _atleast_virtual, + np.transpose: _transpose_virtual, # also np.permute_dims, its alias + np.matrix_transpose: _transpose_virtual, + np.swapaxes: _transpose_virtual, + np.moveaxis: _transpose_virtual, + np.squeeze: _squeeze_virtual, } From e3778f7d4d6ebed622ff1187e1c992d12e0383f0 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 14:47:00 +0200 Subject: [PATCH 36/56] Give tile geometry steps a sign for lazy reversal --- docs/release-notes.md | 1 + tests/test_tiles.py | 169 ++++++++++++++++++++++++++++++++++++++++++ xdas/tiles.py | 165 ++++++++++++++++++++++++++++++++++------- 3 files changed, 309 insertions(+), 26 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index ddebc737..8ced135a 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -3,6 +3,7 @@ ## 0.2.9 (unreleased) ### New Features +- **Tile geometry steps carry a sign.** `steps_k` may now be negative: `flip`, `flipud`, `fliplr`, `rot90` and negative-step slices (`da[::-1]`) become lazy rewrites of the tile geometry instead of bounded reads. Engines are unaffected — they keep receiving ascending source selections, and each part is reversed in memory after decoding (@atrabattoni). - **Tile manifests carry an axis map.** Manifests gain two optional entries — a 1-D `axes` variable (which stored geometry axis each virtual axis presents) and a 0-d `source_ndim` — that make transpose-like operations (`transpose`, `permute_dims`, `matrix_transpose`, `swapaxes`, `moveaxis`), `expand_dims`/`stack`/`np.newaxis` at any position, `squeeze`, and integer indexing all lazy rewrites of the map. Engines now always receive exactly one slice per source axis, in source order — custom `load_tile` implementations no longer need any rank-padding logic — and freshly scanned manifests store neither entry (absent means identity), so existing files are unaffected (@atrabattoni). - **Lazy numpy manipulation routines on tile arrays.** The `split` family (`split`, `array_split`, `vsplit`, `hsplit`, `dsplit`), the `stack` family (`stack`, `vstack`, `hstack`, `dstack`, `column_stack`) and `atleast_1d`/`atleast_2d`/`atleast_3d`, plus `roll`, `tile`, `delete`, and `append`/`insert` between tile arrays, now dispatch on `TileArray` as rewrites of the tile geometry and stay lazy. Cases the tile grid cannot express — axis fusion, element repetition, eager operands — keep materializing as before (@atrabattoni). - **Explicit engine configuration.** The open functions (`open`, `open_dataarray`, `open_mfdataarray`, `open_mfdatatree`) now declare `engine`, `vtype` and `ctype` explicitly, and `engine` accepts a configured `xdas.io.Engine` instance as well as a name. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front; passing them next to the engine name keeps working and now raises a `TypeError` on misspelled or unsupported keywords instead of silently ignoring them (@atrabattoni). diff --git a/tests/test_tiles.py b/tests/test_tiles.py index 10be65fd..f1b13e89 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -1687,6 +1687,175 @@ def test_masked_selection_on_bounded_path(self, stack): npt.assert_array_equal(manifest[mask], reference[mask]) +class TestSignedSteps: + """Negative steps: lazy reversal in the geometry, ascending engine reads.""" + + def test_reversed_slices_fold(self, stack, engine_calls): + manifest, reference = stack + flipped = manifest[::-1] + assert isinstance(flipped, TileArray) + assert engine_calls == [] + npt.assert_array_equal(np.asarray(flipped), reference[::-1]) + + @pytest.mark.parametrize( + "key", + [ + np.s_[::-1, ::-1], + np.s_[::-2], + np.s_[20:5:-3], + np.s_[::-1, 3:1:-1], + np.s_[25:, ::-2], + ], + ) + def test_reversed_slice_values(self, stack, key): + manifest, reference = stack + view = manifest[key] + assert isinstance(view, TileArray) + npt.assert_array_equal(np.asarray(view), reference[key]) + + def test_reversal_composes(self, stack, windowed): + manifest, reference = stack + npt.assert_array_equal(np.asarray(manifest[::2][::-1]), reference[::2][::-1]) + npt.assert_array_equal(np.asarray(manifest[::-1][::2]), reference[::-1][::2]) + npt.assert_array_equal(np.asarray(manifest[::-2][3:8]), reference[::-2][3:8]) + windowed_manifest, windowed_reference = windowed + npt.assert_array_equal( + np.asarray(windowed_manifest[::-1]), windowed_reference[::-1] + ) + flipped = np.transpose(manifest)[::-1, 9:20:2] + assert isinstance(flipped, TileArray) + npt.assert_array_equal(np.asarray(flipped), reference.T[::-1, 9:20:2]) + + def test_double_reversal_leaves_no_trace(self, stack): + manifest, reference = stack + back = manifest[::-1][::-1] + assert isinstance(back, TileArray) + assert "steps_0" not in back.dataset + assert back.equals(manifest) + npt.assert_array_equal(np.asarray(back), reference) + + def test_flip_family(self, stack, engine_calls): + manifest, reference = stack + for flip in [ + lambda a: np.flip(a), + lambda a: np.flip(a, 1), + lambda a: np.flip(a, (0, 1)), + np.flipud, + np.fliplr, + ]: + flipped = flip(manifest) + assert isinstance(flipped, TileArray) + npt.assert_array_equal(np.asarray(flipped), flip(reference)) + assert engine_calls != [] # reads happened, but only at np.asarray + + def test_rot90(self, stack): + manifest, reference = stack + for k in [0, 1, 2, 3, 4, -1]: + rotated = np.rot90(manifest, k) + assert isinstance(rotated, TileArray) + npt.assert_array_equal(np.asarray(rotated), np.rot90(reference, k)) + rotated = np.rot90(manifest, 1, axes=(1, 0)) + assert isinstance(rotated, TileArray) + npt.assert_array_equal(np.asarray(rotated), np.rot90(reference, 1, axes=(1, 0))) + + def test_engine_reads_stay_ascending(self, stack): + """Whatever the reversal, the engine sees ascending selections.""" + manifest, _ = stack + seen = [] + + class AscendingProbe(Engine, name="ascending-probe"): + @staticmethod + def load_tile(path, selection, **params): + seen.append(selection) + widths = tuple( + len(range(entry.start, entry.stop, entry.step or 1)) + for entry in selection + ) + return np.zeros(widths) + + try: + probe = TileArray(manifest.dataset, manifest.dtype, "ascending-probe") + np.asarray(np.flip(probe[::-2, ::-1])) + assert seen and all( + entry.start <= entry.stop and (entry.step or 1) >= 1 + for selection in seen + for entry in selection + ) + finally: + del Engine._registry["ascending-probe"] + + def test_flipped_round_trip(self, stack, tmp_path): + manifest, reference = stack + view = np.flip(manifest, 0)[5:20] + da = wrap(view) + path = str(tmp_path / "flipped.nc") + da.to_netcdf(path) + reopened = xd.open_dataarray(path) + assert isinstance(reopened.data, TileArray) + assert reopened.data.equals(view) + npt.assert_array_equal(reopened.values, reference[::-1][5:20]) + + def test_streaming_reduction_on_flipped(self, stack): + manifest, reference = stack + flipped = np.flip(manifest) + npt.assert_allclose(np.mean(flipped, axis=0), reference[::-1, ::-1].mean(0)) + npt.assert_allclose(np.max(flipped), reference.max()) + + def test_step_validation(self, stack): + manifest, _ = stack + with pytest.raises(ValueError, match="nonzero"): + TileArray( + manifest.dataset.assign( + steps_0=("tile_0", np.zeros(3, dtype=np.int64)) + ), + manifest.dtype, + manifest.engine, + ) + with pytest.raises(ValueError, match="walks out of the source"): + TileArray( + manifest.dataset.assign( + steps_0=("tile_0", np.full(3, -1, dtype=np.int64)) + ), + manifest.dtype, + manifest.engine, + ) + + def test_flip_error_parity(self, stack, line): + manifest, _ = stack + line_manifest, _ = line + with pytest.raises(ValueError, match="repeated"): + np.flip(manifest, (0, 0)) + with pytest.raises(np.exceptions.AxisError): + np.flip(manifest, 5) + with pytest.raises(ValueError, match="different"): + np.rot90(manifest, axes=(0, 0)) + with pytest.raises(ValueError, match="out of range"): + np.rot90(manifest, axes=(0, 5)) + with pytest.raises(ValueError): + np.rot90(line_manifest) + with pytest.raises(ValueError, match=">= 2-d"): + np.fliplr(line_manifest) + + def test_reversed_slice_on_bounded_path(self, stack): + """A reversed slice next to an index array takes the bounded read.""" + manifest, reference = stack + picked = manifest[::-1, [1, 3]] + assert isinstance(picked, np.ndarray) + npt.assert_array_equal(picked, reference[::-1, [1, 3]]) + + def test_flip_dispatch_guards(self, stack): + from xdas import tiles + + manifest, _ = stack + other = np.zeros((3, 4)) + assert tiles._flip_virtual(manifest, np.flip, (other,), {}) is NotImplemented + assert tiles._rot90_virtual(manifest, np.rot90, (other,), {}) is NotImplemented + assert ( + tiles._rot90_virtual(manifest, np.rot90, (manifest, 1.5), {}) + is NotImplemented + ) + + class TestNumpyProtocols: """Direct duck-array protocol behavior, without a wrapping DataArray.""" diff --git a/xdas/tiles.py b/xdas/tiles.py index 145db3d6..0d6a43ff 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -53,11 +53,12 @@ Geometry loads eagerly at construction (tiny); parameters stay folded — a constant occupies one element whatever the grid size — and broadcast -over the grid only as tiles are read. Positive-step slicing, integer -indexing and ``np.newaxis`` fold into the geometry and the axis map -and return a new :class:`TileArray` (as lazy and self-described as its -input); any other indexing reads the bounding box of the selection and -resolves the rest in memory. ``np.asarray`` materializes: tiles are +over the grid only as tiles are read. Slicing (any nonzero step, +negative ones reversing lazily), integer indexing and ``np.newaxis`` +fold into the geometry and the axis map and return a new +:class:`TileArray` (as lazy and self-described as its input); any +other indexing reads the bounding box of the selection and resolves +the rest in memory. ``np.asarray`` materializes: tiles are read one by one by the registered *engine* (``xdas.io.Engine[name]``), whose ``load_tile`` opens each tile's path itself and returns the tile's *source selection* — always one possibly strided slice per @@ -76,9 +77,10 @@ read). On top of slicing and concatenation, the numpy manipulation routines whose effect is a rewrite of the tile geometry or the axis map dispatch lazily too (see ``_LAZY_ROUTINES``): the ``split``, -``stack``, ``atleast`` and transpose (``transpose``, ``swapaxes``, -``moveaxis``, ``matrix_transpose``) families, ``expand_dims`` and -``squeeze`` at any position, ``roll``, ``tile``, ``delete``, and +``stack``, ``atleast``, transpose (``transpose``, ``swapaxes``, +``moveaxis``, ``matrix_transpose``) and flip (``flip``, ``flipud``, +``fliplr``, ``rot90``) families, ``expand_dims`` and ``squeeze`` at +any position, ``roll``, ``tile``, ``delete``, and ``append``/``insert`` between tile arrays. Each falls back to a materializing read for the cases the grid cannot express (axis fusion, element repetition, eager operands). Tile arrays persist @@ -435,12 +437,19 @@ def __init__(self, dataset, dtype, engine): for kind, arrays, bound in ( ("sizes", self._sizes, 1), ("starts", self._starts, 0), - ("steps", self._steps, 1), ): for g, values in enumerate(arrays): if np.any(values < bound): kind_bound = "non-negative" if bound == 0 else "strictly positive" raise ValueError(f"`{kind}_{g}` must be {kind_bound}") + for g in range(ngrid): + if np.any(self._steps[g] == 0): + raise ValueError(f"`steps_{g}` must be nonzero") + # a negative step walks backward from `start`: the last + # sample read must still land inside the source + reach = self._starts[g] + (self._sizes[g] - 1) * self._steps[g] + if np.any(reach < 0): + raise ValueError(f"`steps_{g}` walks out of the source") self._edges = tuple( np.concatenate(([0], np.cumsum(sizes))) for sizes in self._sizes ) @@ -674,18 +683,20 @@ def size(self): def __getitem__(self, key): """Index the array, staying virtual whenever possible. - Positive-step slices fold into the geometry and return a new + Slices of any nonzero step, integers and ``np.newaxis`` fold + into the geometry and the axis map and return a new :class:`TileArray` without touching the sources: ``np.asarray(arr[key])`` equals ``np.asarray(arr)[key]``. Per axis, the overlapping tiles are located by binary search on the running tile sizes and their geometry is trimmed — and, for - stepped slices, decimated — to the selection (steps multiply, - origins compose, one tile stays one tile); tiles the selection - strides over entirely are dropped. The parameters are sliced - through the wrapped dataset, so a lazy array stays lazy. - - Every other key (index arrays, boolean masks, reversed slices, - empty selections) reads the bounding box of the selection and + stepped slices, decimated, negative steps reversing the axis — + to the selection (steps multiply, origins compose, one tile + stays one tile); tiles the selection strides over entirely are + dropped. The parameters are sliced through the wrapped dataset, + so a lazy array stays lazy. + + Every other key (index arrays, boolean masks, empty selections, + all-integer keys) reads the bounding box of the selection and applies the remainder in memory, returning a numpy array. """ key = _normalize_key(key, self.ndim) @@ -708,16 +719,18 @@ def __getitem__(self, key): def _fold(self, key): """Fold slices, integers and new axes into a new array, virtually. - Positive-step slices trim the geometry; an integer pins the - geometry axis at one sample and hides it from the map; ``None`` - inserts a synthetic axis. Raises :class:`_Unfoldable` for other - entries, for empty selections (a grid needs at least one tile) - and for all-integer keys (a scalar is not a tile array); - :meth:`__getitem__` then falls back to a bounded read. + Slices of any nonzero step trim the geometry (a negative step + folds as its ascending twin, then the axis flips); an integer + pins the geometry axis at one sample and hides it from the map; + ``None`` inserts a synthetic axis. Raises :class:`_Unfoldable` + for other entries, for empty selections (a grid needs at least + one tile) and for all-integer keys (a scalar is not a tile + array); :meth:`__getitem__` then falls back to a bounded read. """ indexers = {} assign = {} new_axes = [] + flips = [] entries = (entry for entry in key if entry is not None) for axis, (entry, extent) in enumerate(zip(entries, self.shape)): g = self._axes[axis] @@ -730,10 +743,15 @@ def _fold(self, key): f"index {entry} is out of bounds for axis of size {extent}" ) lo, hi, s = index, index + 1, 1 - elif isinstance(entry, slice) and (entry.step or 1) >= 1: + elif isinstance(entry, slice): lo, hi, s = entry.indices(extent) - if len(range(lo, hi, s)) == 0: + count = len(range(lo, hi, s)) + if count == 0: raise _Unfoldable(f"empty selection along axis {axis}") + if s < 0: + # fold the ascending twin, flip the result axis after + flips.append(len(new_axes)) + lo, hi, s = lo + (count - 1) * s, lo + 1, -s new_axes.append(g) else: raise _Unfoldable( @@ -774,6 +792,8 @@ def _fold(self, key): if new_axes != self._axes: dataset = self._assign_axes(dataset, new_axes, len(self.dims)) result = type(self)(dataset, self.dtype, self.engine) + for axis in flips: + result = result._flip(axis) # np.newaxis entries insert synthetic axes at their output position position = 0 for entry in key: @@ -784,6 +804,34 @@ def _fold(self, key): position += 1 return result + def _flip(self, axis): + """Reverse the array along virtual *axis*, staying virtual. + + The tiles trade places (last first) and each reads its same + source samples backward: the origin moves to the walk's far + end and the step negates. Sources are never touched. + """ + g = self._axes[axis] + dim = self.dims[g] + starts = self._starts[g] + (self._sizes[g] - 1) * self._steps[g] + steps = -self._steps[g] + dataset = self.dataset.isel({dim: slice(None, None, -1)}) + assign = { + f"starts_{g}": (dim, starts[::-1]), + f"steps_{g}": (dim, steps[::-1]), + } + # all-default columns fold away (a double flip leaves no trace) + drop = [ + name + for name, (_, values) in assign.items() + if (name.startswith("starts_") and not values.any()) + or (name.startswith("steps_") and not (values != 1).any()) + ] + assign = {name: entry for name, entry in assign.items() if name not in drop} + dataset = dataset.assign(assign) + dataset = dataset.drop_vars([name for name in drop if name in dataset]) + return type(self)(dataset, self.dtype, self.engine) + @classmethod def concat(cls, arrays, dim=0): """Concatenate tile arrays along axis *dim* into a new array. @@ -961,11 +1009,16 @@ def _read(self): order = [g for g in self._axes if g < rank] order += [g for g in range(rank) if g not in order] for index in np.ndindex(counts): - selection, widths = [], [] + selection, widths, backward = [], [], [] for g in range(rank): first = int(self._starts[g][index[g]]) size = int(self._sizes[g][index[g]]) step = int(self._steps[g][index[g]]) + # the engine always reads ascending; a negative step + # reads the walk's span forward and reverses in memory + if step < 0: + first, step = first + (size - 1) * step, -step + backward.append(g) selection.append(slice(first, first + (size - 1) * step + 1, step)) widths.append(size) selection, widths = tuple(selection), tuple(widths) @@ -986,6 +1039,13 @@ def _read(self): f"part of shape {part.shape} where the array records " f"{self.dtype} parts and the selection has shape {widths}" ) + if backward: + part = part[ + tuple( + slice(None, None, -1) if g in backward else slice(None) + for g in range(rank) + ) + ] part = np.transpose(part, order) out[dest] = part.reshape(tuple(entry.stop - entry.start for entry in dest)) return out @@ -1395,6 +1455,55 @@ def _squeeze_virtual(array, func, args, kwargs): return NotImplemented +def _flip_virtual(array, func, args, kwargs): + """Dispatch the flip family as reversed slices (lazy in the geometry).""" + arguments = _bind(func, args, kwargs) + if arguments is None or next(iter(arguments.values())) is not array: + return NotImplemented + ndim = array.ndim + if func is np.flipud: + axes = (0,) + elif func is np.fliplr: + if ndim < 2: + return NotImplemented + axes = (1,) + else: + axis = arguments["axis"] + if axis is None: + axes = tuple(range(ndim)) + else: + axis = axis if isinstance(axis, (tuple, list)) else (axis,) + axes = tuple(_normalize_axis(entry, ndim) for entry in axis) + if None in axes or len(set(axes)) != len(axes): + return NotImplemented + return array[ + tuple(slice(None, None, -1) if a in axes else slice(None) for a in range(ndim)) + ] + + +def _rot90_virtual(array, func, args, kwargs): + """Dispatch ``numpy.rot90`` as its own flip/transpose composition.""" + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["m"] is not array or array.ndim < 2: + return NotImplemented + try: + turns = operator.index(arguments["k"]) % 4 + except TypeError: + return NotImplemented + axes = tuple(_normalize_axis(entry, array.ndim) for entry in arguments["axes"]) + if None in axes or axes[0] == axes[1]: + return NotImplemented + order = list(range(array.ndim)) + order[axes[0]], order[axes[1]] = order[axes[1]], order[axes[0]] + if turns == 0: + return array[(slice(None),) * array.ndim] + if turns == 1: + return np.transpose(np.flip(array, axes[1]), order) + if turns == 2: + return np.flip(np.flip(array, axes[0]), axes[1]) + return np.flip(np.transpose(array, order), axes[1]) + + def _split_virtual(array, func, args, kwargs): """Split into lazy sub-arrays for the ``numpy.split`` family. @@ -1712,6 +1821,10 @@ def _atleast_virtual(array, func, args, kwargs): np.swapaxes: _transpose_virtual, np.moveaxis: _transpose_virtual, np.squeeze: _squeeze_virtual, + np.flip: _flip_virtual, + np.flipud: _flip_virtual, + np.fliplr: _flip_virtual, + np.rot90: _rot90_virtual, } From d558fd78e8bf15fbf0af1199447872d164fc5120 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 15:05:06 +0200 Subject: [PATCH 37/56] Fold the duplicate geometry defaults, streaming mask and one-shot helpers --- tests/test_tiles.py | 27 +++++++- xdas/tiles.py | 146 ++++++++++++++++++++------------------------ 2 files changed, 90 insertions(+), 83 deletions(-) diff --git a/tests/test_tiles.py b/tests/test_tiles.py index f1b13e89..173fc280 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -269,6 +269,9 @@ def test_validation(self, stack): ) with pytest.raises(ValueError, match="`paths`"): TileArray(dataset.drop_vars("paths"), manifest.dtype, manifest.engine) + bad_starts = dataset.assign(starts_0=("tile_0", np.array([-1, 0, 0]))) + with pytest.raises(ValueError, match="non-negative"): + TileArray(bad_starts, manifest.dtype, manifest.engine) def test_extra_variables_are_params(self, stack): """Any non-geometry manifest variable is a per-tile engine parameter.""" @@ -412,7 +415,7 @@ def test_rootless_manifest_reads(self, tmp_path): assert legacy.equals(manifest) and manifest.equals(legacy) def test_no_common_directory_keeps_paths_whole(self): - from xdas.tiles import _common_root, _split_root + from xdas.tiles import _split_root mixed = np.array([b"rel/f.h5", b"/abs/g.h5"], dtype=object) root, kept = _split_root(mixed) @@ -420,8 +423,26 @@ def test_no_common_directory_keeps_paths_whole(self): empty = np.array([], dtype=object) root, kept = _split_root(empty) assert root == b"" and kept is empty - assert _common_root(["/a/b", ""]) == "" - assert _common_root(["/a/b", "relative"]) == "" + + def test_concat_with_unrelatable_roots_stores_whole_paths(self): + """Roots `commonpath` cannot relate (absolute vs relative) fuse rootless.""" + + def make(root, path): + dataset = xr.Dataset( + { + "sizes_0": ("tile_0", np.array([3])), + "paths": ((), np.asarray(os.fsencode(path))), + "root": ((), np.asarray(os.fsencode(root))), + } + ) + return TileArray(dataset, "float64", ENGINE) + + fused = TileArray.concat([make("/a/b", "f.h5"), make("rel", "g.h5")]) + assert fused.root == "" + assert fused.dataset["paths"].values.tolist() == [ + os.fsencode(os.path.join("/a/b", "f.h5")), + os.fsencode(os.path.join("rel", "g.h5")), + ] def test_no_common_directory_falls_back_rootless(self, tmp_path, monkeypatch): """Paths sharing no directory (several drives) store whole, rootless.""" diff --git a/xdas/tiles.py b/xdas/tiles.py index 0d6a43ff..1130983f 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -216,16 +216,6 @@ def _split_root(paths): return root, _trim(np.strings.slice(paths, cut + 1, None)) -def _common_root(roots): - """Return the deepest directory containing every *root* ("" when none does).""" - if any(not root for root in roots): - return "" - try: - return os.path.commonpath(roots) - except ValueError: - return "" - - def _consumed(entry): """How many data axes one key *entry* consumes, as numpy counts them. @@ -261,12 +251,6 @@ def _normalize_key(key, ndim): return key + (slice(None),) * (ndim - consumed) -def _reinsert_newaxes(key, residual): - """Weave the ``None`` entries of *key* back into the per-axis *residual*.""" - residual = iter(residual) - return tuple(None if entry is None else next(residual) for entry in key) - - def _bounding_key(key, shape): """Split *key* into a positive-step bounding box and a residual key. @@ -329,6 +313,25 @@ def _bounding_key(key, shape): return tuple(box), tuple(residual), empty +def _assign_geometry(dataset, assign): + """Apply geometry *assign* to *dataset*, folding all-default columns away. + + A column of zero starts or unit steps stays derivable and is + dropped rather than stored, so that slicing back to the full + extent, or flipping twice, leaves no trace in the manifest. + """ + drop = [ + name + for name, (_, values) in assign.items() + if (name.startswith("starts_") and not values.any()) + or (name.startswith("steps_") and not (values != 1).any()) + ] + assign = {name: entry for name, entry in assign.items() if name not in drop} + if assign: + dataset = dataset.assign(assign) + return dataset.drop_vars([name for name in drop if name in dataset]) + + def _materialize(value): """Read any :class:`TileArray` in *value*, descending one level.""" if isinstance(value, TileArray): @@ -342,20 +345,6 @@ def _materialize(value): return value -def _to_si(nbytes): - """Render a byte count the way xarray renders its ``Size:`` header. - - Decimal units, no decimals: the repr sits right under that header, - so a base-1024 count would read as a different number. - """ - dividend = float(nbytes) - index = 0 - while dividend >= 1000.0 and index < len(_UNITS) - 1: - dividend /= 1000.0 - index += 1 - return f"{dividend:.0f}{_UNITS[index]}" - - class TileArray(np.lib.mixins.NDArrayOperatorsMixin): """A dense rectilinear grid of file-backed tiles as one virtual array. @@ -434,15 +423,11 @@ def __init__(self, dataset, dtype, engine): self._sizes = self._geometry("sizes", None) self._starts = self._geometry("starts", 0) self._steps = self._geometry("steps", 1) - for kind, arrays, bound in ( - ("sizes", self._sizes, 1), - ("starts", self._starts, 0), - ): - for g, values in enumerate(arrays): - if np.any(values < bound): - kind_bound = "non-negative" if bound == 0 else "strictly positive" - raise ValueError(f"`{kind}_{g}` must be {kind_bound}") for g in range(ngrid): + if np.any(self._sizes[g] < 1): + raise ValueError(f"`sizes_{g}` must be strictly positive") + if np.any(self._starts[g] < 0): + raise ValueError(f"`starts_{g}` must be non-negative") if np.any(self._steps[g] == 0): raise ValueError(f"`steps_{g}` must be nonzero") # a negative step walks backward from `start`: the last @@ -714,7 +699,10 @@ def __getitem__(self, key): # zero-strided: the result is empty, so no value is ever read # and the full shape is never allocated return np.broadcast_to(np.zeros((), self.dtype), self.shape)[key].copy() - return np.asarray(self._fold(box))[_reinsert_newaxes(key, residual)] + # weave the None entries of the key back into the per-axis residual + residual = iter(residual) + outer = tuple(None if entry is None else next(residual) for entry in key) + return np.asarray(self._fold(box))[outer] def _fold(self, key): """Fold slices, integers and new axes into a new array, virtually. @@ -778,16 +766,7 @@ def _fold(self, key): assign[f"steps_{g}"] = (dim, (step * s)[keep]) if not new_axes: raise _Unfoldable("an all-integer key selects a scalar, not a grid") - # all-default starts/steps columns fold away (they stay derivable) - drop = [ - name - for name, (_, values) in assign.items() - if (name.startswith("starts_") and not values.any()) - or (name.startswith("steps_") and not (values != 1).any()) - ] - assign = {name: entry for name, entry in assign.items() if name not in drop} - dataset = self.dataset.isel(indexers).assign(assign) - dataset = dataset.drop_vars([name for name in drop if name in dataset]) + dataset = _assign_geometry(self.dataset.isel(indexers), assign) new_axes = tuple(new_axes) if new_axes != self._axes: dataset = self._assign_axes(dataset, new_axes, len(self.dims)) @@ -815,21 +794,13 @@ def _flip(self, axis): dim = self.dims[g] starts = self._starts[g] + (self._sizes[g] - 1) * self._steps[g] steps = -self._steps[g] - dataset = self.dataset.isel({dim: slice(None, None, -1)}) - assign = { - f"starts_{g}": (dim, starts[::-1]), - f"steps_{g}": (dim, steps[::-1]), - } - # all-default columns fold away (a double flip leaves no trace) - drop = [ - name - for name, (_, values) in assign.items() - if (name.startswith("starts_") and not values.any()) - or (name.startswith("steps_") and not (values != 1).any()) - ] - assign = {name: entry for name, entry in assign.items() if name not in drop} - dataset = dataset.assign(assign) - dataset = dataset.drop_vars([name for name in drop if name in dataset]) + dataset = _assign_geometry( + self.dataset.isel({dim: slice(None, None, -1)}), + { + f"starts_{g}": (dim, starts[::-1]), + f"steps_{g}": (dim, steps[::-1]), + }, + ) return type(self)(dataset, self.dtype, self.engine) @classmethod @@ -904,7 +875,13 @@ def concat(cls, arrays, dim=0): data["axes"] = ("axis", np.asarray(first._axes, dtype=np.int64)) if first._source_ndim != ngrid: data["source_ndim"] = ((), np.asarray(first._source_ndim, dtype=np.int64)) - root = _common_root([array.root for array in arrays]) + # the fused root is the deepest directory containing every input + # root ("" when an input has none or no common directory exists) + roots = [array.root for array in arrays] + try: + root = os.path.commonpath(roots) if all(roots) else "" + except ValueError: + root = "" if root: data["root"] = ((), np.asarray(os.fsencode(root))) for name in ("paths", *first._params): @@ -1164,8 +1141,7 @@ def _reduce_streaming(self, func, args, kwargs): kept = tuple(a for a in range(self.ndim) if a not in axes) out_shape = tuple(self.shape[a] for a in kept) acc = None - filled = np.zeros(out_shape, dtype=bool) - counts = np.zeros(out_shape) if counting else None + counts = np.zeros(out_shape) if counting == "nancount" else None # stream one tile row at a time: the tiling is the only blocking # the array has, and a whole row bounds the memory held at once rest = tuple(slice(0, extent) for extent in self.shape[1:]) @@ -1175,18 +1151,20 @@ def _reduce_streaming(self, func, args, kwargs): partial = np.asarray(block_reduce(block, axis=axes, keepdims=True)) partial = partial.reshape(tuple(block.shape[a] for a in kept)) target = tuple(box[a] for a in kept) - if acc is None: - acc = np.zeros(out_shape, dtype=partial.dtype) - acc[target] = np.where( - filled[target], combine(acc[target], partial), partial - ) - filled[target] = True - if counting == "count": - counts[target] += np.prod([block.shape[a] for a in axes]) - elif counting == "nancount": + if 0 in axes: + # every row reduces into the same output: combine pairwise + acc = partial if acc is None else combine(acc, partial) + else: + # rows land in disjoint output rows: plain assignment + if acc is None: + acc = np.empty(out_shape, dtype=partial.dtype) + acc[target] = partial + if counting == "nancount": counts[target] += np.sum(~np.isnan(block), axis=axes).reshape( partial.shape ) + if counting == "count": + counts = math.prod(self.shape[a] for a in axes) result = acc / counts if counting else acc if dtype is None and counting: dtype = func(np.zeros(1, self.dtype)).dtype @@ -1303,9 +1281,17 @@ def __repr__(self): above. What remains is what only the tiling knows — the volume it stands for, and how many tiles it took. """ + # decimal units, no decimals, the way xarray renders its `Size:` + # header: the repr sits right under it, and a base-1024 count + # would read as a different number + nbytes = float(self.size * self.dtype.itemsize) + index = 0 + while nbytes >= 1000.0 and index < len(_UNITS) - 1: + nbytes /= 1000.0 + index += 1 return ( f"TileArray[{self.engine['name']}] " - f"{_to_si(self.size * self.dtype.itemsize)} ({self.dtype}) " + f"{nbytes:.0f}{_UNITS[index]} ({self.dtype}) " f"{self.ntiles} {'tile' if self.ntiles == 1 else 'tiles'}" ) @@ -1381,10 +1367,10 @@ def _concatenate_virtual(array, func, args, kwargs): def _expand_dims_virtual(array, func, args, kwargs): """Dispatch ``numpy.expand_dims``, delegating to :meth:`TileArray.expand_dims`.""" - kwargs = dict(kwargs) - axis = kwargs.pop("axis", args[1] if len(args) > 1 else None) - if kwargs or len(args) > 2 or args[0] is not array: + arguments = _bind(func, args, kwargs) + if arguments is None or arguments["a"] is not array: return NotImplemented + axis = arguments["axis"] axis = axis if isinstance(axis, tuple) else (axis,) if not all(isinstance(entry, (int, np.integer)) for entry in axis): return NotImplemented From afda1673cdbd53d5757cedfd4626fbd11e37b43c Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 15:05:24 +0200 Subject: [PATCH 38/56] Require equal division from every int-sectioned split but array_split --- tests/test_tiles.py | 4 ++++ xdas/tiles.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_tiles.py b/tests/test_tiles.py index 173fc280..1560fd18 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -1133,6 +1133,10 @@ def test_split_variants(self, stack, line): npt.assert_array_equal(np.asarray(got), expected) with pytest.raises(ValueError, match="3 or more"): np.dsplit(manifest, 1) + # int sections require an equal division for the whole family + # (numpy parity), array_split being the one lenient spelling + with pytest.raises(ValueError, match="equal division"): + np.vsplit(manifest, 2) line_manifest, line_reference = line with pytest.raises(ValueError, match="2 or more"): np.vsplit(line_manifest, 2) diff --git a/xdas/tiles.py b/xdas/tiles.py index 1130983f..74225ac0 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -1522,7 +1522,7 @@ def _split_virtual(array, func, args, kwargs): count = int(sections) if count <= 0: raise ValueError("number sections must be larger than 0.") - if func is np.split and extent % count: + if func is not np.array_split and extent % count: raise ValueError("array split does not result in an equal division") each, extras = divmod(extent, count) sizes = [each + 1] * extras + [each] * (count - extras) From 9adefffb43cd4aaba9690204c78fe8d91a756d1b Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 15:06:40 +0200 Subject: [PATCH 39/56] Condense the 0.2.9 release notes to the user-facing surface --- docs/release-notes.md | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 8ced135a..e02a845c 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -3,26 +3,21 @@ ## 0.2.9 (unreleased) ### New Features -- **Tile geometry steps carry a sign.** `steps_k` may now be negative: `flip`, `flipud`, `fliplr`, `rot90` and negative-step slices (`da[::-1]`) become lazy rewrites of the tile geometry instead of bounded reads. Engines are unaffected — they keep receiving ascending source selections, and each part is reversed in memory after decoding (@atrabattoni). -- **Tile manifests carry an axis map.** Manifests gain two optional entries — a 1-D `axes` variable (which stored geometry axis each virtual axis presents) and a 0-d `source_ndim` — that make transpose-like operations (`transpose`, `permute_dims`, `matrix_transpose`, `swapaxes`, `moveaxis`), `expand_dims`/`stack`/`np.newaxis` at any position, `squeeze`, and integer indexing all lazy rewrites of the map. Engines now always receive exactly one slice per source axis, in source order — custom `load_tile` implementations no longer need any rank-padding logic — and freshly scanned manifests store neither entry (absent means identity), so existing files are unaffected (@atrabattoni). -- **Lazy numpy manipulation routines on tile arrays.** The `split` family (`split`, `array_split`, `vsplit`, `hsplit`, `dsplit`), the `stack` family (`stack`, `vstack`, `hstack`, `dstack`, `column_stack`) and `atleast_1d`/`atleast_2d`/`atleast_3d`, plus `roll`, `tile`, `delete`, and `append`/`insert` between tile arrays, now dispatch on `TileArray` as rewrites of the tile geometry and stay lazy. Cases the tile grid cannot express — axis fusion, element repetition, eager operands — keep materializing as before (@atrabattoni). -- **Explicit engine configuration.** The open functions (`open`, `open_dataarray`, `open_mfdataarray`, `open_mfdatatree`) now declare `engine`, `vtype` and `ctype` explicitly, and `engine` accepts a configured `xdas.io.Engine` instance as well as a name. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front; passing them next to the engine name keeps working and now raises a `TypeError` on misspelled or unsupported keywords instead of silently ignoring them (@atrabattoni). +- **Tile-backed virtual arrays.** The new `xdas.tiles` module exposes file archives as one lazy `TileArray`. Slicing (any step, including negative), integer indexing, `np.newaxis`, concatenation, and the numpy manipulation routines (the `transpose`, `flip`, `split`, `stack` and `atleast` families, `expand_dims`, `squeeze`, `roll`, `tile`, `delete`, `append`/`insert`) all stay lazy; whole-array reductions (`sum`, `mean`, `min`, `max`, …) stream one tile row at a time; reads touch only the tiles the selection overlaps (@atrabattoni). +- **`vtype="tiles"` on every HDF5 engine.** The open functions with `vtype="tiles"` return tile-backed arrays for the asn, febus, terra15, apsensing, prodml and native xdas engines. Silixa and MiniSEED always emit them now (replacing the serialized-dask-graph fallback, with time-axis push-down for Silixa), and Febus defaults to them — one tile per file, where the HDF5 backing needed one virtual mapping per data block. Custom engines add support by implementing `Engine.load_tile(path, selection, **params)` (@atrabattoni). +- Tile-backed arrays round-trip through the native xdas netCDF format: the manifest is stored as a compact `__tiles__` sibling group, relocatable by editing its single root path and directly readable by the 0.3 line (@atrabattoni). +- **Explicit engine configuration.** The open functions declare `engine`, `vtype` and `ctype`, and `engine` also accepts a configured `xdas.io.Engine` instance. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front (@atrabattoni). +- `open_mfdataarray` accepts up to 2 000 000 files with `vtype="tiles"`; the ceiling now depends on the resolved vtype and stays 100 000 for `hdf5` (@atrabattoni). ### Breaking Changes -- Passing a bare read function as `engine` is no longer supported: subclass `xdas.io.Engine` instead (see the data-formats documentation). Combining a configured engine instance with `vtype`, `ctype` or extra engine keywords raises a `ValueError` (@atrabattoni). -- The miniseed `ctype` argument is now honored: it previously routed to an unused attribute and the reader always built interpolated time coordinates. The default is unchanged (@atrabattoni). -- **Tile-backed virtual arrays.** The new `xdas.tiles` module (ported from the 0.3 line) exposes multi-file archives as one lazy `TileArray`: positive-step slicing, concatenation (including along a new dimension), and whole-array reductions all stay lazy, and reads touch only the tiles a selection overlaps. The Silixa TDMS and MiniSEED engines now emit tile-backed data arrays, replacing the serialized-dask-graph fallback; Silixa reads gained time-axis push-down (@atrabattoni). -- Tile-backed data arrays round-trip through the native xdas netCDF format: the tile manifest is stored as a `__tiles__` sibling group (@atrabattoni). -- **Compact manifest path storage.** Tile manifests split the common directory of their source paths into a single 0-d `root` variable and keep only the root-relative rest per tile, and manifest strings are written as fixed-width char arrays instead of variable-length HDF5 strings: manifests shrink in memory and on disk and open faster, and relocating an archive amounts to editing one stored value. Arrays rooted in different directories still concatenate (the fusion is rebased under the deepest directory containing every root), and manifests written by earlier versions reopen unchanged (@atrabattoni). -- **Optional `tiles` vtype for every HDF5 engine.** `open_dataarray`/`open_mfdataarray` with `vtype="tiles"` back the returned array with a lazy `TileArray` instead of an HDF5 virtual source, for the asn, febus, terra15, apsensing, prodml, and native xdas engines. Saved tile views are directly readable by the 0.3 line. Custom engines add tile support by implementing the `load_tile(path, selection, **params)` static half of `xdas.io.Engine` (@atrabattoni). -- **Febus now defaults to `tiles`.** A tile view describes a Febus file as a single tile — the overlap trimming lives in the reader — whereas the HDF5 backing needs one mapping per block, so its manifest grew with the block count as well as the file count. Every other HDF5 engine still defaults to `hdf5` (@atrabattoni). -- `open_mfdataarray` no longer refuses more than 100 000 paths regardless of backing: the ceiling is now taken from the engine's resolved vtype and is far higher for `tiles`, which does not build one HDF5 mapping per file. The error explains the remaining limit — the scan holds one data array per file in memory until they are combined (@atrabattoni). +- Passing a bare read function as `engine` now raises a `TypeError`: subclass `xdas.io.Engine` instead (see the data-formats documentation) (@atrabattoni). +- Misspelled or unsupported keyword arguments passed next to an engine name now raise a `TypeError` instead of being silently ignored, and combining `vtype`, `ctype` or engine keywords with an already configured engine instance raises a `ValueError` (@atrabattoni). ### Deprecations -- Writing dask-backed virtual arrays (`__dask_array__` attribute) is deprecated and emits a `FutureWarning`; existing files still open. No engine emits them any more: the tile-backed engines replace this mechanism (@atrabattoni). +- Writing dask-backed virtual arrays is deprecated and emits a `FutureWarning`; existing files still open, but no engine emits them any more (@atrabattoni). ### Bug Fixes -- Fix data collections holding more than one tile-backed data array being impossible to reopen. The tile manifest lives in a `__tiles__` sibling group, which the reader counted when deciding whether a group held a data array or a nested collection, so every tile-backed array looked one level too deep (@atrabattoni). +- The miniseed `ctype` argument is now honored: it previously routed to an unused attribute and the reader always built interpolated time coordinates. The default is unchanged (@atrabattoni). ## 0.2.8 From 9983540692dc675cad89ee4c1ddd0bdcb9760390 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 15:08:38 +0200 Subject: [PATCH 40/56] Sync the tiles API page with the engine roster --- docs/api/tiles.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index f8e2bff5..59b06d01 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -4,8 +4,10 @@ # xdas.tiles -Lazy tile-backed virtual arrays, the backend of the formats that HDF5 -virtual datasets cannot serve (Silixa TDMS, MiniSEED). +Lazy tile-backed virtual arrays: the only backend of the formats that +HDF5 virtual datasets cannot serve (Silixa TDMS, MiniSEED), the default +one for Febus, and available on request from every other engine +(`vtype="tiles"`). ## TileArray @@ -40,6 +42,7 @@ Methods TileArray.expand_dims TileArray.squeeze TileArray.transpose + TileArray.astype TileArray.equals ``` From b27d0c3d452247c35f74a9d91617df43e64f6b43 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 15:13:31 +0200 Subject: [PATCH 41/56] Lift the file-count ceiling from tiles scans --- docs/release-notes.md | 2 +- docs/user-guide/io/virtual-datasets.md | 6 +++-- tests/test_core.py | 14 ++++-------- xdas/core/routines.py | 31 ++++++++++++-------------- 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index e02a845c..32dd55b6 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -7,7 +7,7 @@ - **`vtype="tiles"` on every HDF5 engine.** The open functions with `vtype="tiles"` return tile-backed arrays for the asn, febus, terra15, apsensing, prodml and native xdas engines. Silixa and MiniSEED always emit them now (replacing the serialized-dask-graph fallback, with time-axis push-down for Silixa), and Febus defaults to them — one tile per file, where the HDF5 backing needed one virtual mapping per data block. Custom engines add support by implementing `Engine.load_tile(path, selection, **params)` (@atrabattoni). - Tile-backed arrays round-trip through the native xdas netCDF format: the manifest is stored as a compact `__tiles__` sibling group, relocatable by editing its single root path and directly readable by the 0.3 line (@atrabattoni). - **Explicit engine configuration.** The open functions declare `engine`, `vtype` and `ctype`, and `engine` also accepts a configured `xdas.io.Engine` instance. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front (@atrabattoni). -- `open_mfdataarray` accepts up to 2 000 000 files with `vtype="tiles"`; the ceiling now depends on the resolved vtype and stays 100 000 for `hdf5` (@atrabattoni). +- `open_mfdataarray` no longer caps the number of files when the resolved vtype is `tiles`; the 100 000 ceiling remains for `hdf5`, which builds one virtual mapping per file (@atrabattoni). ### Breaking Changes - Passing a bare read function as `engine` now raises a `TypeError`: subclass `xdas.io.Engine` instead (see the data-formats documentation) (@atrabattoni). diff --git a/docs/user-guide/io/virtual-datasets.md b/docs/user-guide/io/virtual-datasets.md index 862ea6cd..d3a5d443 100644 --- a/docs/user-guide/io/virtual-datasets.md +++ b/docs/user-guide/io/virtual-datasets.md @@ -182,8 +182,10 @@ tiles as the archive grows. - Building either manifest starts with reading the metadata of every file. That scan is dominated by disk access and is usually the bulk of the total build time, so it is not a criterion for choosing between the two. -- Both keep one data array per file in memory while scanning, so very large file sets must - be opened in batches and combined afterwards, whichever backend is used. +- Both keep one data array per file in memory while scanning — a few kilobytes each for + tiles, much more for HDF5, whose per-file mapping cost caps a single call at 100 000 + files. Sets beyond that (or beyond what memory allows) must be opened in batches and + combined afterwards. - Neither backend helps when a coordinate is not monotonic — for instance when files overlap in time. Label-based selection then falls back to a slow path in both cases, and is better addressed in the data itself. diff --git a/tests/test_core.py b/tests/test_core.py index a92c5f4f..83bc8864 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -30,20 +30,16 @@ def test_open_mfdataarray_file_limit(self, tmp_path, monkeypatch): for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): da.to_netcdf(tmp_path / f"{idx:03}.nc") - monkeypatch.setattr(routines, "MAX_OPEN_FILES", {None: 2, "hdf5": 2}) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) with pytest.raises(NotImplementedError, match="the limit is 2"): xd.open_mfdataarray(tmp_path / "00*.nc") - def test_open_mfdataarray_file_limit_is_higher_for_tiles( - self, tmp_path, monkeypatch - ): + def test_open_mfdataarray_no_file_limit_for_tiles(self, tmp_path, monkeypatch): from xdas.core import routines for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): da.to_netcdf(tmp_path / f"{idx:03}.nc") - monkeypatch.setattr( - routines, "MAX_OPEN_FILES", {None: 2, "hdf5": 2, "tiles": 3} - ) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) da = xd.open_mfdataarray(tmp_path / "00*.nc", engine="xdas", vtype="tiles") assert da.shape == wavelet_wavefronts().shape @@ -53,9 +49,7 @@ def test_open_mfdataarray_file_limit_engine_instance(self, tmp_path, monkeypatch for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): da.to_netcdf(tmp_path / f"{idx:03}.nc") - monkeypatch.setattr( - routines, "MAX_OPEN_FILES", {None: 2, "hdf5": 2, "tiles": 3} - ) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) # the configured instance carries the vtype the limit is keyed on with pytest.raises(NotImplementedError, match="the limit is 2"): xd.open_mfdataarray(tmp_path / "00*.nc", engine=XdasEngine(vtype="hdf5")) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 8ccd74dc..34ad2b0b 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -529,12 +529,11 @@ def defaulttree(depth): return defaultdict(lambda: defaulttree(depth - 1)) -# How many files one call may scan, per virtualization type. Every vtype -# holds one data array per file in memory until they are combined; the hdf5 -# one additionally builds an HDF5 virtual mapping per file, which dominates -# both the memory and the time. Tiles pays neither, so it gets far more room. -# The None entry is the fallback for engines whose vtype cannot be known here. -MAX_OPEN_FILES = {None: 100_000, "hdf5": 100_000, "tiles": 2_000_000} +# How many files one call may scan, for every vtype but "tiles". The hdf5 +# backing builds one HDF5 virtual mapping per file, which dominates both the +# scan memory and the time and stops being practical at this scale. A tiles +# scan retains only a few kilobytes per file, so it gets no ceiling. +MAX_OPEN_FILES = 100_000 def _resolve_engine(engine, vtype, ctype, engine_kwargs): @@ -622,11 +621,10 @@ def open_mfdataarray( FileNotFound If no file can be found. NotImplementedError - If more files are given than the vtype allows in one call. Scanning - keeps one data array per file in memory until they are combined, so the - ceiling is given by `MAX_OPEN_FILES`, which leaves far more room to the - much lighter tiles manifests. Larger sets must be opened in batches and - combined with `combine_by_coords`. + If more than `MAX_OPEN_FILES` files are given with a vtype other than + "tiles", whose scans are the only ones light enough to have no + ceiling. Larger sets must be opened in batches and combined with + `combine_by_coords`, or opened as tiles. """ paths = _ensure_str_paths(paths) if isinstance(paths, str): @@ -642,13 +640,12 @@ def open_mfdataarray( if len(paths) == 0: raise FileNotFoundError("no file to open") engine = _resolve_engine(engine, vtype, ctype, engine_kwargs) - limit = MAX_OPEN_FILES.get(engine.vtype, MAX_OPEN_FILES[None]) - if len(paths) > limit: + if engine.vtype != "tiles" and len(paths) > MAX_OPEN_FILES: raise NotImplementedError( - f"cannot open {len(paths)} files at once: the limit is {limit} for " - f"vtype {engine.vtype!r} because the scan holds one data array per " - "file in memory until they are combined. Open the files in batches " - "and pass the results to `combine_by_coords`." + f"cannot open {len(paths)} files at once with vtype " + f"{engine.vtype!r}: the limit is {MAX_OPEN_FILES}. Open the files " + "in batches and pass the results to `combine_by_coords`, or use " + "`vtype='tiles'`, which has no ceiling." ) max_workers = get_workers_count(parallel) objs = [] From b0ac12fae73e75978be8d1421fc0366b2c9aa1a8 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 19:18:30 +0200 Subject: [PATCH 42/56] Replace Douglas-Peucker with a one-pass sleeve in simplify --- .../coordinates/interpolated-coordinates.md | 8 +- tests/coordinates/test_interp.py | 75 +++++++++++- xdas/coordinates/interp.py | 107 +++++++++++++----- 3 files changed, 154 insertions(+), 36 deletions(-) diff --git a/docs/user-guide/coordinates/interpolated-coordinates.md b/docs/user-guide/coordinates/interpolated-coordinates.md index e5db746b..3dddbed6 100644 --- a/docs/user-guide/coordinates/interpolated-coordinates.md +++ b/docs/user-guide/coordinates/interpolated-coordinates.md @@ -84,8 +84,10 @@ Gaps represent missing data and are generally not problematic; overlaps usually arise from labelling errors and should be resolved. Using the {py:meth}`~xdas.coordinates.InterpCoordinate.simplify` method, -the coordinate can be compressed with controlled accuracy using the -[Ramer–Douglas–Peucker algorithm][RDP]. In the example below, the +the coordinate can be compressed with controlled accuracy using a +one-pass [sleeve algorithm][SDT] (tie points are dropped as long as the +segment joining the surviving neighbours passes within `tolerance` of +them, in a single left-to-right walk). In the example below, the second tie point carries no additional information and is safely discarded: ```{code-cell} @@ -162,4 +164,4 @@ coord.to_index(slice("2023-01-01T00:10:00", "2023-01-01T00:20:00")) ``` [CF]: -[RDP]: +[SDT]: diff --git a/tests/coordinates/test_interp.py b/tests/coordinates/test_interp.py index 562badc4..610925e9 100644 --- a/tests/coordinates/test_interp.py +++ b/tests/coordinates/test_interp.py @@ -392,7 +392,7 @@ def test_simplify_multiple_runs_and_isolated_point(self): ) def test_simplify_keeps_kink(self): - # A genuine kink inside a continuous area forces Douglas-Peucker to keep + # A genuine kink inside a continuous area forces the reduction to keep # the deviating interior point. coord = InterpCoordinate( {"tie_indices": [0, 5, 10], "tie_values": [0.0, 100.0, 0.0]} @@ -1122,7 +1122,7 @@ def test_widen_only_when_needed(self): assert result._is_valid_sampling_interval(s, result.tolerance) def test_widening_beyond_the_budget_never_raises(self): - # Douglas-Peucker bounds how far values move, not how much drift fusing + # The reduction bounds how far values move, not how much drift fusing # a discontinuity exposes, so the required tolerance can exceed the # budget. Real OptoDAS seams: 2 ms late every 10 s at 125 Hz. t0 = np.datetime64("2021-10-27T15:44:10.721999872", "ns") @@ -1215,3 +1215,74 @@ def test_empty(self): coord = InterpCoordinate.from_block(0.0, 0, 2.0, dim="x") assert coord.empty assert coord.sampling_interval == 2.0 + + +class TestSleeve: + """The one-pass reduce stage: same deviation guarantee as the former + Douglas-Peucker, O(n) whatever survives.""" + + def test_deviation_bound_holds_on_jitter(self): + rng = np.random.default_rng(0) + n = 200 + starts = np.arange(n, dtype="int64") * 10 + jitter = rng.integers(-1_500_000, 1_500_000, n) + t0 = np.datetime64("2026-01-01", "ns").astype("i8") + seg = t0 + np.arange(n) * 100_000_000 + jitter + tie_indices = np.empty(2 * n, dtype="int64") + tie_indices[0::2] = starts + tie_indices[1::2] = starts + 9 + tie_values = np.empty(2 * n, dtype="i8") + tie_values[0::2] = seg + tie_values[1::2] = seg + 90_000_000 + coord = InterpCoordinate( + {"tie_indices": tie_indices, "tie_values": tie_values.astype("M8[ns]")}, + "time", + ) + tolerance = np.timedelta64(1_000_000, "ns") + result = coord.simplify(tolerance) + # dropped points stay within tolerance of the simplified curve, and + # surviving values are never moved + deviation = np.abs(result._get_value(coord.tie_indices) - coord.tie_values) + assert deviation.max() <= tolerance + kept = np.isin(result.tie_indices, coord.tie_indices) + assert kept.all() + + def test_every_gap_survives(self): + n = 50 + tie_indices = np.arange(2 * n, dtype="int64") + tie_indices[1::2] = tie_indices[0::2] + 1 + tie_indices = np.cumsum(np.where(np.arange(2 * n) % 2, 1, 9)) + tie_indices -= tie_indices[0] + values = np.arange(2 * n) * 1_000_000_000 + coord = InterpCoordinate( + {"tie_indices": tie_indices, "tie_values": values.astype("M8[ns]")}, + "time", + ) + result = coord.simplify(np.timedelta64(1, "ms")) + assert len(result.tie_indices) == len(coord.tie_indices) + + def test_subunit_tolerance_is_not_truncated(self): + # microsecond values with a nanosecond tolerance: a 1 us seam jitter + # needs a sub-microsecond budget to survive (400 ns keeps it) and a + # 1100 ns one to fuse — both unrepresentable in truncated us + values = np.array([0, 999, 1001, 2000], dtype="M8[us]") + coord = InterpCoordinate( + {"tie_indices": [0, 999, 1000, 1999], "tie_values": values}, "time" + ) + kept = coord.simplify(np.timedelta64(400, "ns")) + fused = coord.simplify(np.timedelta64(1100, "ns")) + assert len(kept.tie_indices) == 4 + assert len(fused.tie_indices) == 2 + + def test_float_values(self): + coord = InterpCoordinate( + {"tie_indices": [0, 9, 10, 19], "tie_values": [0.0, 9.0, 10.5, 19.5]}, + "x", + ) + assert len(coord.simplify(1.0).tie_indices) == 2 + assert len(coord.simplify(0.1).tie_indices) == 4 + + def test_two_ties_pass_through(self): + coord = InterpCoordinate({"tie_indices": [0, 9], "tie_values": [0.0, 9.0]}) + result = coord.simplify(0.0) + assert len(result.tie_indices) == 2 diff --git a/xdas/coordinates/interp.py b/xdas/coordinates/interp.py index c50f85a2..1b19aec2 100644 --- a/xdas/coordinates/interp.py +++ b/xdas/coordinates/interp.py @@ -571,13 +571,14 @@ def to_regular(self, sampling_interval=None, tolerance=None): def simplify(self, tolerance=None, *, reduce=True, regularize=False): """Canonicalise within *tolerance*: drop tie points, then promote to regular. - The *reduce* stage runs Douglas-Peucker to drop tie points whose removal - shifts the curve by no more than *tolerance*. The CF 8.3 structure is - preserved as an emergent property of that bound: real discontinuities are - kept (any spanning line crosses them by far more than *tolerance*), soft - ones are fused into a single ramp, and synchronisation tie points survive - because removing them would, by definition, drift more than *tolerance*. - Surviving values are never moved. + The *reduce* stage runs a one-pass greedy sleeve (see :func:`_sleeve`) + to drop tie points whose removal shifts the curve by no more than + *tolerance*. The CF 8.3 structure is preserved as an emergent property + of that bound: real discontinuities are kept (any spanning line crosses + them by far more than *tolerance*), soft ones are fused into a single + ramp, and synchronisation tie points survive because removing them + would, by definition, drift more than *tolerance*. Surviving values + are never moved. The *regularize* stage promotes the result to *regular* when the surviving continuous segments admit a single ``sampling_interval`` within @@ -597,7 +598,7 @@ def simplify(self, tolerance=None, *, reduce=True, regularize=False): tolerance = self.tolerance tolerance = parse_scalar_delta(tolerance, self.dtype, default_zero=True) if reduce: - tie_indices, tie_values = _douglas_peucker( + tie_indices, tie_values = _sleeve( self.tie_indices, self.tie_values, tolerance ) else: @@ -672,12 +673,25 @@ def _continuous_segments(self): return num[mask], den[mask] -def _douglas_peucker(x, y, epsilon): +def _sleeve(x, y, epsilon): """ - Reduce the piecewise-linear curve *(x, y)* using the Douglas-Peucker algorithm. - - Points are dropped when they deviate less than *epsilon* from the simplified - line connecting their neighbours. + Reduce the piecewise-linear curve *(x, y)* with a one-pass greedy sleeve. + + Points are dropped when the segment connecting the surviving neighbours + passes within *epsilon* of them. The walk is left to right (the direction + acquisition produces tie points): from the current anchor it maintains the + intersection of every dropped point's ±*epsilon* slope cone — the sleeve — + and emits a knot exactly when a candidate leaves it. Knots are original + points, so surviving values are never moved, and any point whose removal + would drift the curve by more than *epsilon* (a discontinuity edge, a + synchronisation tie) empties the sleeve and survives. One pass, O(n) + whatever the number of surviving points — where Douglas-Peucker + degenerates quadratically once discontinuities or jitter make many + points survive. + + Integer and datetime values are compared with exact (arbitrary-precision) + cross-multiplied integer arithmetic, so a zero *epsilon* drops exactly the + collinear points; float values use float arithmetic. Parameters ---------- @@ -693,25 +707,56 @@ def _douglas_peucker(x, y, epsilon): x_simplified : numpy.ndarray y_simplified : numpy.ndarray """ - mask = np.ones(len(x), dtype=bool) - stack = [(0, len(x))] - while stack: - start, stop = stack.pop() - ysimple = forward( - x[start:stop], - x[[start, stop - 1]], - y[[start, stop - 1]], - ) - d = np.abs(y[start:stop] - ysimple) - index = np.argmax(d) - dmax = d[index] - index += start - if dmax > epsilon: - stack.append([start, index + 1]) - stack.append([index, stop]) + if len(x) < 3: + return x, y + if np.issubdtype(y.dtype, np.datetime64): + # exact integer arithmetic, with epsilon brought to the finer of the + # two time units so sub-unit tolerances are not truncated + unit, count = np.datetime_data(y.dtype) + one = np.timedelta64(count, unit) + common = np.promote_types(one.dtype, epsilon.dtype) + scale = int(one.astype(common).view("i8")) + values = (y.view("i8") * scale).tolist() + eps = int(epsilon.astype(common).view("i8")) + margin = one + else: + values = y.tolist() + eps = epsilon + margin = 1 if np.issubdtype(y.dtype, np.integer) else 0.0 + # fast path: one chord spans the whole curve (the fully continuous case, + # resolved vectorized). `forward` rounds to the value unit, so the chord + # is only trusted beyond a one-unit margin; near the boundary the exact + # loop below decides + deviation = np.abs(y - forward(x, x[[0, -1]], y[[0, -1]])) + if deviation.max() + margin <= epsilon: + return x[[0, -1]], y[[0, -1]] + positions = x.tolist() + keep = np.zeros(len(x), dtype=bool) + keep[0] = keep[-1] = True + ax, ay = positions[0], values[0] + # the sleeve: feasible slope interval from the anchor, as exact + # (numerator, positive denominator) rationals; None while unbounded + lo = hi = None + for i in range(1, len(positions)): + dx = positions[i] - ax + dy = values[i] - ay + if (lo is None or lo[0] * dx <= dy * lo[1]) and ( + hi is None or dy * hi[1] <= hi[0] * dx + ): + # the chord anchor -> i passes within epsilon of every dropped + # point; tighten the sleeve with this point's own cone + if lo is None or (dy - eps) * lo[1] > lo[0] * dx: + lo = (dy - eps, dx) + if hi is None or (dy + eps) * hi[1] < hi[0] * dx: + hi = (dy + eps, dx) else: - mask[start + 1 : stop - 1] = False - return x[mask], y[mask] + keep[i - 1] = True + ax, ay = positions[i - 1], values[i - 1] + dx = positions[i] - ax + dy = values[i] - ay + lo = (dy - eps, dx) + hi = (dy + eps, dx) + return x[keep], y[keep] def _chebyshev_center_pair(num, den): From 110771361d84240255e56d56a0b1c1de0097ea0e Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 19:18:42 +0200 Subject: [PATCH 43/56] Stream the multi-file combine and sort tiles at the end --- tests/test_routines.py | 230 +++++++++++++++++++++++++++++++++++++++++ tests/test_tiles.py | 33 ++++++ xdas/__init__.py | 2 + xdas/core/__init__.py | 2 + xdas/core/routines.py | 179 +++++++++++++++++++++++++++++++- xdas/tiles.py | 17 +++ 6 files changed, 458 insertions(+), 5 deletions(-) diff --git a/tests/test_routines.py b/tests/test_routines.py index 2733b1cf..cebb283b 100644 --- a/tests/test_routines.py +++ b/tests/test_routines.py @@ -744,3 +744,233 @@ def test_invalid_type_raises(self): with pytest.raises(TypeError, match="DataCollection"): _get_timeline_dataframe("not_valid") + + +class TestSortby: + def make_archive(self, tmp_path, vtype, pairs=((0, 2), (1, 3))): + """Save 4 time chunks and fuse them losslessly as two runs. + + `concat` sorts whatever it is given, so tile-level disorder is + built the way streamed scans produce it: runs that are internally + ordered but interleave each other. The default pairing yields the + tile order 0, 2, 1, 3. + """ + expected = xd.testing.dummy(dims=("time", "space"), shape=(20, 5)) + chunks = xd.split(expected, 4, "time") + parts = [] + for index, chunk in enumerate(chunks): + path = tmp_path / f"chunk_{index}.nc" + chunk.to_netcdf(path) + parts.append(xd.open_dataarray(path, engine="xdas", vtype=vtype)) + runs = [ + xd.concat([parts[i] for i in pair], "time", tolerance=False) + for pair in pairs + ] + return expected, xd.concat(runs, "time", tolerance=False) + + def test_sorts_tiles_lazily(self, tmp_path): + from xdas.tiles import TileArray + + expected, shuffled = self.make_archive(tmp_path, "tiles") + result = xd.sortby(shuffled, "time") + assert isinstance(result.data, TileArray) + assert result.equals(expected) + assert result["time"].equals(expected["time"]) + + def test_sorts_virtual_stack(self, tmp_path): + from xdas.virtual import VirtualStack + + expected, shuffled = self.make_archive(tmp_path, "hdf5") + result = xd.sortby(shuffled, "time") + assert isinstance(result.data, VirtualStack) + assert result.equals(expected) + + def test_already_sorted_fast_path(self, tmp_path): + expected, arranged = self.make_archive( + tmp_path, "tiles", pairs=((0, 1), (2, 3)) + ) + result = xd.sortby(arranged, "time") + assert result.equals(expected) + # a second sort is a no-op even though the coordinate is simplified + assert xd.sortby(result, "time").equals(expected) + + def test_tolerance_false_skips_simplification(self, tmp_path): + _, shuffled = self.make_archive(tmp_path, "tiles") + result = xd.sortby(shuffled, "time", tolerance=False) + # sorted but not simplified: one tie pair per chunk remains + assert len(result["time"].tie_indices) == 8 + assert bool(np.all(np.diff(result["time"].tie_values.astype("i8")) > 0)) + + def test_eager_data_raises(self): + da = xd.testing.dummy(dims=("time", "space"), shape=(10, 5)) + with pytest.raises(NotImplementedError, match="TileArray or a VirtualStack"): + xd.sortby(da, "time") + + def test_dense_coordinate_raises(self, tmp_path): + _, shuffled = self.make_archive(tmp_path, "tiles") + shuffled["time"] = shuffled["time"].values + with pytest.raises(NotImplementedError, match="interpolated"): + xd.sortby(shuffled, "time") + + def test_simplified_unsorted_raises(self, tmp_path): + # a coordinate whose ties span the first two tiles as one segment + # (the state a prior simplification leaves) while the tile order + # still needs fixing: the exact blockwise gather is impossible + from xdas.coordinates import InterpCoordinate + + _, misaligned = self.make_archive(tmp_path, "tiles") + coord = misaligned["time"] + misaligned["time"] = InterpCoordinate( + { + "tie_indices": np.array([0, 9, 10, 14, 15, 19]), + "tie_values": coord.tie_values[[0, 3, 4, 5, 6, 7]], + }, + "time", + ) + with pytest.raises(NotImplementedError, match="align"): + xd.sortby(misaligned, "time") + + +class TestStreamingCombine: + def save_shuffled(self, tmp_path, nchunk=6): + """Save chunks under names whose lexicographic order shuffles time.""" + expected = xd.testing.dummy(dims=("time", "space"), shape=(30, 5)) + names = ["e", "b", "f", "a", "d", "c"][:nchunk] + for chunk, name in zip(xd.split(expected, nchunk, "time"), names): + chunk.to_netcdf(tmp_path / f"{name}.nc") + return expected + + @pytest.mark.parametrize("vtype", ["tiles", "hdf5"]) + def test_matches_monolithic(self, tmp_path, monkeypatch, vtype): + from xdas.core import routines + + expected = self.save_shuffled(tmp_path) + mono = xd.open_mfdataarray( + tmp_path / "*.nc", engine="xdas", vtype=vtype, parallel=False + ) + monkeypatch.setattr(routines, "BATCH_SIZE", 2) + streamed = xd.open_mfdataarray( + tmp_path / "*.nc", engine="xdas", vtype=vtype, parallel=False + ) + assert streamed.equals(expected) + assert streamed["time"].equals(mono["time"]) + np.testing.assert_array_equal(np.asarray(streamed.data), np.asarray(mono.data)) + + def test_warns_and_recovers_on_corrupted_file(self, tmp_path, monkeypatch): + from xdas.core import routines + + expected = self.save_shuffled(tmp_path) + with (tmp_path / "ba.nc").open("wb") as file: + file.write(b"corrupted") + monkeypatch.setattr(routines, "BATCH_SIZE", 2) + with pytest.warns(RuntimeWarning): + streamed = xd.open_mfdataarray( + tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False + ) + assert streamed.equals(expected) + + def test_groups_interleaved_acquisitions_by_signature(self, tmp_path, monkeypatch): + from xdas.core import routines + + # acquisition A (5 channels) at t0 and t2, B (3 channels) at t1: + # signature grouping fuses A whole where the monolithic time-ordered + # walk would split it around B + wide = xd.testing.dummy(dims=("time", "space"), shape=(20, 5)) + chunks = xd.split(wide, 2, "time") + narrow = xd.testing.dummy(dims=("time", "space"), shape=(10, 3)) + narrow["time"] = narrow["time"] + ( + chunks[1]["time"][0].values - narrow["time"][0].values + ) + chunks[0].to_netcdf(tmp_path / "a.nc") + narrow.to_netcdf(tmp_path / "b.nc") + chunks[1].to_netcdf(tmp_path / "c.nc") + monkeypatch.setattr(routines, "BATCH_SIZE", 2) + streamed = xd.open_mfdataarray( + tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False + ) + assert isinstance(streamed, xd.DataCollection) + assert len(streamed) == 2 + + def test_single_run_squeezes(self, tmp_path, monkeypatch): + from xdas.core import routines + + expected = self.save_shuffled(tmp_path) + monkeypatch.setattr(routines, "BATCH_SIZE", 2) + collection = xd.open_mfdataarray( + tmp_path / "*.nc", + engine="xdas", + vtype="tiles", + parallel=False, + squeeze=False, + ) + assert isinstance(collection, xd.DataCollection) + assert len(collection) == 1 + assert collection[0].equals(expected) + + +class TestStreamingCombineFallbacks: + def test_dim_last_and_plain_name(self, tmp_path, monkeypatch): + from xdas.core import routines + + expected = xd.testing.dummy(dims=("time",), shape=(30,), step=0.01) + names = ["c", "a", "b"] + for chunk, name in zip(xd.split(expected, 3, "time"), names): + chunk.to_netcdf(tmp_path / f"{name}.nc") + monkeypatch.setattr(routines, "BATCH_SIZE", 2) + for dim in ("last", "time"): + result = xd.open_mfdataarray( + tmp_path / "*.nc", + dim=dim, + engine="xdas", + vtype="tiles", + parallel=False, + ) + assert result.equals(expected) + + def test_unsortable_group_falls_back_to_plain_concat(self, tmp_path, monkeypatch): + from xdas.core import routines + + # dense time coordinates: sortby cannot permute them, the group is + # concatenated with the tolerance directly (runs sorted by start) + expected = xd.testing.dummy( + dims=("time", "space"), shape=(30, 5), ctype="dense" + ) + for index, chunk in enumerate(xd.split(expected, 3, "time")): + chunk.to_netcdf(tmp_path / f"chunk_{index}.nc") + monkeypatch.setattr(routines, "BATCH_SIZE", 2) + result = xd.open_mfdataarray( + tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False + ) + assert result.equals(expected) + + def test_no_dim_coordinate(self, tmp_path, monkeypatch): + from xdas.core import routines + + da = xd.DataArray( + np.arange(30.0 * 5).reshape(30, 5), + coords={"space": {"tie_indices": [0, 4], "tie_values": [0.0, 40.0]}}, + dims=("time", "space"), + ) + for index in range(3): + da[10 * index : 10 * (index + 1)].to_netcdf(tmp_path / f"chunk_{index}.nc") + monkeypatch.setattr(routines, "BATCH_SIZE", 2) + result = xd.open_mfdataarray( + tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False + ) + assert result.shape == (30, 5) + + +class TestSortbyMetadataFree: + def test_permutes_without_declared_sampling_interval(self, tmp_path): + from xdas.coordinates import InterpCoordinate + + helper = TestSortby() + expected, shuffled = helper.make_archive(tmp_path, "tiles") + coord = shuffled["time"] + shuffled["time"] = InterpCoordinate( + {"tie_indices": coord.tie_indices, "tie_values": coord.tie_values}, + "time", + ) + result = xd.sortby(shuffled, "time") + assert np.array_equal(result["time"].values, expected["time"].values) + np.testing.assert_array_equal(np.asarray(result.data), expected.values) diff --git a/tests/test_tiles.py b/tests/test_tiles.py index 1560fd18..23288b16 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -2134,3 +2134,36 @@ def test_dask_write_deprecated(self, tmp_path): da.to_netcdf(path, virtual=True) reopened = xd.open_dataarray(path) npt.assert_array_equal(reopened.values, np.zeros((4, NX))) + + +class TestPermuteTiles: + def test_permutes_lazily(self, stack, engine_calls): + manifest, reference = stack + order = [2, 0, 1] + permuted = manifest._permute_tiles(order) + assert engine_calls == [] + expected = np.concatenate( + [reference[lo:hi] for lo, hi in ((17, 29), (0, 10), (10, 17))] + ) + npt.assert_array_equal(np.asarray(permuted), expected) + + def test_folded_params_stay_folded(self, stack): + manifest, _ = stack + arr = TileArray( + manifest.dataset.assign( + constant=((), np.asarray(7)), + record=(("tile_0",), np.arange(3)), + ), + manifest.dtype, + manifest.engine, + ) + permuted = arr._permute_tiles([1, 2, 0]) + assert tuple(permuted.dataset["constant"].dims) == () + npt.assert_array_equal(permuted.dataset["record"].values, [1, 2, 0]) + + def test_rejects_non_permutations(self, stack): + manifest, _ = stack + with pytest.raises(ValueError, match="permutation"): + manifest._permute_tiles([0, 0, 1]) + with pytest.raises(ValueError, match="permutation"): + manifest._permute_tiles([0, 1]) diff --git a/xdas/__init__.py b/xdas/__init__.py index 05536145..fab781c4 100644 --- a/xdas/__init__.py +++ b/xdas/__init__.py @@ -56,6 +56,7 @@ "open_mfdatacollection", "open_mfdatatree", "plot_availability", + "sortby", "split", ] @@ -108,6 +109,7 @@ open_mfdatatree, plot_availability, routines, + sortby, split, ) from .core.methods import * diff --git a/xdas/core/__init__.py b/xdas/core/__init__.py index cb570eb0..d988bb16 100644 --- a/xdas/core/__init__.py +++ b/xdas/core/__init__.py @@ -26,6 +26,7 @@ "open_mfdatacollection", "open_mfdatatree", "plot_availability", + "sortby", "split", ] @@ -48,5 +49,6 @@ open_mfdatacollection, open_mfdatatree, plot_availability, + sortby, split, ) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 34ad2b0b..d7649920 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -535,6 +535,11 @@ def defaulttree(depth): # scan retains only a few kilobytes per file, so it gets no ceiling. MAX_OPEN_FILES = 100_000 +# How many scan products `open_mfdataarray` holds before fusing them into +# compact runs. Bounds the scan memory; a scan that fits in one batch takes +# the exact monolithic path. +BATCH_SIZE = 10_000 + def _resolve_engine(engine, vtype, ctype, engine_kwargs): """Turn the `engine` argument of the open functions into an Engine instance.""" @@ -648,15 +653,26 @@ def open_mfdataarray( "`vtype='tiles'`, which has no ceiling." ) max_workers = get_workers_count(parallel) - objs = [] + objs = [] # pending scan products, drained into `runs` every BATCH_SIZE + runs = [] # per-batch continuous runs (streaming mode only) failures = [] + + def consume(da): + # stream the combine: every BATCH_SIZE scan products are fused into + # compact runs (losslessly: no coordinate simplification) and freed, + # so memory is bounded by the batch, not the archive + objs.append(da) + if len(objs) >= BATCH_SIZE: + runs.extend(combine_by_coords(objs, dim, False, False)) + objs.clear() + if (max_workers == 1) or (engine.name == "miniseed"): # TODO: dirty miniseed fix iterator = ( tqdm(paths, desc="Fetching metadata from files") if verbose else paths ) for path in iterator: try: - objs.append(open_dataarray(path, engine=engine)) + consume(open_dataarray(path, engine=engine)) except Exception as error: # noqa: BLE001 - collected and warned below failures.append((path, error)) warnings.warn(f"could not open {path}: {error}", RuntimeWarning) @@ -681,15 +697,72 @@ def open_mfdataarray( failures.append((path, error)) warnings.warn(f"could not open {path}: {error}", RuntimeWarning) else: - objs.append(obj) - if len(objs) == 0: # there must be failures + consume(obj) + if not objs and not runs: # there must be failures path, error = failures[0] raise RuntimeError( f"could not open any file with engine: " f"{engine.name or type(engine).__name__}; " f"first failure was {path}: {error}" ) from error - return combine_by_coords(objs, dim, tolerance, squeeze, None, verbose) + if not runs: + # a single batch: the exact monolithic path + return combine_by_coords(objs, dim, tolerance, squeeze, None, verbose) + if objs: + runs.extend(combine_by_coords(objs, dim, False, False)) + objs.clear() + return _combine_runs(runs, dim, tolerance, squeeze) + + +def _combine_runs(runs, dim, tolerance, squeeze): + """Fuse the compact runs of a streamed scan into the final collection. + + Runs are grouped by compatibility signature (unlike the monolithic + walk, grouping does not depend on time order, so acquisitions + interleaved in time still fuse into one array each). Each group is + concatenated without simplification — losslessly, whatever the arrival + order — then :func:`sortby` permutes the tiles into coordinate order + and spends the whole *tolerance* budget once, on sorted segments: + the same state, and so the same result, as the monolithic combine. + Groups whose data or coordinate :func:`sortby` cannot permute (eager + data, non-interpolated coordinates) are concatenated with *tolerance* + directly, correct whenever batches do not interleave in time. + """ + if dim == "first": + dim = runs[0].dims[0] + if dim == "last": + dim = runs[0].dims[-1] + bags = [] + for da in runs: + for bag in bags: + try: + bag.append(da) + break + except CompatibilityError: + continue + else: + bag = Bag(dim) + bag.append(da) + bags.append(bag) + results = [] + for bag in bags: + try: + fused = sortby(concat(bag, dim, tolerance=False), dim, tolerance) + except (KeyError, ValueError, NotImplementedError): + fused = concat(bag, dim, tolerance) + results.append(fused) + if all(dim in da.coords for da in results): + results.sort( + key=lambda da: ( + da[dim][0].values + if isinstance(da[dim], AxisCoordinate) + else da[dim].values + ) + ) + collection = DataCollection(results) + if squeeze and len(collection) == 1: + return collection[0] + return collection def open_dataarray(fname, engine=None, vtype=None, ctype=None, **engine_kwargs): @@ -1135,6 +1208,102 @@ def concat( concatenate = concat # TODO: deprecate it +def sortby(da, dim="first", tolerance=None): + """ + Sort a blocked virtual data array along *dim* by coordinate value, lazily. + + The data blocks (the tiles of a :class:`~xdas.tiles.TileArray`, the + sources of a :class:`~xdas.virtual.VirtualStack`) are permuted into + ascending start-value order without reading any of them: the permutation + is a manifest (or source-list) gather, and the coordinate tie points are + gathered blockwise the same way. Ties between equal start values keep + their current order. The reordered coordinate is then simplified with + *tolerance*, spending the accuracy budget once, on sorted segments — + exactly as :func:`concat` does on time-ordered inputs. + + Parameters + ---------- + da : DataArray + The data array to sort. Its data must be a :class:`TileArray` or a + :class:`VirtualStack` blocked along *dim*, and its *dim* coordinate + an interpolated coordinate whose tie points align with the block + boundaries (the state produced by concatenation without + simplification, ``tolerance=False``). + dim : str, optional + The dimension to sort along. Default to "first". + tolerance : float or timedelta64, optional + The tolerance spent by the final coordinate simplification. If None + (default), each coordinate spends its own declared tolerance. Pass + ``False`` to skip simplification entirely. + + Returns + ------- + DataArray + The sorted data array, as lazy as its input. + """ + from ..coordinates import InterpCoordinate + from ..tiles import TileArray + + axis = da.get_axis_num(dim) + dim = da.dims[axis] + coord = da.coords[dim] + if not isinstance(coord, InterpCoordinate): + raise NotImplementedError("can only sort along an interpolated coordinate") + data = da.data + if isinstance(data, TileArray): + sizes = np.asarray(data.chunks[axis]) + elif isinstance(data, VirtualStack) and data.axis == axis: + sizes = np.asarray([source.shape[axis] for source in data.sources]) + else: + raise NotImplementedError( + "can only sort a TileArray or a VirtualStack blocked along `dim`" + ) + edges = np.concatenate(([0], np.cumsum(sizes))) + tie_indices = coord.tie_indices + tie_values = coord.tie_values + order = np.argsort(coord._get_value(edges[:-1]), kind="stable") + if np.array_equal(order, np.arange(len(order))): + sorted_coord = coord + else: + # every block must begin and end on a tie point, so that the blockwise + # gather is exact: the state concatenation without simplification + # leaves; a simplified coordinate can only be verified, not permuted + starts = np.searchsorted(tie_indices, edges[:-1]) + ends = np.searchsorted(tie_indices, edges[1:] - 1, side="right") + if not ( + np.array_equal(tie_indices[starts], edges[:-1]) + and np.array_equal(tie_indices[ends - 1], edges[1:] - 1) + ): + raise NotImplementedError( + "tie points do not align with the block boundaries; sort " + "before simplifying (or concatenate with `tolerance=False`)" + ) + if isinstance(data, TileArray): + data = data._permute_tiles(order, axis) + else: + data = VirtualStack([data.sources[i] for i in order], axis) + # blockwise tie-point gather, fully vectorized: for each block in + # sorted order, its run of tie points, re-offset to its new position + counts = (ends - starts)[order] + offsets = np.cumsum(counts) - counts + gather = np.arange(counts.sum()) - np.repeat(offsets, counts) + gather += np.repeat(starts[order], counts) + new_edges = np.cumsum(sizes[order]) - sizes[order] + shift = np.repeat(new_edges - edges[:-1][order], counts) + parts = { + "tie_indices": tie_indices[gather] + shift, + "tie_values": tie_values[gather], + } + if coord.sampling_interval is not None: + parts["sampling_interval"] = coord.sampling_interval + parts["tolerance"] = coord.tolerance + sorted_coord = InterpCoordinate(parts, dim) + sorted_coord = sorted_coord.simplify(tolerance) + coords = da.coords.copy() + coords[dim] = sorted_coord + return DataArray(data, coords, da.dims, da.name, da.attrs) + + def concat_coords( objs, *, diff --git a/xdas/tiles.py b/xdas/tiles.py index 74225ac0..6b1950fd 100644 --- a/xdas/tiles.py +++ b/xdas/tiles.py @@ -783,6 +783,23 @@ def _fold(self, key): position += 1 return result + def _permute_tiles(self, order, axis=0): + """Reorder the tiles along virtual *axis*, staying virtual. + + One fancy-index over the manifest columns carrying the axis' tile + dimension — geometry, paths and varying parameters reorder together, + folded parameters and absent geometry columns are untouched — so no + per-tile object is ever created. *order* must be a permutation of + the axis' tiles (repeats or subsets would break the tile-to-sample + accounting of the callers). + """ + g = self._axes[axis] + order = np.asarray(order) + if not np.array_equal(np.sort(order), np.arange(len(self._sizes[g]))): + raise ValueError(f"`order` must be a permutation of axis {axis} tiles") + dataset = self.dataset.isel({self.dims[g]: order}) + return type(self)(dataset, self.dtype, self.engine) + def _flip(self, axis): """Reverse the array along virtual *axis*, staying virtual. From bc014cc7b7a6b0fadd8c0982f6c7ea58fcf2cd1b Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Mon, 3 Aug 2026 19:18:42 +0200 Subject: [PATCH 44/56] Document the streamed combine --- docs/api/xdas.md | 1 + docs/release-notes.md | 3 +++ docs/user-guide/io/virtual-datasets.md | 10 ++++++---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/api/xdas.md b/docs/api/xdas.md index 8ebc62ed..8012fb94 100644 --- a/docs/api/xdas.md +++ b/docs/api/xdas.md @@ -32,6 +32,7 @@ concatenate concat_coords get_sampling_interval + sortby split plot_availability ``` diff --git a/docs/release-notes.md b/docs/release-notes.md index 32dd55b6..7fa41027 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,6 +8,9 @@ - Tile-backed arrays round-trip through the native xdas netCDF format: the manifest is stored as a compact `__tiles__` sibling group, relocatable by editing its single root path and directly readable by the 0.3 line (@atrabattoni). - **Explicit engine configuration.** The open functions declare `engine`, `vtype` and `ctype`, and `engine` also accepts a configured `xdas.io.Engine` instance. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front (@atrabattoni). - `open_mfdataarray` no longer caps the number of files when the resolved vtype is `tiles`; the 100 000 ceiling remains for `hdf5`, which builds one virtual mapping per file (@atrabattoni). +- **Streamed multi-file combining.** `open_mfdataarray` now fuses scan results every 10 000 files instead of holding one data array per file until the end, so memory no longer grows with the archive: results are accumulated without coordinate simplification (lossless in any arrival order) and sorted once at the end, giving the same result as before whatever the file naming. Acquisitions interleaved in time now group by compatibility, one array per acquisition, instead of splitting at each alternation (@atrabattoni). +- **`xdas.sortby`.** Sort a tile- or stack-backed data array along a dimension by coordinate value, lazily: the blocks are permuted through the manifest without reading any data. This is how the streamed combine orders shuffled archives, exposed for standalone use (@atrabattoni). +- `simplify` runs in linear time whatever the number of gaps: the reduce stage is now a one-pass sleeve instead of Douglas-Peucker, which degenerated quadratically on gap-rich coordinates (a 100 000-file gappy archive simplified in minutes; now milliseconds). The deviation guarantee is unchanged — dropped tie points stay within `tolerance` of the curve, surviving values never move — though the surviving tie-point selection may differ slightly on jittery axes (@atrabattoni). ### Breaking Changes - Passing a bare read function as `engine` now raises a `TypeError`: subclass `xdas.io.Engine` instead (see the data-formats documentation) (@atrabattoni). diff --git a/docs/user-guide/io/virtual-datasets.md b/docs/user-guide/io/virtual-datasets.md index d3a5d443..0de9ed22 100644 --- a/docs/user-guide/io/virtual-datasets.md +++ b/docs/user-guide/io/virtual-datasets.md @@ -182,10 +182,12 @@ tiles as the archive grows. - Building either manifest starts with reading the metadata of every file. That scan is dominated by disk access and is usually the bulk of the total build time, so it is not a criterion for choosing between the two. -- Both keep one data array per file in memory while scanning — a few kilobytes each for - tiles, much more for HDF5, whose per-file mapping cost caps a single call at 100 000 - files. Sets beyond that (or beyond what memory allows) must be opened in batches and - combined afterwards. +- Scanning combines its results every 10 000 files, so memory does not grow with the + archive. What remains per file is the mapping itself: negligible for tiles (which is + why it has no file-count ceiling), one HDF5 virtual mapping for `hdf5`, whose cost + caps a single call at 100 000 files. +- The combined result does not depend on the scan order: files are sorted by their + coordinate values, not by their names. - Neither backend helps when a coordinate is not monotonic — for instance when files overlap in time. Label-based selection then falls back to a slow path in both cases, and is better addressed in the data itself. From 491c7e948a6d090da9cfb75abd1c48809aeca0e8 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 00:12:44 +0200 Subject: [PATCH 45/56] Bound the scan by one limit the vtype declares The scan ceiling and the streaming batch size were two numbers for one quantity: how many per-file scan products may be held at once. Fold them into MAX_OPEN_FILES and let CONSOLIDATING_VTYPES say which vtypes can shrink a drained batch, instead of naming tiles in the check. Since the batch equals the ceiling, anything that opened in one call before still takes the single-batch path, and a vtype that cannot consolidate never reaches the streaming path at all. open_mfdatacollection had the same ceiling as a bare literal; it now shares the constant. --- docs/release-notes.md | 4 +- docs/user-guide/io/virtual-datasets.md | 9 +++-- tests/test_routines.py | 35 +++++++++++------ xdas/core/routines.py | 54 +++++++++++++------------- 4 files changed, 59 insertions(+), 43 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 7fa41027..7116e427 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -7,8 +7,8 @@ - **`vtype="tiles"` on every HDF5 engine.** The open functions with `vtype="tiles"` return tile-backed arrays for the asn, febus, terra15, apsensing, prodml and native xdas engines. Silixa and MiniSEED always emit them now (replacing the serialized-dask-graph fallback, with time-axis push-down for Silixa), and Febus defaults to them — one tile per file, where the HDF5 backing needed one virtual mapping per data block. Custom engines add support by implementing `Engine.load_tile(path, selection, **params)` (@atrabattoni). - Tile-backed arrays round-trip through the native xdas netCDF format: the manifest is stored as a compact `__tiles__` sibling group, relocatable by editing its single root path and directly readable by the 0.3 line (@atrabattoni). - **Explicit engine configuration.** The open functions declare `engine`, `vtype` and `ctype`, and `engine` also accepts a configured `xdas.io.Engine` instance. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front (@atrabattoni). -- `open_mfdataarray` no longer caps the number of files when the resolved vtype is `tiles`; the 100 000 ceiling remains for `hdf5`, which builds one virtual mapping per file (@atrabattoni). -- **Streamed multi-file combining.** `open_mfdataarray` now fuses scan results every 10 000 files instead of holding one data array per file until the end, so memory no longer grows with the archive: results are accumulated without coordinate simplification (lossless in any arrival order) and sorted once at the end, giving the same result as before whatever the file naming. Acquisitions interleaved in time now group by compatibility, one array per acquisition, instead of splitting at each alternation (@atrabattoni). +- `open_mfdataarray` no longer caps the number of files when the resolved vtype consolidates its scan results, which `tiles` does; the 100 000 ceiling remains for `hdf5`, which builds one virtual mapping per file and so cannot be fused into anything smaller (@atrabattoni). +- **Streamed multi-file combining.** `open_mfdataarray` now fuses scan results every 100 000 files instead of holding one data array per file until the end, so memory no longer grows with the archive: results are accumulated without coordinate simplification (lossless in any arrival order) and sorted once at the end, giving the same result as before whatever the file naming. Acquisitions interleaved in time now group by compatibility, one array per acquisition, instead of splitting at each alternation. Since the batch size is also the ceiling, anything that opened in one call before still takes the single-batch path unchanged (@atrabattoni). - **`xdas.sortby`.** Sort a tile- or stack-backed data array along a dimension by coordinate value, lazily: the blocks are permuted through the manifest without reading any data. This is how the streamed combine orders shuffled archives, exposed for standalone use (@atrabattoni). - `simplify` runs in linear time whatever the number of gaps: the reduce stage is now a one-pass sleeve instead of Douglas-Peucker, which degenerated quadratically on gap-rich coordinates (a 100 000-file gappy archive simplified in minutes; now milliseconds). The deviation guarantee is unchanged — dropped tie points stay within `tolerance` of the curve, surviving values never move — though the surviving tie-point selection may differ slightly on jittery axes (@atrabattoni). diff --git a/docs/user-guide/io/virtual-datasets.md b/docs/user-guide/io/virtual-datasets.md index 0de9ed22..e4adc637 100644 --- a/docs/user-guide/io/virtual-datasets.md +++ b/docs/user-guide/io/virtual-datasets.md @@ -182,10 +182,11 @@ tiles as the archive grows. - Building either manifest starts with reading the metadata of every file. That scan is dominated by disk access and is usually the bulk of the total build time, so it is not a criterion for choosing between the two. -- Scanning combines its results every 10 000 files, so memory does not grow with the - archive. What remains per file is the mapping itself: negligible for tiles (which is - why it has no file-count ceiling), one HDF5 virtual mapping for `hdf5`, whose cost - caps a single call at 100 000 files. +- A scan holds at most 100 000 results at once. Tiles combines them every 100 000 files + into a manifest whose per-file cost is negligible, so the batch is freed and a single + call can scan any number of files. Fusing changes nothing for `hdf5`, which keeps one + HDF5 virtual mapping per file whatever it does, so 100 000 is a ceiling for it instead + of a batch size. - The combined result does not depend on the scan order: files are sorted by their coordinate values, not by their names. - Neither backend helps when a coordinate is not monotonic — for instance when files diff --git a/tests/test_routines.py b/tests/test_routines.py index cebb283b..54ad681a 100644 --- a/tests/test_routines.py +++ b/tests/test_routines.py @@ -840,29 +840,42 @@ def save_shuffled(self, tmp_path, nchunk=6): chunk.to_netcdf(tmp_path / f"{name}.nc") return expected - @pytest.mark.parametrize("vtype", ["tiles", "hdf5"]) - def test_matches_monolithic(self, tmp_path, monkeypatch, vtype): + def test_matches_monolithic(self, tmp_path, monkeypatch): from xdas.core import routines expected = self.save_shuffled(tmp_path) mono = xd.open_mfdataarray( - tmp_path / "*.nc", engine="xdas", vtype=vtype, parallel=False + tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False ) - monkeypatch.setattr(routines, "BATCH_SIZE", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) streamed = xd.open_mfdataarray( - tmp_path / "*.nc", engine="xdas", vtype=vtype, parallel=False + tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False ) assert streamed.equals(expected) assert streamed["time"].equals(mono["time"]) np.testing.assert_array_equal(np.asarray(streamed.data), np.asarray(mono.data)) + def test_non_consolidating_vtype_raises_instead_of_streaming( + self, tmp_path, monkeypatch + ): + from xdas.core import routines + + # the batch size is the ceiling, so a vtype that cannot consolidate + # never reaches the streaming path: it raises at the first batch + self.save_shuffled(tmp_path) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) + with pytest.raises(NotImplementedError, match="cannot be consolidated"): + xd.open_mfdataarray( + tmp_path / "*.nc", engine="xdas", vtype="hdf5", parallel=False + ) + def test_warns_and_recovers_on_corrupted_file(self, tmp_path, monkeypatch): from xdas.core import routines expected = self.save_shuffled(tmp_path) with (tmp_path / "ba.nc").open("wb") as file: file.write(b"corrupted") - monkeypatch.setattr(routines, "BATCH_SIZE", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) with pytest.warns(RuntimeWarning): streamed = xd.open_mfdataarray( tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False @@ -884,7 +897,7 @@ def test_groups_interleaved_acquisitions_by_signature(self, tmp_path, monkeypatc chunks[0].to_netcdf(tmp_path / "a.nc") narrow.to_netcdf(tmp_path / "b.nc") chunks[1].to_netcdf(tmp_path / "c.nc") - monkeypatch.setattr(routines, "BATCH_SIZE", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) streamed = xd.open_mfdataarray( tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False ) @@ -895,7 +908,7 @@ def test_single_run_squeezes(self, tmp_path, monkeypatch): from xdas.core import routines expected = self.save_shuffled(tmp_path) - monkeypatch.setattr(routines, "BATCH_SIZE", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) collection = xd.open_mfdataarray( tmp_path / "*.nc", engine="xdas", @@ -916,7 +929,7 @@ def test_dim_last_and_plain_name(self, tmp_path, monkeypatch): names = ["c", "a", "b"] for chunk, name in zip(xd.split(expected, 3, "time"), names): chunk.to_netcdf(tmp_path / f"{name}.nc") - monkeypatch.setattr(routines, "BATCH_SIZE", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) for dim in ("last", "time"): result = xd.open_mfdataarray( tmp_path / "*.nc", @@ -937,7 +950,7 @@ def test_unsortable_group_falls_back_to_plain_concat(self, tmp_path, monkeypatch ) for index, chunk in enumerate(xd.split(expected, 3, "time")): chunk.to_netcdf(tmp_path / f"chunk_{index}.nc") - monkeypatch.setattr(routines, "BATCH_SIZE", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) result = xd.open_mfdataarray( tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False ) @@ -953,7 +966,7 @@ def test_no_dim_coordinate(self, tmp_path, monkeypatch): ) for index in range(3): da[10 * index : 10 * (index + 1)].to_netcdf(tmp_path / f"chunk_{index}.nc") - monkeypatch.setattr(routines, "BATCH_SIZE", 2) + monkeypatch.setattr(routines, "MAX_OPEN_FILES", 2) result = xd.open_mfdataarray( tmp_path / "*.nc", engine="xdas", vtype="tiles", parallel=False ) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index d7649920..43827f29 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -27,6 +27,16 @@ from .dataarray import DataArray from .datacollection import DataCollection, DataMapping, DataSequence +# How many scan products one call may hold at once: a scan keeps one data array +# per file (~6 KiB) until they are fused. Vtypes that consolidate drain a full +# batch and carry on; for the others this is a hard ceiling. +MAX_OPEN_FILES = 100_000 + +# Vtypes whose concatenation fuses the per-file scan products into one compact +# object, so draining a batch frees memory. An hdf5 stack keeps one virtual +# mapping per source, so batching would free nothing. +CONSOLIDATING_VTYPES = frozenset({"tiles"}) + def open( paths, @@ -260,10 +270,11 @@ def open_mfdatacollection( ) if len(paths) == 0: raise FileNotFoundError("no file to open") - if len(paths) > 100_000: + if len(paths) > MAX_OPEN_FILES: raise NotImplementedError( - "The maximum number of file that can be opened at once is for now limited " - "to 100 000." + f"cannot open {len(paths)} files at once: the limit is " + f"{MAX_OPEN_FILES}, because the scan holds one data collection per " + "file in memory. Open the files in batches and combine the results." ) max_workers = get_workers_count(parallel) if max_workers == 1: @@ -529,18 +540,6 @@ def defaulttree(depth): return defaultdict(lambda: defaulttree(depth - 1)) -# How many files one call may scan, for every vtype but "tiles". The hdf5 -# backing builds one HDF5 virtual mapping per file, which dominates both the -# scan memory and the time and stops being practical at this scale. A tiles -# scan retains only a few kilobytes per file, so it gets no ceiling. -MAX_OPEN_FILES = 100_000 - -# How many scan products `open_mfdataarray` holds before fusing them into -# compact runs. Bounds the scan memory; a scan that fits in one batch takes -# the exact monolithic path. -BATCH_SIZE = 10_000 - - def _resolve_engine(engine, vtype, ctype, engine_kwargs): """Turn the `engine` argument of the open functions into an Engine instance.""" from ..io.core import Engine @@ -626,10 +625,11 @@ def open_mfdataarray( FileNotFound If no file can be found. NotImplementedError - If more than `MAX_OPEN_FILES` files are given with a vtype other than - "tiles", whose scans are the only ones light enough to have no - ceiling. Larger sets must be opened in batches and combined with - `combine_by_coords`, or opened as tiles. + If more than `MAX_OPEN_FILES` files are given with a vtype that does not + consolidate (see `CONSOLIDATING_VTYPES`). A consolidating vtype scans any + number of files, `MAX_OPEN_FILES` at a time; the others hold every scan + product until the end, so larger sets must be opened in batches and + combined with `combine_by_coords`. """ paths = _ensure_str_paths(paths) if isinstance(paths, str): @@ -645,24 +645,26 @@ def open_mfdataarray( if len(paths) == 0: raise FileNotFoundError("no file to open") engine = _resolve_engine(engine, vtype, ctype, engine_kwargs) - if engine.vtype != "tiles" and len(paths) > MAX_OPEN_FILES: + if engine.vtype not in CONSOLIDATING_VTYPES and len(paths) > MAX_OPEN_FILES: + consolidating = ", ".join(repr(name) for name in sorted(CONSOLIDATING_VTYPES)) raise NotImplementedError( f"cannot open {len(paths)} files at once with vtype " - f"{engine.vtype!r}: the limit is {MAX_OPEN_FILES}. Open the files " - "in batches and pass the results to `combine_by_coords`, or use " - "`vtype='tiles'`, which has no ceiling." + f"{engine.vtype!r}: the limit is {MAX_OPEN_FILES}, because its scan " + "products cannot be consolidated into a compact one. Open the files " + "in batches and pass the results to `combine_by_coords`, or use a " + f"vtype that consolidates ({consolidating}), which has no ceiling." ) max_workers = get_workers_count(parallel) - objs = [] # pending scan products, drained into `runs` every BATCH_SIZE + objs = [] # pending scan products, drained into `runs` every MAX_OPEN_FILES runs = [] # per-batch continuous runs (streaming mode only) failures = [] def consume(da): - # stream the combine: every BATCH_SIZE scan products are fused into + # stream the combine: every MAX_OPEN_FILES scan products are fused into # compact runs (losslessly: no coordinate simplification) and freed, # so memory is bounded by the batch, not the archive objs.append(da) - if len(objs) >= BATCH_SIZE: + if len(objs) >= MAX_OPEN_FILES: runs.extend(combine_by_coords(objs, dim, False, False)) objs.clear() From f749e8268f958b7af09b34e2c84fc5558a5f4772 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 15:19:48 +0200 Subject: [PATCH 46/56] Gather the virtual backends into a package xdas/virtual.py becomes xdas/virtual/hdf5.py (the HDF5 virtual dataset backend) and xdas/tiles.py becomes xdas/virtual/tiles.py, under one xdas.virtual package that re-exports everything both modules exposed. Tests re-mirror to tests/virtual/. No behavior change. --- docs/api/tiles.md | 4 +-- docs/release-notes.md | 2 +- docs/user-guide/io/data-formats.md | 4 +-- docs/user-guide/io/virtual-datasets.md | 2 +- tests/io/test_miniseed.py | 2 +- tests/io/test_silixa.py | 2 +- tests/io/test_tiles_vtype.py | 2 +- tests/test_routines.py | 2 +- .../{test_virtual.py => virtual/test_hdf5.py} | 2 +- tests/{ => virtual}/test_tiles.py | 16 +++++---- xdas/__init__.py | 2 -- xdas/core/dataarray.py | 4 +-- xdas/core/routines.py | 5 ++- xdas/io/apsensing.py | 3 +- xdas/io/asn.py | 3 +- xdas/io/core.py | 2 +- xdas/io/febus.py | 3 +- xdas/io/miniseed.py | 2 +- xdas/io/prodml.py | 3 +- xdas/io/silixa.py | 2 +- xdas/io/terra15.py | 3 +- xdas/io/xdas.py | 7 ++-- xdas/virtual/__init__.py | 33 +++++++++++++++++++ xdas/{virtual.py => virtual/hdf5.py} | 2 +- xdas/{ => virtual}/tiles.py | 4 +-- 25 files changed, 71 insertions(+), 45 deletions(-) rename tests/{test_virtual.py => virtual/test_hdf5.py} (99%) rename tests/{ => virtual}/test_tiles.py (99%) create mode 100644 xdas/virtual/__init__.py rename xdas/{virtual.py => virtual/hdf5.py} (99%) rename xdas/{ => virtual}/tiles.py (99%) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index 59b06d01..c71f0521 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -1,8 +1,8 @@ ```{eval-rst} -.. currentmodule:: xdas.tiles +.. currentmodule:: xdas.virtual.tiles ``` -# xdas.tiles +# xdas.virtual.tiles Lazy tile-backed virtual arrays: the only backend of the formats that HDF5 virtual datasets cannot serve (Silixa TDMS, MiniSEED), the default diff --git a/docs/release-notes.md b/docs/release-notes.md index 7116e427..f433afee 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -3,7 +3,7 @@ ## 0.2.9 (unreleased) ### New Features -- **Tile-backed virtual arrays.** The new `xdas.tiles` module exposes file archives as one lazy `TileArray`. Slicing (any step, including negative), integer indexing, `np.newaxis`, concatenation, and the numpy manipulation routines (the `transpose`, `flip`, `split`, `stack` and `atleast` families, `expand_dims`, `squeeze`, `roll`, `tile`, `delete`, `append`/`insert`) all stay lazy; whole-array reductions (`sum`, `mean`, `min`, `max`, …) stream one tile row at a time; reads touch only the tiles the selection overlaps (@atrabattoni). +- **Tile-backed virtual arrays.** The new `xdas.virtual.tiles` module exposes file archives as one lazy `TileArray`. Slicing (any step, including negative), integer indexing, `np.newaxis`, concatenation, and the numpy manipulation routines (the `transpose`, `flip`, `split`, `stack` and `atleast` families, `expand_dims`, `squeeze`, `roll`, `tile`, `delete`, `append`/`insert`) all stay lazy; whole-array reductions (`sum`, `mean`, `min`, `max`, …) stream one tile row at a time; reads touch only the tiles the selection overlaps (@atrabattoni). - **`vtype="tiles"` on every HDF5 engine.** The open functions with `vtype="tiles"` return tile-backed arrays for the asn, febus, terra15, apsensing, prodml and native xdas engines. Silixa and MiniSEED always emit them now (replacing the serialized-dask-graph fallback, with time-axis push-down for Silixa), and Febus defaults to them — one tile per file, where the HDF5 backing needed one virtual mapping per data block. Custom engines add support by implementing `Engine.load_tile(path, selection, **params)` (@atrabattoni). - Tile-backed arrays round-trip through the native xdas netCDF format: the manifest is stored as a compact `__tiles__` sibling group, relocatable by editing its single root path and directly readable by the 0.3 line (@atrabattoni). - **Explicit engine configuration.** The open functions declare `engine`, `vtype` and `ctype`, and `engine` also accepts a configured `xdas.io.Engine` instance. Format-specific parameters (`overlaps`/`offset` for febus, `ignore_last_sample` for miniseed, `swapped_dims` for prodml, `tz` for terra15, `group` for the native format) are engine constructor parameters, validated up front (@atrabattoni). diff --git a/docs/user-guide/io/data-formats.md b/docs/user-guide/io/data-formats.md index e422be04..29f01c8f 100644 --- a/docs/user-guide/io/data-formats.md +++ b/docs/user-guide/io/data-formats.md @@ -134,7 +134,7 @@ da Beside the `hdf5` vtype shown above (an HDF5 virtual source), an engine can offer the `tiles` vtype: `open_dataarray` then backs the data array with a lazy -{py:class}`xdas.tiles.TileArray` describing the file, and the engine implements +{py:class}`xdas.virtual.TileArray` describing the file, and the engine implements the decoding half as a `load_tile` static method — called once per tile touched, with exactly one source-local slice per source axis, in source order (whatever transposes or inserted axes the tile array presents), and the @@ -142,7 +142,7 @@ manifest's engine specification as keyword arguments, returning exactly the selected sub-box: ```{code-cell} -from xdas.tiles import TileArray +from xdas.virtual import TileArray class MyTileEngine(Engine, name="my_tile_engine"): _supported_vtypes = ["hdf5", "tiles"] diff --git a/docs/user-guide/io/virtual-datasets.md b/docs/user-guide/io/virtual-datasets.md index e4adc637..2a205161 100644 --- a/docs/user-guide/io/virtual-datasets.md +++ b/docs/user-guide/io/virtual-datasets.md @@ -91,7 +91,7 @@ When loading large part of a virtual dataset, you might end up with nan values. ## Tile Virtualization With the `tiles` vtype, the mapping is not delegated to HDF5. *Xdas* stores it as a -{py:class}`xdas.tiles.TileArray`: a plain array manifest that records, for each tile, which +{py:class}`xdas.virtual.TileArray`: a plain array manifest that records, for each tile, which file it comes from and which part of that file it contributes. Reading a region resolves which tiles it touches and asks the engine to decode each of them through its `load_tile` method. The manifest is ordinary data, so it can be inspected, sliced and concatenated diff --git a/tests/io/test_miniseed.py b/tests/io/test_miniseed.py index 9974dfd3..71c10f7a 100644 --- a/tests/io/test_miniseed.py +++ b/tests/io/test_miniseed.py @@ -6,7 +6,7 @@ import xdas as xd from xdas.coordinates import Coordinate from xdas.io.miniseed import MiniSEEDEngine, get_band_code, to_stream -from xdas.tiles import TileArray +from xdas.virtual import TileArray def make_network(dirpath, gap=False, samples=100): diff --git a/tests/io/test_silixa.py b/tests/io/test_silixa.py index 5d14563b..68b55eaf 100644 --- a/tests/io/test_silixa.py +++ b/tests/io/test_silixa.py @@ -2,7 +2,7 @@ import numpy.testing as npt from xdas.io import silixa -from xdas.tiles import TileArray +from xdas.virtual import TileArray class FakeTdms: diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index 11fe68a8..8c30a11c 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -6,7 +6,7 @@ import pytest import xdas as xd -from xdas.tiles import TileArray +from xdas.virtual import TileArray def ramp(shape, dtype="float32"): diff --git a/tests/test_routines.py b/tests/test_routines.py index 54ad681a..8d46f041 100644 --- a/tests/test_routines.py +++ b/tests/test_routines.py @@ -769,7 +769,7 @@ def make_archive(self, tmp_path, vtype, pairs=((0, 2), (1, 3))): return expected, xd.concat(runs, "time", tolerance=False) def test_sorts_tiles_lazily(self, tmp_path): - from xdas.tiles import TileArray + from xdas.virtual import TileArray expected, shuffled = self.make_archive(tmp_path, "tiles") result = xd.sortby(shuffled, "time") diff --git a/tests/test_virtual.py b/tests/virtual/test_hdf5.py similarity index 99% rename from tests/test_virtual.py rename to tests/virtual/test_hdf5.py index 503e4c8d..1a31ed62 100644 --- a/tests/test_virtual.py +++ b/tests/virtual/test_hdf5.py @@ -3,7 +3,7 @@ import pytest import xdas as xd -from xdas.virtual import ( +from xdas.virtual.hdf5 import ( Selection, Selectors, SingleSelector, diff --git a/tests/test_tiles.py b/tests/virtual/test_tiles.py similarity index 99% rename from tests/test_tiles.py rename to tests/virtual/test_tiles.py index 23288b16..b2ab6386 100644 --- a/tests/test_tiles.py +++ b/tests/virtual/test_tiles.py @@ -12,7 +12,7 @@ import xdas as xd from xdas.io import Engine -from xdas.tiles import TileArray +from xdas.virtual.tiles import TileArray NX = 5 @@ -415,7 +415,7 @@ def test_rootless_manifest_reads(self, tmp_path): assert legacy.equals(manifest) and manifest.equals(legacy) def test_no_common_directory_keeps_paths_whole(self): - from xdas.tiles import _split_root + from xdas.virtual.tiles import _split_root mixed = np.array([b"rel/f.h5", b"/abs/g.h5"], dtype=object) root, kept = _split_root(mixed) @@ -446,11 +446,13 @@ def make(root, path): def test_no_common_directory_falls_back_rootless(self, tmp_path, monkeypatch): """Paths sharing no directory (several drives) store whole, rootless.""" - import xdas.tiles + import xdas.virtual.tiles data = np.arange(4.0 * NX).reshape(4, NX) _tile_file(tmp_path / "d.h5", data) - monkeypatch.setattr(xdas.tiles, "_split_root", lambda paths: ("", paths)) + monkeypatch.setattr( + xdas.virtual.tiles, "_split_root", lambda paths: ("", paths) + ) manifest = TileArray.from_tiles(str(tmp_path / "d.h5"), (4, NX), "f8", ENGINE) assert manifest.root == "" and "root" not in manifest.dataset npt.assert_array_equal(np.asarray(manifest), data) @@ -1002,7 +1004,7 @@ def test_negative_axis_method(self, stack): npt.assert_array_equal(np.asarray(expanded), reference[np.newaxis]) def test_dispatch_guards(self, stack): - from xdas.tiles import _expand_dims_virtual + from xdas.virtual.tiles import _expand_dims_virtual manifest, _ = stack expand = np.expand_dims @@ -1903,7 +1905,7 @@ def test_boolean_masks(self, stack): manifest, reference = stack mask = reference[:, 0] > 10 npt.assert_array_equal(manifest[mask], reference[mask]) - from xdas.tiles import _bounding_key + from xdas.virtual.tiles import _bounding_key with pytest.raises(NotImplementedError, match="boolean mask"): _bounding_key( @@ -1938,7 +1940,7 @@ def test_concatenate_fallbacks(self, stack): assert casted.dtype == np.float32 out = np.concatenate([manifest, manifest], 0, None) npt.assert_array_equal(out, np.concatenate([reference, reference])) - from xdas.tiles import _concatenate_virtual + from xdas.virtual.tiles import _concatenate_virtual concat = np.concatenate assert _concatenate_virtual(manifest, concat, (), {}) is NotImplemented diff --git a/xdas/__init__.py b/xdas/__init__.py index fab781c4..a1966f7d 100644 --- a/xdas/__init__.py +++ b/xdas/__init__.py @@ -25,7 +25,6 @@ "signal", "synthetics", "testing", - "tiles", "virtual", # classes "Coordinate", @@ -71,7 +70,6 @@ signal, synthetics, testing, - tiles, virtual, ) from .coordinates import ( diff --git a/xdas/core/dataarray.py b/xdas/core/dataarray.py index 37e35f7c..1ed99f1a 100644 --- a/xdas/core/dataarray.py +++ b/xdas/core/dataarray.py @@ -15,7 +15,7 @@ from numpy.lib.mixins import NDArrayOperatorsMixin from ..coordinates import AxisCoordinate, Coordinates -from ..virtual import _to_human +from ..virtual.hdf5 import _to_human HANDLED_NUMPY_FUNCTIONS = {} HANDLED_METHODS = {} @@ -188,7 +188,7 @@ def conjugate(self): @property def data(self): - """The underlying array (numpy, dask, :class:`~xdas.virtual.VirtualArray`, or :class:`~xdas.tiles.TileArray`).""" + """The underlying array (numpy, dask, :class:`~xdas.virtual.VirtualArray`, or :class:`~xdas.virtual.TileArray`).""" return self._data @data.setter diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 43827f29..6249ca50 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -23,7 +23,7 @@ from ..coordinates import AxisCoordinate, Coordinates from ..parallel import get_workers_count -from ..virtual import VirtualSource, VirtualStack +from ..virtual import TileArray, VirtualSource, VirtualStack from .dataarray import DataArray from .datacollection import DataCollection, DataMapping, DataSequence @@ -1214,7 +1214,7 @@ def sortby(da, dim="first", tolerance=None): """ Sort a blocked virtual data array along *dim* by coordinate value, lazily. - The data blocks (the tiles of a :class:`~xdas.tiles.TileArray`, the + The data blocks (the tiles of a :class:`~xdas.virtual.TileArray`, the sources of a :class:`~xdas.virtual.VirtualStack`) are permuted into ascending start-value order without reading any of them: the permutation is a manifest (or source-list) gather, and the coordinate tie points are @@ -1244,7 +1244,6 @@ def sortby(da, dim="first", tolerance=None): The sorted data array, as lazy as its input. """ from ..coordinates import InterpCoordinate - from ..tiles import TileArray axis = da.get_axis_num(dim) dim = da.dims[axis] diff --git a/xdas/io/apsensing.py b/xdas/io/apsensing.py index c2a52e22..94d4e01f 100644 --- a/xdas/io/apsensing.py +++ b/xdas/io/apsensing.py @@ -7,8 +7,7 @@ from ..coordinates import Coordinate from ..core import DataArray -from ..tiles import TileArray -from ..virtual import VirtualSource +from ..virtual import TileArray, VirtualSource from .core import Engine diff --git a/xdas/io/asn.py b/xdas/io/asn.py index 4d991a0e..97175d6c 100644 --- a/xdas/io/asn.py +++ b/xdas/io/asn.py @@ -15,8 +15,7 @@ from ..coordinates import Coordinate, get_sampling_interval from ..core import DataArray, concat_coords -from ..tiles import TileArray -from ..virtual import VirtualSource +from ..virtual import TileArray, VirtualSource from .core import Engine diff --git a/xdas/io/core.py b/xdas/io/core.py index 5938e086..93d27e3e 100644 --- a/xdas/io/core.py +++ b/xdas/io/core.py @@ -124,7 +124,7 @@ def load_tile(path, selection, **kwargs): """Read the selected sub-box of one tile of *path* (abstract). The decode half of the tiles machinery: called on the class by - :class:`~xdas.tiles.TileArray` once per tile touched, with + :class:`~xdas.virtual.TileArray` once per tile touched, with exactly one source-local, possibly strided :class:`slice` per source axis, in source order — whatever virtual arrangement (transposes, inserted axes) the tile array presents — and the diff --git a/xdas/io/febus.py b/xdas/io/febus.py index ac633b48..c8404c23 100644 --- a/xdas/io/febus.py +++ b/xdas/io/febus.py @@ -8,8 +8,7 @@ from ..coordinates import Coordinate from ..core import DataArray, concat, concat_coords -from ..tiles import TileArray -from ..virtual import VirtualSource +from ..virtual import TileArray, VirtualSource from .core import Engine diff --git a/xdas/io/miniseed.py b/xdas/io/miniseed.py index a70bc6d4..7c9546f3 100644 --- a/xdas/io/miniseed.py +++ b/xdas/io/miniseed.py @@ -12,7 +12,7 @@ get_sampling_interval, ) from ..core import DataArray, concat_coords -from ..tiles import TileArray +from ..virtual import TileArray from .core import Engine diff --git a/xdas/io/prodml.py b/xdas/io/prodml.py index 1d9b30a2..70c7b7b4 100644 --- a/xdas/io/prodml.py +++ b/xdas/io/prodml.py @@ -11,8 +11,7 @@ from ..coordinates import Coordinate from ..core import DataArray -from ..tiles import TileArray -from ..virtual import VirtualSource +from ..virtual import TileArray, VirtualSource from .core import Engine _RAWDATA = "/Acquisition/Raw[0]/RawData" diff --git a/xdas/io/silixa.py b/xdas/io/silixa.py index dd9ff82c..b5aa4bd7 100644 --- a/xdas/io/silixa.py +++ b/xdas/io/silixa.py @@ -6,7 +6,7 @@ from ..coordinates import Coordinate from ..core import DataArray -from ..tiles import TileArray +from ..virtual import TileArray from .core import Engine from .tdms import TdmsReader diff --git a/xdas/io/terra15.py b/xdas/io/terra15.py index 376be295..a0f64882 100644 --- a/xdas/io/terra15.py +++ b/xdas/io/terra15.py @@ -7,8 +7,7 @@ from ..coordinates import Coordinate from ..core import DataArray -from ..tiles import TileArray -from ..virtual import VirtualSource +from ..virtual import TileArray, VirtualSource from .core import Engine diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 04d67b1e..231b500f 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -19,8 +19,7 @@ from ..coordinates import Coordinates from ..core import DataArray, DataCollection, DataMapping, DataSequence from ..dask import create_variable, loads -from ..tiles import TileArray -from ..virtual import VirtualArray, VirtualSource +from ..virtual import TileArray, VirtualArray, VirtualSource from .core import Engine TILES_GROUP = "__tiles__" @@ -98,7 +97,7 @@ def open_dataarray(fname, group=None, vtype=None): vtype : str, optional Virtualization backing of the returned data: ``"hdf5"`` (default, an HDF5 virtual source) or ``"tiles"`` (a lazy - :class:`~xdas.tiles.TileArray` over the stored variable). Files + :class:`~xdas.virtual.TileArray` over the stored variable). Files that store a tile manifest reopen as tile arrays regardless. Returns @@ -251,7 +250,7 @@ def save_dataarray( elif isinstance(da.data, DaskArray): warnings.warn( "writing dask-backed virtual arrays is deprecated; the " - "tile-backed engines (xdas.tiles) replace them", + "tile-backed engines (xdas.virtual.tiles) replace them", FutureWarning, ) variable = create_variable( diff --git a/xdas/virtual/__init__.py b/xdas/virtual/__init__.py new file mode 100644 index 00000000..322f2ee8 --- /dev/null +++ b/xdas/virtual/__init__.py @@ -0,0 +1,33 @@ +""" +Virtual (lazy) array backends over on-disk sources. + +Two backends coexist, selected by the ``vtype`` of the open functions: +:mod:`xdas.virtual.hdf5` exposes HDF5 virtual datasets +(:class:`VirtualSource`, :class:`VirtualStack`, :class:`VirtualLayout` +under the :class:`VirtualArray` base) and :mod:`xdas.virtual.tiles` +exposes tile manifests (:class:`TileArray`). +""" + +__all__ = [ + "Selection", + "Selectors", + "SingleSelector", + "SliceSelector", + "TileArray", + "VirtualArray", + "VirtualLayout", + "VirtualSource", + "VirtualStack", +] + +from .hdf5 import ( + Selection, + Selectors, + SingleSelector, + SliceSelector, + VirtualArray, + VirtualLayout, + VirtualSource, + VirtualStack, +) +from .tiles import TileArray diff --git a/xdas/virtual.py b/xdas/virtual/hdf5.py similarity index 99% rename from xdas/virtual.py rename to xdas/virtual/hdf5.py index 17748539..25ee9499 100644 --- a/xdas/virtual.py +++ b/xdas/virtual/hdf5.py @@ -1,5 +1,5 @@ """ -Virtual (lazy) array types for deferred HDF5/NetCDF4 access. +The HDF5 virtual dataset backend (``vtype="hdf5"``). Includes :class:`VirtualArray` base, :class:`VirtualSource` for a single dataset slice, and :class:`VirtualStack` for concatenating sources along diff --git a/xdas/tiles.py b/xdas/virtual/tiles.py similarity index 99% rename from xdas/tiles.py rename to xdas/virtual/tiles.py index 6b1950fd..fdcbcfca 100644 --- a/xdas/tiles.py +++ b/xdas/virtual/tiles.py @@ -407,7 +407,7 @@ def __init__(self, dataset, dtype, engine): if not isinstance(engine, dict) or "name" not in engine: raise ValueError("the engine specification must have a `name` key") # imported here: xdas.io imports this module at package init - from .io.core import Engine + from ..io.core import Engine Engine[engine["name"]] # fail fast on unregistered engines self._engine = engine @@ -962,7 +962,7 @@ def _grid_values(self, name): @functools.cached_property def _engine_impl(self): """The ``(load_tile, spec)`` of the engine specification.""" - from .io.core import Engine + from ..io.core import Engine spec = dict(self.engine) name = spec.pop("name") From 519e6cae5dcfcc45eee56f3839e88401dd086ca2 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 15:21:14 +0200 Subject: [PATCH 47/56] Register the virtual backends by vtype VirtualBackend is a marker base whose whole job is naming: backends register by passing vtype= in the class definition and are retrieved with VirtualBackend[vtype], the same registry fashion as Engine (by name) and Coordinate (by ctype). The duck-array contract stays informal: the backends share no implementation. The consolidates flag declares whether concatenation fuses scan products into one compact object. --- docs/api/virtual.md | 13 +++++++- tests/virtual/test_core.py | 37 ++++++++++++++++++++++ xdas/virtual/__init__.py | 6 +++- xdas/virtual/core.py | 63 ++++++++++++++++++++++++++++++++++++++ xdas/virtual/hdf5.py | 4 ++- xdas/virtual/tiles.py | 8 ++++- 6 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 tests/virtual/test_core.py create mode 100644 xdas/virtual/core.py diff --git a/docs/api/virtual.md b/docs/api/virtual.md index 3c1e40a1..56e7842f 100644 --- a/docs/api/virtual.md +++ b/docs/api/virtual.md @@ -4,9 +4,20 @@ # xdas.virtual +## VirtualBackend + +Marker base and `vtype` registry for the virtual array backends. + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary + + VirtualBackend +``` + ## VirtualArray -Base class for all virtual array types. +Base class of the HDF5 virtual dataset backend (`vtype="hdf5"`). Attributes diff --git a/tests/virtual/test_core.py b/tests/virtual/test_core.py new file mode 100644 index 00000000..fb608e90 --- /dev/null +++ b/tests/virtual/test_core.py @@ -0,0 +1,37 @@ +import pytest + +from xdas.virtual import ( + TileArray, + VirtualArray, + VirtualBackend, + VirtualSource, + VirtualStack, +) + + +class TestVirtualBackend: + def test_lookup(self): + assert VirtualBackend["hdf5"] is VirtualArray + assert VirtualBackend["tiles"] is TileArray + + def test_unknown_vtype_raises_key_error(self): + with pytest.raises(KeyError, match="no virtual backend registered"): + VirtualBackend["netcdf"] + + def test_registry_holds_only_named_backends(self): + assert set(VirtualBackend._registry) == {"hdf5", "tiles"} + + def test_subclasses_inherit_vtype_without_reregistering(self): + assert VirtualSource.vtype == "hdf5" + assert VirtualStack.vtype == "hdf5" + assert VirtualBackend["hdf5"] is VirtualArray + + def test_consolidates(self): + assert TileArray.consolidates + assert not VirtualArray.consolidates + assert not VirtualBackend.consolidates + + def test_isinstance_covers_both_backends(self): + source = VirtualSource("path.h5", "data", (2, 3), "f8") + assert isinstance(source, VirtualBackend) + assert isinstance(VirtualStack([source]), VirtualBackend) diff --git a/xdas/virtual/__init__.py b/xdas/virtual/__init__.py index 322f2ee8..a4b10714 100644 --- a/xdas/virtual/__init__.py +++ b/xdas/virtual/__init__.py @@ -5,7 +5,9 @@ :mod:`xdas.virtual.hdf5` exposes HDF5 virtual datasets (:class:`VirtualSource`, :class:`VirtualStack`, :class:`VirtualLayout` under the :class:`VirtualArray` base) and :mod:`xdas.virtual.tiles` -exposes tile manifests (:class:`TileArray`). +exposes tile manifests (:class:`TileArray`). Both register on the +:class:`VirtualBackend` base and are retrieved with +``VirtualBackend[vtype]``. """ __all__ = [ @@ -15,11 +17,13 @@ "SliceSelector", "TileArray", "VirtualArray", + "VirtualBackend", "VirtualLayout", "VirtualSource", "VirtualStack", ] +from .core import VirtualBackend from .hdf5 import ( Selection, Selectors, diff --git a/xdas/virtual/core.py b/xdas/virtual/core.py new file mode 100644 index 00000000..dec03a4c --- /dev/null +++ b/xdas/virtual/core.py @@ -0,0 +1,63 @@ +"""Registry base class :class:`VirtualBackend` for the virtual array backends.""" + +from typing import ClassVar + + +class VirtualBackend: + """ + Marker base and registry for the virtual array backends. + + A virtual backend is a lazy, numpy-like duck array whose values stay + on disk: it reports ``shape`` and ``dtype`` without reading, slices + lazily through ``__getitem__``, and materializes through + ``__array__``. That contract is informal — the backends share no + implementation and differ beyond it (blocking, persistence, + concatenation), so this base only names them: subclasses register by + passing ``vtype=`` in the class definition and are retrieved with + the ``VirtualBackend[vtype]`` syntax — the same registry fashion as + :class:`~xdas.io.Engine` (by name) and + :class:`~xdas.coordinates.Coordinate` (by ctype). + + Attributes + ---------- + vtype : str or None + The registered name of the backend, inherited by its subclasses + (``VirtualSource.vtype`` is ``"hdf5"``). ``None`` on this base. + consolidates : bool + Whether concatenating scan products of this backend fuses them + into one compact object, so that multi-file scans can drain + batches and keep memory bounded. Default ``False``: an HDF5 + stack keeps one virtual mapping per source, so batching would + free nothing. + + Examples + -------- + >>> from xdas.virtual import TileArray, VirtualArray, VirtualBackend + + >>> VirtualBackend["tiles"] is TileArray + True + >>> VirtualBackend["hdf5"] is VirtualArray + True + + >>> VirtualBackend["netcdf"] + Traceback (most recent call last): + KeyError: "no virtual backend registered under 'netcdf'; available: ['hdf5', 'tiles']" + """ + + _registry: ClassVar[dict] = {} + vtype: ClassVar[str | None] = None + consolidates: ClassVar[bool] = False + + def __init_subclass__(cls, *, vtype=None, **kwargs): + super().__init_subclass__(**kwargs) + if vtype is not None: + cls.vtype = vtype + VirtualBackend._registry[vtype] = cls + + def __class_getitem__(cls, item): + if item in cls._registry: + return cls._registry[item] + raise KeyError( + f"no virtual backend registered under {item!r}; " + f"available: {sorted(cls._registry)}" + ) diff --git a/xdas/virtual/hdf5.py b/xdas/virtual/hdf5.py index 25ee9499..08cebbaa 100644 --- a/xdas/virtual/hdf5.py +++ b/xdas/virtual/hdf5.py @@ -13,8 +13,10 @@ import h5py import numpy as np +from .core import VirtualBackend -class VirtualArray: + +class VirtualArray(VirtualBackend, vtype="hdf5"): """ Abstract base class for lazy array objects backed by HDF5/NetCDF4 files. diff --git a/xdas/virtual/tiles.py b/xdas/virtual/tiles.py index fdcbcfca..1e7dc8dd 100644 --- a/xdas/virtual/tiles.py +++ b/xdas/virtual/tiles.py @@ -105,6 +105,8 @@ import numpy as np import xarray as xr +from .core import VirtualBackend + TILE_PREFIX = "tile_" """Prefix of the tile-grid dimensions of a manifest dataset.""" @@ -345,7 +347,7 @@ def _materialize(value): return value -class TileArray(np.lib.mixins.NDArrayOperatorsMixin): +class TileArray(VirtualBackend, np.lib.mixins.NDArrayOperatorsMixin, vtype="tiles"): """A dense rectilinear grid of file-backed tiles as one virtual array. Numpy-like duck array over the *manifest dataset* described in the @@ -387,6 +389,10 @@ class TileArray(np.lib.mixins.NDArrayOperatorsMixin): settings (``vtype``, ``ctype``) do not belong in it. """ + # manifest concatenation fuses scan products into one compact array, + # so multi-file scans can drain batches (see VirtualBackend) + consolidates = True + def __init__(self, dataset, dtype, engine): # canonical string dtype is fixed-width bytes: str-valued # variables (hand-built or legacy stored manifests) recode here From 3aae8f3b306bc1ef09f748cc9947d37cfdcf8ac8 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 15:23:33 +0200 Subject: [PATCH 48/56] Look up backend capabilities from the registry The consolidating vtypes are no longer a hardcoded set in the routines: the multi-file scan asks the registered backend's consolidates flag, and the save-side virtual detection covers any registered backend instead of enumerating the classes. --- xdas/core/routines.py | 21 ++++++++++++--------- xdas/io/xdas.py | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 6249ca50..696d8e77 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -23,7 +23,7 @@ from ..coordinates import AxisCoordinate, Coordinates from ..parallel import get_workers_count -from ..virtual import TileArray, VirtualSource, VirtualStack +from ..virtual import TileArray, VirtualBackend, VirtualSource, VirtualStack from .dataarray import DataArray from .datacollection import DataCollection, DataMapping, DataSequence @@ -32,11 +32,6 @@ # batch and carry on; for the others this is a hard ceiling. MAX_OPEN_FILES = 100_000 -# Vtypes whose concatenation fuses the per-file scan products into one compact -# object, so draining a batch frees memory. An hdf5 stack keeps one virtual -# mapping per source, so batching would free nothing. -CONSOLIDATING_VTYPES = frozenset({"tiles"}) - def open( paths, @@ -626,7 +621,7 @@ def open_mfdataarray( If no file can be found. NotImplementedError If more than `MAX_OPEN_FILES` files are given with a vtype that does not - consolidate (see `CONSOLIDATING_VTYPES`). A consolidating vtype scans any + consolidate (see `VirtualBackend.consolidates`). A consolidating vtype scans any number of files, `MAX_OPEN_FILES` at a time; the others hold every scan product until the end, so larger sets must be opened in batches and combined with `combine_by_coords`. @@ -645,8 +640,16 @@ def open_mfdataarray( if len(paths) == 0: raise FileNotFoundError("no file to open") engine = _resolve_engine(engine, vtype, ctype, engine_kwargs) - if engine.vtype not in CONSOLIDATING_VTYPES and len(paths) > MAX_OPEN_FILES: - consolidating = ", ".join(repr(name) for name in sorted(CONSOLIDATING_VTYPES)) + backend = VirtualBackend._registry.get(engine.vtype) + if ( + not (backend is not None and backend.consolidates) + and len(paths) > MAX_OPEN_FILES + ): + consolidating = ", ".join( + repr(vtype) + for vtype, cls in sorted(VirtualBackend._registry.items()) + if cls.consolidates + ) raise NotImplementedError( f"cannot open {len(paths)} files at once with vtype " f"{engine.vtype!r}: the limit is {MAX_OPEN_FILES}, because its scan " diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 231b500f..e511142c 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -19,7 +19,7 @@ from ..coordinates import Coordinates from ..core import DataArray, DataCollection, DataMapping, DataSequence from ..dask import create_variable, loads -from ..virtual import TileArray, VirtualArray, VirtualSource +from ..virtual import TileArray, VirtualArray, VirtualBackend, VirtualSource from .core import Engine TILES_GROUP = "__tiles__" @@ -199,7 +199,7 @@ def save_dataarray( fname = str(fname) if virtual is None: - virtual = isinstance(da.data, (VirtualArray, DaskArray, TileArray)) + virtual = isinstance(da.data, (VirtualBackend, DaskArray)) # initialize dataset = xr.Dataset(attrs={"Conventions": "CF-1.9"}) From a10cf8b73b65a66ea6787c14a045853ee5ec3b17 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 15:23:33 +0200 Subject: [PATCH 49/56] Validate vtype names against the backend registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any non-None vtype must name a registered VirtualBackend before the per-engine support check, so a typo fails fast with the registered list — even through AutoEngine, which has no supported list of its own. --- tests/io/test_generic.py | 12 ++++++++++++ xdas/io/core.py | 13 ++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/io/test_generic.py b/tests/io/test_generic.py index 839ae364..28f44a95 100644 --- a/tests/io/test_generic.py +++ b/tests/io/test_generic.py @@ -16,6 +16,18 @@ def test_invalid_vtype_raises_value_error(self): with pytest.raises(ValueError, match="vtype must be None or a string"): Engine["asn"](vtype=42) + def test_unregistered_vtype_raises_key_error(self): + with pytest.raises(KeyError, match="no virtual backend registered"): + Engine["asn"](vtype="netcdf") + + def test_auto_engine_validates_vtype_upfront(self): + with pytest.raises(KeyError, match="no virtual backend registered"): + AutoEngine(vtype="netcdf") + + def test_registered_but_unsupported_vtype_still_engine_checked(self): + with pytest.raises(NotImplementedError, match="not supported by"): + Engine["silixa"](vtype="hdf5") + def test_dict_ctype_fills_missing_keys(self): engine = Engine["asn"](ctype={"time": "interpolated"}) assert engine.ctype["time"] == "interpolated" diff --git a/xdas/io/core.py b/xdas/io/core.py index 93d27e3e..59ee5273 100644 --- a/xdas/io/core.py +++ b/xdas/io/core.py @@ -8,6 +8,8 @@ import socket from typing import ClassVar +from ..virtual import VirtualBackend + class Engine: """ @@ -44,7 +46,8 @@ class Engine: Notes ----- Subclasses should define class attributes: - - `_supported_vtypes` (list): List of supported virtualization types + - `_supported_vtypes` (list): List of supported virtualization types, each + the name of a registered :class:`~xdas.virtual.VirtualBackend` - `_supported_ctypes` (dict): Maps component names to lists of supported coordinate types @@ -137,14 +140,14 @@ def load_tile(path, selection, **kwargs): raise NotImplementedError def _parse_vtype(self, vtype): + if vtype is not None: + if not isinstance(vtype, str): + raise ValueError("vtype must be None or a string") + VirtualBackend[vtype] # fail fast on unregistered vtypes if self._supported_vtypes is None: return vtype if vtype is None: vtype = self._supported_vtypes[0] - elif isinstance(vtype, str): - pass - else: - raise ValueError("vtype must be None or a string") if vtype not in self._supported_vtypes: raise NotImplementedError( f"vtype '{vtype}' is not supported by {self.__class__.__name__}" From 13df14109a9fb3f03d8cd714fba5fd660cace405 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 15:49:48 +0200 Subject: [PATCH 50/56] Repair three tiles imports the package move missed 'from xdas import tiles' kept resolving through the editable install's finder to the pre-move main checkout, so the suite stayed green until the move was merged there. --- tests/virtual/test_tiles.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/virtual/test_tiles.py b/tests/virtual/test_tiles.py index b2ab6386..12de5034 100644 --- a/tests/virtual/test_tiles.py +++ b/tests/virtual/test_tiles.py @@ -1403,7 +1403,7 @@ def test_error_parity(self, stack, line): def test_lazy_rewrite_guards(self, stack): """Handlers step aside for calls that are not theirs to rewrite.""" - from xdas import tiles + from xdas.virtual import tiles manifest, _ = stack other = np.zeros(3) @@ -1689,7 +1689,7 @@ def test_map_error_parity(self, stack, line): def test_map_dispatch_guards(self, stack): """Handlers step aside for calls that are not theirs to rewrite.""" - from xdas import tiles + from xdas.virtual import tiles manifest, _ = stack other = np.zeros((3, 4)) @@ -1871,7 +1871,7 @@ def test_reversed_slice_on_bounded_path(self, stack): npt.assert_array_equal(picked, reference[::-1, [1, 3]]) def test_flip_dispatch_guards(self, stack): - from xdas import tiles + from xdas.virtual import tiles manifest, _ = stack other = np.zeros((3, 4)) From 9b7bc60c08a5a8b5fb784ce971b5f34e1334afd9 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 15:49:48 +0200 Subject: [PATCH 51/56] Dispatch the plain-variable wrap through the registry Both backends gain a from_variable classmethod that exposes one stored HDF5 variable as their lazy array (a virtual source; a single tile decoded by the generic 'xdas' engine), so the native format's open path becomes VirtualBackend[vtype].from_variable(variable) instead of branching on the vtype literal. --- docs/api/tiles.md | 1 + docs/api/virtual.md | 1 + tests/virtual/test_core.py | 15 +++++++++++++++ xdas/io/xdas.py | 14 ++++---------- xdas/virtual/core.py | 9 ++++++--- xdas/virtual/hdf5.py | 16 ++++++++++++++++ xdas/virtual/tiles.py | 24 ++++++++++++++++++++++++ 7 files changed, 67 insertions(+), 13 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index c71f0521..40b46fe6 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -37,6 +37,7 @@ Methods :toctree: ../_autosummary TileArray.from_tiles + TileArray.from_variable TileArray.to_dataset TileArray.concat TileArray.expand_dims diff --git a/docs/api/virtual.md b/docs/api/virtual.md index 56e7842f..93104ba2 100644 --- a/docs/api/virtual.md +++ b/docs/api/virtual.md @@ -39,6 +39,7 @@ Methods .. autosummary:: :toctree: ../_autosummary + VirtualArray.from_variable VirtualArray.to_dataset ``` diff --git a/tests/virtual/test_core.py b/tests/virtual/test_core.py index fb608e90..9a65beb2 100644 --- a/tests/virtual/test_core.py +++ b/tests/virtual/test_core.py @@ -1,3 +1,5 @@ +import h5py +import numpy as np import pytest from xdas.virtual import ( @@ -35,3 +37,16 @@ def test_isinstance_covers_both_backends(self): source = VirtualSource("path.h5", "data", (2, 3), "f8") assert isinstance(source, VirtualBackend) assert isinstance(VirtualStack([source]), VirtualBackend) + + def test_from_variable_dispatches_to_each_backend(self, tmp_path): + data = np.arange(6.0).reshape(2, 3) + path = tmp_path / "source.h5" + with h5py.File(path, "w") as file: + file.create_dataset("data", data=data) + with h5py.File(path) as file: + source = VirtualBackend["hdf5"].from_variable(file["data"]) + tiled = VirtualBackend["tiles"].from_variable(file["data"]) + assert isinstance(source, VirtualSource) + assert isinstance(tiled, TileArray) + np.testing.assert_array_equal(np.asarray(source), data) + np.testing.assert_array_equal(np.asarray(tiled), data) diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index e511142c..67f70cac 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -19,7 +19,7 @@ from ..coordinates import Coordinates from ..core import DataArray, DataCollection, DataMapping, DataSequence from ..dask import create_variable, loads -from ..virtual import TileArray, VirtualArray, VirtualBackend, VirtualSource +from ..virtual import TileArray, VirtualArray, VirtualBackend from .core import Engine TILES_GROUP = "__tiles__" @@ -152,15 +152,9 @@ def open_dataarray(fname, group=None, vtype=None): if group: file = file[group] variable = file["__values__" if name is None else name] - if vtype == "tiles": - data = TileArray.from_tiles( - str(fname), - variable.shape, - variable.dtype, - {"name": "xdas", "dataset": variable.name}, - ) - else: - data = VirtualSource(variable) + data = VirtualBackend["hdf5" if vtype is None else vtype].from_variable( + variable + ) # pack everything return DataArray( diff --git a/xdas/virtual/core.py b/xdas/virtual/core.py index dec03a4c..5d057296 100644 --- a/xdas/virtual/core.py +++ b/xdas/virtual/core.py @@ -10,9 +10,12 @@ class VirtualBackend: A virtual backend is a lazy, numpy-like duck array whose values stay on disk: it reports ``shape`` and ``dtype`` without reading, slices lazily through ``__getitem__``, and materializes through - ``__array__``. That contract is informal — the backends share no - implementation and differ beyond it (blocking, persistence, - concatenation), so this base only names them: subclasses register by + ``__array__``; a ``from_variable`` classmethod wraps one stored + HDF5 variable, so open paths dispatch + ``VirtualBackend[vtype].from_variable(...)``. That contract is + informal — the backends share no implementation and differ beyond + it (blocking, persistence, concatenation), so this base only names + them: subclasses register by passing ``vtype=`` in the class definition and are retrieved with the ``VirtualBackend[vtype]`` syntax — the same registry fashion as :class:`~xdas.io.Engine` (by name) and diff --git a/xdas/virtual/hdf5.py b/xdas/virtual/hdf5.py index 08cebbaa..ae241bda 100644 --- a/xdas/virtual/hdf5.py +++ b/xdas/virtual/hdf5.py @@ -47,6 +47,22 @@ def to_dataset(self, file_or_group, name): """Write this virtual array as an HDF5 dataset (abstract — must be overridden).""" raise NotImplementedError + @classmethod + def from_variable(cls, variable): + """ + Expose one stored HDF5 variable as a lazy virtual source. + + Parameters + ---------- + variable : h5py.Dataset + The open variable to wrap. + + Returns + ------- + VirtualSource + """ + return VirtualSource(variable) + @property def ndim(self): """Number of dimensions.""" diff --git a/xdas/virtual/tiles.py b/xdas/virtual/tiles.py index 1e7dc8dd..72cb95ce 100644 --- a/xdas/virtual/tiles.py +++ b/xdas/virtual/tiles.py @@ -595,6 +595,30 @@ def from_tiles(cls, paths, sizes, dtype, engine, *, attrs=None, **params): dataset = xr.Dataset(data, attrs=dict(attrs or {})) return cls(dataset, dtype, engine) + @classmethod + def from_variable(cls, variable): + """ + Expose one stored HDF5 variable as a single-tile array. + + The whole variable is one tile, decoded by the generic "xdas" + engine (a plain h5py read of the named dataset). + + Parameters + ---------- + variable : h5py.Dataset + The open variable to wrap. + + Returns + ------- + TileArray + """ + return cls.from_tiles( + variable.file.filename, + variable.shape, + variable.dtype, + {"name": "xdas", "dataset": variable.name}, + ) + def to_dataset(self): """Encode this tile array as its manifest dataset. From 486321f9893f187af1693ea774fb2c3987fc1bc5 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 15:53:25 +0200 Subject: [PATCH 52/56] Let the backends write their own stored form create_variable/finalize_save form the persistence pair of the VirtualBackend contract: the first writes the variable inside the open h5netcdf handle (an HDF5 virtual dataset; a placeholder carrying the engine specification), the second appends what outlives it once the handle closes (nothing; the tile manifest as a sibling group). The native format's save path loses its per-backend branches, and the TILES_GROUP convention moves to the tiles module with the code that writes it. --- docs/api/tiles.md | 2 ++ tests/virtual/test_core.py | 6 ++++ xdas/io/xdas.py | 28 ++++-------------- xdas/virtual/core.py | 31 +++++++++++++++++--- xdas/virtual/tiles.py | 58 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 26 deletions(-) diff --git a/docs/api/tiles.md b/docs/api/tiles.md index 40b46fe6..dec6c782 100644 --- a/docs/api/tiles.md +++ b/docs/api/tiles.md @@ -39,6 +39,8 @@ Methods TileArray.from_tiles TileArray.from_variable TileArray.to_dataset + TileArray.create_variable + TileArray.finalize_save TileArray.concat TileArray.expand_dims TileArray.squeeze diff --git a/tests/virtual/test_core.py b/tests/virtual/test_core.py index 9a65beb2..990a0f8d 100644 --- a/tests/virtual/test_core.py +++ b/tests/virtual/test_core.py @@ -38,6 +38,12 @@ def test_isinstance_covers_both_backends(self): assert isinstance(source, VirtualBackend) assert isinstance(VirtualStack([source]), VirtualBackend) + def test_persistence_contract_defaults(self): + backend = VirtualBackend() + with pytest.raises(NotImplementedError): + backend.create_variable(None, "data") + assert backend.finalize_save("path.nc") is None + def test_from_variable_dispatches_to_each_backend(self, tmp_path): data = np.arange(6.0).reshape(2, 3) path = tmp_path / "source.h5" diff --git a/xdas/io/xdas.py b/xdas/io/xdas.py index 67f70cac..a2e0858b 100644 --- a/xdas/io/xdas.py +++ b/xdas/io/xdas.py @@ -19,12 +19,10 @@ from ..coordinates import Coordinates from ..core import DataArray, DataCollection, DataMapping, DataSequence from ..dask import create_variable, loads -from ..virtual import TileArray, VirtualArray, VirtualBackend +from ..virtual import TileArray, VirtualBackend +from ..virtual.tiles import TILES_GROUP from .core import Engine -TILES_GROUP = "__tiles__" -"""Name of the sibling group holding a virtual variable's tile manifest.""" - class XdasEngine(Engine, name="xdas"): """ @@ -229,18 +227,10 @@ def save_dataarray( else: if encoding is not None: raise ValueError("cannot use `encoding` with in virtual mode") - if isinstance(da.data, VirtualArray): + if isinstance(da.data, VirtualBackend): variable = da.data.create_variable( file, variable_name, da.dims, da.dtype ) - elif isinstance(da.data, TileArray): - # the placeholder variable already records the dtype (as any - # typed store would, e.g. zarr array metadata): only the - # engine needs the by-value sidecar - variable = file.create_variable(variable_name, da.dims, da.dtype) - variable.attrs["__tile_array__"] = json.dumps( - {"engine": da.data.engine} - ) elif isinstance(da.data, DaskArray): warnings.warn( "writing dask-backed virtual arrays is deprecated; the " @@ -262,15 +252,9 @@ def save_dataarray( # write metadata dataset.to_netcdf(fname, mode="a", group=group, engine="h5netcdf") - # write the tile manifest as a sibling group - if virtual and isinstance(da.data, TileArray): - manifest = da.data.to_dataset() - # strings are fixed-width bytes by construction and land on disk - # as char arrays; only stale open-time encodings need clearing - for name in list(manifest.variables): - manifest[name].encoding.clear() - location = TILES_GROUP if group is None else f"{group}/{TILES_GROUP}" - manifest.to_netcdf(fname, mode="a", group=location, engine="h5netcdf") + # append what of the stored form outlives the variable (the tile manifest) + if virtual and isinstance(da.data, VirtualBackend): + da.data.finalize_save(fname, group) def open_datacollection(fname, group=None): diff --git a/xdas/virtual/core.py b/xdas/virtual/core.py index 5d057296..b962f01c 100644 --- a/xdas/virtual/core.py +++ b/xdas/virtual/core.py @@ -12,10 +12,12 @@ class VirtualBackend: lazily through ``__getitem__``, and materializes through ``__array__``; a ``from_variable`` classmethod wraps one stored HDF5 variable, so open paths dispatch - ``VirtualBackend[vtype].from_variable(...)``. That contract is - informal — the backends share no implementation and differ beyond - it (blocking, persistence, concatenation), so this base only names - them: subclasses register by + ``VirtualBackend[vtype].from_variable(...)``, and the + :meth:`create_variable`/:meth:`finalize_save` pair writes the + backend's stored form, so save paths need no per-backend branch. + The rest of the contract is informal — the backends share no + implementation and differ beyond it (blocking, concatenation), so + this base only names them: subclasses register by passing ``vtype=`` in the class definition and are retrieved with the ``VirtualBackend[vtype]`` syntax — the same registry fashion as :class:`~xdas.io.Engine` (by name) and @@ -64,3 +66,24 @@ def __class_getitem__(cls, item): f"no virtual backend registered under {item!r}; " f"available: {sorted(cls._registry)}" ) + + def create_variable(self, file, name, dims=None, dtype=None): + """ + Write this array as variable *name* of an open h5netcdf *file*. + + The first half of the persistence contract (abstract — each + backend writes its own stored form): an HDF5 virtual dataset + for the hdf5 backend, a placeholder variable carrying the + engine specification for the tiles backend. + """ + raise NotImplementedError + + def finalize_save(self, fname, group=None): + """ + Append what of the stored form outlives the variable. + + The second half of the persistence contract, called once the + file handle of :meth:`create_variable` is closed. Default: the + variable is the whole stored form, nothing to append — the + tiles backend appends its manifest as a sibling group. + """ diff --git a/xdas/virtual/tiles.py b/xdas/virtual/tiles.py index 72cb95ce..15d15c52 100644 --- a/xdas/virtual/tiles.py +++ b/xdas/virtual/tiles.py @@ -110,6 +110,9 @@ TILE_PREFIX = "tile_" """Prefix of the tile-grid dimensions of a manifest dataset.""" +TILES_GROUP = "__tiles__" +"""Name of the sibling group holding a stored tile array's manifest.""" + _UNITS = ("B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") """Decimal byte units, as xarray spells them in its ``Size:`` header.""" @@ -636,6 +639,61 @@ def to_dataset(self): """ return self.dataset.copy() + def create_variable(self, file, name, dims=None, dtype=None): + """ + Create the placeholder variable of the stored form. + + The placeholder records the dtype (as any typed store would, + e.g. zarr array metadata) and carries the engine specification + by value in a ``__tile_array__`` attribute; the manifest itself + is appended by :meth:`finalize_save` once *file* is closed. + + Parameters + ---------- + file : h5netcdf.File or h5netcdf.Group + Open writable file or group. + name : str + Variable name to create inside *file*. + dims : sequence of str, optional + Dimension names for the variable. + dtype : dtype-like, optional + Element type of the placeholder. Default to :attr:`dtype`. + + Returns + ------- + variable + The newly created file variable. + """ + variable = file.create_variable( + name, dims, self.dtype if dtype is None else dtype + ) + variable.attrs["__tile_array__"] = json.dumps({"engine": self.engine}) + return variable + + def finalize_save(self, fname, group=None): + """ + Append the manifest as a sibling group of the stored variable. + + The second half of the stored form, written natively by xarray + once the file handle of :meth:`create_variable` is closed: the + manifest dataset lands in a ``__tiles__`` group next to the + placeholder variable. + + Parameters + ---------- + fname : str + Path of the file holding the placeholder variable. + group : str, optional + Group of the placeholder variable within the file. + """ + manifest = self.to_dataset() + # strings are fixed-width bytes by construction and land on disk + # as char arrays; only stale open-time encodings need clearing + for name in list(manifest.variables): + manifest[name].encoding.clear() + location = TILES_GROUP if group is None else f"{group}/{TILES_GROUP}" + manifest.to_netcdf(fname, mode="a", group=location, engine="h5netcdf") + def _geometry(self, kind, default): """Load the eager 1-D ``{kind}_k`` arrays (*default* where absent).""" arrays = [] From 775503a136ed9678c7bbbac957b963e13d6658f9 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 16:10:46 +0200 Subject: [PATCH 53/56] Declare the whole backend contract on the base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VirtualBackend now states every member a backend implements — shape, dtype, __getitem__, __array__, from_variable, and the persistence pair — and hosts the properties derived from them (ndim, size, nbytes, empty), shared by both backends instead of living on the hdf5 base. TileArray exposes shape and dtype as read-only properties over the values its constructor computes. --- tests/virtual/test_core.py | 24 +++++++++- xdas/virtual/core.py | 95 ++++++++++++++++++++++++++++++++------ xdas/virtual/hdf5.py | 47 ++----------------- xdas/virtual/tiles.py | 22 +++++---- 4 files changed, 121 insertions(+), 67 deletions(-) diff --git a/tests/virtual/test_core.py b/tests/virtual/test_core.py index 990a0f8d..658ab3cf 100644 --- a/tests/virtual/test_core.py +++ b/tests/virtual/test_core.py @@ -38,12 +38,34 @@ def test_isinstance_covers_both_backends(self): assert isinstance(source, VirtualBackend) assert isinstance(VirtualStack([source]), VirtualBackend) - def test_persistence_contract_defaults(self): + def test_contract_stubs(self): backend = VirtualBackend() + with pytest.raises(NotImplementedError): + _ = backend.shape + with pytest.raises(NotImplementedError): + _ = backend.dtype + with pytest.raises(NotImplementedError): + backend[0] + with pytest.raises(NotImplementedError): + backend.__array__() + with pytest.raises(NotImplementedError): + VirtualBackend.from_variable(None) with pytest.raises(NotImplementedError): backend.create_variable(None, "data") assert backend.finalize_save("path.nc") is None + def test_derived_properties_shared_by_both_backends(self): + source = VirtualSource("path.h5", "data", (2, 3), np.dtype("f8")) + assert source.ndim == 2 + assert source.size == 6 + assert source.nbytes == 48 + assert not source.empty + tiled = TileArray.from_tiles("path.h5", (2, 3), "f8", "xdas") + assert tiled.ndim == 2 + assert tiled.size == 6 + assert tiled.nbytes == 48 + assert not tiled.empty + def test_from_variable_dispatches_to_each_backend(self, tmp_path): data = np.arange(6.0).reshape(2, 3) path = tmp_path / "source.h5" diff --git a/xdas/virtual/core.py b/xdas/virtual/core.py index b962f01c..5422e492 100644 --- a/xdas/virtual/core.py +++ b/xdas/virtual/core.py @@ -1,23 +1,28 @@ """Registry base class :class:`VirtualBackend` for the virtual array backends.""" +import math from typing import ClassVar class VirtualBackend: """ - Marker base and registry for the virtual array backends. - - A virtual backend is a lazy, numpy-like duck array whose values stay - on disk: it reports ``shape`` and ``dtype`` without reading, slices - lazily through ``__getitem__``, and materializes through - ``__array__``; a ``from_variable`` classmethod wraps one stored - HDF5 variable, so open paths dispatch - ``VirtualBackend[vtype].from_variable(...)``, and the - :meth:`create_variable`/:meth:`finalize_save` pair writes the - backend's stored form, so save paths need no per-backend branch. - The rest of the contract is informal — the backends share no - implementation and differ beyond it (blocking, concatenation), so - this base only names them: subclasses register by + Base class and registry for the virtual array backends. + + A virtual backend is a lazy, numpy-like duck array whose values + stay on disk. The contract, declared here: + + - :attr:`shape` and :attr:`dtype` answer without reading, with + :attr:`ndim`, :attr:`size`, :attr:`nbytes` and :attr:`empty` + derived from them; + - ``__getitem__`` selects lazily and ``__array__`` materializes; + - :meth:`from_variable` wraps one stored HDF5 variable, so open + paths dispatch ``VirtualBackend[vtype].from_variable(...)``; + - the :meth:`create_variable`/:meth:`finalize_save` pair writes + the backend's stored form, so save paths need no per-backend + branch. + + Beyond the contract the backends share no implementation and + differ freely (blocking, concatenation). Subclasses register by passing ``vtype=`` in the class definition and are retrieved with the ``VirtualBackend[vtype]`` syntax — the same registry fashion as :class:`~xdas.io.Engine` (by name) and @@ -67,6 +72,42 @@ def __class_getitem__(cls, item): f"available: {sorted(cls._registry)}" ) + # --- the contract, implemented by every backend --- + + @property + def shape(self): + """Tuple of array dimensions (abstract — must be overridden).""" + raise NotImplementedError + + @property + def dtype(self): + """NumPy dtype of the array elements (abstract — must be overridden).""" + raise NotImplementedError + + def __getitem__(self, key): + """Select lazily, returning an array of the same kind (abstract).""" + raise NotImplementedError + + def __array__(self, dtype=None, copy=None): + """Materialize as a numpy array (abstract — must be overridden).""" + raise NotImplementedError + + @classmethod + def from_variable(cls, variable): + """ + Expose one stored HDF5 variable as this backend's lazy array. + + The open half of the dispatch (abstract — each backend wraps + its own way): a virtual source pointing at the variable for the + hdf5 backend, a single tile covering it for the tiles backend. + + Parameters + ---------- + variable : h5py.Dataset + The open variable to wrap. + """ + raise NotImplementedError + def create_variable(self, file, name, dims=None, dtype=None): """ Write this array as variable *name* of an open h5netcdf *file*. @@ -87,3 +128,31 @@ def finalize_save(self, fname, group=None): variable is the whole stored form, nothing to append — the tiles backend appends its manifest as a sibling group. """ + + # --- derived from the contract, shared by every backend --- + + @property + def ndim(self): + """Number of dimensions.""" + return len(self.shape) + + @property + def size(self): + """Total number of elements.""" + if self.shape: + return math.prod(self.shape) + else: + return 0 + + @property + def nbytes(self): + """Total number of bytes occupied by the array elements.""" + if self.shape: + return self.size * self.dtype.itemsize + else: + return 0 + + @property + def empty(self): + """``True`` if the array contains no elements.""" + return self.size == 0 diff --git a/xdas/virtual/hdf5.py b/xdas/virtual/hdf5.py index ae241bda..a044ab83 100644 --- a/xdas/virtual/hdf5.py +++ b/xdas/virtual/hdf5.py @@ -20,29 +20,14 @@ class VirtualArray(VirtualBackend, vtype="hdf5"): """ Abstract base class for lazy array objects backed by HDF5/NetCDF4 files. - Subclasses must implement :meth:`shape`, :meth:`dtype`, :meth:`__getitem__`, - :meth:`__array__`, and :meth:`to_dataset`. + Subclasses must implement the :class:`~xdas.virtual.VirtualBackend` + contract (:attr:`shape`, :attr:`dtype`, ``__getitem__``, + ``__array__``) plus :meth:`to_dataset`. """ def __repr__(self): return f"{self.__class__.__name__}: {_to_human(self.nbytes)} ({self.dtype})" - def __getitem__(self, key): - raise NotImplementedError - - def __array__(self, dtype=None, copy=None): - raise NotImplementedError - - @property - def shape(self): - """Tuple of array dimensions (abstract — must be overridden).""" - raise NotImplementedError - - @property - def dtype(self): - """NumPy dtype of the array elements (abstract — must be overridden).""" - raise NotImplementedError - def to_dataset(self, file_or_group, name): """Write this virtual array as an HDF5 dataset (abstract — must be overridden).""" raise NotImplementedError @@ -63,32 +48,6 @@ def from_variable(cls, variable): """ return VirtualSource(variable) - @property - def ndim(self): - """Number of dimensions.""" - return len(self.shape) - - @property - def size(self): - """Total number of elements.""" - if self.shape: - return np.prod(self.shape) - else: - return 0 - - @property - def empty(self): - """``True`` if the array contains no elements.""" - return self.size == 0 - - @property - def nbytes(self): - """Total number of bytes occupied by the array elements.""" - if self.shape: - return self.size * self.dtype.itemsize - else: - return 0 - def create_variable(self, file, name, dims=None, dtype=None): """ Write this virtual array into *file* and register it as a named variable. diff --git a/xdas/virtual/tiles.py b/xdas/virtual/tiles.py index 15d15c52..301c8800 100644 --- a/xdas/virtual/tiles.py +++ b/xdas/virtual/tiles.py @@ -420,8 +420,8 @@ def __init__(self, dataset, dtype, engine): Engine[engine["name"]] # fail fast on unregistered engines self._engine = engine - self.dtype = np.dtype(dtype) - if self.dtype.byteorder == ">": + self._dtype = np.dtype(dtype) + if self._dtype.byteorder == ">": raise ValueError("only little-endian or single-byte dtypes are supported") ngrid = 0 while f"sizes_{ngrid}" in dataset: @@ -481,8 +481,7 @@ def __init__(self, dataset, dtype, engine): raise ValueError(f"axis {g} is synthetic or hidden: sizes must be 1") if g not in self._axes and len(self._sizes[g]) != 1: raise ValueError(f"hidden axis {g} must hold a single tile") - self.ndim = len(self._axes) - self.shape = tuple(int(self._edges[g][-1]) for g in self._axes) + self._shape = tuple(int(self._edges[g][-1]) for g in self._axes) if "paths" not in dataset: raise ValueError("a tile array needs a `paths` variable") if "root" in dataset: @@ -743,16 +742,21 @@ def _assign_axes(self, dataset, axes, ngrid): assign["source_ndim"] = ((), np.asarray(self._source_ndim, dtype=np.int64)) return dataset.assign(assign) if assign else dataset + @property + def shape(self): + """Tuple of int: array dimensions, summed from the tile sizes.""" + return self._shape + + @property + def dtype(self): + """numpy.dtype: element type of the sources as the engine decodes them.""" + return self._dtype + @property def ntiles(self): """int: total number of tiles in the grid.""" return math.prod(len(sizes) for sizes in self._sizes) - @property - def size(self): - """int: total number of elements.""" - return math.prod(self.shape) - def __getitem__(self, key): """Index the array, staying virtual whenever possible. From 72b14937f353a13fe40c884107a70edd13691f17 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 16:21:38 +0200 Subject: [PATCH 54/56] Promote VirtualBackend to an ABC With the whole contract declared on the base and every member overridden at class level by both backends, abstractness is now enforceable: the stubs become docstring-only abstract methods, as Coordinate already does. VirtualArray stays the (now formally) abstract hdf5 family base; its concrete forms and TileArray instantiate unchanged. --- tests/virtual/test_core.py | 22 +++++++-------------- tests/virtual/test_hdf5.py | 17 ++++++----------- xdas/virtual/core.py | 39 +++++++++++++++++++------------------- 3 files changed, 33 insertions(+), 45 deletions(-) diff --git a/tests/virtual/test_core.py b/tests/virtual/test_core.py index 658ab3cf..a63b071a 100644 --- a/tests/virtual/test_core.py +++ b/tests/virtual/test_core.py @@ -38,21 +38,13 @@ def test_isinstance_covers_both_backends(self): assert isinstance(source, VirtualBackend) assert isinstance(VirtualStack([source]), VirtualBackend) - def test_contract_stubs(self): - backend = VirtualBackend() - with pytest.raises(NotImplementedError): - _ = backend.shape - with pytest.raises(NotImplementedError): - _ = backend.dtype - with pytest.raises(NotImplementedError): - backend[0] - with pytest.raises(NotImplementedError): - backend.__array__() - with pytest.raises(NotImplementedError): - VirtualBackend.from_variable(None) - with pytest.raises(NotImplementedError): - backend.create_variable(None, "data") - assert backend.finalize_save("path.nc") is None + def test_base_is_abstract(self): + with pytest.raises(TypeError, match="abstract"): + VirtualBackend() + + def test_finalize_save_defaults_to_nothing(self): + source = VirtualSource("path.h5", "data", (2, 3), np.dtype("f8")) + assert source.finalize_save("path.nc") is None def test_derived_properties_shared_by_both_backends(self): source = VirtualSource("path.h5", "data", (2, 3), np.dtype("f8")) diff --git a/tests/virtual/test_hdf5.py b/tests/virtual/test_hdf5.py index 1a31ed62..2d9596a0 100644 --- a/tests/virtual/test_hdf5.py +++ b/tests/virtual/test_hdf5.py @@ -389,18 +389,13 @@ def test_get_indexer(self): class TestVirtualArrayAbstract: - def test_abstract_stubs(self): - va = VirtualArray() - with pytest.raises(NotImplementedError): - _ = va[0] - with pytest.raises(NotImplementedError): - va.__array__() - with pytest.raises(NotImplementedError): - _ = va.shape - with pytest.raises(NotImplementedError): - _ = va.dtype + def test_cannot_instantiate(self): + with pytest.raises(TypeError, match="abstract"): + VirtualArray() + + def test_to_dataset_stub(self): with pytest.raises(NotImplementedError): - va.to_dataset(None, None) + VirtualArray.to_dataset(None, None, None) class TestVirtualStackExtra: diff --git a/xdas/virtual/core.py b/xdas/virtual/core.py index 5422e492..6bab281c 100644 --- a/xdas/virtual/core.py +++ b/xdas/virtual/core.py @@ -1,12 +1,13 @@ """Registry base class :class:`VirtualBackend` for the virtual array backends.""" import math +from abc import ABC, abstractmethod from typing import ClassVar -class VirtualBackend: +class VirtualBackend(ABC): """ - Base class and registry for the virtual array backends. + Abstract base class and registry for the virtual array backends. A virtual backend is a lazy, numpy-like duck array whose values stay on disk. The contract, declared here: @@ -75,49 +76,49 @@ def __class_getitem__(cls, item): # --- the contract, implemented by every backend --- @property + @abstractmethod def shape(self): - """Tuple of array dimensions (abstract — must be overridden).""" - raise NotImplementedError + """Tuple of array dimensions.""" @property + @abstractmethod def dtype(self): - """NumPy dtype of the array elements (abstract — must be overridden).""" - raise NotImplementedError + """NumPy dtype of the array elements.""" + @abstractmethod def __getitem__(self, key): - """Select lazily, returning an array of the same kind (abstract).""" - raise NotImplementedError + """Select lazily, returning an array of the same kind.""" + @abstractmethod def __array__(self, dtype=None, copy=None): - """Materialize as a numpy array (abstract — must be overridden).""" - raise NotImplementedError + """Materialize as a numpy array (numpy array protocol).""" @classmethod + @abstractmethod def from_variable(cls, variable): """ Expose one stored HDF5 variable as this backend's lazy array. - The open half of the dispatch (abstract — each backend wraps - its own way): a virtual source pointing at the variable for the - hdf5 backend, a single tile covering it for the tiles backend. + The open half of the dispatch — each backend wraps its own way: + a virtual source pointing at the variable for the hdf5 backend, + a single tile covering it for the tiles backend. Parameters ---------- variable : h5py.Dataset The open variable to wrap. """ - raise NotImplementedError + @abstractmethod def create_variable(self, file, name, dims=None, dtype=None): """ Write this array as variable *name* of an open h5netcdf *file*. - The first half of the persistence contract (abstract — each - backend writes its own stored form): an HDF5 virtual dataset - for the hdf5 backend, a placeholder variable carrying the - engine specification for the tiles backend. + The first half of the persistence contract — each backend + writes its own stored form: an HDF5 virtual dataset for the + hdf5 backend, a placeholder variable carrying the engine + specification for the tiles backend. """ - raise NotImplementedError def finalize_save(self, fname, group=None): """ From c992d9bf8ea5ad44349b2f67607e7eaaaaab84c3 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 17:07:21 +0200 Subject: [PATCH 55/56] Pin the febus reference read to the hdf5 vtype The block-crossing test built its reference with the engine default, which f08de4f flipped to tiles: the test compared the tiles path against itself, and the legacy per-block loop lost its coverage. Also flatten a double negative in the open ceiling check. --- tests/io/test_tiles_vtype.py | 3 ++- xdas/core/routines.py | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/io/test_tiles_vtype.py b/tests/io/test_tiles_vtype.py index 8c30a11c..f6e9c3ff 100644 --- a/tests/io/test_tiles_vtype.py +++ b/tests/io/test_tiles_vtype.py @@ -114,7 +114,8 @@ def test_febus_block_crossing_reads(tmp_path): path = str(tmp_path / "febus.h5") make_febus_file(path) kwargs = {"overlaps": (1, 1), "offset": 0} - expected = xd.open_dataarray(path, engine="febus", **kwargs).values + # the hdf5 vtype is the independent reference (tiles is the febus default) + expected = xd.open_dataarray(path, engine="febus", vtype="hdf5", **kwargs).values result = xd.open_dataarray(path, engine="febus", vtype="tiles", **kwargs) assert result.data.engine["block_size"] == 12 assert result.data.engine["overlaps"] == [1, 1] diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 696d8e77..844feaa3 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -641,10 +641,7 @@ def open_mfdataarray( raise FileNotFoundError("no file to open") engine = _resolve_engine(engine, vtype, ctype, engine_kwargs) backend = VirtualBackend._registry.get(engine.vtype) - if ( - not (backend is not None and backend.consolidates) - and len(paths) > MAX_OPEN_FILES - ): + if (backend is None or not backend.consolidates) and len(paths) > MAX_OPEN_FILES: consolidating = ", ".join( repr(vtype) for vtype, cls in sorted(VirtualBackend._registry.items()) From a07c786f8a0f28933ec41eb90983ade1cb1f0292 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Tue, 4 Aug 2026 17:32:53 +0200 Subject: [PATCH 56/56] Drop Python 3.10 support np.strings.slice (used by the tiles manifest) only exists in numpy>=2.3, which itself requires Python>=3.11. Python 3.10 reaches EOL in October 2026. --- .github/workflows/tests.yaml | 2 +- docs/release-notes.md | 1 + pyproject.toml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b67a3372..2170a002 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/docs/release-notes.md b/docs/release-notes.md index f433afee..b5cfc0e4 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -13,6 +13,7 @@ - `simplify` runs in linear time whatever the number of gaps: the reduce stage is now a one-pass sleeve instead of Douglas-Peucker, which degenerated quadratically on gap-rich coordinates (a 100 000-file gappy archive simplified in minutes; now milliseconds). The deviation guarantee is unchanged — dropped tie points stay within `tolerance` of the curve, surviving values never move — though the surviving tie-point selection may differ slightly on jittery axes (@atrabattoni). ### Breaking Changes +- Python 3.10 support is dropped and the numpy requirement is raised to 2.3: the tile manifests use `np.strings` routines introduced in numpy 2.3, which itself requires Python 3.11+. Python 3.10 reaches end of life in October 2026 (@atrabattoni). - Passing a bare read function as `engine` now raises a `TypeError`: subclass `xdas.io.Engine` instead (see the data-formats documentation) (@atrabattoni). - Misspelled or unsupported keyword arguments passed next to an engine name now raise a `TypeError` instead of being silently ignored, and combining `vtype`, `ctype` or engine keywords with an already configured engine instance raises a `ValueError` (@atrabattoni). diff --git a/pyproject.toml b/pyproject.toml index 6d56f87d..321a3ec6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "xdas" dynamic = ["version"] -requires-python = ">= 3.10" +requires-python = ">= 3.11" authors = [ { name = "Alister Trabattoni", email = "alister.trabattoni@gmail.com" }, ] @@ -17,7 +17,7 @@ dependencies = [ "loky", "msgpack", "numba", - "numpy>=2.1", + "numpy>=2.3", "obspy", "pandas", "plotly",