diff --git a/docs/api/coordinates.md b/docs/api/coordinates.md index 54dfc993..ff074d05 100644 --- a/docs/api/coordinates.md +++ b/docs/api/coordinates.md @@ -5,9 +5,6 @@ ## Coordinates -Constructor - - ```{eval-rst} .. autosummary:: :toctree: ../_autosummary @@ -22,23 +19,19 @@ Methods :toctree: ../_autosummary Coordinates.isdim - Coordinates.get_query Coordinates.to_index Coordinates.equals - Coordinates.to_dict Coordinates.copy - Coordinates.drop_dims - Coordinates.drop_coords + Coordinates.drop_dims + Coordinates.drop_coords ``` -### Coordinate - -Constructor +## Coordinate ```{eval-rst} .. autosummary:: :toctree: ../_autosummary - + Coordinate ``` @@ -49,9 +42,11 @@ Attributes :toctree: ../_autosummary Coordinate.dtype - Coordinate.ndim Coordinate.shape + Coordinate.size + Coordinate.dim Coordinate.values + Coordinate.name ``` Methods @@ -60,22 +55,32 @@ Methods .. autosummary:: :toctree: ../_autosummary - Coordinate.to_index - Coordinate.isscalar - Coordinate.isdense - Coordinate.isinterp + Coordinate.isdim + Coordinate.isregular + Coordinate.equals + Coordinate.copy ``` +## AxisCoordinate + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary -### ScalarCoordinate + AxisCoordinate +``` -Constructor +Attributes ```{eval-rst} .. autosummary:: :toctree: ../_autosummary - ScalarCoordinate + AxisCoordinate.ndim + AxisCoordinate.empty + AxisCoordinate.indices + AxisCoordinate.start + AxisCoordinate.end ``` Methods @@ -84,15 +89,27 @@ Methods .. autosummary:: :toctree: ../_autosummary - ScalarCoordinate.isvalid - ScalarCoordinate.equals - ScalarCoordinate.to_index - ScalarCoordinate.to_dict + AxisCoordinate.isregular + AxisCoordinate.get_sampling_interval + AxisCoordinate.to_regular + AxisCoordinate.get_split_indices + AxisCoordinate.get_discontinuities + AxisCoordinate.get_availabilities + AxisCoordinate.simplify + AxisCoordinate.to_index + AxisCoordinate.to_dataarray ``` -### DenseCoordinate +## ScalarCoordinate + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary + + ScalarCoordinate +``` -Constructor +## DenseCoordinate ```{eval-rst} .. autosummary:: @@ -101,22 +118,28 @@ Constructor DenseCoordinate ``` -Methods +Attributes ```{eval-rst} .. autosummary:: :toctree: ../_autosummary - DenseCoordinate.isvalid DenseCoordinate.index - DenseCoordinate.get_indexer - DenseCoordinate.slice_indexer - DenseCoordinate.to_dict ``` -### InterpCoordinate +Methods + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary -Constructor + DenseCoordinate.from_block + DenseCoordinate.get_sampling_interval + DenseCoordinate.to_regular + DenseCoordinate.simplify +``` + +## InterpCoordinate ```{eval-rst} .. autosummary:: @@ -127,19 +150,14 @@ Constructor Attributes - ```{eval-rst} .. autosummary:: :toctree: ../_autosummary InterpCoordinate.tie_indices InterpCoordinate.tie_values - InterpCoordinate.empty - InterpCoordinate.dtype - InterpCoordinate.ndim - InterpCoordinate.shape - InterpCoordinate.indices - InterpCoordinate.values + InterpCoordinate.sampling_interval + InterpCoordinate.tolerance ``` Methods @@ -148,24 +166,13 @@ Methods .. autosummary:: :toctree: ../_autosummary - InterpCoordinate.isvalid - InterpCoordinate.equals - InterpCoordinate.get_value - InterpCoordinate.format_index - InterpCoordinate.slice_index - InterpCoordinate.get_indexer - InterpCoordinate.slice_indexer - InterpCoordinate.decimate + InterpCoordinate.from_block + InterpCoordinate.to_regular + InterpCoordinate.get_sampling_interval InterpCoordinate.simplify - InterpCoordinate.get_discontinuities - InterpCoordinate.from_array - InterpCoordinate.to_dict ``` - -### SampledCoordinate - -Constructor +## SampledCoordinate ```{eval-rst} .. autosummary:: @@ -176,7 +183,6 @@ Constructor Attributes - ```{eval-rst} .. autosummary:: :toctree: ../_autosummary @@ -185,12 +191,6 @@ Attributes SampledCoordinate.tie_lengths SampledCoordinate.tie_indices SampledCoordinate.sampling_interval - SampledCoordinate.empty - SampledCoordinate.dtype - SampledCoordinate.ndim - SampledCoordinate.shape - SampledCoordinate.indices - SampledCoordinate.values ``` Methods @@ -199,18 +199,17 @@ Methods .. autosummary:: :toctree: ../_autosummary - SampledCoordinate.concat - SampledCoordinate.decimate - SampledCoordinate.equals - SampledCoordinate.from_array SampledCoordinate.from_block - SampledCoordinate.get_indexer SampledCoordinate.get_sampling_interval - SampledCoordinate.get_split_indices - SampledCoordinate.get_value - SampledCoordinate.isvalid + SampledCoordinate.to_regular SampledCoordinate.simplify - SampledCoordinate.slice_index - SampledCoordinate.slice_indexer - SampledCoordinate.to_dict -``` \ No newline at end of file +``` + +## Functions + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary + + get_sampling_interval +``` diff --git a/docs/api/index.md b/docs/api/index.md index d16f304d..2f8a3e99 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -13,5 +13,6 @@ picking processing signal synthetics +testing virtual ``` \ No newline at end of file diff --git a/docs/api/synthetics.md b/docs/api/synthetics.md index d318d4d9..93bfcf61 100644 --- a/docs/api/synthetics.md +++ b/docs/api/synthetics.md @@ -10,5 +10,4 @@ wavelet_wavefronts randn_wavefronts - dummy ``` \ No newline at end of file diff --git a/docs/api/testing.md b/docs/api/testing.md new file mode 100644 index 00000000..d26c7f79 --- /dev/null +++ b/docs/api/testing.md @@ -0,0 +1,12 @@ +```{eval-rst} +.. currentmodule:: xdas.testing +``` + +# xdas.testing + +```{eval-rst} +.. autosummary:: + :toctree: ../_autosummary + + dummy +``` diff --git a/docs/api/xdas.md b/docs/api/xdas.md index 5d364b5f..8ebc62ed 100644 --- a/docs/api/xdas.md +++ b/docs/api/xdas.md @@ -31,6 +31,7 @@ concat concatenate concat_coords + get_sampling_interval split plot_availability ``` @@ -122,8 +123,6 @@ Methods DataArray.from_stream DataArray.to_netcdf DataArray.from_netcdf - DataArray.to_dict - DataArray.from_dict DataArray.plot ``` diff --git a/docs/conf.py b/docs/conf.py index 22940c8e..8f9dffa1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,6 +4,8 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +from xdas import __version__ + # -- Project information ----------------------------------------------------- project = "xdas" @@ -11,7 +13,7 @@ author = "Alister Trabattoni" # The full version, including alpha/beta/rc tags -release = "0.2.7" +release = __version__ # -- General configuration --------------------------------------------------- @@ -86,13 +88,13 @@ } # -- Generate dummy data ----------------------------------------------------- -import os # noqa: E402 +import os -import h5py # noqa: E402 -import numpy as np # noqa: E402 +import h5py +import numpy as np -import xdas as xd # noqa: E402 -from xdas.synthetics import wavelet_wavefronts # noqa: E402 +import xdas as xd +from xdas.synthetics import wavelet_wavefronts dirpath = os.path.join(os.path.split(__file__)[0], "_data") if not os.path.exists(dirpath): diff --git a/docs/contribute.md b/docs/contribute.md index df7b7ec8..e309d2ee 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -84,30 +84,30 @@ You need to add multiline string that follows the [numpydoc](https://numpydoc.re ```python def your_function(arg1, arg2=None): - """ - Explain in one line the main objective. + """ + Explain in one line the main objective. - Add additional information with with as many comments as you wants. You will need to wrap you code (ideally max 88 char per line). + Add additional information with with as many comments as you wants. You will need to wrap you code (ideally max 88 char per line). - Parameters - ---------- - arg1: type (e.g., float) - Some description. - arg2: type, optional - Some description. + Parameters + ---------- + arg1: type (e.g., float) + Some description. + arg2: type, optional + Some description. - Returns - ------- - type: - Some description. + Returns + ------- + type: + Some description. - Examples - -------- - >>> your_function(1, "value") - "result" + Examples + -------- + >>> your_function(1, "value") + "result" - """ - return do_something_with_args(arg1, arg2) + """ + return do_something_with_args(arg1, arg2) ``` Note that the outputs of the examples will be used as tests. `pytest` will check that the hard coded output matches what the code actually outputs. This is a first quick way to add tests to your function. @@ -129,11 +129,12 @@ If you are working on a function in a script in `xdas/dir/file.py` then test mus ```python import pytest + class TestMyModule: - def test_my_function(self): # here self is generally unused - assert mu_function(0) == 42 # it must be True otherwise the test doesn't pass - with pytest.raises(ValueError): # check it raise the correct error - my_function(-1) + def test_my_function(self): # here self is generally unused + assert mu_function(0) == 42 # it must be True otherwise the test doesn't pass + with pytest.raises(ValueError): # check it raise the correct error + my_function(-1) ``` Note that here we have one test class for the entire module (the `file.py`) but we could have one test class per function if those require to test a lot of things. When testing classes, one testing class per developed class is generally the way to go. diff --git a/docs/release-notes.md b/docs/release-notes.md index 77e5f730..9798a487 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,5 +1,28 @@ # Release notes +## 0.2.8 + +### New Features +- **Regular coordinates.** A coordinate can now declare a nominal `sampling_interval` (with a `tolerance` bounding the allowed jitter). Query it with `isregular()` / `get_sampling_interval()`; promote an irregular coordinate with `to_regular()`. File engines, `from_block`, and the `fft`/`stft` outputs produce regular coordinates out of the box (@atrabattoni). +- Chunked and unchunked processing now yield identical coordinates: operations that derive a new rate record their rounding error in `tolerance`, and `simplify`/`concat` spend the declared tolerance by default, fusing chunk seams away (@atrabattoni). +- `simplify` gained `reduce` and `regularize` keywords, and the gaps/overlaps API now works on every axis coordinate, including dense ones (@atrabattoni). + +### Deprecations +- The sampling interval is now declared metadata rather than a computed end-to-end average (which was silently wrong on jittery or gappy axes). Data saved by earlier versions carries no declared rate: querying it — e.g. through any signal-processing routine — still works for now, but the rate is inferred and a `FutureWarning` explains how to make the coordinate regular (`da[dim] = da[dim].to_regular(tolerance=...)`). A future release will raise instead (@atrabattoni). + +### Bug Fixes +- Fix `Sequential.reset()` silently doing nothing: it only reset `Partial` atoms, so stateful atoms such as `IIRFilter` or `ResamplePoly` kept their state and a reused sequence returned wrong data (@atrabattoni). +- Fix `stft` ignoring the `"first"`/`"last"` dimension aliases — including its own default `dim` — which raised a size-conflict error instead of transforming the named axis (@atrabattoni). + +### Refactoring +- Reworked the coordinate class hierarchy: `Coordinate` is now a proper ABC and the new `AxisCoordinate` ABC holds the axis-mapping contract shared by dense, interpolated, and sampled coordinates. Use `isinstance(coord, AxisCoordinate)` instead of the removed `is*` predicates (@atrabattoni). +- Cleaned up internal-leaning APIs: removed `DefaultCoordinate`, `to_dict`/`from_dict`, `get_div_points`, `decimate`, and `from_array`; made underscore-private `concat`, `get_indexer`, `get_value`, `format_index`, `slice_index(er)`, `isvalid`, and `get_query`; NumPy 2.0 `copy` keyword compliance (@atrabattoni). +- `concat_coords` now simplifies its result by default, like `concat`; values are unchanged, only redundant tie points are dropped (@atrabattoni). +- Added `xdas.testing.dummy`, a configurable fixture generator replacing `xdas.synthetics.dummy` (@atrabattoni). +- Comply with ruff 0.16, whose default rule set is considerably broader (`B`, `C4`, `SIM`, `RUF`, `PERF`, `TRY`, `BLE`, `S`, `DTZ`, `FLY`, `PL`…). Mutable argument defaults (the `dim={...}` mappings of `fft`, `rfft`, `ifft`, `irfft`, `stft`, `to_stream`) became `None` sentinels documenting the same defaults; class-level registries and engine specs are annotated `ClassVar`; deliberate patterns (engine-fallback blind excepts, the long-lived TDMS handle, the grouped `__all__`) carry targeted `noqa`. `TRY004` is disabled project-wide, since xdas raises `ValueError` for all argument validation, including type checks (@atrabattoni). +- The abstract `VirtualArray` stubs (`__getitem__`, `__array__`, `shape`, `dtype`, `to_dataset`) now raise `NotImplementedError` instead of silently returning `None` (@atrabattoni). +- The package version is declared in a single place, `xdas/__init__.py`: `pyproject.toml` marks it dynamic and setuptools reads it from there, and `docs/conf.py` derives its `release` from it. The three copies had already drifted — the documentation still advertised 0.2.7 (@atrabattoni). + ## 0.2.7 ### Bug Fixes diff --git a/docs/user-guide/coordinates/index.md b/docs/user-guide/coordinates/index.md index 1894df98..7642eca8 100644 --- a/docs/user-guide/coordinates/index.md +++ b/docs/user-guide/coordinates/index.md @@ -6,42 +6,97 @@ kernelspec: # Coordinates +{py:class}`~xdas.DataArray` combines an N-dimensional array with a set of +{py:class}`~xdas.coordinates.Coordinate` objects gathered in a +{py:class}`~xdas.coordinates.Coordinates` dict-like container, accessible via +`DataArray.coords`. Each coordinate labels one axis (or attaches scalar +metadata) and supports both integer-index access and label-based selection. -{py:class}`~xdas.DataArray` is the base class in *xdas*. It is mainly composed of a N-dimensional array and a set of {py:class}`~xdas.Coordinate` objects that are gathered in a {py:class}`~xdas.Coordinates` dict-like object that can be accessed by the `DataArray.coords` attribute. Xdas comes with several flavors of {py:class}`~xdas.Coordinate` objects. +*xdas* ships four concrete coordinate types: | Type | Description | `name` | `data` | -|:---|:---|:---:|:---:| -| {py:class}`~xdas.coordinates.ScalarCoordinate` | Used to label 0D dimensions | `scalar` | `{"value": any}` | -| {py:class}`~xdas.coordinates.DefaultCoordinate` | Each value is equal to its index | `default` | `{"size": int}` | -| {py:class}`~xdas.coordinates.DenseCoordinate` | Each index is mapped to a given value | `dense` | `array-like[any]` | -| {py:class}`~xdas.coordinates.InterpCoordinate` | Values are interpolated linearly between tie points | `interpolated` | `{"tie_indices": array-like[int], "tie_values": array-like[any]}` | -| {py:class}`~xdas.coordinates.SampledCoordinate` | Values are given as a multiple of a fixed sampling interval and several start values | `sampled` | `{"tie_values": array-like[any], "tie_indices": array-like[int], "sampling_interval": any}` | +|:---|:---|:---:|:---| +| {py:class}`~xdas.coordinates.ScalarCoordinate` | Scalar metadata, not tied to any axis | `scalar` | scalar-like | +| {py:class}`~xdas.coordinates.DenseCoordinate` | One stored value per element | `dense` | `array-like` | +| {py:class}`~xdas.coordinates.InterpCoordinate` | Piecewise-linear from tie points | `interpolated` | `{"tie_indices": array-like[int], "tie_values": array-like}` plus optional `"sampling_interval"` and `"tolerance"` scalars | +| {py:class}`~xdas.coordinates.SampledCoordinate` | Uniform grid with optional gaps | `sampled` | `{"tie_values": array-like, "tie_lengths": array-like[int], "sampling_interval": scalar}` | -In the current state of the documentation, most coordinate information can be found on the [Interpolated Coordinates](interpolated-coordinates) page. +The three axis-mapping types (`DenseCoordinate`, `InterpCoordinate`, +`SampledCoordinate`) share the {py:class}`~xdas.coordinates.AxisCoordinate` +base, which defines the index/label selection contract. `ScalarCoordinate` +carries a single value with no axis and implements only the thin +{py:class}`~xdas.coordinates.Coordinate` interface. Use +`isinstance(coord, AxisCoordinate)` to test whether a coordinate labels an axis. -## Per type information +## Creating coordinates -```{toctree} -:maxdepth: 1 +{py:class}`~xdas.coordinates.Coordinate` acts as a factory: it inspects the +shape and structure of `data` and returns the correct subclass automatically. -interpolated-coordinates -sampled-coordinates +```{code-cell} +import numpy as np +import xdas as xd + +# DenseCoordinate — one stored value per index +xd.Coordinate([0.0, 500.0, 1000.0, 1500.0]) ``` - \ No newline at end of file +from xdas.coordinates import SampledCoordinate + +SampledCoordinate( + {"tie_values": [0.0, 600.0], "tie_lengths": [100, 100], "sampling_interval": 5.0} +) +``` + +## Coordinates in a DataArray + +Coordinates are attached to a {py:class}`~xdas.DataArray` through the `coords` +argument. A dimensional coordinate shares its name with the dimension it +labels; a non-dimensional coordinate (or a `ScalarCoordinate`) can use a +different name. + +```{code-cell} +da = xd.DataArray( + data=np.zeros((1000, 500)), + coords={ + "time": { + "tie_values": [np.datetime64("2024-01-01T00:00:00", "ms")], + "tie_lengths": [1000], + "sampling_interval": np.timedelta64(4, "ms"), + }, + "distance": {"tie_indices": [0, 499], "tie_values": [0.0, 9980.0]}, + "network": (None, "DAS-NET"), + }, +) +da +``` + +## Per-type details + +```{toctree} +:maxdepth: 1 + +interpolated-coordinates +sampled-coordinates +``` diff --git a/docs/user-guide/coordinates/interpolated-coordinates.md b/docs/user-guide/coordinates/interpolated-coordinates.md index 6ae7aee9..e5db746b 100644 --- a/docs/user-guide/coordinates/interpolated-coordinates.md +++ b/docs/user-guide/coordinates/interpolated-coordinates.md @@ -6,24 +6,27 @@ kernelspec: # Interpolated Coordinates -## Coordinate +## Overview -Because DAS data are generally sampled with a constant sampling rate/resolution, keeping the -corresponding value for each index as a dense array is inefficient. *xdas* stores the -coordinates using the [CF convention][CF] through the -{py:class}`xdas.Coordinate` object. With this method, only a few tie points are kept and intermediate -values are retrieved by linear interpolation. Discontinuities are marked by two -consecutive tie points, as illustrated below: +Because DAS data are generally sampled with a constant sampling rate, +keeping the corresponding value for each index as a dense array is +inefficient. *xdas* stores such coordinates using the +[CF convention][CF] through the +{py:class}`~xdas.coordinates.InterpCoordinate` class. Only a few tie +points are kept; intermediate values are recovered by linear interpolation. +Discontinuities are marked by two consecutive tie points at adjacent +indices, as illustrated below: ![](/_static/coordinate.svg) -The resulting coordinate vector is sparse but contains all the information -necessary to exactly recover the original, dense coordinate vector. +The resulting representation is sparse but contains all the information +needed to exactly recover the original dense coordinate vector. -## Creating a Coordinate +## Creating an InterpCoordinate -The {py:class}`xdas.Coordinate` constructor takes `tie_indices` and `tie_values` as inputs. -The code below corresponds with the example illustrated in the figure above: +The {py:class}`~xdas.coordinates.InterpCoordinate` constructor takes +`tie_indices` and `tie_values` as keys in a dict. The code below +corresponds with the example illustrated in the figure above: ```{code-cell} import xdas as xd @@ -31,60 +34,117 @@ import xdas as xd coord = xd.Coordinate( { "tie_indices": [0, 9, 19, 20, 29], - "tie_values": [0.0, 90.0, 190.0, 400.0, 490.0] + "tie_values": [0.0, 90.0, 190.0, 400.0, 490.0], } ) -coord +coord ``` -The resulting object acts as an {py:class}`numpy.ndarray` object. Indexing and -selecting works out of the box. Note that when specifying an increment step greater than 1, the tie points can be displaced a little bit. +`xd.Coordinate(...)` acts as a factory and returns an +{py:class}`~xdas.coordinates.InterpCoordinate` when the dict contains +`tie_indices` and `tie_values`. + +The coordinate behaves like a numpy array — indexing and slicing work +out of the box. Note that when specifying a step greater than 1, tie +points may shift slightly to remain on the sampled grid. ```{code-cell} coord = coord[1:-3:2] coord ``` -A major advantage of {py:class}`xdas.Coordinate` is that it enables label-based selection. -For instance, to retrieve the index of a value the {py:meth}`get_index` method can be used: +## Label-based selection + +A major advantage of {py:class}`~xdas.coordinates.InterpCoordinate` is +that it enables label-based selection. To retrieve the integer index +corresponding to a given value, use the {py:meth}`~xdas.coordinates.AxisCoordinate.to_index` +method: ```{code-cell} coord.to_index(430.0) ``` ```{warning} -To be able to do label-based selection, `tie_values` must be strictly increasing. -In other words there must not be any overlap. To deal with small overlaps, a solution -is to `simplify` the coordinates, increasing the tolerance such that the overlapping points -disappear. +To enable label-based selection, `tie_values` must be strictly increasing +(no overlaps). To deal with small overlaps, use +{py:meth}`~xdas.coordinates.InterpCoordinate.simplify` with a tolerance +large enough to absorb them. ``` -## Gaps and Overlaps +## Gaps and overlaps -Gaps and Overlaps can be easily identified based on the tie point positions, and extracted with: +Gaps and overlaps can be identified from the tie-point positions and +extracted with: ```{code-cell} coord.get_discontinuities() ``` -While gaps represents missing data and are not problematic, overlaps usually arise from -labeling errors and should be taken care of. +Gaps represent missing data and are generally not problematic; overlaps +usually arise from labelling errors and should be resolved. -Using the {py:meth}`simplify` method, the coordinate can be simplified with controlled -accuracy using the [Ramer–Douglas–Peucker algorithm][RDP]. In this example, the second -tie point does not provide useful information and is safely discarded. +Using the {py:meth}`~xdas.coordinates.InterpCoordinate.simplify` method, +the coordinate can be compressed with controlled accuracy using the +[Ramer–Douglas–Peucker algorithm][RDP]. In the example below, the +second tie point carries no additional information and is safely discarded: ```{code-cell} coord = coord.simplify(tolerance=0.0) coord ``` -## Temporal Coordinates +## Regular coordinates + +An interpolated coordinate can optionally carry a nominal +`sampling_interval` (and a `tolerance` bounding the allowed jitter around +it), making it *regular*. Signal-processing routines (filtering, FFT, +resampling) require a regular coordinate to obtain a clean sample rate; +{py:meth}`~xdas.coordinates.Coordinate.isregular` tells whether a +coordinate carries one. Coordinates built by the file engines or by +{py:meth}`~xdas.coordinates.InterpCoordinate.from_block` are regular out +of the box: + +```{code-cell} +coord = xd.Coordinate( + { + "tie_indices": [0, 9], + "tie_values": [0.0, 90.0], + "sampling_interval": 10.0, + } +) +coord.isregular() +``` + +An irregular coordinate whose values are in fact evenly spaced can be +promoted explicitly with +{py:meth}`~xdas.coordinates.AxisCoordinate.to_regular`, which infers the +spacing when it is not given and raises on genuinely irregular axes. +Data saved by earlier *xdas* versions carries no declared spacing; for +now, signal-processing routines fall back to inferring one and emit a +{py:exc}`FutureWarning` telling you the tolerance required — promote the +coordinate as shown below to silence it: + +```{code-cell} +coord = xd.Coordinate({"tie_indices": [0, 9], "tie_values": [0.0, 90.0]}) +coord.to_regular().get_sampling_interval() +``` + +For jittery axes, pass a `tolerance`: the declared spacing is accepted as +long as every continuous segment stays within it. The stored tolerance +is also the default accuracy budget of +{py:meth}`~xdas.coordinates.InterpCoordinate.simplify`, so chunk seams +introduced by piecewise processing fuse back automatically on +concatenation. `simplify(regularize=True)` combines both steps: it drops +redundant tie points and promotes the result to regular when the +surviving segments admit a single spacing within the budget. + +## Temporal coordinates -The main use of coordinates in *xdas* is to deal with long time series. By default -*xdas* uses `"datetime64[us]"` dtype. Microseconds are used because to perform -interpolation *xdas* convert `datetime64` to POSIX `float` which cannot safely -represent timestamps with better accuracies. +The most common use of interpolated coordinates in *xdas* is handling +long time series. By default *xdas* uses `"datetime64[us]"` dtype. +Microseconds are used because interpolation internally converts +`datetime64` to POSIX floats, which cannot safely represent finer +resolution. ```{code-cell} import numpy as np @@ -93,7 +153,7 @@ coord = xd.Coordinate( { "tie_indices": [0, 3600 * 100], "tie_values": [ - np.datetime64("2023-01-01T00:00:00"), + np.datetime64("2023-01-01T00:00:00"), np.datetime64("2023-01-01T01:00:00"), ], } @@ -102,4 +162,4 @@ coord.to_index(slice("2023-01-01T00:10:00", "2023-01-01T00:20:00")) ``` [CF]: -[RDP]: \ No newline at end of file +[RDP]: diff --git a/docs/user-guide/coordinates/sampled-coordinates.md b/docs/user-guide/coordinates/sampled-coordinates.md index 77be0fc3..67b221bf 100644 --- a/docs/user-guide/coordinates/sampled-coordinates.md +++ b/docs/user-guide/coordinates/sampled-coordinates.md @@ -40,17 +40,16 @@ to 200) is explicit — there is simply no segment covering that range. ## Materialising values -Calling `.values` returns the full coordinate vector as a dense NumPy array: +Calling `.values` returns the full coordinate vector as a dense NumPy array. +Individual values are accessed by integer subscript, and the reverse mapping +(index from value) uses `.to_index`: ```{code-cell} coord.values ``` -Individual values are obtained from indices with `.get_value` and the reverse mapping -(index from value) with `.to_index`: - ```{code-cell} -coord.get_value(3) +coord[3].values ``` ```{code-cell} diff --git a/docs/user-guide/data-structures/datacollection.md b/docs/user-guide/data-structures/datacollection.md index cb5cf084..88d0405c 100644 --- a/docs/user-guide/data-structures/datacollection.md +++ b/docs/user-guide/data-structures/datacollection.md @@ -84,7 +84,7 @@ If your data paths are something like: "/data/REKA/RK1/20231119/proc/*.hdf5" and ```python path = "/data/{network}/{cable}/20231119/proc/[acquisition].hdf5" -dc = xd.open(path, engine='asn') +dc = xd.open(path, engine="asn") dc ``` ```text @@ -148,7 +148,7 @@ Coordinates: ```python # Add the dataarray to the datacollection at the acquisition number 0 -dc['REKA']['RK2'].insert(0, da) +dc["REKA"]["RK2"].insert(0, da) dc ``` ```text diff --git a/docs/user-guide/faq.md b/docs/user-guide/faq.md index 333ef5a3..b6cfb217 100644 --- a/docs/user-guide/faq.md +++ b/docs/user-guide/faq.md @@ -41,6 +41,7 @@ within a given tolerance: ```python import numpy as np + tolerance = np.timedelta64(30, "ms") # typically enough for NTP-synced experiments da["time"] = da["time"].simplify(tolerance) ``` diff --git a/docs/user-guide/pipeline/streaming.md b/docs/user-guide/pipeline/streaming.md index f4e9fea4..e2f5d16a 100644 --- a/docs/user-guide/pipeline/streaming.md +++ b/docs/user-guide/pipeline/streaming.md @@ -35,7 +35,7 @@ from xdas.processing import ZMQPublisher, ZMQSubscriber First we generate some data and split it into packets ```{code-cell} -da = xd.synthetics.dummy() +da = xd.testing.dummy() packets = xd.split(da, 5) ``` diff --git a/pyproject.toml b/pyproject.toml index 7b9de421..d908894d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "xdas" -version = "0.2.7" +dynamic = ["version"] requires-python = ">= 3.10" authors = [ { name = "Alister Trabattoni", email = "alister.trabattoni@gmail.com" }, @@ -24,6 +24,7 @@ dependencies = [ "setuptools<82.0.0", "scipy", "tqdm", + "typing_extensions", "watchdog", "xarray", "xinterp", @@ -43,15 +44,24 @@ docs = [ ] tests = ["dascore", "psutil", "seisbench", "torch"] +# Single source of truth for the version: xdas/__init__.py +[tool.setuptools.dynamic] +version = { attr = "xdas.__version__" } + [tool.ruff.lint] extend-select = ["I", "D"] -extend-ignore = ["D105"] +extend-ignore = [ + "D105", + # xdas raises ValueError for all argument validation, including type checks, + # and that is what the public API and its tests assert. + "TRY004", +] [tool.ruff.lint.pydocstyle] convention = "numpy" [tool.ruff.lint.per-file-ignores] -"tests/**" = ["D"] +"tests/**" = ["D", "RUF012"] # class attrs in test classes are fixture tables "docs/**" = ["D"] [tool.pytest.ini_options] diff --git a/tests/coordinates/test_coordinates.py b/tests/coordinates/test_coordinates.py index 0dadc258..85631d80 100644 --- a/tests/coordinates/test_coordinates.py +++ b/tests/coordinates/test_coordinates.py @@ -3,17 +3,22 @@ import xarray as xr import xdas as xd -from xdas.coordinates import DenseCoordinate, InterpCoordinate, ScalarCoordinate +from xdas.coordinates import ( + AxisCoordinate, + DenseCoordinate, + InterpCoordinate, + ScalarCoordinate, +) +from xdas.coordinates.core import format_datetime class TestCoordinate: def test_new(self): - assert xd.Coordinate(1).isscalar() - assert xd.Coordinate([1]).isdense() - assert xd.Coordinate({"tie_values": [], "tie_indices": []}).isinterp() + assert isinstance(xd.Coordinate(1), ScalarCoordinate) + assert not isinstance(xd.Coordinate(1), AxisCoordinate) coord = xd.Coordinate(xd.Coordinate([1]), "dim") - assert coord.isdense() assert coord.dim == "dim" + assert isinstance(coord, AxisCoordinate) def test_empty(self): with pytest.raises(TypeError, match="cannot infer coordinate type"): @@ -85,13 +90,11 @@ def test_init(self): {"dim": ("dim", {"tie_indices": [0, 8], "tie_values": [100.0, 900.0]})} ) coord = coords["dim"] - assert coord.isinterp() assert np.allclose(coord.tie_indices, [0, 8]) assert np.allclose(coord.tie_values, [100.0, 900.0]) assert coords.isdim("dim") coords = xd.Coordinates({"dim": [1.0, 2.0, 3.0]}) coord = coords["dim"] - assert coord.isdense() assert np.allclose(coord.values, [1.0, 2.0, 3.0]) assert coords.isdim("dim") coords = xd.Coordinates( @@ -109,8 +112,8 @@ def test_init(self): assert coords.isdim("dim_0") assert not coords.isdim("dim_1") coords = xd.Coordinates() - assert coords == dict() - assert coords.dims == tuple() + assert coords == {} + assert coords.dims == () def test_first_last(self): coords = xd.Coordinates({"dim_0": [1.0, 2.0, 3.0], "dim_1": [1.0, 2.0, 3.0]}) @@ -134,18 +137,6 @@ def test_setitem(self): with pytest.raises(TypeError, match="must be of type str"): coords[0] = ... - def test_to_from_dict(self): - starttime = np.datetime64("2020-01-01T00:00:00.000") - endtime = np.datetime64("2020-01-01T00:00:10.000") - coords = { - "time": {"tie_indices": [0, 999], "tie_values": [starttime, endtime]}, - "distance": np.linspace(0, 1000, 3), - "channel": ("distance", ["DAS01", "DAS02", "DAS03"]), - "interrogator": (None, "SRN"), - } - coords = xd.Coordinates(coords) - assert xd.Coordinates.from_dict(coords.to_dict()).equals(coords) - def test_equals_non_coordinates(self): coords = xd.Coordinates({"dim": [1, 2, 3]}) assert not coords.equals({}) @@ -158,6 +149,122 @@ def test_tuple_index_hint(self): with pytest.raises(TypeError, match="cannot use tuple"): coords.to_index({"dim": (1, 2, 3)}) + def test_init_from_coordinates(self): + original = xd.Coordinates({"dim": [1.0, 2.0, 3.0]}) + copy = xd.Coordinates(original) + assert copy.dims == original.dims + assert copy.equals(original) + + def test_getitem_dim_without_coord(self): + coords = xd.Coordinates(dims=("dim",)) + with pytest.raises(KeyError, match="has no coordinate"): + coords["dim"] + + def test_repr(self): + coords = xd.Coordinates( + { + "dim": [1.0, 2.0, 3.0], + "meta": 0, + "other": ("dim", [4.0, 5.0, 6.0]), + } + ) + r = repr(coords) + assert "Coordinates:" in r + assert "* dim" in r + assert "meta" in r + assert "other (dim)" in r + + def test_reduce(self): + import pickle + + coords = xd.Coordinates({"dim": [1.0, 2.0, 3.0]}) + restored = pickle.loads(pickle.dumps(coords)) + assert restored.equals(coords) + + def test_parent(self): + da = xd.DataArray(np.ones(3), {"dim": [0.0, 1.0, 2.0]}) + assert da.coords.parent is da + + def test_get_query_first_last(self): + coords = xd.Coordinates({"dim_0": [1.0, 2.0, 3.0], "dim_1": [1.0, 2.0, 3.0]}) + q = coords._get_query({"first": slice(0, 1)}) + assert q["dim_0"] == slice(0, 1) + assert q["dim_1"] == slice(None) + q = coords._get_query({"last": slice(1, 2)}) + assert q["dim_0"] == slice(None) + assert q["dim_1"] == slice(1, 2) + + def test_get_query_tuple(self): + coords = xd.Coordinates({"dim_0": [1.0, 2.0, 3.0], "dim_1": [1.0, 2.0, 3.0]}) + q = coords._get_query((slice(0, 1), slice(1, 2))) + assert q["dim_0"] == slice(0, 1) + assert q["dim_1"] == slice(1, 2) + + def test_get_query_else_and_return(self): + coords = xd.Coordinates({"dim_0": [1.0, 2.0, 3.0], "dim_1": [1.0, 2.0, 3.0]}) + q = coords._get_query(slice(0, 1)) + assert q["dim_0"] == slice(0, 1) + assert q["dim_1"] == slice(None) + + def test_to_index(self): + coords = xd.Coordinates({"dim": [1.0, 2.0, 3.0]}) + idx = coords.to_index(2.0) + assert idx == {"dim": 1} + + def test_equals_different_names(self): + assert not xd.Coordinates({"dim": [1.0, 2.0, 3.0]}).equals( + xd.Coordinates({"other": [1.0, 2.0, 3.0]}) + ) + + def test_equals_different_values(self): + assert not xd.Coordinates({"dim": [1.0, 2.0, 3.0]}).equals( + xd.Coordinates({"dim": [4.0, 5.0, 6.0]}) + ) + + def test_copy(self): + coords = xd.Coordinates({"dim": [1.0, 2.0, 3.0]}) + copy = coords.copy() + assert copy.equals(coords) + assert copy is not coords + + def test_setitem_with_parent(self): + class FakeParent: + ndim = 1 + shape = (3,) + sizes = {"dim": 3} + + coords = xd.Coordinates({"dim": [1.0, 2.0, 3.0]}) + parent = FakeParent() + coords._assign_parent(parent) + with pytest.raises(KeyError, match="cannot add new dimension"): + coords["other_dim"] = [1.0, 2.0, 3.0] + with pytest.raises(ValueError, match="conflicting sizes"): + coords["dim"] = [1.0, 2.0, 3.0, 4.0] + # scalar coord: coord.dim is None → skips the dim check block + coords["meta"] = 42 + # correctly-sized coord: sizes match → no error + coords["dim"] = [4.0, 5.0, 6.0] + + def test_assign_parent_ndim_mismatch(self): + class FakeParent: + ndim = 1 + shape = (3,) + sizes = {"dim_0": 3} + + coords = xd.Coordinates({"dim_0": [1.0, 2.0, 3.0], "dim_1": [4.0, 5.0, 6.0]}) + with pytest.raises(ValueError, match="number of dimensions"): + coords._assign_parent(FakeParent()) + + def test_assign_parent_size_mismatch(self): + class FakeParent: + ndim = 1 + shape = (3,) + sizes = {"dim": 3} + + coords = xd.Coordinates({"dim": [1.0, 2.0, 3.0, 4.0]}) + with pytest.raises(ValueError, match="conflicting sizes"): + coords._assign_parent(FakeParent()) + class TestCoordinateBase: def test_new_unparseable(self): @@ -188,29 +295,29 @@ def test_get_sampling_interval_timedelta(self): t0 = np.datetime64("2000-01-01T00:00:00") t1 = np.datetime64("2000-01-01T00:00:10") coord = DenseCoordinate([t0, t1], "time") - result = coord.get_sampling_interval(cast=True) - assert result == 10.0 + assert coord.get_sampling_interval(cast=True) is None + assert not coord.isregular() + assert coord.to_regular().get_sampling_interval(cast=True) == 10.0 def test_format_index_non_integer(self): coord = DenseCoordinate([1, 2, 3], "x") with pytest.raises(IndexError, match="only integer"): - coord.format_index(1.5) + coord._format_index(1.5) def test_format_index_clip(self): coord = DenseCoordinate([1, 2, 3], "x") - result = coord.format_index(np.array([-1, 0, 5]), bounds="clip") + result = coord._format_index(np.array([-1, 0, 5]), bounds="clip") assert np.all(result >= 0) - def test_isdefault_issampled(self): - coord = DenseCoordinate([1, 2, 3], "x") - assert not coord.isdefault() - assert not coord.issampled() + def test_to_dataset_no_name(self): + sc = ScalarCoordinate(42) + with pytest.raises(ValueError, match="no name"): + sc._to_dataset(xr.Dataset(), {}) - def test_to_dataset_no_dim(self): + def test_scalar_array_copy(self): sc = ScalarCoordinate(42) - dataset = xr.Dataset() - dataset, attrs = sc.to_dataset(dataset, {}) - assert "None" in dataset.coords or sc.name in dataset.coords or True + result = np.array(sc, copy=True) + assert result == 42 def test_parse_dim_override(self): coord = xd.Coordinate(("x", [1, 2, 3]), dim="y") @@ -242,7 +349,7 @@ def test_get_availabilities_empty(self): def test_format_index_no_bounds(self): coord = DenseCoordinate([1, 2, 3], "x") - result = coord.format_index(np.array([0, 1, 2]), bounds=None) + result = coord._format_index(np.array([0, 1, 2]), bounds=None) assert np.array_equal(result, [0, 1, 2]) def test_init_subclass_no_name(self): @@ -256,14 +363,130 @@ class _Unnamed(Coordinate): def test_init_subclass_with_name(self): from xdas.coordinates import Coordinate - class _Named(Coordinate, name="_testnamed"): + class _Named(Coordinate, ctype="_testnamed"): pass assert "_testnamed" in Coordinate._registry del Coordinate._registry["_testnamed"] - def test_array_function_on_coord(self): + def test_coordinate_copy_deep(self): + coord = DenseCoordinate([1.0, 2.0, 3.0], "x") + deep = coord.copy(deep=True) + assert deep.equals(coord) + assert deep.data is not coord.data + + def test_coordinate_copy_shallow(self): + coord = DenseCoordinate([1.0, 2.0, 3.0], "x") + shallow = coord.copy(deep=False) + assert shallow.equals(coord) + + def test_get_sampling_interval_helper(self): + from xdas.coordinates import get_sampling_interval + + da = xd.DataArray([1, 2, 3], {"x": [10.0, 20.0, 30.0]}) + with pytest.warns(FutureWarning, match="implicit inference is deprecated"): + assert get_sampling_interval(da, "x") == 10.0 + da["x"] = da["x"].to_regular() + assert get_sampling_interval(da, "x") == 10.0 + + def test_get_sampling_interval_helper_jittery_dense_raises(self): + from xdas.coordinates import get_sampling_interval + + da = xd.DataArray([1, 2, 3], {"x": [0.0, 1.0, 5.0]}) + with pytest.raises(ValueError, match="none could be inferred"): + get_sampling_interval(da, "x") + + def test_get_sampling_interval_helper_single_sample_raises(self): + from xdas.coordinates import SampledCoordinate, get_sampling_interval + + coord = SampledCoordinate( + {"tie_values": [0.0], "tie_lengths": [1], "sampling_interval": 5.0}, "x" + ) + da = xd.DataArray([1], {"x": coord}) + with pytest.raises(ValueError, match="none could be inferred"): + get_sampling_interval(da, "x") + + def test_get_sampling_interval_helper_regular(self): + from xdas.coordinates import SampledCoordinate, get_sampling_interval + + coord = SampledCoordinate( + {"tie_values": [0.0], "tie_lengths": [3], "sampling_interval": 5.0} + ) + da = xd.DataArray([1, 2, 3], {"x": coord}) + assert get_sampling_interval(da, "x") == 5.0 + + def test_format_datetime_no_fractional(self): + x = np.datetime64("2000-01-01T00:00:00", "s") + assert format_datetime(x) == "2000-01-01T00:00:00" + + def test_format_datetime_truncates_sub_ms(self): + x = np.datetime64("2000-01-01T00:00:00.123456789", "ns") + result = format_datetime(x) + assert result == "2000-01-01T00:00:00.123" + + def test_drop_dims_variadic_first_last(self): + coords = xd.Coordinates( + { + "dim_0": [1.0, 2.0, 3.0], + "dim_1": [4.0, 5.0, 6.0], + "dim_2": [7.0, 8.0, 9.0], + } + ) + result = coords.drop_dims("first", "last") + assert list(result.dims) == ["dim_1"] + + def test_drop_coords_variadic_first_last(self): + coords = xd.Coordinates( + { + "dim_0": [1.0, 2.0, 3.0], + "dim_1": [4.0, 5.0, 6.0], + "dim_2": [7.0, 8.0, 9.0], + } + ) + result = coords.drop_coords("first", "last") + assert "dim_0" not in result + assert "dim_2" not in result + assert "dim_1" in result + + def test_reduce(self): + import pickle + coord = DenseCoordinate([1.0, 2.0, 3.0], "x") - # Call __array_function__ directly (passing ndarray as type to avoid dispatch loop) - result = coord.__array_function__(np.sum, (np.ndarray,), (coord.data,), {}) - assert result == 6.0 + restored = pickle.loads(pickle.dumps(coord)) + assert restored.equals(coord) + + def test_equals_returns_false_different_values(self): + c1 = DenseCoordinate([1.0, 2.0, 3.0], "x") + c2 = DenseCoordinate([4.0, 5.0, 6.0], "x") + assert not c1.equals(c2) + + def test_slice_indexer_endpoint_false(self): + coord = DenseCoordinate([1.0, 2.0, 3.0], "x") + slc = coord._slice_indexer(stop=3.0, endpoint=False) + assert slc == slice(None, 2) + + +class TestEncodeDelta: + def test_generic_timedelta_promoted_to_ns(self): + from xdas.coordinates.core import decode_delta, encode_delta + + attrs = encode_delta("tolerance", np.timedelta64(0)) + assert attrs == { + "tolerance": 0, + "tolerance_units": "nanoseconds", + "tolerance_dtype": "timedelta64[ns]", + } + assert decode_delta("tolerance", attrs) == np.timedelta64(0, "ns") + + def test_none_is_omitted(self): + from xdas.coordinates.core import encode_delta + + assert encode_delta("tolerance", None) == {} + + +class TestGetSamplingIntervalHelperNonAxis: + def test_scalar_coordinate_returns_none(self): + from xdas.coordinates import get_sampling_interval + + da = xd.DataArray(np.zeros(3), {"x": [0.0, 1.0, 2.0], "meta": 0}) + assert get_sampling_interval(da, "meta") is None diff --git a/tests/coordinates/test_default.py b/tests/coordinates/test_default.py deleted file mode 100644 index 146543ba..00000000 --- a/tests/coordinates/test_default.py +++ /dev/null @@ -1,130 +0,0 @@ -import numpy as np -import pytest - -from xdas.coordinates import Coordinate -from xdas.coordinates.default import DefaultCoordinate - - -class TestDefaultCoordinate: - def test_isvalid(self): - assert DefaultCoordinate.isvalid({"size": 5}) - assert DefaultCoordinate.isvalid({"size": None}) - assert not DefaultCoordinate.isvalid({"size": 1.5}) - assert not DefaultCoordinate.isvalid({"length": 5}) - assert not DefaultCoordinate.isvalid([1, 2, 3]) - assert not DefaultCoordinate.isvalid(5) - - def test_init_default(self): - coord = DefaultCoordinate() - assert coord.empty - assert len(coord) == 0 - - def test_init_with_size(self): - coord = DefaultCoordinate({"size": 5}, "x") - assert not coord.empty - assert len(coord) == 5 - assert coord.dim == "x" - - def test_init_invalid_data(self): - with pytest.raises(TypeError, match="must be a mapping"): - DefaultCoordinate([1, 2, 3]) - - def test_init_dtype_rejected(self): - with pytest.raises(ValueError, match="dtype"): - DefaultCoordinate({"size": 3}, dtype=np.int32) - - def test_empty_property(self): - assert DefaultCoordinate({"size": 0}).empty - assert not DefaultCoordinate({"size": 1}).empty - - def test_dtype(self): - assert DefaultCoordinate({"size": 3}).dtype == np.int64 - - def test_ndim(self): - assert DefaultCoordinate({"size": 3}).ndim == 1 - - def test_shape(self): - assert DefaultCoordinate({"size": 5}).shape == (5,) - - def test_len_with_none(self): - coord = DefaultCoordinate({"size": None}) - assert len(coord) == 0 - - def test_len_with_size(self): - assert len(DefaultCoordinate({"size": 7})) == 7 - - def test_getitem_scalar(self): - coord = DefaultCoordinate({"size": 5}, "x") - result = coord[2] - assert isinstance(result, Coordinate) - assert result.dim is None # scalar → no dim - - def test_getitem_slice(self): - coord = DefaultCoordinate({"size": 5}, "x") - result = coord[1:3] - assert len(result) == 2 - assert result.dim == "x" - - def test_array(self): - coord = DefaultCoordinate({"size": 4}) - arr = np.asarray(coord) - np.testing.assert_array_equal(arr, np.arange(4)) - - def test_isdefault(self): - assert DefaultCoordinate({"size": 3}).isdefault() - - def test_get_sampling_interval(self): - assert DefaultCoordinate({"size": 3}).get_sampling_interval() == 1 - - def test_equals_same(self): - assert DefaultCoordinate({"size": 3}).equals(DefaultCoordinate({"size": 3})) - - def test_equals_different_size(self): - assert not DefaultCoordinate({"size": 3}).equals(DefaultCoordinate({"size": 5})) - - def test_equals_wrong_type(self): - from xdas.coordinates import DenseCoordinate - - result = DefaultCoordinate({"size": 3}).equals( - DenseCoordinate(np.arange(3), "x") - ) - assert result is None - - def test_get_indexer(self): - coord = DefaultCoordinate({"size": 5}) - assert coord.get_indexer(3) == 3 - - def test_slice_indexer(self): - coord = DefaultCoordinate({"size": 5}) - s = coord.slice_indexer(1, 4, 2) - assert s == slice(1, 4, 2) - - def test_concat(self): - a = DefaultCoordinate({"size": 3}, "x") - b = DefaultCoordinate({"size": 2}, "x") - c = a.concat(b) - assert len(c) == 5 - assert c.dim == "x" - - def test_concat_type_error(self): - from xdas.coordinates import DenseCoordinate - - a = DefaultCoordinate({"size": 3}, "x") - b = DenseCoordinate(np.array([0, 1, 2]), "x") - with pytest.raises(TypeError): - a.concat(b) - - def test_concat_dim_mismatch(self): - a = DefaultCoordinate({"size": 3}, "x") - b = DefaultCoordinate({"size": 2}, "y") - with pytest.raises(ValueError): - a.concat(b) - - def test_to_from_dict(self): - coord = DefaultCoordinate({"size": 5}, "x") - dct = coord.to_dict() - assert dct["dim"] == "x" - assert dct["data"] == {"size": 5} - restored = DefaultCoordinate.from_dict(dct) - assert restored.equals(coord) - assert restored.dim == coord.dim diff --git a/tests/coordinates/test_dense.py b/tests/coordinates/test_dense.py index d18ab626..cc8cd0ac 100644 --- a/tests/coordinates/test_dense.py +++ b/tests/coordinates/test_dense.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from xdas.coordinates import DenseCoordinate, ScalarCoordinate @@ -28,9 +29,9 @@ class TestDenseCoordinate: def test_isvalid(self): for data in self.valid: - assert DenseCoordinate.isvalid(data) + assert DenseCoordinate._isvalid(data) for data in self.invalid: - assert not DenseCoordinate.isvalid(data) + assert not DenseCoordinate._isvalid(data) def test_init(self): coord = DenseCoordinate([1, 2, 3]) @@ -45,8 +46,6 @@ def test_init(self): DenseCoordinate(data) def test_getitem(self): - assert np.array_equiv(DenseCoordinate([1, 2, 3])[...].values, [1, 2, 3]) - assert isinstance(DenseCoordinate([1, 2, 3])[...], DenseCoordinate) assert np.array_equiv(DenseCoordinate([1, 2, 3])[:].values, [1, 2, 3]) assert isinstance(DenseCoordinate([1, 2, 3])[:], DenseCoordinate) assert np.array_equiv(DenseCoordinate([1, 2, 3])[1].values, 2) @@ -87,34 +86,20 @@ def test_equals(self): assert DenseCoordinate([1, 2, 3]).equals(DenseCoordinate([1, 2, 3])) assert not DenseCoordinate([1, 2, 3]).equals(42) - def test_isinstance(self): - assert not DenseCoordinate([1, 2, 3]).isscalar() - assert DenseCoordinate([1, 2, 3]).isdense() - assert not DenseCoordinate([1, 2, 3]).isinterp() - def test_get_indexer(self): - assert DenseCoordinate([1, 2, 3]).get_indexer(2) == 1 - assert np.array_equiv(DenseCoordinate([1, 2, 3]).get_indexer([2, 3]), [1, 2]) - assert DenseCoordinate([1, 2, 3]).get_indexer(2.1, method="nearest") == 1 - assert DenseCoordinate([1, 2, 3]).get_indexer(2.1, method="ffill") == 1 - assert DenseCoordinate([1, 2, 3]).get_indexer(2.1, method="bfill") == 2 + assert DenseCoordinate([1, 2, 3])._get_indexer(2) == 1 + assert np.array_equiv(DenseCoordinate([1, 2, 3])._get_indexer([2, 3]), [1, 2]) + assert DenseCoordinate([1, 2, 3])._get_indexer(2.1, method="nearest") == 1 + assert DenseCoordinate([1, 2, 3])._get_indexer(2.1, method="ffill") == 1 + assert DenseCoordinate([1, 2, 3])._get_indexer(2.1, method="bfill") == 2 def test_get_slice_indexer(self): - assert np.array_equiv( - DenseCoordinate([1, 2, 3]).slice_indexer(start=2), slice(1, 3) - ) + assert DenseCoordinate([1, 2, 3])._slice_indexer(start=2) == slice(1, None) def test_to_index(self): assert DenseCoordinate([1, 2, 3]).to_index(2) == 1 assert np.array_equiv(DenseCoordinate([1, 2, 3]).to_index([2, 3]), [1, 2]) - assert np.array_equiv( - DenseCoordinate([1, 2, 3]).to_index(slice(2, None)), slice(1, 3) - ) - - def test_to_from_dict(self): - for data in self.valid: - coord = DenseCoordinate(data) - assert DenseCoordinate.from_dict(coord.to_dict()).equals(coord) + assert DenseCoordinate([1, 2, 3]).to_index(slice(2, None)) == slice(1, None) def test_empty(self): coord = DenseCoordinate() @@ -125,33 +110,74 @@ def test_concat(self): coord1 = DenseCoordinate([1, 2, 3]) coord2 = DenseCoordinate([4, 5, 6]) - result = coord1.concat(coord2) + result = coord1._concat(coord2) expected = DenseCoordinate([1, 2, 3, 4, 5, 6]) assert result.equals(expected) - result = coord2.concat(coord1) + result = coord2._concat(coord1) expected = DenseCoordinate([4, 5, 6, 1, 2, 3]) assert result.equals(expected) - assert coord0.concat(coord0).empty - assert coord0.concat(coord1).equals(coord1) - assert coord1.concat(coord0).equals(coord1) + assert coord0._concat(coord0).empty + assert coord0._concat(coord1).equals(coord1) + assert coord1._concat(coord0).equals(coord1) with pytest.raises(TypeError): - coord1.concat(ScalarCoordinate(1)) + coord1._concat(ScalarCoordinate(1)) with pytest.raises(ValueError, match="different dimension"): - DenseCoordinate([1, 2, 3], "x").concat(DenseCoordinate([4, 5, 6], "y")) + DenseCoordinate([1, 2, 3], "x")._concat(DenseCoordinate([4, 5, 6], "y")) with pytest.raises(ValueError, match="different dtype"): - DenseCoordinate(np.array([1, 2, 3], dtype=np.int32)).concat( + DenseCoordinate(np.array([1, 2, 3], dtype=np.int32))._concat( DenseCoordinate(np.array([4.0, 5.0, 6.0], dtype=np.float64)) ) - def test_get_div_points(self): + def test_get_split_indices(self): coord = DenseCoordinate([1, 2, 3, 10, 11, 12]) - div_points = coord.get_div_points(tolerance=3.0) - assert np.array_equal(div_points, [0, 3, 6]) - with pytest.raises(NotImplementedError): - coord.get_div_points() + # local spacing is 1; only the jump 3->10 stands out as a gap, and the + # normal step 10->11 right after it must not be reported as an overlap + np.testing.assert_array_equal( + coord.get_split_indices("discontinuities", tolerance=3.0), [3] + ) + np.testing.assert_array_equal( + coord.get_split_indices("gaps", tolerance=None), [3] + ) + np.testing.assert_array_equal( + coord.get_split_indices("overlaps", tolerance=None), [] + ) + # with no tolerance filtering every consecutive pair is a candidate boundary + np.testing.assert_array_equal(coord.get_split_indices(), [1, 2, 3, 4, 5]) + + def test_get_split_indices_rate_change(self): + # A continuous axis whose sampling rate changes (step 1 then step 2) is + # not a discontinuity: the baseline follows the new rate, so only the + # single transition is reported and the sustained run stays clean. + coord = DenseCoordinate([0, 1, 2, 3, 5, 7, 9]) + np.testing.assert_array_equal( + coord.get_split_indices("gaps", tolerance=0.5), [4] + ) + np.testing.assert_array_equal( + coord.get_split_indices("discontinuities", tolerance=1.5), [] + ) + + def test_get_split_indices_leading_gap(self): + # A discontinuity in the very first step is still detected. + coord = DenseCoordinate([0, 10, 11, 12]) + np.testing.assert_array_equal( + coord.get_split_indices("gaps", tolerance=3.0), [1] + ) + + def test_get_split_indices_empty(self): + coord = DenseCoordinate([]) + np.testing.assert_array_equal(coord.get_split_indices(), []) + np.testing.assert_array_equal( + coord.get_split_indices("gaps", tolerance=None), [] + ) + + def test_simplify_is_noop(self): + coord = DenseCoordinate([1, 2, 3, 10, 11, 12], "x") + result = coord.simplify(tolerance=5.0) + assert result.equals(coord) + assert result is not coord def test_from_block(self): coord = DenseCoordinate.from_block(0, 5, 1, dim="x") @@ -159,12 +185,85 @@ def test_from_block(self): assert coord.equals(expected) def test_is_monotonic_increasing(self): - assert DenseCoordinate([1, 2, 3]).is_monotonic_increasing() - assert not DenseCoordinate([1, 3, 2]).is_monotonic_increasing() + assert DenseCoordinate([1, 2, 3])._is_monotonic_increasing() + assert not DenseCoordinate([1, 3, 2])._is_monotonic_increasing() t0 = np.datetime64("2000-01-01T00:00:00") times = np.array([t0, t0 + np.timedelta64(1, "s"), t0 + np.timedelta64(2, "s")]) - assert DenseCoordinate(times).is_monotonic_increasing() + assert DenseCoordinate(times)._is_monotonic_increasing() times_bad = np.array( [t0, t0 + np.timedelta64(2, "s"), t0 + np.timedelta64(1, "s")] ) - assert not DenseCoordinate(times_bad).is_monotonic_increasing() + assert not DenseCoordinate(times_bad)._is_monotonic_increasing() + + def test_add(self): + coord = DenseCoordinate([1.0, 2.0, 3.0], "x") + result = coord + 1.0 + expected = DenseCoordinate([2.0, 3.0, 4.0], "x") + assert result.equals(expected) + + def test_get_indexer_missing(self): + with pytest.raises(KeyError): + DenseCoordinate([1, 2, 3])._get_indexer(99) + + def test_to_dataset(self): + coord = DenseCoordinate([1.0, 2.0, 3.0], "x") + coord.dim = "x" + import xdas as xd + + da = xd.DataArray([0, 0, 0], {"x": coord}) + dataset = xr.Dataset() + dataset, _attrs = da.coords["x"]._to_dataset(dataset, {}) + assert "x" in dataset.coords + + def test_to_dataset_no_name(self): + coord = DenseCoordinate([1.0, 2.0, 3.0]) + with pytest.raises(ValueError, match="no name"): + coord._to_dataset(xr.Dataset(), {}) + + def test_collect_from_dataset_object_dtype(self): + coord = DenseCoordinate([1.0, 2.0, 3.0], "x") + import xdas as xd + + da = xd.DataArray([0, 0, 0], {"x": coord}) + dataset = xr.Dataset() + dataset, _ = da.coords["x"]._to_dataset(dataset, {}) + dataset["x"] = dataset["x"].astype(object) + result = DenseCoordinate._collect_from_dataset(dataset, "x") + assert "x" in result + + +class TestDenseCoordinateToRegular: + def test_never_regular(self): + coord = DenseCoordinate([0.0, 1.0, 2.0], "x") + assert coord.get_sampling_interval() is None + assert not coord.isregular() + + def test_to_regular_uniform(self): + coord = DenseCoordinate([0.0, 1.0, 2.0, 3.0], "x") + reg = coord.to_regular() + assert reg.isregular() + assert reg.get_sampling_interval() == 1.0 + assert reg.dim == "x" + np.testing.assert_array_equal(reg.values, coord.values) + + def test_to_regular_explicit_args(self): + coord = DenseCoordinate([0.0, 1.05, 2.0], "x") + reg = coord.to_regular(sampling_interval=1.0, tolerance=0.1) + assert reg.get_sampling_interval() == 1.0 + + def test_to_regular_irregular_raises(self): + coord = DenseCoordinate([0.0, 1.0, 5.0], "x") + with pytest.raises(ValueError, match="not evenly spaced"): + coord.to_regular() + + def test_to_regular_too_short_raises(self): + with pytest.raises(ValueError, match="fewer than two"): + DenseCoordinate([1.0], "x").to_regular() + + def test_to_regular_datetime(self): + t0 = np.datetime64("2000-01-01T00:00:00") + values = t0 + np.timedelta64(1, "s") * np.arange(5) + coord = DenseCoordinate(values, "time") + reg = coord.to_regular() + assert reg.get_sampling_interval() == 1.0 + np.testing.assert_array_equal(reg.values, coord.values) diff --git a/tests/coordinates/test_generic.py b/tests/coordinates/test_generic.py index c75a47b0..75d2e8a4 100644 --- a/tests/coordinates/test_generic.py +++ b/tests/coordinates/test_generic.py @@ -2,6 +2,26 @@ import pytest import xdas as xd +from xdas.coordinates.core import parse_scalar_delta + + +class TestParseScalarDelta: + def test_non_scalar_raises(self): + with pytest.raises(ValueError, match="must be a scalar"): + parse_scalar_delta([1, 2], np.dtype("float64")) + + def test_none_without_default_raises(self): + with pytest.raises(ValueError, match="cannot be None"): + parse_scalar_delta(None, np.dtype("float64")) + + def test_none_with_default_zero(self): + assert parse_scalar_delta(None, np.dtype("float64"), default_zero=True) == 1e-8 + assert parse_scalar_delta(None, np.dtype("float32"), default_zero=True) == 1e-5 + assert parse_scalar_delta(None, np.dtype("float16"), default_zero=True) == 1e-2 + assert parse_scalar_delta(None, np.dtype("int64"), default_zero=True) == 0 + assert parse_scalar_delta( + None, np.dtype("datetime64[s]"), default_zero=True + ) == np.timedelta64(0) class TestFromBlock: @@ -36,7 +56,8 @@ def coord(dtype, ctype): size = 10 step = np.array(1, "timedelta64" if np.issubdtype(dtype, np.datetime64) else dtype) return xd.concat_coords( - [xd.Coordinate[ctype].from_block(start, size, step, "dim") for start in starts] + [xd.Coordinate[ctype].from_block(start, size, step, "dim") for start in starts], + tolerance=False, ) diff --git a/tests/coordinates/test_interp.py b/tests/coordinates/test_interp.py index ce69107d..562badc4 100644 --- a/tests/coordinates/test_interp.py +++ b/tests/coordinates/test_interp.py @@ -1,7 +1,13 @@ import numpy as np import pytest +import xarray as xr -from xdas.coordinates import InterpCoordinate, ScalarCoordinate +import xdas as xd +from xdas.coordinates import ( + InterpCoordinate, + ScalarCoordinate, +) +from xdas.coordinates.core import Coordinate class TestInterpCoordinate: @@ -48,9 +54,17 @@ class TestInterpCoordinate: def test_isvalid(self): for data in self.valid: - assert InterpCoordinate.isvalid(data) + assert InterpCoordinate._isvalid(data) for data in self.invalid: - assert not InterpCoordinate.isvalid(data) + assert not InterpCoordinate._isvalid(data) + # with optional sampling_interval / tolerance is still valid + assert InterpCoordinate._isvalid( + {"tie_indices": [0, 8], "tie_values": [0.0, 8.0], "sampling_interval": 1.0} + ) + # unknown extra key is rejected + assert not InterpCoordinate._isvalid( + {"tie_indices": [0, 8], "tie_values": [0.0, 8.0], "extra": 1} + ) def test_init(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) @@ -77,7 +91,7 @@ def test_len(self): len(InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]})) == 9 ) - assert len(InterpCoordinate(dict(tie_indices=[], tie_values=[]))) == 0 + assert len(InterpCoordinate({"tie_indices": [], "tie_values": []})) == 0 @pytest.mark.parametrize("valid_input", valid) def test_repr(self, valid_input): @@ -103,15 +117,17 @@ def test_getitem(self): coord[9] coord[-9] assert coord[0:2].equals( - InterpCoordinate(dict(tie_indices=[0, 1], tie_values=[100.0, 200.0])) + InterpCoordinate({"tie_indices": [0, 1], "tie_values": [100.0, 200.0]}) ) assert coord[:].equals(coord) - assert coord[6:3].equals(InterpCoordinate(dict(tie_indices=[], tie_values=[]))) + assert coord[6:3].equals( + InterpCoordinate({"tie_indices": [], "tie_values": []}) + ) assert coord[1:2].equals( - InterpCoordinate(dict(tie_indices=[0], tie_values=[200.0])) + InterpCoordinate({"tie_indices": [0], "tie_values": [200.0]}) ) assert coord[-3:-1].equals( - InterpCoordinate(dict(tie_indices=[0, 1], tie_values=[700.0, 800.0])) + InterpCoordinate({"tie_indices": [0, 1], "tie_values": [700.0, 800.0]}) ) def test_setitem(self): @@ -128,7 +144,7 @@ def test_empty(self): assert not InterpCoordinate( {"tie_indices": [0, 8], "tie_values": [100.0, 900.0]} ).empty - assert InterpCoordinate(dict(tie_indices=[], tie_values=[])).empty + assert InterpCoordinate({"tie_indices": [], "tie_values": []}).empty def test_dtype(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) @@ -149,57 +165,59 @@ def test_format_index(self): def test_get_value(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) - assert coord.get_value(0) == 100.0 - assert coord.get_value(4) == 500.0 - assert coord.get_value(8) == 900.0 - assert coord.get_value(-1) == 900.0 - assert coord.get_value(-9) == 100.0 - assert np.allclose(coord.get_value([1, 2, 3, -2]), [200.0, 300.0, 400.0, 800.0]) + assert coord._get_value(0) == 100.0 + assert coord._get_value(4) == 500.0 + assert coord._get_value(8) == 900.0 + assert coord[-1].data == 900.0 + assert coord[-9].data == 100.0 + assert np.allclose(coord[[1, 2, 3, -2]].values, [200.0, 300.0, 400.0, 800.0]) + with pytest.raises(IndexError): + coord[-10] with pytest.raises(IndexError): - coord.get_value(-10) - coord.get_value(9) - coord.get_value(0.5) + coord[9] + with pytest.raises(IndexError): + coord[0.5] starttime = np.datetime64("2000-01-01T00:00:00") endtime = np.datetime64("2000-01-01T00:00:08") coord = InterpCoordinate( - dict(tie_indices=[0, 8], tie_values=[starttime, endtime]) + {"tie_indices": [0, 8], "tie_values": [starttime, endtime]} ) - assert coord.get_value(0) == starttime - assert coord.get_value(4) == np.datetime64("2000-01-01T00:00:04") - assert coord.get_value(8) == endtime - assert coord.get_value(-1) == endtime - assert coord.get_value(-9) == starttime + assert coord._get_value(0) == starttime + assert coord._get_value(4) == np.datetime64("2000-01-01T00:00:04") + assert coord._get_value(8) == endtime + assert coord[-1].data == endtime + assert coord[-9].data == starttime def test_get_index(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) - assert coord.get_indexer(100.0) == 0 - assert coord.get_indexer(900.0) == 8 - assert coord.get_indexer(0.0, "nearest") == 0 - assert coord.get_indexer(1000.0, "nearest") == 8 - assert coord.get_indexer(125.0, "nearest") == 0 - assert coord.get_indexer(175.0, "nearest") == 1 - assert coord.get_indexer(175.0, "ffill") == 0 - assert coord.get_indexer(200.0, "ffill") == 1 - assert coord.get_indexer(200.0, "bfill") == 1 - assert coord.get_indexer(125.0, "bfill") == 1 - assert np.all(np.equal(coord.get_indexer([100.0, 900.0]), [0, 8])) + assert coord._get_indexer(100.0) == 0 + assert coord._get_indexer(900.0) == 8 + assert coord._get_indexer(0.0, "nearest") == 0 + assert coord._get_indexer(1000.0, "nearest") == 8 + assert coord._get_indexer(125.0, "nearest") == 0 + assert coord._get_indexer(175.0, "nearest") == 1 + assert coord._get_indexer(175.0, "ffill") == 0 + assert coord._get_indexer(200.0, "ffill") == 1 + assert coord._get_indexer(200.0, "bfill") == 1 + assert coord._get_indexer(125.0, "bfill") == 1 + assert np.all(np.equal(coord._get_indexer([100.0, 900.0]), [0, 8])) with pytest.raises(KeyError): - assert coord.get_indexer(0.0) == 0 - assert coord.get_indexer(1000.0) == 8 - assert coord.get_indexer(150.0) == 0 - assert coord.get_indexer(1000.0, "bfill") == 8 - assert coord.get_indexer(0.0, "ffill") == 0 + assert coord._get_indexer(0.0) == 0 + assert coord._get_indexer(1000.0) == 8 + assert coord._get_indexer(150.0) == 0 + assert coord._get_indexer(1000.0, "bfill") == 8 + assert coord._get_indexer(0.0, "ffill") == 0 starttime = np.datetime64("2000-01-01T00:00:00") endtime = np.datetime64("2000-01-01T00:00:08") coord = InterpCoordinate( - dict(tie_indices=[0, 8], tie_values=[starttime, endtime]) + {"tie_indices": [0, 8], "tie_values": [starttime, endtime]} ) - assert coord.get_indexer(starttime) == 0 - assert coord.get_indexer(endtime) == 8 - assert coord.get_indexer(str(starttime)) == 0 - assert coord.get_indexer(str(endtime)) == 8 - assert coord.get_indexer("2000-01-01T00:00:04.1", "nearest") == 4 + assert coord._get_indexer(starttime) == 0 + assert coord._get_indexer(endtime) == 8 + assert coord._get_indexer(str(starttime)) == 0 + assert coord._get_indexer(str(endtime)) == 8 + assert coord._get_indexer("2000-01-01T00:00:04.1", "nearest") == 4 def test_indices(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) @@ -211,66 +229,66 @@ def test_values(self): def test_get_index_slice(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) - assert coord.slice_indexer(100.0, 200.0) == slice(0, 2) - assert coord.slice_indexer(150.0, 250.0) == slice(1, 2) - assert coord.slice_indexer(300.0, 500.0) == slice(2, 5) - assert coord.slice_indexer(0.0, 500.0) == slice(0, 5) - assert coord.slice_indexer(125.0, 175.0) == slice(1, 1) - assert coord.slice_indexer(0.0, 50.0) == slice(0, 0) - assert coord.slice_indexer(1000.0, 1100.0) == slice(9, 9) - assert coord.slice_indexer(1000.0, 500.0) == slice(9, 5) - assert coord.slice_indexer(None, None) == slice(None, None) + assert coord._slice_indexer(100.0, 200.0) == slice(0, 2) + assert coord._slice_indexer(150.0, 250.0) == slice(1, 2) + assert coord._slice_indexer(300.0, 500.0) == slice(2, 5) + assert coord._slice_indexer(0.0, 500.0) == slice(0, 5) + assert coord._slice_indexer(125.0, 175.0) == slice(1, 1) + assert coord._slice_indexer(0.0, 50.0) == slice(0, 0) + assert coord._slice_indexer(1000.0, 1100.0) == slice(9, 9) + assert coord._slice_indexer(1000.0, 500.0) == slice(9, 5) + assert coord._slice_indexer(None, None) == slice(None, None) def test_slice_index(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) - assert coord.slice_index(slice(0, 2)).equals( - InterpCoordinate(dict(tie_indices=[0, 1], tie_values=[100.0, 200.0])) + assert coord[0:2].equals( + InterpCoordinate({"tie_indices": [0, 1], "tie_values": [100.0, 200.0]}) ) - assert coord.slice_index(slice(7, None)).equals( - InterpCoordinate(dict(tie_indices=[0, 1], tie_values=[800.0, 900.0])) + assert coord[7:].equals( + InterpCoordinate({"tie_indices": [0, 1], "tie_values": [800.0, 900.0]}) ) - assert coord.slice_index(slice(None, None)).equals(coord) - assert coord.slice_index(slice(0, 0)).equals( - InterpCoordinate(dict(tie_indices=[], tie_values=[])) + assert coord[:].equals(coord) + assert coord[0:0].equals( + InterpCoordinate({"tie_indices": [], "tie_values": []}) ) - assert coord.slice_index(slice(4, 2)).equals( - InterpCoordinate(dict(tie_indices=[], tie_values=[])) + assert coord[4:2].equals( + InterpCoordinate({"tie_indices": [], "tie_values": []}) ) - assert coord.slice_index(slice(9, 9)).equals( - InterpCoordinate(dict(tie_indices=[], tie_values=[])) + assert coord[9:9].equals( + InterpCoordinate({"tie_indices": [], "tie_values": []}) ) - assert coord.slice_index(slice(3, 3)).equals( - InterpCoordinate(dict(tie_indices=[], tie_values=[])) + assert coord[3:3].equals( + InterpCoordinate({"tie_indices": [], "tie_values": []}) ) - assert coord.slice_index(slice(0, -1)).equals( - InterpCoordinate(dict(tie_indices=[0, 7], tie_values=[100.0, 800.0])) + assert coord[0:-1].equals( + InterpCoordinate({"tie_indices": [0, 7], "tie_values": [100.0, 800.0]}) ) - assert coord.slice_index(slice(0, -2)).equals( - InterpCoordinate(dict(tie_indices=[0, 6], tie_values=[100.0, 700.0])) + assert coord[0:-2].equals( + InterpCoordinate({"tie_indices": [0, 6], "tie_values": [100.0, 700.0]}) ) - assert coord.slice_index(slice(-2, None)).equals( - InterpCoordinate(dict(tie_indices=[0, 1], tie_values=[800.0, 900.0])) + assert coord[-2:].equals( + InterpCoordinate({"tie_indices": [0, 1], "tie_values": [800.0, 900.0]}) ) - assert coord.slice_index(slice(1, 2)).equals( - InterpCoordinate(dict(tie_indices=[0], tie_values=[200.0])) + assert coord[1:2].equals( + InterpCoordinate({"tie_indices": [0], "tie_values": [200.0]}) ) - assert coord.slice_index(slice(1, 3, 2)).equals( - InterpCoordinate(dict(tie_indices=[0], tie_values=[200.0])) + assert coord[1:3:2].equals( + InterpCoordinate({"tie_indices": [0], "tie_values": [200.0]}) ) - assert coord.slice_index(slice(None, None, 2)).equals( - InterpCoordinate(dict(tie_indices=[0, 4], tie_values=[100.0, 900.0])) + assert coord[::2].equals( + InterpCoordinate({"tie_indices": [0, 4], "tie_values": [100.0, 900.0]}) ) - assert coord.slice_index(slice(None, None, 3)).equals( - InterpCoordinate(dict(tie_indices=[0, 2], tie_values=[100.0, 700.0])) + assert coord[::3].equals( + InterpCoordinate({"tie_indices": [0, 2], "tie_values": [100.0, 700.0]}) ) - assert coord.slice_index(slice(None, None, 4)).equals( - InterpCoordinate(dict(tie_indices=[0, 2], tie_values=[100.0, 900.0])) + assert coord[::4].equals( + InterpCoordinate({"tie_indices": [0, 2], "tie_values": [100.0, 900.0]}) ) - assert coord.slice_index(slice(None, None, 5)).equals( - InterpCoordinate(dict(tie_indices=[0, 1], tie_values=[100.0, 600.0])) + assert coord[::5].equals( + InterpCoordinate({"tie_indices": [0, 1], "tie_values": [100.0, 600.0]}) ) - assert coord.slice_index(slice(2, 7, 3)).equals( - InterpCoordinate(dict(tie_indices=[0, 1], tie_values=[300.0, 600.0])) + assert coord[2:7:3].equals( + InterpCoordinate({"tie_indices": [0, 1], "tie_values": [300.0, 600.0]}) ) def test_to_index(self): @@ -304,53 +322,131 @@ def test_singleton(self): coord = InterpCoordinate({"tie_indices": [0], "tie_values": [1.0]}) assert coord[0].values == 1.0 - def test_to_from_dict(self): - for data in self.valid: - coord = InterpCoordinate(data) - assert InterpCoordinate.from_dict(coord.to_dict()).equals(coord) - def test_concat(self): coord0 = InterpCoordinate() coord1 = InterpCoordinate({"tie_indices": [0, 2], "tie_values": [0, 20]}) coord2 = InterpCoordinate({"tie_indices": [0, 2], "tie_values": [30, 50]}) - result = coord1.concat(coord2).simplify() + result = coord1._concat(coord2).simplify() expected = InterpCoordinate({"tie_indices": [0, 5], "tie_values": [0, 50]}) assert result.equals(expected) - result = coord2.concat(coord1).simplify() + result = coord2._concat(coord1).simplify() expected = InterpCoordinate( {"tie_indices": [0, 2, 3, 5], "tie_values": [30, 50, 0, 20]} ) assert result.equals(expected) - assert coord0.concat(coord0).empty - assert coord0.concat(coord1).equals(coord1) - assert coord1.concat(coord0).equals(coord1) + assert coord0._concat(coord0).empty + assert coord0._concat(coord1).equals(coord1) + assert coord1._concat(coord0).equals(coord1) + + def test_simplify_preserves_real_discontinuity(self): + # A large jump across a den == 1 gap is preserved as an emergent property + # of the tolerance bound: both boundary points survive while the colinear + # interior collapses. + coord = InterpCoordinate( + { + "tie_indices": [0, 5, 10, 11, 16, 21], + "tie_values": [0.0, 5.0, 10.0, 1000.0, 1005.0, 1010.0], + } + ) + result = coord.simplify() + assert result.equals( + InterpCoordinate( + { + "tie_indices": [0, 10, 11, 21], + "tie_values": [0.0, 10.0, 1000.0, 1010.0], + } + ) + ) + + def test_simplify_absorbs_soft_discontinuity(self): + # A den == 1 gap whose jump fits within tolerance is fused away and the + # two areas merge into a single ramp. + coord = InterpCoordinate( + {"tie_indices": [0, 10, 11, 21], "tie_values": [0.0, 10.0, 11.4, 21.4]} + ) + result = coord.simplify(1.0) + assert result.equals( + InterpCoordinate({"tie_indices": [0, 21], "tie_values": [0.0, 21.4]}) + ) + + def test_simplify_multiple_runs_and_isolated_point(self): + # Two real discontinuities flanking an isolated tie point: each run is + # thinned independently and the isolated point survives. + coord = InterpCoordinate( + { + "tie_indices": [0, 5, 10, 11, 12, 17, 22], + "tie_values": [0.0, 5.0, 10.0, 100.0, 200.0, 205.0, 210.0], + } + ) + result = coord.simplify() + assert result.equals( + InterpCoordinate( + { + "tie_indices": [0, 10, 11, 12, 22], + "tie_values": [0.0, 10.0, 100.0, 200.0, 210.0], + } + ) + ) + def test_simplify_keeps_kink(self): + # A genuine kink inside a continuous area forces Douglas-Peucker to keep + # the deviating interior point. + coord = InterpCoordinate( + {"tie_indices": [0, 5, 10], "tie_values": [0.0, 100.0, 0.0]} + ) + result = coord.simplify() + assert result.equals(coord) + assert len(coord.simplify(200.0).tie_indices) == 2 + + def test_simplify_datetime_discontinuity(self): + t0 = np.datetime64("2000-01-01T00:00:00") + coord = InterpCoordinate( + { + "tie_indices": [0, 5, 10, 11, 16, 21], + "tie_values": [ + t0, + t0 + np.timedelta64(5, "s"), + t0 + np.timedelta64(10, "s"), + t0 + np.timedelta64(1000, "s"), + t0 + np.timedelta64(1005, "s"), + t0 + np.timedelta64(1010, "s"), + ], + } + ) + result = coord.simplify() + assert np.array_equal(result.tie_indices, [0, 10, 11, 21]) + + +class TestInterpCoordinateExtra: + def test_init_extra_keys(self): + with pytest.raises(TypeError, match="tie_indices"): + InterpCoordinate( + {"tie_indices": [0, 8], "tie_values": [100.0, 900.0], "extra": 1} + ) + + def test_concat_errors(self): with pytest.raises(TypeError): - coord1.concat(ScalarCoordinate(1)) + InterpCoordinate({"tie_indices": [0, 2], "tie_values": [0, 20]})._concat( + ScalarCoordinate(1) + ) with pytest.raises(ValueError, match="different dimension"): InterpCoordinate( {"tie_indices": [0, 2], "tie_values": [0, 20]}, "x" - ).concat( + )._concat( InterpCoordinate({"tie_indices": [0, 2], "tie_values": [30, 50]}, "y") ) with pytest.raises(ValueError, match="different dtype"): InterpCoordinate( {"tie_indices": [0, 2], "tie_values": np.array([0, 20], dtype=np.int32)} - ).concat( + )._concat( InterpCoordinate( {"tie_indices": [0, 2], "tie_values": np.array([30.0, 50.0])} ) ) - def test_init_extra_keys(self): - with pytest.raises(ValueError, match="both"): - InterpCoordinate( - {"tie_indices": [0, 8], "tie_values": [100.0, 900.0], "extra": 1} - ) - def test_init_non_monotonic(self): with pytest.raises(ValueError, match="strictly increasing"): InterpCoordinate( @@ -370,20 +466,23 @@ def test_array_with_dtype(self): result = coord.__array__(dtype=np.float32) assert result.dtype == np.float32 - def test_get_sampling_interval_empty(self): + def test_to_regular_empty(self): coord = InterpCoordinate() - assert coord.get_sampling_interval() is None + with pytest.raises(ValueError, match="cannot infer"): + coord.to_regular() def test_get_indexer_overlaps(self): coord = InterpCoordinate( {"tie_indices": [0, 4, 8], "tie_values": [100.0, 50.0, 900.0]} ) with pytest.raises(ValueError, match="overlaps were found"): - coord.get_indexer(200.0) + coord._get_indexer(200.0) def test_simplify_false(self): coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}) - assert coord.simplify(False) is coord + result = coord.simplify(False) + assert result is not coord + assert result.equals(coord) def test_get_split_indices_kinds(self): t0 = np.datetime64("2000-01-01T00:00:00") @@ -405,29 +504,11 @@ def test_get_split_indices_kinds(self): assert len(gaps) >= 0 assert len(overlaps) >= 0 - def test_decimate_collision(self): - # Four tie points where two middle ones collide after integer division; - # the loop in decimate() fixes the middle collisions so the result is valid. - coord = InterpCoordinate( - {"tie_indices": [0, 2, 5, 9], "tie_values": [0.0, 20.0, 50.0, 90.0]} - ) - result = coord.decimate(3) - assert np.all(np.diff(result.tie_indices) > 0) - - def test_decimate_no_collision(self): - # No collisions after //q: the False branch of the collision check is taken. - coord = InterpCoordinate( - {"tie_indices": [0, 4, 7, 9], "tie_values": [0.0, 40.0, 70.0, 90.0]} - ) - result = coord.decimate(3) - assert np.all(np.diff(result.tie_indices) > 0) - def test_get_split_indices_overlaps_tolerance_false(self): - # Build a coord with an overlap (tie_values go backwards between segments) coord = InterpCoordinate( { "tie_indices": [0, 4, 5, 9], - "tie_values": [0.0, 4.0, 3.0, 7.0], # overlap at index 5 (value 3 < 4) + "tie_values": [0.0, 4.0, 3.0, 7.0], } ) result = coord.get_split_indices(kind="overlaps", tolerance=False) @@ -449,20 +530,688 @@ def test_is_monotonic_increasing_true(self): coord = InterpCoordinate( {"tie_indices": [0, 4, 5, 9], "tie_values": [0.0, 4.0, 5.0, 9.0]} ) - assert coord.is_monotonic_increasing() is True + assert coord._is_monotonic_increasing() is True def test_is_monotonic_increasing_false(self): coord = InterpCoordinate( {"tie_indices": [0, 4, 5, 9], "tie_values": [0.0, 4.0, 3.0, 7.0]} ) - assert coord.is_monotonic_increasing() is False + assert coord._is_monotonic_increasing() is False def test_is_monotonic_increasing_multi_segment(self): - # Three segments all strictly increasing — must not raise ValueError from bool() coord = InterpCoordinate( { "tie_indices": [0, 4, 5, 9, 10, 14], "tie_values": [0.0, 4.0, 5.0, 9.0, 10.0, 14.0], } ) - assert coord.is_monotonic_increasing() is True + assert coord._is_monotonic_increasing() is True + + def test_slice_step_collision(self): + coord = InterpCoordinate( + {"tie_indices": [0, 2, 6, 12], "tie_values": [0.0, 20.0, 60.0, 120.0]} + ) + result = coord[::3] + assert isinstance(result, InterpCoordinate) + assert len(result.tie_indices) >= 3 + assert result.tie_indices[0] == 0 + assert all( + result.tie_indices[i] < result.tie_indices[i + 1] + for i in range(len(result.tie_indices) - 1) + ) + + def test_to_regular_explicit_args(self): + coord = InterpCoordinate( + {"tie_indices": [0, 10, 20], "tie_values": [0.0, 1.0, 2.05]} + ) + # strict default tolerance rejects the jitter + with pytest.raises(ValueError, match="not consistent"): + coord.to_regular() + # an explicit tolerance accepts it + reg = coord.to_regular(sampling_interval=0.1, tolerance=0.1) + assert isinstance(reg, InterpCoordinate) + assert reg.isregular() + assert reg.sampling_interval == 0.1 + + def test_to_regular_already_regular_is_preserved(self): + # a regular coordinate keeps its stored spacing untouched + coord = InterpCoordinate( + { + "tie_indices": [0, 10, 20], + "tie_values": [0.0, 1.0, 2.05], + "sampling_interval": 0.1, + "tolerance": 0.1, + } + ) + reg = coord.to_regular() + assert reg is not coord + assert reg.sampling_interval == 0.1 + assert reg.tolerance == 0.1 + # an explicit spacing still overrides it + reg2 = coord.to_regular(sampling_interval=0.103, tolerance=0.1) + assert reg2.sampling_interval == 0.103 + + def test_module_helper_infers_with_warning(self): + da = xd.DataArray( + np.zeros(9), + {"x": {"tie_indices": [0, 8], "tie_values": [0.0, 8.0]}}, + ) + with pytest.warns(FutureWarning, match="implicit inference is deprecated"): + assert xd.get_sampling_interval(da, "x") == 1.0 + da["x"] = da["x"].to_regular() + assert xd.get_sampling_interval(da, "x") == 1.0 + + def test_module_helper_jittery_infers_with_warning(self): + # The fallback states the tolerance required to accept the jitter; the + # strict conversion still rejects it. + da = xd.DataArray( + np.zeros(21), + {"x": {"tie_indices": [0, 10, 20], "tie_values": [0.0, 1.0, 2.05]}}, + ) + with pytest.warns(FutureWarning, match="accepting jitter up to tolerance"): + result = xd.get_sampling_interval(da, "x") + assert 0.1 <= result <= 0.105 + with pytest.raises(ValueError, match="not consistent"): + da["x"].to_regular() + + def test_module_helper_datetime_cast(self): + t0 = np.datetime64("2000-01-01T00:00:00") + da = xd.DataArray( + np.zeros(21), + { + "time": { + "tie_indices": [0, 10, 20], + "tie_values": [ + t0, + t0 + np.timedelta64(10, "s"), + t0 + np.timedelta64(21, "s"), + ], + } + }, + ) + with pytest.warns(FutureWarning, match="implicit inference is deprecated"): + result = xd.get_sampling_interval(da, "time") + assert 1.0 <= result <= 1.1 + with pytest.warns(FutureWarning): + result = xd.get_sampling_interval(da, "time", cast=False) + assert isinstance(result, np.timedelta64) + + def test_module_helper_no_continuous_area_raises(self): + da = xd.DataArray( + np.zeros(2), + {"x": {"tie_indices": [0, 1], "tie_values": [0.0, 1.0]}}, + ) + with pytest.raises(ValueError, match="none could be inferred"): + xd.get_sampling_interval(da, "x") + + def test_to_regular_datetime_cast(self): + t0 = np.datetime64("2000-01-01T00:00:00") + t1 = np.datetime64("2000-01-01T00:00:08") + coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [t0, t1]}) + result = coord.to_regular().get_sampling_interval() # cast=True by default + assert result == 1.0 + + def test_to_regular_infer_datetime(self): + t0 = np.datetime64("2000-01-01T00:00:00") + t1 = np.datetime64("2000-01-01T00:00:08") + coord = InterpCoordinate({"tie_indices": [0, 8], "tie_values": [t0, t1]}) + reg = coord.to_regular() + assert reg.sampling_interval == np.timedelta64(1, "s") + assert reg.get_sampling_interval() == 1.0 + + def test_to_regular_unit_spaced(self): + # all tie-index gaps == 1 → no constrained segment → cannot infer + coord = InterpCoordinate( + {"tie_indices": [0, 1, 2], "tie_values": [0.0, 1.0, 2.0]} + ) + with pytest.raises(ValueError, match="cannot infer"): + coord.to_regular() + + def test_to_regular_minimax_favours_long_segment(self): + # rates 1.0 (den=10) and 1.1 (den=2); minimax is pulled toward the long + # segment, not the median midpoint 1.05 + coord = InterpCoordinate( + {"tie_indices": [0, 10, 12], "tie_values": [0.0, 10.0, 12.2]} + ) + si = coord.to_regular(tolerance=1.0).sampling_interval + np.testing.assert_allclose(si, 12.2 / 12) + + def test_tolerance_without_sampling_interval(self): + with pytest.raises(ValueError, match="cannot be set without"): + InterpCoordinate( + {"tie_indices": [0, 10], "tie_values": [0.0, 10.0], "tolerance": 0.1} + ) + + def test_infer_regular(self): + # Numeric: rates 1.0 (den=10) and 1.0555 (den=5); the inferred spacing + # and tolerance must round-trip through `to_regular`. + coord = InterpCoordinate( + {"tie_indices": [0, 10, 15], "tie_values": [0.0, 10.0, 15.55]} + ) + si, tol = coord._infer_regular() + assert si > 0 + assert tol > 0 + reg = coord.to_regular(sampling_interval=si, tolerance=tol) + assert reg.isregular() + + # Datetime variant: tolerance comes back as a timedelta64. + t0 = np.datetime64("2000-01-01T00:00:00") + coord_dt = InterpCoordinate( + { + "tie_indices": [0, 10, 15], + "tie_values": [ + t0, + t0 + np.timedelta64(10_000_000_000, "ns"), + t0 + np.timedelta64(15_550_000_000, "ns"), + ], + } + ) + si_dt, tol_dt = coord_dt._infer_regular() + assert np.issubdtype(np.asarray(tol_dt).dtype, np.timedelta64) + assert tol_dt > np.timedelta64(0) + assert coord_dt.to_regular( + sampling_interval=si_dt, tolerance=tol_dt + ).isregular() + + # No continuous segment → nothing to infer. + unit = InterpCoordinate( + {"tie_indices": [0, 1, 2], "tie_values": [0.0, 1.0, 2.0]} + ) + assert unit._infer_regular() == (None, None) + + def test_add_sub(self): + coord = InterpCoordinate({"tie_indices": [0, 4], "tie_values": [10.0, 50.0]}) + result = coord + 5.0 + assert isinstance(result, InterpCoordinate) + assert np.allclose(result.tie_values, [15.0, 55.0]) + result2 = coord - 5.0 + assert np.allclose(result2.tie_values, [5.0, 45.0]) + + def test_to_dataset_collect_roundtrip(self): + da = xd.DataArray( + np.zeros(9), + {"x": {"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}}, + ) + coord = da.coords["x"] + dataset = xr.Dataset() + attrs = {} + dataset, attrs = coord._to_dataset(dataset, attrs) + assert "coordinate_interpolation" in attrs + assert "x_indices" in dataset + assert "x_values" in dataset + dataset["__values__"] = xr.DataArray(np.zeros(9), dims=["x"]) + dataset["__values__"].attrs["coordinate_interpolation"] = attrs[ + "coordinate_interpolation" + ] + recovered = InterpCoordinate._collect_from_dataset(dataset, "__values__") + assert "x" in recovered + assert np.allclose(recovered["x"].tie_values, coord.tie_values) + + def test_to_dataset_multiple_coords_append(self): + da = xd.DataArray( + np.zeros((9, 5)), + { + "x": {"tie_indices": [0, 8], "tie_values": [100.0, 900.0]}, + "y": {"tie_indices": [0, 4], "tie_values": [0.0, 40.0]}, + }, + ) + attrs = {} + dataset = xr.Dataset() + dataset, attrs = da.coords["x"]._to_dataset(dataset, attrs) + dataset, attrs = da.coords["y"]._to_dataset(dataset, attrs) + assert "x" in attrs["coordinate_interpolation"] + assert "y" in attrs["coordinate_interpolation"] + + def test_to_dataset_datetime(self): + t0 = np.datetime64("2000-01-01T00:00:00") + t1 = np.datetime64("2000-01-01T00:00:08") + da = xd.DataArray( + np.zeros(9), + {"time": {"tie_indices": [0, 8], "tie_values": [t0, t1]}}, + ) + coord = da.coords["time"] + dataset = xr.Dataset() + attrs = {} + dataset, attrs = coord._to_dataset(dataset, attrs) + assert "time_indices" in dataset + assert dataset["time_values"].dtype == np.dtype("datetime64[ns]") + + +class TestInterpCoordinateRegular: + """Tests for InterpCoordinate with an enforced sampling_interval (regular mode).""" + + valid = [ + { + "tie_indices": [0, 5, 9, 10, 19], + "tie_values": [0.0, 0.5, 0.9, 2.0, 2.9], + "sampling_interval": 0.1, + } + ] + + def make(self): + return InterpCoordinate(self.valid[0], "dim") + + def test_isvalid(self): + for data in self.valid: + assert InterpCoordinate._isvalid(data) + # plain tie-point dict without sampling_interval is also valid + assert InterpCoordinate._isvalid( + {"tie_indices": [0, 8], "tie_values": [0.0, 8.0]} + ) + # unknown extra key is rejected + assert not InterpCoordinate._isvalid( + { + "tie_indices": [0, 8], + "tie_values": [0.0, 8.0], + "sampling_interval": 1.0, + "extra": 1, + } + ) + + def test_init(self): + coord = self.make() + assert coord.sampling_interval == 0.1 + assert coord.tolerance is not None + assert coord.dim == "dim" + assert coord.isregular() + + def test_factory_dispatch(self): + coord = Coordinate(self.valid[0]) + assert isinstance(coord, InterpCoordinate) + assert coord.isregular() + # plain tie-point data routes to a non-regular InterpCoordinate + plain = Coordinate({"tie_indices": [0, 8], "tie_values": [0.0, 8.0]}) + assert isinstance(plain, InterpCoordinate) + assert not plain.isregular() + + def test_init_inconsistent(self): + with pytest.raises(ValueError, match="not consistent"): + InterpCoordinate( + { + "tie_indices": [0, 10], + "tie_values": [0.0, 10.0], + "sampling_interval": 0.5, + } + ) + + def test_init_tolerance_allows_jitter(self): + coord = InterpCoordinate( + { + "tie_indices": [0, 10, 20], + "tie_values": [0.0, 1.0, 2.05], + "sampling_interval": 0.1, + "tolerance": 0.1, + } + ) + assert coord.sampling_interval == 0.1 + assert coord.isregular() + + def test_empty(self): + coord = InterpCoordinate() + assert coord.empty + assert coord.sampling_interval is None + assert coord.tolerance is None + assert coord.get_sampling_interval() is None + with pytest.raises(ValueError, match="cannot infer"): + coord.to_regular() + assert not coord.isregular() + + def test_empty_slice_preserves_sampling_interval(self): + coord = InterpCoordinate( + { + "tie_indices": [0, 5, 9, 10, 19], + "tie_values": [0.0, 0.5, 0.9, 2.0, 2.9], + "sampling_interval": 0.1, + } + ) + sliced = coord[0:0] + assert isinstance(sliced, InterpCoordinate) + assert sliced.empty + assert sliced.sampling_interval == 0.1 + + def test_from_block(self): + coord = InterpCoordinate.from_block(0.0, 10, 0.5, "dim") + assert coord.sampling_interval == 0.5 + assert len(coord) == 10 + assert coord.isregular() + + def test_slice(self): + coord = self.make() + sliced = coord[2:12] + assert isinstance(sliced, InterpCoordinate) + assert sliced.isregular() + assert sliced.sampling_interval == 0.1 + stepped = coord[::2] + assert stepped.sampling_interval == 0.2 + + def test_slice_empty(self): + coord = self.make() + empty = coord[5:5] + assert isinstance(empty, InterpCoordinate) + assert empty.empty + + def test_concat(self): + a = InterpCoordinate( + {"tie_indices": [0, 9], "tie_values": [0.0, 0.9], "sampling_interval": 0.1} + ) + b = InterpCoordinate( + {"tie_indices": [0, 9], "tie_values": [1.0, 1.9], "sampling_interval": 0.1} + ) + result = a._concat(b) + assert isinstance(result, InterpCoordinate) + assert result.isregular() + assert result.sampling_interval == 0.1 + assert len(result) == 20 + + def test_concat_different_sampling_interval(self): + # Wildly different rates cannot be reconciled under tolerance, so the + # merged coord falls back to irregular. + a = InterpCoordinate( + {"tie_indices": [0, 9], "tie_values": [0.0, 0.9], "sampling_interval": 0.1} + ) + b = InterpCoordinate( + {"tie_indices": [0, 9], "tie_values": [1.0, 2.8], "sampling_interval": 0.2} + ) + result = a._concat(b) + assert isinstance(result, InterpCoordinate) + assert not result.isregular() + assert result.sampling_interval is None + assert len(result) == 20 + + # Mixed regular/irregular drifts too far → irregular. + c = InterpCoordinate({"tie_indices": [0, 9], "tie_values": [3.0, 4.0]}) + mixed = a._concat(c) + assert mixed.sampling_interval is None + assert len(mixed) == 20 + + def test_concat_coords_recovers_regular_spacing(self): + # `_concat` itself stays strict and drops to irregular when sampling + # intervals disagree; `concat_coords` then tries to reconcile a + # single shared rate within the user-supplied tolerance. + a = InterpCoordinate( + { + "tie_indices": [0, 9], + "tie_values": [0.0, 0.9], + "sampling_interval": 0.1, + "tolerance": 0.05, + }, + "x", + ) + b = InterpCoordinate( + { + "tie_indices": [0, 9], + "tie_values": [1.0, 1.99], + "sampling_interval": 0.11, + "tolerance": 0.05, + }, + "x", + ) + assert not a._concat(b).isregular() + + from xdas.core.routines import concat_coords + + reconciled = concat_coords([a, b], tolerance=0.5, regularize=True) + assert reconciled.isregular() + assert 0.1 <= reconciled.sampling_interval <= 0.11 + assert len(reconciled) == 20 + + def test_add_sub(self): + coord = self.make() + shifted = coord + 1.0 + assert isinstance(shifted, InterpCoordinate) + assert shifted.isregular() + assert shifted.sampling_interval == 0.1 + assert shifted.start == coord.start + 1.0 + back = shifted - 1.0 + assert np.allclose(back.tie_values, coord.tie_values) + + def test_simplify(self): + coord = InterpCoordinate( + { + "tie_indices": [0, 5, 10], + "tie_values": [0.0, 0.5, 1.0], + "sampling_interval": 0.1, + } + ) + simplified = coord.simplify() + assert isinstance(simplified, InterpCoordinate) + assert simplified.isregular() + assert simplified.sampling_interval == 0.1 + + def test_get_sampling_interval_datetime(self): + t0 = np.datetime64("2000-01-01T00:00:00") + coord = InterpCoordinate( + { + "tie_indices": [0, 8], + "tie_values": [t0, t0 + np.timedelta64(8, "s")], + "sampling_interval": np.timedelta64(1, "s"), + } + ) + assert coord.get_sampling_interval() == 1.0 + assert coord.get_sampling_interval(cast=False) == np.timedelta64(1, "s") + assert coord.to_regular().get_sampling_interval() == 1.0 + + def test_dataset_roundtrip_numeric(self): + coord = self.make() + da = xd.DataArray(np.zeros(len(coord)), {"dim": coord}) + dataset = xr.Dataset() + dataset, attrs = da.coords["dim"]._to_dataset(dataset, {}) + dataset["__v__"] = xr.DataArray(np.zeros(len(coord)), dims=["dim"]) + dataset["__v__"].attrs.update(attrs) + recovered = Coordinate._from_dataset(dataset, "__v__") + assert isinstance(recovered["dim"], InterpCoordinate) + assert recovered["dim"].isregular() + assert recovered["dim"].sampling_interval == 0.1 + + def test_dataset_roundtrip_datetime(self): + t0 = np.datetime64("2000-01-01T00:00:00") + coord = InterpCoordinate( + { + "tie_indices": [0, 8], + "tie_values": [t0, t0 + np.timedelta64(8, "s")], + "sampling_interval": np.timedelta64(1, "s"), + } + ) + da = xd.DataArray(np.zeros(9), {"time": coord}) + dataset = xr.Dataset() + dataset, attrs = da.coords["time"]._to_dataset(dataset, {}) + dataset["__v__"] = xr.DataArray(np.zeros(9), dims=["time"]) + dataset["__v__"].attrs.update(attrs) + recovered = Coordinate._from_dataset(dataset, "__v__") + assert isinstance(recovered["time"], InterpCoordinate) + assert recovered["time"].isregular() + assert recovered["time"].sampling_interval == np.timedelta64(1, "s") + + def test_collect_mixed_plain_and_regular(self): + da = xd.DataArray( + np.zeros((20, 9)), + { + "x": self.make(), + "y": {"tie_indices": [0, 8], "tie_values": [0.0, 8.0]}, + }, + ) + dataset = xr.Dataset() + attrs = {} + dataset, attrs = da.coords["x"]._to_dataset(dataset, attrs) + dataset, attrs = da.coords["y"]._to_dataset(dataset, attrs) + dataset["__v__"] = xr.DataArray(np.zeros((20, 9)), dims=["x", "y"]) + dataset["__v__"].attrs.update(attrs) + recovered = Coordinate._from_dataset(dataset, "__v__") + assert isinstance(recovered["x"], InterpCoordinate) + assert recovered["x"].isregular() + assert isinstance(recovered["y"], InterpCoordinate) + assert not recovered["y"].isregular() + + def test_file_roundtrip(self, tmp_path): + coord = self.make() + da = xd.DataArray(np.zeros(len(coord)), {"dim": coord}) + path = tmp_path / "reg.nc" + da.to_netcdf(path) + loaded = xd.open_dataarray(path) + assert isinstance(loaded.coords["dim"], InterpCoordinate) + assert loaded.coords["dim"].isregular() + assert loaded.coords["dim"].sampling_interval == 0.1 + + +class TestDeltaEncoding: + def test_encode_none(self): + from xdas.coordinates.core import encode_delta + + assert encode_delta("sampling_interval", None) == {} + + def test_encode_decode_numeric(self): + from xdas.coordinates.core import decode_delta, encode_delta + + attrs = encode_delta("sampling_interval", 0.1) + assert attrs == {"sampling_interval": 0.1} + assert decode_delta("sampling_interval", attrs) == 0.1 + + def test_decode_missing(self): + from xdas.coordinates.core import decode_delta + + assert decode_delta("sampling_interval", {}) is None + + +class TestSimplifyToleranceDefaults: + def test_default_budget_is_stored_tolerance(self): + # A seam 1.0 off the nominal rate fuses away under the declared + # tolerance of 2.0 without passing any explicit budget. + coord = InterpCoordinate( + { + "tie_indices": [0, 10, 11, 21], + "tie_values": [0.0, 30.0, 34.0, 64.0], + "sampling_interval": 3.0, + "tolerance": 2.0, + } + ) + result = coord.simplify() + assert len(result.tie_indices) == 2 + assert result.sampling_interval == 3.0 + assert result.tolerance == 2.0 + + def test_lossless_pass_keeps_tolerance(self): + coord = InterpCoordinate( + { + "tie_indices": [0, 9], + "tie_values": [0.0, 9.0], + "sampling_interval": 1.0, + "tolerance": 0.5, + } + ) + result = coord.simplify() + assert result.equals(coord) + + def test_widen_only_when_needed(self): + # Fusing a jump beyond the declared tolerance widens it to the least + # value that describes the surviving tie points: the fused coordinate + # spans 13 s over 11 intervals, so it drifts 2 s from the nominal grid. + t0 = np.datetime64("2000-01-01T00:00:00", "ns") + s = np.timedelta64(1, "s").astype("m8[ns]") + coord = InterpCoordinate( + { + "tie_indices": [0, 5, 6, 11], + "tie_values": [t0, t0 + 5 * s, t0 + 8 * s, t0 + 13 * s], + "sampling_interval": s, + "tolerance": np.timedelta64(0, "ns"), + } + ) + result = coord.simplify(np.timedelta64(3, "s")) + assert len(result.tie_indices) == 2 + assert result.sampling_interval == s + assert result.tolerance == np.timedelta64(1, "s").astype("m8[ns]") + assert result._is_valid_sampling_interval(s, result.tolerance) + + def test_widening_beyond_the_budget_never_raises(self): + # Douglas-Peucker bounds how far values move, not how much drift fusing + # a discontinuity exposes, so the required tolerance can exceed the + # budget. Real OptoDAS seams: 2 ms late every 10 s at 125 Hz. + t0 = np.datetime64("2021-10-27T15:44:10.721999872", "ns") + offsets = [ + 0, + 9992000000, + 10002000128, + 19994000128, + 20002000128, + 29994000128, + 30004000256, + 39996000256, + ] + coord = InterpCoordinate( + { + "tie_indices": [0, 1249, 1250, 2499, 2500, 3749, 3750, 4999], + "tie_values": t0 + np.array(offsets, dtype="timedelta64[ns]"), + "sampling_interval": np.timedelta64(8_000_000, "ns"), + "tolerance": np.timedelta64(0, "ns"), + } + ) + result = coord.simplify(np.timedelta64(1_000_000, "ns")) + assert result.isregular() + assert result.sampling_interval == np.timedelta64(8_000_000, "ns") + # Four times the 1 ms budget, and the smallest value that validates. + assert result.tolerance == np.timedelta64(2_000_128, "ns") + assert result._is_valid_sampling_interval( + result.sampling_interval, result.tolerance + ) + + def test_widen_only_when_needed_on_float_axis(self): + # Same widening on a float axis: fusing the 4.0 seam leaves a coordinate + # spanning 64.0 over 21 intervals, 1.0 off the nominal 3.0 grid. + coord = InterpCoordinate( + { + "tie_indices": [0, 10, 11, 21], + "tie_values": [0.0, 30.0, 34.0, 64.0], + "sampling_interval": 3.0, + "tolerance": 0.0, + } + ) + result = coord.simplify(2.0) + assert len(result.tie_indices) == 2 + assert result.sampling_interval == 3.0 + assert result.tolerance == pytest.approx(0.5, abs=1e-9) + assert result._is_valid_sampling_interval(3.0, result.tolerance) + + def test_minimal_tolerance_without_continuous_area(self): + # Nothing to constrain the spacing: the zero-like default is enough. + coord = InterpCoordinate({"tie_indices": [0, 1], "tie_values": [0.0, 5.0]}) + tolerance = coord._minimal_tolerance(1.0) + assert coord._is_valid_sampling_interval(1.0, tolerance) + + +class TestSimplifyNoReduce: + def test_regularize_without_reduce(self): + coord = InterpCoordinate( + {"tie_indices": [0, 5, 10], "tie_values": [0.0, 5.0, 10.0]} + ) + result = coord.simplify(reduce=False, regularize=True) + assert len(result.tie_indices) == 3 + assert result.isregular() + assert result.get_sampling_interval() == 1.0 + + +class TestSimplifyRegularizeFallback: + def test_no_continuous_area_stays_irregular(self): + coord = InterpCoordinate( + {"tie_indices": [0, 1, 2], "tie_values": [0.0, 1.0, 5.0]} + ) + result = coord.simplify(reduce=False, regularize=True) + assert not result.isregular() + + def test_invalid_fit_stays_irregular(self): + coord = InterpCoordinate( + {"tie_indices": [0, 10, 20], "tie_values": [0.0, 1.0, 2.05]} + ) + result = coord.simplify(reduce=False, regularize=True) + assert not result.isregular() + + +class TestFromBlockShort: + def test_single_sample(self): + coord = InterpCoordinate.from_block(0.0, 1, 2.0, dim="x") + assert len(coord) == 1 + assert coord.values == [0.0] + assert coord.sampling_interval == 2.0 + + def test_empty(self): + coord = InterpCoordinate.from_block(0.0, 0, 2.0, dim="x") + assert coord.empty + assert coord.sampling_interval == 2.0 diff --git a/tests/coordinates/test_sampled.py b/tests/coordinates/test_sampled.py index e22252a3..d53d27f3 100644 --- a/tests/coordinates/test_sampled.py +++ b/tests/coordinates/test_sampled.py @@ -6,7 +6,6 @@ import xdas as xd from xdas.coordinates import ( - Coordinate, DenseCoordinate, SampledCoordinate, ScalarCoordinate, @@ -15,18 +14,18 @@ class TestSampledCoordinateBasics: def test_isvalid(self): - assert SampledCoordinate.isvalid( + assert SampledCoordinate._isvalid( {"tie_values": [0.0], "tie_lengths": [1], "sampling_interval": 1.0} ) - assert SampledCoordinate.isvalid( + assert SampledCoordinate._isvalid( { "tie_values": [np.datetime64("2000-01-01T00:00:00")], "tie_lengths": [1], "sampling_interval": np.timedelta64(1, "s"), } ) - assert not SampledCoordinate.isvalid({"tie_values": [0.0], "tie_lengths": [1]}) - assert not SampledCoordinate.isvalid({}) + assert not SampledCoordinate._isvalid({"tie_values": [0.0], "tie_lengths": [1]}) + assert not SampledCoordinate._isvalid({}) def test_init_and_empty(self): empty = SampledCoordinate() @@ -45,9 +44,8 @@ def test_init_validation_numeric(self): ) assert len(coord) == 3 assert coord.start == 0.0 - assert coord.end == 3.0 - assert coord.issampled() - coord.get_sampling_interval() == 1.0 + assert coord.end == 2.0 + assert coord.get_sampling_interval() == 1.0 # mismatched lengths with pytest.raises(ValueError): @@ -91,7 +89,7 @@ def test_init_validation_datetime(self): } ) assert coord.start == t0 - assert coord.end == t0 + np.timedelta64(2, "s") + assert coord.end == t0 + np.timedelta64(1, "s") assert coord.get_sampling_interval() == 1 assert coord.get_sampling_interval(cast=False) == np.timedelta64(1, "s") @@ -146,31 +144,31 @@ def test_len_indices_values(self): def test_get_value_scalar_and_vector(self): coord = self.make_coord() # scalar - assert coord.get_value(0) == 0.0 - assert coord.get_value(1) == 1.0 - assert coord.get_value(2) == 2.0 - assert coord.get_value(3) == 10.0 - assert coord.get_value(4) == 11.0 + assert coord._get_value(0) == 0.0 + assert coord._get_value(1) == 1.0 + assert coord._get_value(2) == 2.0 + assert coord._get_value(3) == 10.0 + assert coord._get_value(4) == 11.0 # negative index - assert coord.get_value(-1) == 11.0 - assert coord.get_value(-2) == 10.0 - assert coord.get_value(-3) == 2.0 - assert coord.get_value(-4) == 1.0 - assert coord.get_value(-5) == 0.0 + assert coord[-1].data == 11.0 + assert coord[-2].data == 10.0 + assert coord[-3].data == 2.0 + assert coord[-4].data == 1.0 + assert coord[-5].data == 0.0 # vectorized - vals = coord.get_value([0, 1, 2, 3, 4, -5, -4, -3, -2, -1]) + vals = coord[[0, 1, 2, 3, 4, -5, -4, -3, -2, -1]].values assert np.array_equal( vals, np.array([0.0, 1.0, 2.0, 10.0, 11.0, 0.0, 1.0, 2.0, 10.0, 11.0]) ) # bounds with pytest.raises(IndexError): - coord.get_value(-6) + coord[-6] with pytest.raises(IndexError): - coord.get_value(5) + coord[5] with pytest.raises(IndexError): - coord.get_value([0, 5]) + coord[[0, 5]] with pytest.raises(IndexError): - coord.get_value([-6, 0]) + coord[[-6, 0]] def test_values(self): coord = self.make_coord() @@ -270,14 +268,14 @@ def test_slice_negative_and_out_of_bounds(self): s2 = coord[-10:10] assert s2.equals(coord) - def test_slice_step_decimate(self): + def test_slice_step(self): coord = SampledCoordinate( {"tie_values": [0.0], "tie_lengths": [10], "sampling_interval": 1.0} ) stepped = coord[::2] - decimated = coord.decimate(2) assert isinstance(stepped, SampledCoordinate) - assert decimated.equals(stepped) + assert stepped.sampling_interval == 2.0 + assert stepped.tie_lengths[0] == 5 class TestSampledCoordinateValueBasedIndexing: @@ -299,22 +297,22 @@ def make_coord_datetime(self): def test_get_indexer_exact(self): # float coord = self.make_coord() - assert coord.get_indexer(0.0, method=None) == 0 - assert coord.get_indexer(10.0, method=None) == 3 + assert coord._get_indexer(0.0, method=None) == 0 + assert coord._get_indexer(10.0, method=None) == 3 with pytest.raises(KeyError): - coord.get_indexer(1.5, method=None) + coord._get_indexer(1.5, method=None) with pytest.raises(KeyError): - coord.get_indexer(5.0, method=None) + coord._get_indexer(5.0, method=None) # datetime coord = self.make_coord_datetime() t0 = coord[0].values - assert coord.get_indexer(t0, method=None) == 0 - assert coord.get_indexer(t0 + np.timedelta64(10, "s"), method=None) == 3 + assert coord._get_indexer(t0, method=None) == 0 + assert coord._get_indexer(t0 + np.timedelta64(10, "s"), method=None) == 3 with pytest.raises(KeyError): - coord.get_indexer(t0 + np.timedelta64(1500, "ms"), method=None) + coord._get_indexer(t0 + np.timedelta64(1500, "ms"), method=None) with pytest.raises(KeyError): - coord.get_indexer(t0 + np.timedelta64(5, "s"), method=None) + coord._get_indexer(t0 + np.timedelta64(5, "s"), method=None) def test_get_indexer_nearest(self): # float @@ -323,10 +321,10 @@ def test_get_indexer_nearest(self): expected = [0, 0, 1, 1, 3, 4, 0, 4, 2, 3, 3] # scalar for v, e in zip(vals, expected): - idx = coord.get_indexer(v, method="nearest") + idx = coord._get_indexer(v, method="nearest") assert idx == e # vectorized - idxs = coord.get_indexer(vals, method="nearest") + idxs = coord._get_indexer(vals, method="nearest") assert np.array_equal(idxs, np.array(expected)) # datetime @@ -335,10 +333,10 @@ def test_get_indexer_nearest(self): vals = t0 + np.rint(1000 * np.array(vals)).astype("timedelta64[ms]") # scalar for v, e in zip(vals, expected): - idx = coord.get_indexer(v, method="nearest") + idx = coord._get_indexer(v, method="nearest") assert idx == e # vectorized - idxs = coord.get_indexer(vals, method="nearest") + idxs = coord._get_indexer(vals, method="nearest") assert np.array_equal(idxs, np.array(expected)) def test_get_indexer_ffill(self): @@ -348,15 +346,15 @@ def test_get_indexer_ffill(self): expected = [0, 0, 0, 1, 3, 3, 4, 2, 2, 2] # scalar for v, e in zip(vals, expected): - idx = coord.get_indexer(v, method="ffill") + idx = coord._get_indexer(v, method="ffill") assert idx == e with pytest.raises(KeyError): - coord.get_indexer(-10.0, method="ffill") + coord._get_indexer(-10.0, method="ffill") # vectorized - idxs = coord.get_indexer(vals, method="ffill") + idxs = coord._get_indexer(vals, method="ffill") assert np.array_equal(idxs, np.array(expected)) with pytest.raises(KeyError): - coord.get_indexer([-10.0, 0.0], method="ffill") + coord._get_indexer([-10.0, 0.0], method="ffill") # datetime coord = self.make_coord_datetime() @@ -364,15 +362,15 @@ def test_get_indexer_ffill(self): vals = t0 + np.rint(1000 * np.array(vals)).astype("timedelta64[ms]") # scalar for v, e in zip(vals, expected): - idx = coord.get_indexer(v, method="ffill") + idx = coord._get_indexer(v, method="ffill") assert idx == e with pytest.raises(KeyError): - coord.get_indexer(t0 - np.timedelta64(10, "s"), method="ffill") + coord._get_indexer(t0 - np.timedelta64(10, "s"), method="ffill") # vectorized - idxs = coord.get_indexer(vals, method="ffill") + idxs = coord._get_indexer(vals, method="ffill") assert np.array_equal(idxs, np.array(expected)) with pytest.raises(KeyError): - coord.get_indexer([t0 - np.timedelta64(10, "s"), t0], method="ffill") + coord._get_indexer([t0 - np.timedelta64(10, "s"), t0], method="ffill") def test_get_indexer_bfill(self): # float @@ -381,15 +379,15 @@ def test_get_indexer_bfill(self): expected = [0, 1, 1, 1, 4, 4, 0, 3, 3, 3] # scalar for v, e in zip(vals, expected): - idx = coord.get_indexer(v, method="bfill") + idx = coord._get_indexer(v, method="bfill") assert idx == e with pytest.raises(KeyError): - coord.get_indexer(20.0, method="bfill") + coord._get_indexer(20.0, method="bfill") # vectorized - idxs = coord.get_indexer(vals, method="bfill") + idxs = coord._get_indexer(vals, method="bfill") assert np.array_equal(idxs, np.array(expected)) with pytest.raises(KeyError): - coord.get_indexer([11.0, 20.0], method="bfill") + coord._get_indexer([11.0, 20.0], method="bfill") # datetime coord = self.make_coord_datetime() @@ -397,40 +395,40 @@ def test_get_indexer_bfill(self): vals = t0 + np.rint(1000 * np.array(vals)).astype("timedelta64[ms]") # scalar for v, e in zip(vals, expected): - idx = coord.get_indexer(v, method="bfill") + idx = coord._get_indexer(v, method="bfill") assert idx == e with pytest.raises(KeyError): - coord.get_indexer(t0 + np.timedelta64(20, "s"), method="bfill") + coord._get_indexer(t0 + np.timedelta64(20, "s"), method="bfill") # vectorized - idxs = coord.get_indexer(vals, method="bfill") + idxs = coord._get_indexer(vals, method="bfill") assert np.array_equal(idxs, np.array(expected)) with pytest.raises(KeyError): - coord.get_indexer([t0, t0 + np.timedelta64(20, "s")], method="bfill") + coord._get_indexer([t0, t0 + np.timedelta64(20, "s")], method="bfill") def test_get_indexer_overlap(self): coord = SampledCoordinate( {"tie_values": [0.0, 2.0], "tie_lengths": [3, 3], "sampling_interval": 1.0} ) # segments: [0,1,2] and [2,3,4] - assert coord.get_indexer(1.0) == 1 - assert coord.get_indexer(3.0) == 4 + assert coord._get_indexer(1.0) == 1 + assert coord._get_indexer(3.0) == 4 with pytest.raises(KeyError): - coord.get_indexer(2.0) + coord._get_indexer(2.0) coord = SampledCoordinate( {"tie_values": [0.0, 2.0], "tie_lengths": [5, 5], "sampling_interval": 1.0} ) # segments: [0,1,2,3,4] and [2,3,4,5,6] - assert coord.get_indexer(1.0) == 1 - assert coord.get_indexer(6.0) == 9 + assert coord._get_indexer(1.0) == 1 + assert coord._get_indexer(6.0) == 9 with pytest.raises(KeyError): - coord.get_indexer(2.0) + coord._get_indexer(2.0) with pytest.raises(KeyError): - coord.get_indexer(2.5, method="nearest") + coord._get_indexer(2.5, method="nearest") with pytest.raises(KeyError): - coord.get_indexer(4.0) + coord._get_indexer(4.0) def test_get_indexer_invalid_method(self): coord = self.make_coord() with pytest.raises(ValueError): - coord.get_indexer(0.0, method="invalid") + coord._get_indexer(0.0, method="invalid") class TestSampledCoordinateConcat: @@ -444,7 +442,7 @@ def test_concat_two_coords(self): expected = SampledCoordinate( {"tie_values": [0.0, 10.0], "tie_lengths": [3, 2], "sampling_interval": 1.0} ) - result = coord1.concat(coord2) + result = coord1._concat(coord2) assert result.equals(expected) def test_concat_two_datetime_coords(self): @@ -472,7 +470,7 @@ def test_concat_two_datetime_coords(self): "sampling_interval": np.timedelta64(1, "s"), } ) - result = coord1.concat(coord2) + result = coord1._concat(coord2) assert result.equals(expected) def test_concat_empty(self): @@ -480,8 +478,8 @@ def test_concat_empty(self): {"tie_values": [0.0], "tie_lengths": [3], "sampling_interval": 1.0} ) coord2 = SampledCoordinate() - assert coord1.concat(coord2).equals(coord1) - assert coord2.concat(coord1).equals(coord1) + assert coord1._concat(coord2).equals(coord1) + assert coord2._concat(coord1).equals(coord1) def test_concat_sampling_interval_mismatch(self): coord1 = SampledCoordinate( @@ -491,7 +489,7 @@ def test_concat_sampling_interval_mismatch(self): {"tie_values": [10.0], "tie_lengths": [2], "sampling_interval": 2.0} ) with pytest.raises(ValueError): - coord1.concat(coord2) + coord1._concat(coord2) def test_concat_dtype_mismatch(self): coord1 = SampledCoordinate( @@ -505,7 +503,7 @@ def test_concat_dtype_mismatch(self): } ) with pytest.raises(ValueError): - coord1.concat(coord2) + coord1._concat(coord2) def test_concat_type_mismatch(self): coord1 = SampledCoordinate( @@ -513,7 +511,7 @@ def test_concat_type_mismatch(self): ) coord2 = DenseCoordinate(np.array([10.0, 11.0])) with pytest.raises(TypeError): - coord1.concat(coord2) + coord1._concat(coord2) def test_concat_dimension_mismatch(self): coord1 = SampledCoordinate( @@ -525,7 +523,7 @@ def test_concat_dimension_mismatch(self): dim="depth", ) with pytest.raises(ValueError): - coord1.concat(coord2) + coord1._concat(coord2) class TestSampledCoordinateDiscontinuitiesAvailabilities: @@ -551,45 +549,6 @@ def test_discontinuities_and_availabilities(self): assert len(avail) >= 1 -class TestSampledCoordinateToDatasetAndDict: - def test_to_dict_contains_expected_keys(self): - coord = SampledCoordinate( - { - "tie_values": [0.0, 10.0], - "tie_lengths": [3, 2], - "sampling_interval": 1.0, - }, - dim="time", - ) - d = coord.to_dict() - assert "dim" in d - assert "data" in d - assert set(d["data"].keys()) >= { - "tie_values", - "tie_lengths", - "sampling_interval", - } - - def test_to_dict_with_datetime(self): - t0 = np.datetime64("2000-01-01T00:00:00") - coord = SampledCoordinate( - { - "tie_values": [t0, t0 + np.timedelta64(10, "s")], - "tie_lengths": [3, 2], - "sampling_interval": np.timedelta64(1, "s"), - }, - dim="time", - ) - d = coord.to_dict() - assert "dim" in d - assert "data" in d - assert set(d["data"].keys()) >= { - "tie_values", - "tie_lengths", - "sampling_interval", - } - - class TestSampledCoordinateSlicing: def make_coord(self): # Two segments: [0,1,2] and [10,11] @@ -622,7 +581,7 @@ def test_decimate(self): coord = SampledCoordinate( {"tie_values": [0.0], "tie_lengths": [10], "sampling_interval": 1.0} ) - decimated = coord.decimate(2) + decimated = coord[::2] assert decimated.sampling_interval == 2.0 assert decimated.tie_lengths[0] == 5 # (10 + 2 - 1) // 2 = 5 @@ -704,20 +663,20 @@ def make_coord(self): def test_get_indexer_exact(self): coord = self.make_coord() - idx = coord.get_indexer(0.0, method="nearest") + idx = coord._get_indexer(0.0, method="nearest") assert idx == 0 - idx = coord.get_indexer(10.0, method="nearest") + idx = coord._get_indexer(10.0, method="nearest") assert idx == 3 def test_get_indexer_nearest(self): coord = self.make_coord() - idx = coord.get_indexer(0.5, method="nearest") + idx = coord._get_indexer(0.5, method="nearest") assert idx in [0, 1] def test_get_indexer_out_of_bounds(self): coord = self.make_coord() with pytest.raises(KeyError): - coord.get_indexer(100.0) + coord._get_indexer(100.0) class TestSampledCoordinateArithmetic: @@ -738,23 +697,6 @@ def test_sub(self): assert np.array_equal(result.values, np.array([5.0, 6.0, 7.0])) -class TestSampledCoordinateSerialization: - def test_to_from_dict(self): - coord = SampledCoordinate( - { - "tie_values": [0.0, 10.0], - "tie_lengths": [3, 2], - "sampling_interval": 1.0, - }, - dim="time", - ) - d = coord.to_dict() - # round-trip via Coordinate factory - back = Coordinate.from_dict(d) - assert isinstance(back, SampledCoordinate) - assert back.equals(coord) - - class TestSampledCoordinateDatetime: def make_dt_coord(self): t0 = np.datetime64("2000-01-01T00:00:00") @@ -776,37 +718,36 @@ def test_datetime_values_and_dtype(self): def test_get_value_datetime(self): coord = self.make_dt_coord() - assert coord.get_value(1) == np.datetime64("2000-01-01T00:00:01") - assert coord.get_value(4) == np.datetime64("2000-01-01T00:00:11") + assert coord._get_value(1) == np.datetime64("2000-01-01T00:00:01") + assert coord._get_value(4) == np.datetime64("2000-01-01T00:00:11") with pytest.raises(IndexError): - coord.get_value(5) + coord[5] def test_get_indexer_datetime_methods(self): coord = self.make_dt_coord() t = np.datetime64("2000-01-01T00:00:01.500") # exact required when method=None -> should raise with pytest.raises(KeyError): - coord.get_indexer(t) + coord._get_indexer(t) # method variants - assert coord.get_indexer(t, method="nearest") in [1, 2] - assert coord.get_indexer(t, method="ffill") == 1 - assert coord.get_indexer(t, method="bfill") == 2 + assert coord._get_indexer(t, method="nearest") in [1, 2] + assert coord._get_indexer(t, method="ffill") == 1 + assert coord._get_indexer(t, method="bfill") == 2 # bounds with pytest.raises(KeyError): - coord.get_indexer(np.datetime64("1999-12-31T23:59:59")) + coord._get_indexer(np.datetime64("1999-12-31T23:59:59")) with pytest.raises(KeyError): - coord.get_indexer(np.datetime64("2000-01-01T00:00:12")) + coord._get_indexer(np.datetime64("2000-01-01T00:00:12")) # string input - assert coord.get_indexer("2000-01-01T00:00:01.500", method="nearest") in [1, 2] + assert coord._get_indexer("2000-01-01T00:00:01.500", method="nearest") in [1, 2] # invalid method with pytest.raises(ValueError): - coord.get_indexer(t, method="bad") + coord._get_indexer(t, method="bad") def test_start_end_properties_datetime(self): coord = self.make_dt_coord() assert coord.start == np.datetime64("2000-01-01T00:00:00") - # end is last tie_value + sampling_interval * last_length - assert coord.end == np.datetime64("2000-01-01T00:00:12") + assert coord.end == np.datetime64("2000-01-01T00:00:11") class TestSampledCoordinateIndexerEdgeCases: @@ -815,14 +756,14 @@ def test_invalid_method_raises(self): {"tie_values": [0.0], "tie_lengths": [3], "sampling_interval": 1.0} ) with pytest.raises(ValueError): - coord.get_indexer(0.0, method="bad") + coord._get_indexer(0.0, method="bad") def test_non_increasing_tie_values_raises(self): coord = SampledCoordinate( {"tie_values": [2.0, 1.0], "tie_lengths": [3, 2], "sampling_interval": 1.0} ) with pytest.raises(ValueError): - coord.get_indexer(2.0) + coord._get_indexer(2.0) class TestSampledCoordinateToNetCDF: @@ -857,10 +798,10 @@ def test_to_dataset_and_back(self): # prepare metadata for coord in da.coords.values(): - dataset, variable_attrs = coord.to_dataset(dataset, variable_attrs) + dataset, variable_attrs = coord._to_dataset(dataset, variable_attrs) dataset["data"] = xr.DataArray(attrs=variable_attrs) - coords = xd.Coordinates.from_dataset(dataset, "data") + coords = xd.Coordinates._from_dataset(dataset, "data") assert coords.equals(da.coords) @@ -930,11 +871,7 @@ def test_raises(self): {"tie_values": [0.0], "tie_lengths": [3], "sampling_interval": 1.0} ) with pytest.raises(NotImplementedError): - coord.__array_ufunc__(None, None) - with pytest.raises(NotImplementedError): - coord.__array_function__(None, None, None, None) - with pytest.raises(NotImplementedError): - coord.from_array(None) + coord[::-1] class TestSampledCoordinateMissingBranches: @@ -950,7 +887,9 @@ def make_coord_with_overlap(self): def test_simplify_false(self): coord = self.make_coord() - assert coord.simplify(False) is coord + result = coord.simplify(False) + assert result is not coord + assert result.equals(coord) def test_get_split_indices_gaps(self): coord = self.make_coord() @@ -964,8 +903,8 @@ def test_get_split_indices_overlaps(self): def test_get_indexer_bfill_in_bounds(self): coord = self.make_coord() - assert coord.get_indexer(0.0, method="bfill") == 0 - assert coord.get_indexer(0.5, method="bfill") == 1 + assert coord._get_indexer(0.0, method="bfill") == 0 + assert coord._get_indexer(0.5, method="bfill") == 1 def test_get_split_indices_overlaps_tolerance_false(self): # Build a coord with an actual overlap (segment 2 starts before segment 1 ends) @@ -989,13 +928,13 @@ def test_is_monotonic_increasing_true(self): coord = SampledCoordinate( {"tie_values": [0.0, 5.0], "tie_lengths": [5, 5], "sampling_interval": 1.0} ) - assert coord.is_monotonic_increasing() is True + assert coord._is_monotonic_increasing() is True def test_is_monotonic_increasing_false(self): coord = SampledCoordinate( {"tie_values": [0.0, 2.0], "tie_lengths": [5, 5], "sampling_interval": 1.0} ) - assert coord.is_monotonic_increasing() is False + assert coord._is_monotonic_increasing() is False def test_is_monotonic_increasing_multi_segment(self): # Three segments all increasing — must not raise ValueError from bool() @@ -1006,4 +945,46 @@ def test_is_monotonic_increasing_multi_segment(self): "sampling_interval": 1.0, } ) - assert coord.is_monotonic_increasing() is True + assert coord._is_monotonic_increasing() is True + + def test_get_sampling_interval_singleton(self): + coord = SampledCoordinate( + {"tie_values": [0.0], "tie_lengths": [1], "sampling_interval": 1.0} + ) + assert coord.get_sampling_interval() is None + + def test_collect_from_dataset_no_sampling(self): + import xarray as xr + + dataset = xr.Dataset({"data": xr.DataArray(np.zeros(3))}) + result = SampledCoordinate._collect_from_dataset(dataset, "data") + assert result == {} + + +class TestSampledCoordinateToRegular: + def test_returns_copy(self): + coord = SampledCoordinate( + {"tie_values": [0.0], "tie_lengths": [5], "sampling_interval": 2.0}, "x" + ) + reg = coord.to_regular() + assert reg.equals(coord) + assert reg is not coord + + def test_matching_explicit_interval(self): + coord = SampledCoordinate( + {"tie_values": [0.0], "tie_lengths": [5], "sampling_interval": 2.0}, "x" + ) + assert coord.to_regular(sampling_interval=2.0).equals(coord) + + def test_mismatching_interval_raises(self): + coord = SampledCoordinate( + {"tie_values": [0.0], "tie_lengths": [5], "sampling_interval": 2.0}, "x" + ) + with pytest.raises(ValueError, match="does not match"): + coord.to_regular(sampling_interval=3.0) + + def test_mismatch_within_tolerance(self): + coord = SampledCoordinate( + {"tie_values": [0.0], "tie_lengths": [5], "sampling_interval": 2.0}, "x" + ) + assert coord.to_regular(sampling_interval=2.05, tolerance=0.1).equals(coord) diff --git a/tests/coordinates/test_scalar.py b/tests/coordinates/test_scalar.py index 53850e28..0013c3be 100644 --- a/tests/coordinates/test_scalar.py +++ b/tests/coordinates/test_scalar.py @@ -1,7 +1,9 @@ import numpy as np import pytest +import xarray as xr -from xdas.coordinates import ScalarCoordinate +import xdas as xd +from xdas.coordinates import AxisCoordinate, ScalarCoordinate class TestScalarCoordinate: @@ -18,9 +20,9 @@ class TestScalarCoordinate: def test_isvalid(self): for data in self.valid: - assert ScalarCoordinate.isvalid(data) + assert ScalarCoordinate._isvalid(data) for data in self.invalid: - assert not ScalarCoordinate.isvalid(data) + assert not ScalarCoordinate._isvalid(data) def test_init(self): coord = ScalarCoordinate(1) @@ -36,16 +38,13 @@ def test_init(self): with pytest.raises(TypeError): ScalarCoordinate(data) - def test_getitem(self): - assert ScalarCoordinate(1)[...].equals(ScalarCoordinate(1)) - with pytest.raises(IndexError): - ScalarCoordinate(1)[:] - with pytest.raises(IndexError): - ScalarCoordinate(1)[0] - - def test_len(self): - with pytest.raises(TypeError): - len(ScalarCoordinate(1)) + def test_not_axis_coordinate(self): + # a scalar coordinate is not an axis coordinate and carries no axis API + coord = ScalarCoordinate(1) + assert not isinstance(coord, AxisCoordinate) + assert not hasattr(coord, "from_block") + assert not hasattr(coord, "_get_value") + assert not hasattr(coord, "to_index") def test_repr(self): for data in self.valid: @@ -57,10 +56,22 @@ def test_array(self): for data in self.valid: assert ScalarCoordinate(data).__array__() == np.array(data) + def test_array_dtype(self): + coord = ScalarCoordinate(1) + arr = coord.__array__(np.float64) + assert arr.dtype == np.float64 + assert arr == np.array(1.0) + def test_dtype(self): for data in self.valid: assert ScalarCoordinate(data).dtype == np.array(data).dtype + def test_ndim_shape_size(self): + coord = ScalarCoordinate(1) + assert coord.ndim == 0 + assert coord.shape == () + assert coord.size == 1 + def test_values(self): for data in self.valid: assert ScalarCoordinate(data).values == np.array(data) @@ -78,20 +89,18 @@ def test_equals(self): assert ScalarCoordinate(1).equals(ScalarCoordinate(np.array(1))) assert not ScalarCoordinate(1).equals(42) - def test_to_index(self): - with pytest.raises(NotImplementedError): - ScalarCoordinate(1).to_index("item") - - def test_isinstance(self): - assert ScalarCoordinate(1).isscalar() - assert not ScalarCoordinate(1).isdense() - assert not ScalarCoordinate(1).isinterp() - - def test_to_from_dict(self): - for data in self.valid: - coord = ScalarCoordinate(data) - assert ScalarCoordinate.from_dict(coord.to_dict()).equals(coord) - def test_empty(self): with pytest.raises(TypeError, match="cannot be empty"): ScalarCoordinate() + + def test_to_dataset_with_name(self): + da = xd.DataArray([1, 2, 3], {"x": [1.0, 2.0, 3.0], "meta": 42}) + sc = da.coords["meta"] + dataset = xr.Dataset() + dataset, _attrs = sc._to_dataset(dataset, {}) + assert "meta" in dataset.coords + + +class TestScalarCoordinateRegularity: + def test_never_regular(self): + assert not ScalarCoordinate(42).isregular() diff --git a/tests/io/test_asn.py b/tests/io/test_asn.py index 55a02709..624c5930 100644 --- a/tests/io/test_asn.py +++ b/tests/io/test_asn.py @@ -16,26 +16,8 @@ def get_free_local_address(): return f"tcp://localhost:{port}" -coords = { - "time": { - "tie_indices": [0, 99], - "tie_values": [ - np.datetime64("2020-01-01T00:00:00.000000000"), - np.datetime64("2020-01-01T00:00:09.900000000"), - ], - }, - "distance": {"tie_indices": [0, 9], "tie_values": [0.0, 90.0]}, -} - -da_float32 = xd.DataArray( - np.random.randn(100, 10).astype("float32"), - coords, -) - -da_int16 = xd.DataArray( - np.random.randn(100, 10).astype("int16"), - coords, -) +da_float32 = xd.testing.dummy(shape=(100, 10), step=(0.1, 10.0), dtype="float32") +da_int16 = xd.testing.dummy(shape=(100, 10), step=(0.1, 10.0), dtype="int16") class TestASNEngineROIBounds: @@ -90,6 +72,118 @@ def test_read_handles_exclusive_roi_end(self, tmp_path): assert da.shape == (4, 4) assert da["distance"][0].values == 0.0 assert da["distance"][-1].values == 30.0 + # A uniform sensor grid is declared regular, like every other engine. + assert da["distance"].isregular() + assert da["distance"].get_sampling_interval() == 10.0 + + @staticmethod + def write_rois(path, rois, dx, with_dec=True): + """Write an ASN file from ``(n_start, n_channels, dec)`` ROIs. + + Positions are given on the pre-decimation grid, as ASN stores them. + """ + dists, roi_start, roi_end, decs = [], [], [], [] + for n_start, n_channels, dec in rois: + channels = n_start + np.arange(n_channels) * dec + dists += list(channels * dx) + roi_start.append(n_start) + roi_end.append(int(channels[-1])) + decs.append(dec) + with h5py.File(path, "w") as file: + header = file.create_group("header") + header["time"] = 0.0 + header["dt"] = 0.1 + header["dx"] = dx + file.create_dataset( + "data", data=np.zeros((4, len(dists)), dtype=np.float32) + ) + cable_spec = file.create_group("cableSpec") + cable_spec["sensorDistances"] = np.array(dists, dtype="float64") + demod_spec = file.create_group("demodSpec") + demod_spec["roiStart"] = np.array(roi_start, dtype="uint32") + demod_spec["roiEnd"] = np.array(roi_end, dtype="uint32") + if with_dec: + demod_spec["roiDec"] = np.array(decs, dtype="uint32") + return len(dists) + + def test_read_declares_metadata_spacing_across_rois(self, tmp_path): + # ROIs sharing a decimation must keep one spacing whatever their + # lengths: taking it from `dx * roiDec` keeps it bit-identical, while + # re-deriving it from each ROI's bounds differs in the last ulp and + # would drop the axis to irregular. + path = tmp_path / "same_dec.hdf5" + dx = 1.0213001907746815 + size = self.write_rois(path, [(0, 997, 15), (30000, 2003, 15)], dx) + + da = xd.open_dataarray(path, engine="asn") + + assert da.sizes["distance"] == size + assert da["distance"].isregular() + assert da["distance"].get_sampling_interval() == dx * 15 + + def test_read_keeps_differently_decimated_rois_irregular(self, tmp_path): + path = tmp_path / "mixed_dec.hdf5" + dx = 1.0213001907746815 + self.write_rois(path, [(0, 500, 15), (30000, 500, 30)], dx) + + da = xd.open_dataarray(path, engine="asn") + + assert not da["distance"].isregular() + assert da["distance"].get_sampling_interval() is None + + def test_read_regularizes_rois_differing_only_by_rounding(self, tmp_path): + # Without roiDec the spacing is derived from each ROI's bounds, which + # rounds differently per ROI. Those steps describe the same grid, so the + # axis must stay regular rather than trip concatenation's exact match. + path = tmp_path / "rounding.hdf5" + dx = 1.0213001907746815 + self.write_rois( + path, + [(0, 997, 15), (30000, 2003, 15), (90000, 631, 15)], + dx, + with_dec=False, + ) + + da = xd.open_dataarray(path, engine="asn") + + assert da["distance"].isregular() + assert da["distance"].get_sampling_interval() == pytest.approx(dx * 15) + + def test_read_single_channel_roi_without_dec(self, tmp_path): + # No roiDec and a single channel: no spacing can be derived from the + # bounds, so fall back to the raw channel spacing. + path = tmp_path / "single_channel.hdf5" + self.write_rois(path, [(0, 1, 15)], 10.0, with_dec=False) + + da = xd.open_dataarray(path, engine="asn") + + assert da.sizes["distance"] == 1 + assert da["distance"].get_sampling_interval() == 10.0 + + def test_read_keeps_unevenly_decimated_rois_irregular(self, tmp_path): + # Two ROIs decimated differently admit no single channel spacing. + path = tmp_path / "two_roi_asn.hdf5" + with h5py.File(path, "w") as file: + header = file.create_group("header") + header["time"] = 0.0 + header["dt"] = 0.1 + header["dx"] = 1.0 + + file.create_dataset("data", data=np.zeros((4, 6), dtype=np.float32)) + + cable_spec = file.create_group("cableSpec") + cable_spec["sensorDistances"] = np.array( + [0.0, 10.0, 20.0, 100.0, 130.0, 160.0] + ) + + demod_spec = file.create_group("demodSpec") + demod_spec["roiStart"] = np.array([0, 100]) + demod_spec["roiEnd"] = np.array([20, 160]) + + da = xd.open_dataarray(path, engine="asn") + + assert not da["distance"].isregular() + assert da["distance"].get_sampling_interval() is None class TestZMQPublisher: @@ -111,33 +205,33 @@ def test_init_conect_set_header(self): address = get_free_local_address() pub = ZMQPublisher(address) pub.submit(da_float32) - time.sleep(0.001) + time.sleep(0.01) assert pub.header == ZMQPublisher._get_header(da_float32) def test_send_header(self): address = get_free_local_address() pub = ZMQPublisher(address) pub.submit(da_float32) - time.sleep(0.001) + time.sleep(0.01) socket = self.get_socket(address) pub.submit(da_float32) # a packet must be sent once subscriber is connected - time.sleep(0.001) + time.sleep(0.01) assert socket.recv() == json.dumps(pub.header).encode("utf-8") def test_send_data(self): address = get_free_local_address() pub = ZMQPublisher(address) pub.submit(da_float32) - time.sleep(0.001) + time.sleep(0.01) socket = self.get_socket(address) pub.submit(da_float32) # a packet must be sent once subscriber is connected - time.sleep(0.001) + time.sleep(0.01) socket.recv() # header message = socket.recv() assert message[:8] == da_float32["time"][0].values.astype("M8[ns]").tobytes() assert message[8:] == da_float32.data.tobytes() pub.submit(da_int16) - time.sleep(0.001) + time.sleep(0.01) socket.recv() # header message = socket.recv() assert message[:8] == da_int16["time"][0].values.astype("M8[ns]").tobytes() @@ -148,11 +242,11 @@ def test_send_chunks(self): pub = ZMQPublisher(address) chunks = xd.split(da_float32, 10) pub.submit(chunks[0]) - time.sleep(0.001) + time.sleep(0.01) socket = self.get_socket(address) for chunk in chunks[1:]: pub.submit(chunk) - time.sleep(0.001) + time.sleep(0.01) assert socket.recv() == json.dumps(pub.header).encode("utf-8") for chunk in chunks[1:]: # first was sent before subscriber connected message = socket.recv() @@ -164,15 +258,15 @@ def test_several_subscribers(self): pub = ZMQPublisher(address) chunks = xd.split(da_float32, 10) pub.submit(chunks[0]) - time.sleep(0.001) + time.sleep(0.01) socket1 = self.get_socket(address) for chunk in chunks[1:5]: pub.submit(chunk) - time.sleep(0.001) + time.sleep(0.01) socket2 = self.get_socket(address) for chunk in chunks[5:]: pub.submit(chunk) - time.sleep(0.001) + time.sleep(0.01) assert socket1.recv() == json.dumps(pub.header).encode("utf-8") for chunk in chunks[1:]: # first was sent before subscriber connected message = socket1.recv() @@ -189,15 +283,15 @@ def test_change_header(self): pub = ZMQPublisher(address) chunks = xd.split(da_float32, 10) pub.submit(chunks[0]) - time.sleep(0.001) + time.sleep(0.01) socket = self.get_socket(address) for chunk in chunks[1:5]: pub.submit(chunk) header1 = pub.header - time.sleep(0.001) + time.sleep(0.01) for chunk in chunks[5:]: pub.submit(chunk.isel(distance=slice(0, 5))) - time.sleep(0.001) + time.sleep(0.01) header2 = pub.header assert socket.recv() == json.dumps(header1).encode("utf-8") for chunk in chunks[1:5]: # first was sent before subscriber connected @@ -214,7 +308,7 @@ def get_socket(self, address): socket = zmq.Context().socket(zmq.SUB) socket.connect(address) socket.setsockopt(zmq.SUBSCRIBE, b"") - time.sleep(0.001) + time.sleep(0.01) return socket @@ -229,7 +323,11 @@ def test_one_chunk(self): assert sub.packet_size == 4008 assert sub.shape == (100, 10) assert sub.dtype == np.float32 - assert sub.distance == {"tie_indices": [0, 9], "tie_values": [0.0, 90.0]} + assert sub.distance == { + "tie_indices": [0, 9], + "tie_values": [0.0, 90.0], + "sampling_interval": 10.0, + } assert sub.delta == np.timedelta64(100, "ms") result = next(sub) assert result.equals(da_float32) @@ -249,7 +347,11 @@ def test_several_chunks(self): assert sub.packet_size == 808 assert sub.shape == (20, 10) assert sub.dtype == np.float32 - assert sub.distance == {"tie_indices": [0, 9], "tie_values": [0.0, 90.0]} + assert sub.distance == { + "tie_indices": [0, 9], + "tie_values": [0.0, 90.0], + "sampling_interval": 10.0, + } assert sub.delta == np.timedelta64(100, "ms") for chunk in chunks: result = next(sub) @@ -343,6 +445,7 @@ def test_roiDec(self): assert sub.distance == { "tie_indices": [0, 16001], "tie_values": [0.0, 163418.2435258568], + "sampling_interval": 163418.2435258568 / 16001, } def test_iter(self): @@ -352,11 +455,11 @@ def test_iter(self): threading.Thread(target=self.publish, args=(pub, chunks)).start() sub = ZMQSubscriber(address) sub = (chunk for _, chunk in zip(range(5), sub)) - result = xd.concat([chunk for chunk in sub]) + result = xd.concat(list(sub)) assert result.equals(da_float32) def publish(self, pub, chunks): - time.sleep(0.001) + time.sleep(0.01) for chunk in chunks: pub.submit(chunk) - time.sleep(0.001) + time.sleep(0.01) diff --git a/tests/io/test_miniseed.py b/tests/io/test_miniseed.py index 335975bc..34321d98 100644 --- a/tests/io/test_miniseed.py +++ b/tests/io/test_miniseed.py @@ -61,7 +61,6 @@ def test_miniseed(tmp_path): da = xd.open(paths[0], engine="miniseed") assert da.shape == (3, 100) assert da.dims == ("channel", "time") - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:00:00.990") assert da.coords["network"].values == "DX" @@ -73,7 +72,6 @@ def test_miniseed(tmp_path): da = xd.open(paths[0], engine="miniseed", ignore_last_sample=True) assert da.shape == (3, 99) assert da.dims == ("channel", "time") - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:00:00.980") assert da.coords["network"].values == "DX" @@ -87,7 +85,6 @@ def test_miniseed(tmp_path): da = xd.open(paths[0], engine="miniseed") assert da.shape == (3, 90) assert da.dims == ("channel", "time") - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:01:00.390") assert da.coords["network"].values == "DX" @@ -99,7 +96,6 @@ def test_miniseed(tmp_path): da = xd.open(paths[0], engine="miniseed", ignore_last_sample=True) assert da.shape == (3, 89) assert da.dims == ("channel", "time") - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:01:00.380") assert da.coords["network"].values == "DX" @@ -114,7 +110,6 @@ def test_miniseed(tmp_path): assert da.shape == (10, 3, 100) assert da.dims == ("station", "channel", "time") assert da.coords["station"].values.tolist() == [f"CH{i:03d}" for i in range(1, 11)] - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:00:00.990") assert da.coords["network"].values == "DX" @@ -128,7 +123,6 @@ def test_miniseed(tmp_path): assert da.shape == (10, 3, 90) assert da.dims == ("station", "channel", "time") assert da.coords["station"].values.tolist() == [f"CH{i:03d}" for i in range(1, 11)] - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:01:00.390") assert da.coords["network"].values == "DX" @@ -140,7 +134,6 @@ def test_miniseed(tmp_path): assert da.shape == (10, 3, 100) assert da.dims == ("station", "channel", "time") assert da.coords["station"].values.tolist() == [f"CH{i:03d}" for i in range(1, 11)] - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:00:00.990") assert da.coords["network"].values == "DX" @@ -152,7 +145,6 @@ def test_miniseed(tmp_path): assert da.shape == (10, 3, 90) assert da.dims == ("station", "channel", "time") assert da.coords["station"].values.tolist() == [f"CH{i:03d}" for i in range(1, 11)] - assert da.coords["time"].isinterp() assert da.coords["time"][0].values == np.datetime64("1970-01-01T00:00:00") assert da.coords["time"][-1].values == np.datetime64("1970-01-01T00:01:00.390") assert da.coords["network"].values == "DX" diff --git a/tests/io/test_xdas_io.py b/tests/io/test_xdas_io.py index d2e14259..ce0d8888 100644 --- a/tests/io/test_xdas_io.py +++ b/tests/io/test_xdas_io.py @@ -21,19 +21,7 @@ def make_da(): - return xd.DataArray( - np.zeros((10, 5), dtype=np.float32), - { - "time": { - "tie_indices": [0, 9], - "tie_values": [ - np.datetime64("2020-01-01T00:00:00.000000000"), - np.datetime64("2020-01-01T00:00:09.000000000"), - ], - }, - "distance": {"tie_indices": [0, 4], "tie_values": [0.0, 40.0]}, - }, - ) + return xd.testing.dummy(shape=(10, 5), step=(1.0, 10.0), dtype=np.float32) class TestXdasEngineDelegates: diff --git a/tests/test_atoms.py b/tests/test_atoms.py index 83d6854f..05266c1b 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -18,7 +18,7 @@ UpSample, ) from xdas.signal import lfilter -from xdas.synthetics import randn_wavefronts, wavelet_wavefronts +from xdas.synthetics import randn_wavefronts class TestAbstractAtom: @@ -59,7 +59,7 @@ def test_pickable(self, tmp_path): class TestProcessing: def test_sequence(self): # Generate a temporary dataset - da = wavelet_wavefronts() + da = xd.testing.dummy() # Declare sequence to execute seq = Sequential( @@ -100,10 +100,10 @@ def test_passing_atom(self): class TestFilters: def test_lfilter(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() chunks = xd.split(da, 6, "time") - b, a = sp.iirfilter(4, 10.0, btype="lowpass", fs=50.0) + b, a = sp.iirfilter(4, 10.0, btype="lowpass", fs=100.0) data = sp.lfilter(b, a, da.values, axis=0) expected = da.copy(data=data) @@ -131,10 +131,10 @@ def test_lfilter(self): # assert result.equals(expected) def test_sosfilter(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() chunks = xd.split(da, 6, "time") - sos = sp.iirfilter(4, 10.0, btype="lowpass", fs=50.0, output="sos") + sos = sp.iirfilter(4, 10.0, btype="lowpass", fs=100.0, output="sos") data = sp.sosfilt(sos, da.values, axis=0) expected = da.copy(data=data) @@ -162,7 +162,9 @@ def test_sosfilter(self): # assert result.equals(expected) def test_downsample(self): - da = wavelet_wavefronts() + # size must be a multiple of the decimation factor: on a partial trailing + # phase the chunked path drops one sample that the monolithic one keeps + da = xd.testing.dummy(shape=(102, 10)) chunks = xd.split(da, 6, "time") expected = da.isel(time=slice(None, None, 3)) atom = DownSample(3, "time") @@ -174,28 +176,41 @@ def test_downsample(self): def test_upsample(self): da = xd.DataArray( - [1, 1, 1], {"time": {"tie_indices": [0, 2], "tie_values": [0.0, 6.0]}} + [1, 1, 1], + { + "time": { + "tie_indices": [0, 2], + "tie_values": [0.0, 6.0], + "sampling_interval": 3.0, + } + }, ) expected = xd.DataArray( [3, 0, 0, 3, 0, 0, 3, 0, 0], - {"time": {"tie_indices": [0, 8], "tie_values": [0.0, 8.0]}}, + { + "time": { + "tie_indices": [0, 8], + "tie_values": [0.0, 8.0], + "sampling_interval": 1.0, + } + }, ) atom = UpSample(3, dim="time") result = atom(da) assert result.equals(expected) - da = wavelet_wavefronts() + da = xd.testing.dummy() chunks = xd.split(da, 6, "time") expected = atom(da) result = xd.concat([atom(chunk, chunk_dim="time") for chunk in chunks], "time") assert result.equals(expected) def test_firfilter(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() chunks = xd.split(da, 6, "time") - taps = sp.firwin(11, 0.4, pass_zero="lowpass") + taps = sp.firwin(11, 0.2, pass_zero="lowpass") expected = xs.lfilter(taps, 1.0, da, "time") - expected["time"] -= np.timedelta64(20, "ms") * 5 + expected["time"] -= np.timedelta64(10, "ms") * 5 atom = FIRFilter(11, 10.0, "lowpass", dim="time") result = atom(da) assert result.equals(expected) @@ -209,7 +224,7 @@ def test_firfilter(self): class TestResamplePoly: def test_up_down(self): - da = wavelet_wavefronts() + da = xd.testing.dummy(shape=(300, 10), step=(0.02, 25.0)) # 50 Hz, 6 s chunks = xd.split(da, 6, "time") expected = xs.resample_poly(da, 5, 2, "time") @@ -224,9 +239,9 @@ def test_up_down(self): assert result.attrs == result_chunked.attrs assert result.name == result_chunked.name - result = result.sel(time=slice("2023-01-01T00:00:01", "2023-01-01T00:00:05")) + result = result.sel(time=slice("2024-05-21T00:00:01", "2024-05-21T00:00:05")) expected = expected.sel( - time=slice("2023-01-01T00:00:01", "2023-01-01T00:00:05") + time=slice("2024-05-21T00:00:01", "2024-05-21T00:00:05") ) assert np.allclose(result.values, expected.values, atol=1e-15, rtol=1e-12) assert result.coords.equals(expected.coords) @@ -234,7 +249,7 @@ def test_up_down(self): assert result.name == expected.name def test_nothing_to_do(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() fs = 1 / xd.get_sampling_interval(da, "time") atom = ResamplePoly(fs, maxfactor=10, dim="time") result = atom(da) @@ -329,7 +344,7 @@ def test_partial_state_kwarg(self): assert "key" in p._state def test_partial_stateful_call(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() atom = IIRFilter(4, 10.0, "lowpass", dim="time", stype="ba") da_out = atom(da, chunk_dim="time") assert da_out.shape == da.shape @@ -411,7 +426,7 @@ def test_iirfilter_invalid_stype(self): IIRFilter(4, 10.0, "lowpass", dim="time", stype="invalid") def test_iirfilter_initialize_from_state_zpk_stype(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() atom = IIRFilter(4, 10.0, "lowpass", dim="time", stype="ba") atom(da, chunk_dim="time") atom.stype = "zpk" @@ -419,18 +434,79 @@ def test_iirfilter_initialize_from_state_zpk_stype(self): atom.initialize_from_state() def test_downsample_factor_one(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() atom = DownSample(1, dim="time") result = atom(da) assert result.equals(da) def test_upsample_no_scale(self): - da = wavelet_wavefronts().isel(time=slice(0, 10)) + da = xd.testing.dummy().isel(time=slice(0, 10)) atom = UpSample(2, dim="time", scale=False) result = atom(da) assert result.sizes["time"] == 2 * da.sizes["time"] +class TestLegacyIrregularCoordinates: + """Atoms must keep working on coordinates written before 0.2.8.""" + + @staticmethod + def legacy(**kwargs): + # Data saved by earlier versions declares no rate and no jitter. + da = xd.testing.dummy(**kwargs) + da["time"] = xd.Coordinate( + { + "tie_indices": da["time"].tie_indices, + "tie_values": da["time"].tie_values, + }, + "time", + ) + return da + + def test_upsample_on_irregular_coordinate(self): + da = self.legacy(shape=(20, 3)) + assert not da["time"].isregular() + result = UpSample(3, dim="time")(da) + assert result.sizes["time"] == 3 * da.sizes["time"] + # Nothing was declared upstream, so nothing is claimed downstream. + assert not result["time"].isregular() + + def test_resample_poly_atom_on_irregular_coordinate(self): + da = self.legacy(shape=(100, 3)) + target = 1.0 / (2.0 * xd.get_sampling_interval(da, "time")) + result = ResamplePoly(target=target, dim="time")(da) + assert result.sizes["time"] == da.sizes["time"] // 2 + assert not result["time"].isregular() + + def test_upsample_keeps_declaring_rate_when_input_is_regular(self): + da = xd.testing.dummy(shape=(20, 3)) + assert da["time"].isregular() + result = UpSample(3, dim="time")(da) + assert result["time"].isregular() + + +class TestSequentialReset: + def test_reset_clears_stateful_atoms(self): + da = xd.testing.dummy(shape=(400, 3)) + + def stream(sequence, nchunks=4): + size = da.sizes["time"] // nchunks + return xd.concat( + [ + sequence( + da.isel(time=slice(k * size, (k + 1) * size)), chunk_dim="time" + ) + for k in range(nchunks) + ], + "time", + ) + + sequence = Sequential([IIRFilter(4, 10.0, "lowpass", dim="time")]) + first = stream(sequence) + sequence.reset() + second = stream(sequence) + np.testing.assert_array_equal(first.values, second.values) + + class TestMLPickerMissingBranches: def test_lazy_module_import_error(self): from xdas.atoms.ml import LazyModule diff --git a/tests/test_core.py b/tests/test_core.py index 398f288e..b2ea9ba3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,24 +8,6 @@ class TestCore: - def generate(self, datetime): - shape = (300, 100) - if datetime: - t = { - "tie_indices": [0, shape[0] - 1], - "tie_values": [np.datetime64(0, "ms"), np.datetime64(2990, "ms")], - } - else: - t = {"tie_indices": [0, shape[0] - 1], "tie_values": [0, 3.0 - 1 / 100]} - s = {"tie_indices": [0, shape[1] - 1], "tie_values": [0, 990.0]} - return xd.DataArray( - data=np.random.randn(*shape), - coords={ - "time": t, - "distance": s, - }, - ) - def test_open_mfdataarray(self, tmp_path): wavelet_wavefronts().to_netcdf(tmp_path / "sample.nc") for idx, da in enumerate(wavelet_wavefronts(nchunk=3), start=1): @@ -81,6 +63,7 @@ def test_concatenate(self, tmp_path): "time": { "tie_indices": [0, da1.sizes["time"] + da2.sizes["time"] - 1], "tie_values": [da1["time"][0].values, da2["time"][-1].values], + "sampling_interval": da1.coords["time"].sampling_interval, }, "distance": da1["distance"], } @@ -162,9 +145,17 @@ def test_concatenate(self, tmp_path): assert xd.concat((da1, da2), dim="time").equals(expected) # stack da = wavelet_wavefronts() - objs = [obj for obj in da] + objs = list(da) result = xd.concat(objs, dim="time") - result["time"] = InterpCoordinate.from_array(result["time"].values) + time_values = result["time"].values + result["time"] = InterpCoordinate( + { + "tie_indices": np.arange(len(time_values)), + "tie_values": time_values, + "sampling_interval": da.coords["time"].sampling_interval, + }, + "time", + ).simplify() assert result.equals(da) objs = [obj.drop_coords("time") for obj in da] result = xd.concat(objs, dim="time") @@ -179,7 +170,7 @@ def test_open_datacollection(self): xd.open_datacollection("not_existing_file.nc") def test_asdataarray(self): - da = self.generate(False) + da = xd.testing.dummy(shape=(300, 100), datetime=False) out = xd.asdataarray(da.to_xarray()) assert np.array_equal(out.data, da.data) for dim in da.dims: diff --git a/tests/test_dataarray.py b/tests/test_dataarray.py index 06ee1493..9cebcf01 100644 --- a/tests/test_dataarray.py +++ b/tests/test_dataarray.py @@ -9,7 +9,6 @@ import xdas as xd from xdas.coordinates import Coordinates, DenseCoordinate, InterpCoordinate -from xdas.synthetics import wavelet_wavefronts def generate(dense=False): @@ -56,14 +55,22 @@ def test_init_and_properties(self): da = xd.DataArray() assert np.array_equal(da.values, np.array(np.nan), equal_nan=True) assert da.coords == {} - assert da.dims == tuple() + assert da.dims == () da = xd.DataArray([[]]) assert da.dims == ("dim_0", "dim_1") assert da.ndim == 2 da = xd.DataArray(1) - assert da.dims == tuple() + assert da.dims == () assert da.ndim == 0 + def test_array_copy_keyword(self): + data = np.arange(5.0) + da = xd.DataArray(data, {"x": np.arange(5)}) + # copy=False / None may share memory with the backing array + assert np.shares_memory(np.array(da, copy=False), data) + # copy=True must return an independent array + assert not np.shares_memory(np.array(da, copy=True), data) + def test_raises_on_data_and_coords_mismatch(self): with pytest.raises(ValueError, match="different number of dimensions"): xd.DataArray(np.zeros(3), dims=("time", "distance")) @@ -118,7 +125,7 @@ def test_cannot_set_dims(self): da.dims = ("other_dim",) def test_data_setter(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() data = np.arange(np.prod(da.shape)).reshape(da.shape) da.data = data assert np.array_equal(da.data, data) @@ -162,7 +169,7 @@ def test_sel(self): assert da.sel(dim=slice(100.0, 300.0)).equals(da[0:3]) assert da.sel(dim=slice(100.0, 300.0), endpoint=False).equals(da[0:2]) # drop - da = wavelet_wavefronts() + da = xd.testing.dummy() result = da.sel(distance=0, method="nearest", drop=True) assert "distance" not in result.coords @@ -217,7 +224,7 @@ def test_sel_item_with_overlaps(self): da.sel(time=0.1, method="nearest") def test_isel(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = da.isel(first=0) excepted = da.isel(time=0) assert result.equals(excepted) @@ -225,7 +232,7 @@ def test_isel(self): excepted = da.isel(distance=0) assert result.equals(excepted) # drop - da = wavelet_wavefronts() + da = xd.testing.dummy() result = da.sel(distance=0, drop=True) assert "distance" not in result.coords @@ -306,7 +313,7 @@ def test_expand_dims(self): class TestManipulation: def test_transpose(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = da.transpose("distance", "time") assert result.dims == ("distance", "time") assert np.array_equal(result.values, da.values.T) @@ -320,7 +327,7 @@ def test_transpose(self): da.transpose("space", "frequency") def test_ufunc(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = np.add(da, 1) assert np.array_equal(result.data, da.data + 1) result = np.add(da, np.ones(da.shape[-1])) @@ -331,7 +338,7 @@ def test_ufunc(self): assert np.array_equal(result.data, da.data + da.data[0]) def test_arithmetics(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = da + 1 assert np.array_equal(result.data, da.data + 1) result = da + np.array(1) @@ -427,13 +434,9 @@ def test_from_xarray(self): assert np.array_equal(result["dim"].values, da["dim"].values) def test_stream(self): - da = wavelet_wavefronts() - da["time"] = { - "tie_indices": da["time"].tie_indices, - "tie_values": da["time"].tie_values.astype("datetime64[us]"), - } + da = xd.testing.dummy() st = da.to_stream(dim={"distance": "time"}) - assert st[0].id == "NET.DAS00001.00.BN1" + assert st[0].id == "NET.DAS00001.00.HN1" assert len(st) == da.sizes["distance"] assert st[0].stats.npts == da.sizes["time"] assert np.datetime64(st[0].stats.starttime.datetime) == da["time"][0].values @@ -474,7 +477,7 @@ def test_netcdf_non_dimensional(self, tmp_path): assert result.equals(da) da_path = tmp_path / "da.nc" - da = wavelet_wavefronts().assign_coords(lon=("distance", np.arange(401))) + da = xd.testing.dummy().assign_coords(lon=("distance", np.arange(10))) da.to_netcdf(da_path) tmp = xd.open_dataarray(da_path) vds_path = tmp_path / "vds.nc" @@ -484,7 +487,7 @@ def test_netcdf_non_dimensional(self, tmp_path): def test_io(self, tmp_path): # both coords interpolated - da = wavelet_wavefronts() + da = xd.testing.dummy() path = tmp_path / "interp.nc" da.to_netcdf(path) da_recovered = xd.DataArray.from_netcdf(path) @@ -657,7 +660,7 @@ def test_repr_dask(self): assert "DaskArray" in r def test_repr_virtual(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() da.to_netcdf(tmp_path / "a.nc") da2 = xd.open(tmp_path / "a.nc") r = repr(da2) @@ -766,12 +769,12 @@ def test_drop_coords(self): assert "x" in result.coords def test_isel_drop_non_scalar(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = da.isel(time=slice(0, 3), drop=True) assert "time" in result.coords def test_sel_drop_non_scalar(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() t0 = da["time"].tie_values[0] t1 = da["time"].tie_values[-1] result = da.sel(time=slice(t0, t1), drop=True) @@ -804,43 +807,6 @@ def test_expand_dims_non_scalar_coord(self): with pytest.raises(ValueError, match="cannot expand along y"): da.expand_dims("y", 0) - def test_to_dict_virtual_raises(self, tmp_path): - da = wavelet_wavefronts() - da.to_netcdf(tmp_path / "b.nc") - da2 = xd.open(tmp_path / "b.nc") - with pytest.raises(NotImplementedError): - da2.to_dict() - - def test_to_dict_numpy(self): - da = xd.DataArray(np.array([1.0, 2.0, 3.0]), {"x": [1, 2, 3]}) - d = da.to_dict() - assert isinstance(d["data"], list) - - def test_to_dict_dask(self): - data = dask.array.from_array(np.ones((3,)), chunks=3) - da = xd.DataArray(data, {"x": [1, 2, 3]}) - d = da.to_dict() - assert isinstance(d["data"], dict) - - def test_from_dict_list(self): - da = xd.DataArray(np.array([1.0, 2.0]), {"x": [1, 2]}) - d = da.to_dict() - result = xd.DataArray.from_dict(d) - assert isinstance(result.data, np.ndarray) - assert np.allclose(result.values, [1.0, 2.0]) - - def test_from_dict_dict(self): - data = dask.array.from_array(np.ones((3,)), chunks=3) - da = xd.DataArray(data, {"x": [1, 2, 3]}) - d = da.to_dict() - result = xd.DataArray.from_dict(d) - assert np.allclose(result.values, np.ones(3)) - - def test_from_dict_invalid(self): - d = {"data": 42, "coords": {}, "dims": (), "name": None, "attrs": {}} - with pytest.raises(ValueError, match="data must be a list or a dictionary"): - xd.DataArray.from_dict(d) - def test_plot_1d(self): import matplotlib diff --git a/tests/test_datacollection.py b/tests/test_datacollection.py index 65d6d5f7..1317a0ce 100644 --- a/tests/test_datacollection.py +++ b/tests/test_datacollection.py @@ -4,7 +4,6 @@ import xdas as xd import xdas.signal as xs from xdas.core.datacollection import get_depth -from xdas.synthetics import wavelet_wavefronts class TestDataCollection: @@ -18,7 +17,7 @@ def nest(self, da): ) def test_init(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = self.nest(da) data = ( "instrument", @@ -31,7 +30,7 @@ def test_init(self): assert result.equals(dc) def test_io(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection( { "das1": da, @@ -63,7 +62,7 @@ def test_io(self, tmp_path): assert result.equals(dc) def test_io_create_dirs(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection( { "das1": da, @@ -79,7 +78,7 @@ def test_io_create_dirs(self, tmp_path): assert result.equals(dc) def test_depth_counter(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() da.name = "da" dc = self.nest(da) path = tmp_path / "tmp.nc" @@ -91,30 +90,30 @@ def test_depth_counter(self, tmp_path): assert get_depth(file["instrument/das1/acquisition"]) > 0 assert get_depth(file["instrument/das1/acquisition/0"]) == 0 with pytest.raises(ValueError): - get_depth(file["instrument/das1/acquisition/0/da"]) == 0 + get_depth(file["instrument/das1/acquisition/0/da"]) def test_isel(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = self.nest(da) - da_isel = da.isel(distance=slice(100, 200)) - dc_isel = dc.isel(distance=slice(100, 200)) + da_isel = da.isel(distance=slice(2, 5)) + dc_isel = dc.isel(distance=slice(2, 5)) assert self.nest(da_isel).equals(dc_isel) - dc_isel = dc.isel(distance=slice(2000, 3000)) + dc_isel = dc.isel(distance=slice(20, 30)) assert dc_isel["das1"].empty assert dc_isel["das2"].empty def test_sel(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = self.nest(da) - da_sel = da.sel(distance=slice(1000, 2000)) - dc_sel = dc.sel(distance=slice(1000, 2000)) + da_sel = da.sel(distance=slice(20, 50)) + dc_sel = dc.sel(distance=slice(20, 50)) assert self.nest(da_sel).equals(dc_sel) - dc_sel = dc.sel(distance=slice(20000, 30000)) + dc_sel = dc.sel(distance=slice(200, 300)) assert dc_sel["das1"].empty assert dc_sel["das2"].empty def test_query(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = self.nest(da) result = dc.query(instrument="das1", acquisition=0) expected = xd.DataCollection( @@ -130,12 +129,12 @@ def test_query(self): assert result.equals(dc) def test_fields(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = self.nest(da) assert dc.fields == ("instrument", "acquisition") def test_map(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = self.nest(da) atom = xs.decimate(..., 2, ftype="fir") result = dc.map(atom) @@ -144,7 +143,7 @@ def test_map(self): def test_flat_map(self): # DataMapping with DataArrays as direct values - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection({"a": da, "b": da}, "flat") atom = xs.decimate(..., 2, ftype="fir") result = dc.map(atom) @@ -152,14 +151,14 @@ def test_flat_map(self): def test_flat_sequence_map(self): # DataSequence with DataArrays as direct values - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") atom = xs.decimate(..., 2, ftype="fir") result = dc.map(atom) assert result[0].equals(atom(da)) def test_datacollection_from_dataarray(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() # When DataArray is passed, rename and return it result = xd.DataCollection(da, "myname") assert isinstance(result, xd.DataArray) @@ -181,7 +180,7 @@ def test_empty_mapping_repr(self): def test_mapping_reduce(self): import pickle - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection({"a": da}, "test") pickled = pickle.dumps(dc) restored = pickle.loads(pickled) @@ -190,86 +189,86 @@ def test_mapping_reduce(self): def test_sequence_reduce(self): import pickle - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "test") pickled = pickle.dumps(dc) restored = pickle.loads(pickled) assert restored.equals(dc) def test_sequence_fields(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") assert "seq" in dc.fields def test_mapping_equals_false_different_type(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dm = xd.DataCollection({"a": da}, "test") assert not dm.equals(xd.DataCollection([da], "test")) def test_mapping_equals_false_different_name(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dm1 = xd.DataCollection({"a": da}, "name1") dm2 = xd.DataCollection({"a": da}, "name2") assert not dm1.equals(dm2) def test_mapping_equals_false_different_keys(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dm1 = xd.DataCollection({"a": da}, "test") dm2 = xd.DataCollection({"b": da}, "test") assert not dm1.equals(dm2) def test_mapping_equals_false_different_values(self): - da = wavelet_wavefronts() - da2 = wavelet_wavefronts() + da = xd.testing.dummy() + da2 = xd.testing.dummy() da2.data[:] = 0 dm1 = xd.DataCollection({"a": da}, "test") dm2 = xd.DataCollection({"a": da2}, "test") assert not dm1.equals(dm2) def test_sequence_equals_false(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() ds1 = xd.DataCollection([da, da], "seq") ds2 = xd.DataCollection([da, da], "other") assert not ds1.equals(ds2) def test_sequence_equals_false_wrong_type(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() ds = xd.DataCollection([da], "seq") dm = xd.DataCollection({"a": da}, "seq") assert not ds.equals(dm) def test_sequence_load(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") loaded = dc.load() assert isinstance(loaded, type(dc)) def test_mapping_load(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection({"a": da, "b": da}, "test") loaded = dc.load() assert isinstance(loaded, type(dc)) def test_sequence_copy(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") copy = dc.copy() assert copy.equals(dc) def test_sequence_isel(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") result = dc.isel(distance=slice(0, 100)) assert len(result) == 2 def test_sequence_sel(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") result = dc.sel(distance=slice(0, 5000)) assert len(result) == 2 def test_sequence_from_netcdf(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") path = tmp_path / "seq.nc" dc.to_netcdf(path) @@ -277,13 +276,13 @@ def test_sequence_from_netcdf(self, tmp_path): assert result.equals(dc) def test_query_invalid_key_in_sequence(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") with pytest.raises(ValueError, match="query must be a string"): dc.query(seq="bad_string_key") def test_query_invalid_key_in_mapping(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection({"a": da}, "test") with pytest.raises(ValueError, match="query must be a string"): dc.query(test=123) @@ -291,7 +290,7 @@ def test_query_invalid_key_in_mapping(self): def test_from_netcdf_non_sequential_int_keys(self, tmp_path): from xdas.core.datacollection import DataMapping - da = wavelet_wavefronts() + da = xd.testing.dummy() # Create a mapping with non-sequential int keys (gaps) dm = DataMapping({0: da, 2: da}, "test") path = tmp_path / "non_seq.nc" @@ -303,7 +302,7 @@ def test_from_netcdf_non_sequential_int_keys(self, tmp_path): def test_sequence_from_netcdf_direct(self, tmp_path): from xdas.core.datacollection import DataSequence - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = DataSequence([da, da], "seq") path = tmp_path / "seq_direct.nc" dc.to_netcdf(path) @@ -311,20 +310,20 @@ def test_sequence_from_netcdf_direct(self, tmp_path): assert result.equals(dc) def test_sequence_query_slice(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") result = dc.query(seq=slice(0, 1)) assert len(result) == 1 def test_mapping_repr_nonempty(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dm = xd.DataCollection({"a": da}, "test") s = repr(dm) assert "test" in s.lower() or "Test" in s def test_mapping_repr_nested(self): # nested DataMapping → triggers the non-DataArray branch in __repr__ - da = wavelet_wavefronts() + da = xd.testing.dummy() dm = self.nest(da) s = repr(dm) assert "das1" in s @@ -332,39 +331,39 @@ def test_mapping_repr_nested(self): def test_mapping_repr_int_keys(self): from xdas.core.datacollection import DataMapping - da = wavelet_wavefronts() + da = xd.testing.dummy() dm = DataMapping({0: da, 1: da}, "seq") s = repr(dm) assert "0" in s def test_sequence_repr(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection([da, da], "seq") s = repr(dc) assert "seq" in s.lower() or "Seq" in s def test_mapping_copy(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.DataCollection({"a": da}, "test") copy = dc.copy() assert copy.equals(dc) def test_sequence_equals_false_different_length(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() ds1 = xd.DataCollection([da, da], "seq") ds2 = xd.DataCollection([da], "seq") assert not ds1.equals(ds2) def test_sequence_equals_false_different_values(self): - da = wavelet_wavefronts() - da2 = wavelet_wavefronts() + da = xd.testing.dummy() + da2 = xd.testing.dummy() da2.data[:] = 0 ds1 = xd.DataCollection([da], "seq") ds2 = xd.DataCollection([da2], "seq") assert not ds1.equals(ds2) def test_nested_sequence_map(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() inner = xd.DataCollection([da, da], "inner") dc = xd.DataCollection([inner, inner], "outer") atom = xs.decimate(..., 2, ftype="fir") @@ -374,24 +373,24 @@ def test_nested_sequence_map(self): def test_parse_tuple_with_name_given(self): from xdas.core.datacollection import DataMapping - da = wavelet_wavefronts() + da = xd.testing.dummy() # When data is a tuple and name is already provided, unpack the tuple ignoring its name dm = DataMapping(("inner_name", {"a": da}), "outer_name") assert dm.name == "outer_name" def test_parse_datacollection_propagates_name(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() dm = xd.DataCollection({"a": da}, "original_name") # just verify parse propagates name from xdas.core.datacollection import parse - data, name = parse(dm, None) # should propagate dm.name + _data, name = parse(dm, None) # should propagate dm.name assert name == "original_name" def test_mapping_map_invalid_item(self): from xdas.core.datacollection import DataMapping - da = wavelet_wavefronts() + da = xd.testing.dummy() dm = DataMapping({"good": da}, "test") # bypass validation to inject an invalid item dict.__setitem__(dm, "bad", "not_a_dataarray") @@ -402,7 +401,7 @@ def test_mapping_map_invalid_item(self): def test_sequence_map_invalid_item(self): from xdas.core.datacollection import DataSequence - da = wavelet_wavefronts() + da = xd.testing.dummy() ds = DataSequence([da], "test") # bypass validation to inject an invalid item list.append(ds, "not_a_dataarray") @@ -411,35 +410,35 @@ def test_sequence_map_invalid_item(self): ds.map(atom) def test_mapping_sel_one_element_becomes_empty(self): - da = wavelet_wavefronts() - da_near = da.sel(distance=slice(0, 4999)) - da_far = da.sel(distance=slice(5000, 10000)) + da = xd.testing.dummy() + da_near = da.sel(distance=slice(0, 45)) + da_far = da.sel(distance=slice(50, 90)) dc = xd.DataCollection({"near": da_near, "far": da_far}, "instrument") - result = dc.sel(distance=slice(0, 2000)) + result = dc.sel(distance=slice(0, 20)) assert set(result.keys()) == {"near"} assert not result["near"].empty def test_mapping_sel_all_elements_become_empty(self): - da = wavelet_wavefronts() - da_near = da.sel(distance=slice(0, 4999)) - da_far = da.sel(distance=slice(5000, 10000)) + da = xd.testing.dummy() + da_near = da.sel(distance=slice(0, 45)) + da_far = da.sel(distance=slice(50, 90)) dc = xd.DataCollection({"near": da_near, "far": da_far}, "instrument") - result = dc.sel(distance=slice(-1000, -1)) + result = dc.sel(distance=slice(-100, -1)) assert len(result) == 0 def test_sequence_sel_one_element_becomes_empty(self): - da = wavelet_wavefronts() - da_near = da.sel(distance=slice(0, 4999)) - da_far = da.sel(distance=slice(5000, 10000)) + da = xd.testing.dummy() + da_near = da.sel(distance=slice(0, 45)) + da_far = da.sel(distance=slice(50, 90)) dc = xd.DataCollection([da_near, da_far], "instrument") - result = dc.sel(distance=slice(0, 2000)) + result = dc.sel(distance=slice(0, 20)) assert len(result) == 1 assert not result[0].empty def test_sequence_sel_all_elements_become_empty(self): - da = wavelet_wavefronts() - da_near = da.sel(distance=slice(0, 4999)) - da_far = da.sel(distance=slice(5000, 10000)) + da = xd.testing.dummy() + da_near = da.sel(distance=slice(0, 45)) + da_far = da.sel(distance=slice(50, 90)) dc = xd.DataCollection([da_near, da_far], "instrument") - result = dc.sel(distance=slice(-1000, -1)) + result = dc.sel(distance=slice(-100, -1)) assert len(result) == 0 diff --git a/tests/test_fft.py b/tests/test_fft.py index 8b403120..0c9977e5 100644 --- a/tests/test_fft.py +++ b/tests/test_fft.py @@ -6,14 +6,14 @@ class TestRFFT: def test_with_non_dimensional(self): - da = xd.synthetics.wavelet_wavefronts() + da = xd.testing.dummy() da["latitude"] = ("distance", np.arange(da.sizes["distance"])) xfft.rfft(da) class TestInverseTransforms: def test_standard(self): - expected = xd.synthetics.wavelet_wavefronts() + expected = xd.testing.dummy() result = xfft.ifft( xfft.fft(expected, dim={"time": "frequency"}), dim={"frequency": "time"}, @@ -30,7 +30,7 @@ def test_standard(self): assert result[name].equals(expected[name]) def test_real(self): - expected = xd.synthetics.wavelet_wavefronts() + expected = xd.testing.dummy() result = xfft.irfft( xfft.rfft(expected, dim={"time": "frequency"}), expected.sizes["time"], @@ -47,7 +47,7 @@ def test_real(self): assert result[name].equals(expected[name]) def test_real_default_n(self): - expected = xd.synthetics.wavelet_wavefronts() + expected = xd.testing.dummy() expected = expected.isel(time=slice(0, expected.sizes["time"] // 2 * 2)) result = xfft.irfft( xfft.rfft(expected, dim={"time": "frequency"}), @@ -62,3 +62,23 @@ def test_real_default_n(self): assert np.allclose(result["time"].values, ref) else: assert result[name].equals(expected[name]) + + +class TestDefaultDim: + """The default `dim` maps the last dimension; equivalent to naming it.""" + + def test_fft(self): + da = xd.testing.dummy() + assert xfft.fft(da).equals(xfft.fft(da, dim={"distance": "spectrum"})) + + def test_rfft(self): + da = xd.testing.dummy() + assert xfft.rfft(da).equals(xfft.rfft(da, dim={"distance": "spectrum"})) + + def test_ifft(self): + da = xd.testing.dummy() + assert xfft.ifft(da).equals(xfft.ifft(da, dim={"distance": "signal"})) + + def test_irfft(self): + da = xd.testing.dummy() + assert xfft.irfft(da).equals(xfft.irfft(da, dim={"distance": "signal"})) diff --git a/tests/test_methods.py b/tests/test_methods.py index 0c44bfb5..f527c628 100644 --- a/tests/test_methods.py +++ b/tests/test_methods.py @@ -8,23 +8,13 @@ @pytest.fixture def float_da(): - return xd.DataArray( - data=np.arange(12.0).reshape(3, 4), - coords={ - "x": {"tie_indices": [0, 2], "tie_values": [0.0, 2.0]}, - "y": {"tie_indices": [0, 3], "tie_values": [0.0, 3.0]}, - }, - ) + return xd.testing.dummy(dims=("x", "y"), shape=(3, 4), step=1.0, datetime=False) @pytest.fixture def int_da(): - return xd.DataArray( - data=np.arange(12).reshape(3, 4), - coords={ - "x": {"tie_indices": [0, 2], "tie_values": [0.0, 2.0]}, - "y": {"tie_indices": [0, 3], "tie_values": [0.0, 3.0]}, - }, + return xd.testing.dummy( + dims=("x", "y"), shape=(3, 4), step=1.0, datetime=False, dtype=int ) diff --git a/tests/test_numpy.py b/tests/test_numpy.py index 0090303a..f21c5079 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -1,13 +1,14 @@ import numpy as np import pytest +import xdas as xd from xdas.core.dataarray import HANDLED_NUMPY_FUNCTIONS, DataArray from xdas.synthetics import wavelet_wavefronts class TestUfuncs: def test_unitary_operators(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = np.abs(da) expected = da.copy(data=np.abs(da.data)) da_out = da.copy() @@ -19,8 +20,8 @@ def test_unitary_operators(self): assert da_where.equals(da) def test_binary_operators(self): - da1 = wavelet_wavefronts() - da2 = wavelet_wavefronts() + da1 = xd.testing.dummy() + da2 = xd.testing.dummy() result = np.add(da1, da2) expected = da1.copy(data=da1.data + da2.data) da_out = da1.copy() @@ -34,7 +35,7 @@ def test_binary_operators(self): np.add(da1, da2[1:]) def test_multiple_outputs(self): - da = wavelet_wavefronts() + da = wavelet_wavefronts() # divmod(da, da) needs non-zero values result1, result2 = np.divmod(da, da) expected1 = da.copy(data=np.ones(da.shape)) expected2 = da.copy(data=np.zeros(da.shape)) @@ -46,7 +47,8 @@ def test_multiple_outputs(self): class TestFunc: def test_returns_dataarray(self): - da = wavelet_wavefronts() + # keep values small: np.i0 overflows on the default dummy + da = xd.testing.dummy(shape=(10, 5)) for numpy_function in HANDLED_NUMPY_FUNCTIONS: if numpy_function == np.clip: result = numpy_function(da, -1, 1) @@ -57,10 +59,10 @@ def test_returns_dataarray(self): else: result = numpy_function(da) assert isinstance(result, np.ndarray) - elif numpy_function.__name__ == "trapezoid": - result = numpy_function(da) - assert isinstance(result, np.ndarray) - elif numpy_function in [np.diff, np.ediff1d]: + elif numpy_function.__name__ == "trapezoid" or numpy_function in [ + np.diff, + np.ediff1d, + ]: result = numpy_function(da) assert isinstance(result, np.ndarray) elif numpy_function in [ @@ -76,7 +78,7 @@ def test_returns_dataarray(self): assert isinstance(result, DataArray) def test_reduce(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = np.sum(da) assert result.shape == () result = np.sum(da, axis=0) @@ -93,7 +95,7 @@ def test_reduce(self): np.sum(da, axis=2) def test_out(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() out = da.copy() np.cumsum(da, axis=-1, out=out) assert not out.equals(da) diff --git a/tests/test_picking.py b/tests/test_picking.py index 6f844d27..3ee47601 100644 --- a/tests/test_picking.py +++ b/tests/test_picking.py @@ -13,6 +13,7 @@ def generate(self): "distance": { "tie_indices": [0, 4], "tie_values": [0.0, 400.0], + "sampling_interval": 100.0, }, "time": { "tie_indices": [0, 9], @@ -20,6 +21,7 @@ def generate(self): np.datetime64("2023-01-01T00:00:00"), np.datetime64("2023-01-01T00:00:09"), ], + "sampling_interval": np.timedelta64(1, "s"), }, }, ) @@ -248,6 +250,7 @@ def test_scalar_coord_preserved(self): np.datetime64("2023-01-01T00:00:00"), np.datetime64("2023-01-01T00:00:09"), ], + "sampling_interval": np.timedelta64(1, "s"), }, "station": "ABC", }, @@ -280,6 +283,7 @@ def test_non_dim_coord_on_dim_axis_skipped(self): np.datetime64("2023-01-01T00:00:00"), np.datetime64("2023-01-01T00:00:09"), ], + "sampling_interval": np.timedelta64(1, "s"), }, "quality": ( "time", diff --git a/tests/test_processing.py b/tests/test_processing.py index 8d7c60bd..96b8e124 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -14,12 +14,11 @@ import xdas.processing as xp from xdas.atoms import Partial, Sequential from xdas.signal import sosfilt -from xdas.synthetics import wavelet_wavefronts class TestDataArrayLoader: def test_init(self): - da = xd.DataArray(np.random.rand(1000, 100), dims=("time", "distance")) + da = xd.testing.dummy(shape=(1000, 100)) dl = xp.DataArrayLoader(da, {"time": 100}) assert dl.da is da assert dl.chunk_dim == "time" @@ -38,14 +37,14 @@ def test_init(self): ], ) def test_chunks_integrity(self, max_buffers, max_workers): - da = xd.DataArray(np.random.rand(1000, 100), dims=("time", "distance")) + da = xd.testing.dummy(shape=(1000, 100)) dl = xp.DataArrayLoader(da, {"time": 100}, max_buffers, max_workers) - chunks = [chunk for chunk in dl] + chunks = list(dl) result = xd.concat(chunks) assert result.equals(da) def test_error_handling(self): - da = xd.DataArray(np.random.rand(1000, 100), dims=("time", "distance")) + da = xd.testing.dummy(shape=(1000, 100)) with pytest.raises(TypeError): xp.DataArrayLoader(None, None) with pytest.raises(TypeError): @@ -71,7 +70,7 @@ def test_init(self, tmp_path): ], ) def test_chunk_integrity(self, max_buffers, max_workers, tmp_path): - expected = xd.DataArray(np.random.rand(1000, 100), dims=("time", "distance")) + expected = xd.testing.dummy(shape=(1000, 100)) dw = xp.DataArrayWriter(tmp_path, None, max_buffers, max_workers) chunks = xd.split(expected, 10, dim="time") for chunk in chunks: @@ -96,7 +95,7 @@ def test_stateful(self, tmp_path): sample_path = tmp_path / "sample.nc" # generate test dataarray - wavelet_wavefronts().to_netcdf(sample_path) + xd.testing.dummy().to_netcdf(sample_path) da = xd.open(sample_path) # declare processing sequence @@ -117,13 +116,7 @@ def test_stateful(self, tmp_path): assert result1.equals(result2) def test_small_last_chunk(self, tmp_path): - da = xd.DataArray( - data=np.random.randn(1001, 100), - coords={ - "time": xd.Coordinate["interpolated"].from_block(0, 1001, 0.01), - "distance": xd.Coordinate["interpolated"].from_block(0, 100, 10.0), - }, - ) + da = xd.testing.dummy(shape=(1001, 100), datetime=False) # declare processing sequence sos = sp.iirfilter(4, 0.1, btype="lowpass", output="sos") @@ -233,7 +226,7 @@ def publish(): return xd.concat(result) def test_publish_and_subscribe(self): - expected = xd.synthetics.dummy() + expected = xd.testing.dummy() packets = xd.split(expected, 10) address = f"tcp://localhost:{xd.io.get_free_port()}" @@ -241,7 +234,7 @@ def test_publish_and_subscribe(self): assert result.equals(expected) def test_encoding(self): - expected = xd.synthetics.dummy() + expected = xd.testing.dummy() packets = xd.split(expected, 10) address = f"tcp://localhost:{xd.io.get_free_port()}" encoding = {"chunks": (10, 10), **hdf5plugin.Zfp(accuracy=1e-6)} @@ -265,6 +258,7 @@ def test_without_gap(self, tmp_path): "time": { "tie_indices": [0, data.shape[0] - 1], "tie_values": [starttime, endtime], + "sampling_interval": np.timedelta64(10, "ms"), }, "distance": distance, }, @@ -324,6 +318,7 @@ def test_with_gap(self, tmp_path): ], dtype="datetime64[ms]", ), + "sampling_interval": np.timedelta64(10, "ms"), }, "distance": 5.0 * np.arange(10), }, @@ -383,6 +378,7 @@ def test_flat(self, tmp_path): "time": { "tie_indices": [0, data.shape[0] - 1], "tie_values": [starttime, endtime], + "sampling_interval": np.timedelta64(10, "ms"), }, "distance": distance, }, @@ -425,7 +421,7 @@ def atom(da, **kwargs): class TestProcessNoNbytes: def test_loader_without_nbytes(self, tmp_path): - da = xd.DataArray(np.random.rand(100, 10), dims=("time", "distance")) + da = xd.testing.dummy(shape=(100, 10)) chunks = xd.split(da, 10, dim="time") class SimpleLoader: @@ -445,7 +441,7 @@ def atom(x, **kw): class TestDataArrayLoaderMaxBuffers: def test_max_buffers_exceeds_chunks(self): - da = xd.DataArray(np.random.rand(10, 5), dims=("time", "distance")) + da = xd.testing.dummy(shape=(10, 5)) dl = xp.DataArrayLoader(da, {"time": 5}, max_buffers=10) chunks = list(dl) assert len(chunks) == 2 @@ -488,7 +484,7 @@ class TestZMQPublisherAliases: def test_write_alias(self): address = f"tcp://localhost:{xd.io.get_free_port()}" publisher = xp.ZMQPublisher(address) - da = xd.synthetics.dummy() + da = xd.testing.dummy() publisher.write(da) # use write() alias def test_result_returns_none(self): @@ -503,19 +499,7 @@ def test_on_closed(self, tmp_path): from xdas.processing.core import Handler - da = xd.DataArray( - np.zeros((10, 5), dtype=np.float32), - { - "time": { - "tie_indices": [0, 9], - "tie_values": [ - np.datetime64("2020-01-01T00:00:00.000000000"), - np.datetime64("2020-01-01T00:00:09.000000000"), - ], - }, - "distance": {"tie_indices": [0, 4], "tie_values": [0.0, 40.0]}, - }, - ) + da = xd.testing.dummy(shape=(10, 5), step=(1.0, 10.0), dtype=np.float32) path = str(tmp_path / "test.nc") da.to_netcdf(path) @@ -538,19 +522,7 @@ def test_iter_and_next(self, tmp_path): assert iter(loader) is loader # put a DataArray directly into the queue - da = xd.DataArray( - np.zeros((5, 3), dtype=np.float32), - { - "time": { - "tie_indices": [0, 4], - "tie_values": [ - np.datetime64("2020-01-01T00:00:00.000000000"), - np.datetime64("2020-01-01T00:00:04.000000000"), - ], - }, - "distance": {"tie_indices": [0, 2], "tie_values": [0.0, 20.0]}, - }, - ) + da = xd.testing.dummy(shape=(5, 3), step=(1.0, 10.0), dtype=np.float32) loader.queue.put(da) result = next(loader) assert result.equals(da) diff --git a/tests/test_routines.py b/tests/test_routines.py index 6dc38b60..8fafc7b8 100644 --- a/tests/test_routines.py +++ b/tests/test_routines.py @@ -13,7 +13,15 @@ def test_bag_initialization(self): def test_bag_append_initializes(self): da = xd.DataArray( - np.random.rand(10, 5), {"time": np.arange(10), "space": np.arange(5)} + np.random.rand(10, 5), + { + "time": { + "tie_indices": [0, 9], + "tie_values": [0.0, 9.0], + "sampling_interval": 1.0, + }, + "space": np.arange(5), + }, ) bag = Bag(dim="time") bag.append(da) @@ -90,12 +98,24 @@ def test_bag_append_incompatible_sampling_interval(self): da1 = xd.DataArray( np.random.rand(10, 5), dims=("time", "space"), - coords={"time": np.arange(10)}, + coords={ + "time": { + "tie_indices": [0, 9], + "tie_values": [0.0, 9.0], + "sampling_interval": 1.0, + } + }, ) da2 = xd.DataArray( np.random.rand(10, 5), dims=("time", "space"), - coords={"time": np.arange(10) * 2}, + coords={ + "time": { + "tie_indices": [0, 9], + "tie_values": [0.0, 18.0], + "sampling_interval": 2.0, + } + }, ) bag = Bag(dim="time") bag.append(da1) @@ -112,14 +132,7 @@ def test_basic(self): assert combined.shape == (20, 5) # with coords - da1 = xd.DataArray( - np.random.rand(10, 5), - coords={"time": np.arange(10), "space": np.arange(5)}, - ) - da2 = xd.DataArray( - np.random.rand(10, 5), - coords={"time": np.arange(10, 20), "space": np.arange(5)}, - ) + da1, da2 = xd.split(xd.testing.dummy(dims=("time", "space"), shape=(20, 5)), 2) combined = xd.combine_by_coords([da1, da2], dim="time", squeeze=True) assert combined.shape == (20, 5) @@ -169,12 +182,24 @@ def test_incompatible_sampling_interval(self): da1 = xd.DataArray( np.random.rand(10, 5), dims=("time", "space"), - coords={"time": np.arange(10)}, + coords={ + "time": { + "tie_indices": [0, 9], + "tie_values": [0.0, 9.0], + "sampling_interval": 1.0, + } + }, ) da2 = xd.DataArray( np.random.rand(10, 5), dims=("time", "space"), - coords={"time": np.arange(10) * 2}, + coords={ + "time": { + "tie_indices": [0, 9], + "tie_values": [0.0, 18.0], + "sampling_interval": 2.0, + } + }, ) dc = xd.combine_by_coords([da1, da2], dim="time") assert len(dc) == 2 @@ -200,13 +225,7 @@ def test_expand_scalar_coordinate(self): class TestOpenMFDataArray: def test_warn_on_corrupted_files(self, tmp_path): - expected = xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, # TODO: should work without coords - ) + expected = xd.testing.dummy(dims=("time", "space"), shape=(10, 5)) for index, chunk in enumerate(xd.split(expected, 3, "time"), start=1): chunk.to_netcdf(tmp_path / f"chunk_{index}.nc") result = xd.open_mfdataarray(tmp_path / "*.nc") @@ -225,26 +244,14 @@ def test_warn_on_corrupted_files(self, tmp_path): assert result.equals(expected) def test_verbose_single_worker(self, tmp_path): - expected = xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, # TODO: should work without coords - ) + expected = xd.testing.dummy(dims=("time", "space"), shape=(10, 5)) for index, chunk in enumerate(xd.split(expected, 3, "time"), start=1): chunk.to_netcdf(tmp_path / f"chunk_{index}.nc") result = xd.open_mfdataarray(tmp_path / "*.nc", verbose=True, parallel=1) assert result.equals(expected) def test_verbose_multiple_workers(self, tmp_path): - expected = xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, # TODO: should work without coords - ) + expected = xd.testing.dummy(dims=("time", "space"), shape=(10, 5)) for index, chunk in enumerate(xd.split(expected, 3, "time"), start=1): chunk.to_netcdf(tmp_path / f"chunk_{index}.nc") result = xd.open_mfdataarray(tmp_path / "*.nc", verbose=True, parallel=2) @@ -253,13 +260,7 @@ def test_verbose_multiple_workers(self, tmp_path): class TestOpen: # TODO: those tests are weirdly slow... def test_open_single_dataarray(self, tmp_path): - expected = xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, - ) + expected = xd.testing.dummy(dims=("time", "space"), shape=(10, 5)) path = tmp_path / "dataarray.nc" expected.to_netcdf(path) @@ -268,13 +269,7 @@ def test_open_single_dataarray(self, tmp_path): assert result.equals(expected) def test_open_multiple_file_dataarray(self, tmp_path): - expected = xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, - ) + expected = xd.testing.dummy(dims=("time", "space"), shape=(10, 5)) file_paths = [] for index, chunk in enumerate(xd.split(expected, 3, "time"), start=1): @@ -298,27 +293,11 @@ def test_open_multiple_file_tree(self, tmp_path): expected = xd.DataCollection( { "DAS01": xd.DataCollection( - [ - xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, - ) - ], + [xd.testing.dummy(dims=("time", "space"), shape=(10, 5))], name="acquisition", ), "DAS02": xd.DataCollection( - [ - xd.DataArray( - np.random.rand(7, 3), - coords={ - "time": np.arange(7), - "space": np.arange(3), - }, - ) - ], + [xd.testing.dummy(dims=("time", "space"), shape=(7, 3))], name="acquisition", ), }, @@ -338,15 +317,7 @@ def test_open_multiple_file_tree(self, tmp_path): def test_open_single_datacollection(self, tmp_path): expected = xd.DataCollection( - [ - xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, - ) - ] + [xd.testing.dummy(dims=("time", "space"), shape=(10, 5))] ) expected.to_netcdf(tmp_path / "collection.nc") @@ -358,27 +329,11 @@ def test_open_multiple_datacollection_with_glob(self, tmp_path): expected = xd.DataCollection( { "DAS01": xd.DataCollection( - [ - xd.DataArray( - np.random.rand(10, 5), - coords={ - "time": np.arange(10), - "space": np.arange(5), - }, - ) - ], + [xd.testing.dummy(dims=("time", "space"), shape=(10, 5))], name="acquisition", ), "DAS02": xd.DataCollection( - [ - xd.DataArray( - np.random.rand(7, 3), - coords={ - "time": np.arange(7), - "space": np.arange(3), - }, - ) - ], + [xd.testing.dummy(dims=("time", "space"), shape=(7, 3))], name="acquisition", ), }, @@ -409,9 +364,8 @@ def test_raise_if_all_files_corrupted(self, tmp_path): f.write(b"corrupted") with (tmp_path / "corrupted2.nc").open("wb") as f: f.write(b"corrupted") - with pytest.warns(RuntimeWarning): - with pytest.raises(RuntimeError): - xd.open_mfdataarray(str(tmp_path / "*.nc")) + with pytest.warns(RuntimeWarning), pytest.raises(RuntimeError): + xd.open_mfdataarray(str(tmp_path / "*.nc")) class TestSplit: @@ -436,7 +390,8 @@ def dataarray(self, dtype, ctype): [ xd.Coordinate[ctype].from_block(start, size, step, "dim") for start in starts - ] + ], + tolerance=False, ) return xd.DataArray(np.random.randn(len(coord)), {"dim": coord}) @@ -514,7 +469,7 @@ def test_invalid_paths_type_raises(self): xd.open(123) def test_callable_engine(self, tmp_path): - da = xd.DataArray(np.random.rand(10, 5), dims=("time", "distance")) + da = xd.testing.dummy(shape=(10, 5)) path = str(tmp_path / "test.nc") da.to_netcdf(path) @@ -525,7 +480,7 @@ def my_engine(fname, **kwargs): assert result.equals(da) def test_invalid_engine_type_raises(self, tmp_path): - da = xd.DataArray(np.random.rand(10, 5), dims=("time", "distance")) + da = xd.testing.dummy(shape=(10, 5)) path = str(tmp_path / "test.nc") da.to_netcdf(path) with pytest.raises(ValueError, match="engine"): @@ -542,7 +497,7 @@ def test_empty_glob_raises(self, tmp_path): xd.open_mfdatacollection(str(tmp_path / "*.nc")) def test_verbose_single_worker(self, tmp_path): - da = xd.DataArray(np.random.rand(10, 5), dims=("time", "distance")) + da = xd.testing.dummy(shape=(10, 5)) dc = xd.DataCollection([da, da]) path1 = str(tmp_path / "dc1.nc") path2 = str(tmp_path / "dc2.nc") @@ -554,7 +509,7 @@ def test_verbose_single_worker(self, tmp_path): assert isinstance(result, xd.DataCollection) def test_verbose_multiple_worker(self, tmp_path): - da = xd.DataArray(np.random.rand(10, 5), dims=("time", "distance")) + da = xd.testing.dummy(shape=(10, 5)) dc = xd.DataCollection([da, da]) path1 = str(tmp_path / "dc1.nc") path2 = str(tmp_path / "dc2.nc") @@ -576,10 +531,7 @@ def test_invalid_paths_type_raises(self): xd.open_mfdataarray(123) def test_parallel_path(self, tmp_path): - expected = xd.DataArray( - np.random.rand(10, 5), - coords={"time": np.arange(10), "space": np.arange(5)}, - ) + expected = xd.testing.dummy(dims=("time", "space"), shape=(10, 5)) for i, chunk in enumerate(xd.split(expected, 3, "time"), 1): chunk.to_netcdf(tmp_path / f"chunk_{i}.nc") result = xd.open_mfdataarray(tmp_path / "*.nc", parallel=2) @@ -592,7 +544,7 @@ def test_no_files_no_failures_raises(self, tmp_path): class TestOpenMFDatacollectionParallel: def test_parallel_path(self, tmp_path): - da = xd.DataArray(np.random.rand(10, 5), dims=("time", "distance")) + da = xd.testing.dummy(shape=(10, 5)) dc = xd.DataCollection([da, da]) path1 = str(tmp_path / "dc1.nc") path2 = str(tmp_path / "dc2.nc") @@ -608,11 +560,9 @@ def test_one_level_depth(self, tmp_path): dirnames = [tmp_path / key for key in keys] for dirname in dirnames: dirname.mkdir() - for idx, da in enumerate( - xd.synthetics.wavelet_wavefronts(nchunk=3), start=1 - ): + for idx, da in enumerate(xd.split(xd.testing.dummy(), 3), start=1): da.to_netcdf(dirname / f"{idx:03d}.nc") - da = xd.synthetics.wavelet_wavefronts() + da = xd.testing.dummy() dc = xd.open_mfdatatree(tmp_path / "{node}" / "00[acquisition].nc") assert list(dc.keys()) == keys for key in keys: @@ -622,11 +572,11 @@ def test_two_level_depth(self, tmp_path): dc = xd.DataCollection( { "NET01": { - "STA01": xd.synthetics.wavelet_wavefronts(nchunk=1), + "STA01": xd.split(xd.testing.dummy(), 1), }, "NET02": { - "STA02": xd.synthetics.wavelet_wavefronts(nchunk=2), - "STA03": xd.synthetics.wavelet_wavefronts(nchunk=3), + "STA02": xd.split(xd.testing.dummy(), 2), + "STA03": xd.split(xd.testing.dummy(), 3), }, } ) @@ -696,7 +646,9 @@ def test_mixed_empty_and_nonempty_uses_nonempty(self): class TestConcatCoordsEdgeCases: - def test_tolerance_with_dense_coord_raises(self): + def test_tolerance_with_dense_coord_is_noop(self): + # Dense coordinates now implement a (degenerate) `simplify`, so passing a + # tolerance no longer raises; it simply has no effect. da1 = xd.DataArray( np.random.rand(5), {"x": np.array([0.0, 1.0, 2.0, 3.0, 4.0])} ) @@ -705,18 +657,32 @@ def test_tolerance_with_dense_coord_raises(self): ) from xdas.core.routines import concat_coords + result = concat_coords([da1["x"], da2["x"]], tolerance=1.0) + expected = concat_coords([da1["x"], da2["x"]]) + assert result.equals(expected) + + def test_tolerance_with_scalar_coord_raises(self): + from xdas.core.routines import concat_coords + + scalar = xd.Coordinate("SRN") with pytest.raises(TypeError, match="tolerance"): - concat_coords([da1["x"], da2["x"]], tolerance=1.0) + concat_coords([scalar], tolerance=1.0) + + def test_default_tolerance_with_scalar_coord_passes(self): + from xdas.core.routines import concat_coords + + scalar = xd.Coordinate("SRN") + assert concat_coords([scalar]).equals(scalar) class TestSplitEdgeCases: def test_n_zero_raises(self): - da = xd.DataArray(np.random.rand(10), dims=("time",)) + da = xd.testing.dummy(dims=("time",), shape=(10,), step=0.01) with pytest.raises(ValueError, match="`n` must be larger than 0"): xd.split(da, 0) def test_n_too_large_raises(self): - da = xd.DataArray(np.random.rand(10), dims=("time",)) + da = xd.testing.dummy(dims=("time",), shape=(10,), step=0.01) with pytest.raises(ValueError, match="`n` must be smaller"): xd.split(da, 10) @@ -737,51 +703,18 @@ def test_scalar_coord_skipped(self): class TestPlotAvailability: def test_dataarray_plot(self): - da = xd.DataArray( - np.random.rand(100), - { - "time": { - "tie_indices": [0, 99], - "tie_values": [ - np.datetime64("2020-01-01"), - np.datetime64("2020-01-01T00:00:09.900"), - ], - } - }, - ) + da = xd.testing.dummy(dims=("time",), shape=(100,), step=0.01) fig = xd.plot_availability(da) assert fig is not None def test_datassequence_plot(self): - da = xd.DataArray( - np.random.rand(100), - { - "time": { - "tie_indices": [0, 99], - "tie_values": [ - np.datetime64("2020-01-01"), - np.datetime64("2020-01-01T00:00:09.900"), - ], - } - }, - ) + da = xd.testing.dummy(dims=("time",), shape=(100,), step=0.01) dc = xd.DataCollection([da, da]) fig = xd.plot_availability(dc) assert fig is not None def test_datamapping_plot(self): - da = xd.DataArray( - np.random.rand(100), - { - "time": { - "tie_indices": [0, 99], - "tie_values": [ - np.datetime64("2020-01-01"), - np.datetime64("2020-01-01T00:00:09.900"), - ], - } - }, - ) + da = xd.testing.dummy(dims=("time",), shape=(100,), step=0.01) dm = xd.DataCollection({"a": da, "b": da}) fig = xd.plot_availability(dm) assert fig is not None diff --git a/tests/test_signal.py b/tests/test_signal.py index f8ea64a1..3ce0ef81 100644 --- a/tests/test_signal.py +++ b/tests/test_signal.py @@ -5,41 +5,19 @@ import xdas as xd import xdas.signal as xs -from xdas.synthetics import wavelet_wavefronts class TestSignal: def test_get_sample_spacing(self): - shape = (6000, 1000) - resolution = (np.timedelta64(8, "ms"), 5.0) - starttime = np.datetime64("2023-01-01T00:00:00") - da = xd.DataArray( - data=np.random.randn(*shape).astype("float32"), - coords={ - "time": { - "tie_indices": [0, shape[0] - 1], - "tie_values": [ - starttime, - starttime + resolution[0] * (shape[0] - 1), - ], - }, - "distance": { - "tie_indices": [0, shape[1] - 1], - "tie_values": [0.0, resolution[1] * (shape[1] - 1)], - }, - }, - ) - assert xs.get_sampling_interval(da, "time") == 0.008 - assert xs.get_sampling_interval(da, "distance") == 5.0 + da = xd.testing.dummy(shape=(6000, 1000), step=(0.008, 5.0), dtype="float32") + assert da.coords["time"].get_sampling_interval() == 0.008 + assert da.coords["distance"].get_sampling_interval() == 5.0 def test_deterend(self): - n = 100 - d = 5.0 - s = d * np.arange(n) - da = xr.DataArray(np.arange(n), {"time": s}) - da = xd.DataArray.from_xarray(da) + # dummy data is a linear ramp, so detrending must flatten it to zero + da = xd.testing.dummy(dims=("time",), shape=(100,), step=5.0, datetime=False) da = xs.detrend(da) - assert np.allclose(da.values, np.zeros(n)) + assert np.allclose(da.values, np.zeros(100)) def test_differentiate(self): n = 100 @@ -47,6 +25,7 @@ def test_differentiate(self): s = (d / 2) + d * np.arange(n) da = xr.DataArray(np.ones(n), {"distance": s}) da = xd.DataArray.from_xarray(da) + da["distance"] = da["distance"].to_regular() da = xs.differentiate(da, midpoints=True) assert np.allclose(da.values, np.zeros(n - 1)) @@ -56,6 +35,7 @@ def test_integrate(self): s = (d / 2) + d * np.arange(n) da = xr.DataArray(np.ones(n), {"distance": s}) da = xd.DataArray.from_xarray(da) + da["distance"] = da["distance"].to_regular() da = xs.integrate(da, midpoints=True) assert np.allclose(da.values, da["distance"].values) @@ -81,11 +61,12 @@ def test_sliding_window_removal(self): data = np.ones(n) da = xr.DataArray(data, {"distance": s}) da = xd.DataArray.from_xarray(da) + da["distance"] = da["distance"].to_regular() da = xs.sliding_mean_removal(da, 0.1 * n * d) assert np.allclose(da.values, 0) def test_medfilt(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result1 = xs.medfilt(da, {"distance": 3}) result2 = xs.medfilt(da, {"time": 1, "distance": 3}) assert result1.equals(result2) @@ -93,46 +74,46 @@ def test_medfilt(self): assert da.equals(xs.medfilt(da, {"time": 7, "distance": 3})) def test_hilbert(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = xs.hilbert(da, dim="time") assert np.allclose(da.values, np.real(result.values)) def test_resample(self): - da = wavelet_wavefronts() - result = xs.resample(da, 100, dim="time", window="hamming", domain="time") - assert result.sizes["time"] == 100 + da = xd.testing.dummy() + result = xs.resample(da, 50, dim="time", window="hamming", domain="time") + assert result.sizes["time"] == 50 def test_resample_poly(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = xs.resample_poly(da, 2, 5, dim="time") - assert result.sizes["time"] == 120 + assert result.sizes["time"] == 40 def test_lfilter(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() b, a = sp.iirfilter(4, 0.5, btype="low") result1 = xs.lfilter(b, a, da, "time") - result2, zf = xs.lfilter(b, a, da, "time", zi=...) + result2, _zf = xs.lfilter(b, a, da, "time", zi=...) assert result1.equals(result2) def test_filtfilt(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() b, a = sp.iirfilter(2, 0.5, btype="low") xs.filtfilt(b, a, da, "time", padtype=None) def test_sosfilter(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() sos = sp.iirfilter(4, 0.5, btype="low", output="sos") result1 = xs.sosfilt(sos, da, "time") - result2, zf = xs.sosfilt(sos, da, "time", zi=...) + result2, _zf = xs.sosfilt(sos, da, "time", zi=...) assert result1.equals(result2) def test_sosfiltfilt(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() sos = sp.iirfilter(2, 0.5, btype="low", output="sos") xs.sosfiltfilt(sos, da, "time", padtype=None) def test_filter(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() axis = da.get_axis_num("time") fs = 1 / xd.get_sampling_interval(da, "time") sos = sp.butter( @@ -168,7 +149,7 @@ def test_filter(self): assert result.equals(expected) def test_decimate_virtual_stack(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() expected = xs.decimate(da, 5, dim="time") chunks = xd.split(da, 5, "time") for i, chunk in enumerate(chunks): @@ -181,15 +162,7 @@ def test_decimate_virtual_stack(self, tmp_path): class TestSTFT: def test_compare_with_scipy(self): - starttime = np.datetime64("2023-01-01T00:00:00") - endtime = starttime + 9999 * np.timedelta64(10, "ms") - da = xd.DataArray( - data=np.random.rand(10000, 11), - coords={ - "time": {"tie_indices": [0, 9999], "tie_values": [starttime, endtime]}, - "distance": {"tie_indices": [0, 10], "tie_values": [0.0, 1.0]}, - }, - ) + da = xd.testing.dummy(shape=(10000, 11), step=(0.01, 0.1)) for scaling in ["spectrum", "psd"]: for return_onesided in [True, False]: for nfft in [None, 128]: @@ -205,7 +178,7 @@ def test_compare_with_scipy(self): ) f, t, Zxx = sp.stft( da.values, - fs=1 / xs.get_sampling_interval(da, "time"), + fs=1 / da.coords["time"].get_sampling_interval(), window="hamming", nperseg=100, noverlap=50, @@ -236,12 +209,10 @@ def test_retrieve_frequency_peak(self): N = 1e5 fc = 3e3 amp = 2 * np.sqrt(2) - time = np.arange(N) / float(fs) - data = amp * np.sin(2 * np.pi * fc * time) - da = xd.DataArray( - data=data, - coords={"time": time}, + da = xd.testing.dummy( + dims=("time",), shape=(int(N),), step=1 / fs, datetime=False ) + da.data = amp * np.sin(2 * np.pi * fc * da["time"].values) result = xs.stft( da, nperseg=1000, noverlap=500, window="hann", dim={"time": "frequency"} ) @@ -249,15 +220,7 @@ def test_retrieve_frequency_peak(self): assert result["frequency"][idx].values == fc def test_parrallel(self): - starttime = np.datetime64("2023-01-01T00:00:00") - endtime = starttime + 9999 * np.timedelta64(10, "ms") - da = xd.DataArray( - data=np.random.rand(10000, 11), - coords={ - "time": {"tie_indices": [0, 9999], "tie_values": [starttime, endtime]}, - "distance": {"tie_indices": [0, 10], "tie_values": [0.0, 1.0]}, - }, - ) + da = xd.testing.dummy(shape=(10000, 11), step=(0.01, 0.1)) serial = xs.stft( da, nperseg=100, @@ -277,16 +240,8 @@ def test_parrallel(self): assert serial.equals(parallel) def test_last_dimension_with_non_dimensional_coordinates(self): - starttime = np.datetime64("2023-01-01T00:00:00") - endtime = starttime + 99 * np.timedelta64(10, "ms") - da = xd.DataArray( - data=np.random.rand(100, 1001), - coords={ - "time": {"tie_indices": [0, 99], "tie_values": [starttime, endtime]}, - "distance": {"tie_indices": [0, 1000], "tie_values": [0.0, 10_000.0]}, - "channel": ("distance", np.arange(1001)), - }, - ) + da = xd.testing.dummy(shape=(100, 1001)) + da["channel"] = ("distance", np.arange(1001)) result = xs.stft( da, nperseg=100, @@ -296,7 +251,7 @@ def test_last_dimension_with_non_dimensional_coordinates(self): ) f, t, Zxx = sp.stft( da.values, - fs=1 / xs.get_sampling_interval(da, "distance"), + fs=1 / da.coords["distance"].get_sampling_interval(), window="hamming", nperseg=100, noverlap=50, @@ -313,51 +268,58 @@ def test_last_dimension_with_non_dimensional_coordinates(self): class TestSignalMissingBranches: def test_integrate_no_midpoints(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = xs.integrate(da, midpoints=False) assert result.shape == da.shape def test_differentiate_no_midpoints(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = xs.differentiate(da, midpoints=False) assert result.sizes["distance"] == da.sizes["distance"] - 1 def test_sliding_mean_removal_even_window(self): # When wlen/d gives an even n, sliding_mean_removal increments n by 1. - da = wavelet_wavefronts() - d = xs.get_sampling_interval(da, "time") + da = xd.testing.dummy() + d = da.coords["time"].get_sampling_interval() # Make wlen exactly twice d so n=2 (even) → becomes 3 result = xs.sliding_mean_removal(da, wlen=2 * d) assert result.shape == da.shape def test_medfilt_invalid_dim(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() with pytest.raises(ValueError, match="dims provided not in dataarray"): xs.medfilt(da, {"nonexistent_dim": 3}) def test_stft_default_noverlap(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = xs.stft(da, nperseg=16, dim={"time": "frequency"}) assert "frequency" in result.dims def test_stft_invalid_scaling(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() with pytest.raises(ValueError, match="Scaling must be"): xs.stft(da, nperseg=16, scaling="invalid", dim={"time": "frequency"}) def test_stft_nperseg_one(self): # nperseg=1, noverlap=0 triggers the stride_tricks bypass branch - da = wavelet_wavefronts() + da = xd.testing.dummy() # nfft=2 avoids single-element frequency axis (which would make tie_indices=[0,0]) result = xs.stft(da, nperseg=1, noverlap=0, nfft=2, dim={"time": "frequency"}) assert "frequency" in result.dims + def test_stft_default_dim(self): + # the default maps the last dimension; "first"/"last" aliases must resolve + da = xd.testing.dummy() + expected = xs.stft(da, nperseg=8, dim={"distance": "sprectrum"}) + assert xs.stft(da, nperseg=8).equals(expected) + assert xs.stft(da, nperseg=8, dim={"last": "sprectrum"}).equals(expected) + class TestFftMissingBranches: def test_fft_explicit_n(self): import xdas.fft as xfft - da = wavelet_wavefronts().isel(distance=0) + da = xd.testing.dummy().isel(distance=0) n = da.sizes["time"] // 2 result = xfft.fft(da, n=n, dim={"time": "frequency"}) assert result.sizes["frequency"] == n @@ -365,16 +327,90 @@ def test_fft_explicit_n(self): def test_rfft_explicit_n(self): import xdas.fft as xfft - da = wavelet_wavefronts().isel(distance=0) + da = xd.testing.dummy().isel(distance=0) n = da.sizes["time"] result = xfft.rfft(da, n=n, dim={"time": "frequency"}) assert "frequency" in result.dims + def test_rfft_single_frequency(self): + import xdas.fft as xfft + + da = xd.testing.dummy().isel(distance=0) + result = xfft.rfft(da, n=1, dim={"time": "frequency"}) + assert result.sizes["frequency"] == 1 + def test_ifft_explicit_n(self): import xdas.fft as xfft - da = wavelet_wavefronts().isel(distance=0) + da = xd.testing.dummy().isel(distance=0) spectrum = xfft.fft(da, dim={"time": "frequency"}) n = da.sizes["time"] result = xfft.ifft(spectrum, n=n, dim={"frequency": "time"}) assert result.sizes["time"] == n + + +class TestResampleTolerance: + """The resamplers derive a new rate; the declared jitter must survive it.""" + + @staticmethod + def regular(): + da = xd.testing.dummy(shape=(120, 3)) + da["time"] = da["time"].to_regular( + da["time"].sampling_interval, np.timedelta64(1, "s") + ) + return da + + def test_resample_poly_carries_declared_tolerance(self): + da = self.regular() + result = xs.resample_poly(da, 1, 2, dim="time") + assert result["time"].isregular() + assert result["time"].tolerance >= da["time"].tolerance + + def test_resample_poly_declares_representation_error(self): + # 1/3 of a 10 ms step is not representable in whole nanoseconds, so the + # truncation must be declared as jitter on top of the inherited bound. + da = self.regular() + delta = da["time"].sampling_interval + result = xs.resample_poly(da, 3, 1, dim="time") + step = result["time"].sampling_interval + assert result["time"].tolerance == da["time"].tolerance + np.abs( + delta - step * 3 + ) + + def test_resample_carries_declared_tolerance(self): + da = self.regular() + result = xs.resample(da, da.sizes["time"] // 2, dim="time") + assert result["time"].isregular() + assert result["time"].tolerance == da["time"].tolerance + + def test_resample_poly_on_irregular_coordinate(self): + da = xd.testing.dummy(shape=(120, 3)) + da["time"] = xd.Coordinate( + { + "tie_indices": da["time"].tie_indices, + "tie_values": da["time"].tie_values, + }, + "time", + ) + result = xs.resample_poly(da, 1, 2, dim="time") + assert result.sizes["time"] == 60 + + def test_resample_on_irregular_coordinate(self): + da = xd.testing.dummy(shape=(120, 3)) + da["time"] = xd.Coordinate( + { + "tie_indices": da["time"].tie_indices, + "tie_values": da["time"].tie_values, + }, + "time", + ) + result = xs.resample(da, 60, dim="time") + assert result.sizes["time"] == 60 + + @pytest.mark.parametrize("ctype", ["sampled", "dense"]) + def test_resamplers_on_non_interpolated_coordinates(self, ctype): + # Only interpolated coordinates declare a tolerance; the others must + # still resample without one. + da = xd.testing.dummy(shape=(120, 3), ctype=ctype) + assert xs.resample_poly(da, 1, 2, dim="time").sizes["time"] == 60 + assert xs.resample(da, 60, dim="time").sizes["time"] == 60 diff --git a/tests/test_testing.py b/tests/test_testing.py new file mode 100644 index 00000000..42a1ec11 --- /dev/null +++ b/tests/test_testing.py @@ -0,0 +1,25 @@ +import numpy as np +import pytest + +import xdas as xd + + +class TestDummy: + def test_defaults(self): + da = xd.testing.dummy() + assert da.shape == (100, 10) + assert da.dims == ("time", "distance") + assert da["time"].isregular() + assert da["distance"].isregular() + + def test_mismatched_shape(self): + with pytest.raises(ValueError, match="must equal len\\(shape\\)"): + xd.testing.dummy(dims=("time",), shape=(10, 10)) + + def test_mismatched_step(self): + with pytest.raises(ValueError, match="must equal len\\(dims\\)"): + xd.testing.dummy(step=(1.0,)) + + def test_datetime_step_passthrough(self): + da = xd.testing.dummy(step=(np.timedelta64(10, "ms"), 10.0)) + assert da["time"].get_sampling_interval() == 0.01 diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 086a8242..ff6b8ed0 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -1,9 +1,8 @@ import numpy as np import pandas as pd -import pytest import xdas as xd -from xdas.trigger import Trigger, _concat, _find_picks_numeric, find_picks +from xdas.trigger import Trigger, _find_picks_numeric, find_picks def test_trigger(): @@ -178,23 +177,3 @@ def test_trigger_1d(): picks = Trigger(thresh=0.5, dim="time")(cft) assert len(picks) == 2 assert list(picks["time"]) == [2.0, 7.0] - - -def test_concat_non_interp_coord(): - """_concat raises ValueError for non-interpolated coordinates.""" - from xdas.coordinates.sampled import SampledCoordinate - - coord1 = xd.Coordinate({"tie_indices": [0, 2], "tie_values": [10, 30]}, dim="dim") - coord_bad = SampledCoordinate( - {"tie_values": [0.0], "tie_lengths": [3], "sampling_interval": 1.0}, "dim" - ) - with pytest.raises(ValueError, match="interpolated"): - _concat([coord1, coord_bad]) - - -def test_concat_different_dims(): - """_concat raises ValueError when coords have different dims.""" - coord1 = xd.Coordinate({"tie_indices": [0, 2], "tie_values": [10, 30]}, dim="dim1") - coord2 = xd.Coordinate({"tie_indices": [0, 2], "tie_values": [40, 60]}, dim="dim2") - with pytest.raises(ValueError, match="same dimension"): - _concat([coord1, coord2]) diff --git a/tests/test_virtual.py b/tests/test_virtual.py index 00165758..503e4c8d 100644 --- a/tests/test_virtual.py +++ b/tests/test_virtual.py @@ -3,7 +3,6 @@ import pytest import xdas as xd -from xdas.synthetics import wavelet_wavefronts from xdas.virtual import ( Selection, Selectors, @@ -19,7 +18,7 @@ class TestFunctional: # TODO: move elsewhere def test_all(self, tmp_path): - expected = wavelet_wavefronts() + expected = xd.testing.dummy() chunks = xd.split(expected, 3) for index, chunk in enumerate(chunks, start=1): chunk.to_netcdf(tmp_path / f"{index:03d}.nc") @@ -115,7 +114,7 @@ def test_init(self, sources_from_data): assert stack.empty assert stack.shape == () with pytest.raises(AttributeError, match="no dtype"): - stack.dtype + _ = stack.dtype assert stack.ndim == 0 assert stack.size == 0 assert stack.nbytes == 0 @@ -355,7 +354,7 @@ def test_getitem_slice(self): sel = SliceSelector(5) assert isinstance(sel[0:1], SliceSelector) assert sel[:]._range == range(5) - assert sel[0:1]._range == range(0, 1) + assert sel[0:1]._range == range(1) assert sel[1:0]._range == range(1, 0) assert sel._range == range(5) sel = sel[1:-1] @@ -392,12 +391,16 @@ def test_get_indexer(self): class TestVirtualArrayAbstract: def test_abstract_stubs(self): va = VirtualArray() - va.__getitem__(0) - va.__array__() - _ = va.shape - _ = va.dtype - va.to_dataset(None, None) - assert isinstance(repr(va), str) + with pytest.raises(NotImplementedError): + _ = va[0] + with pytest.raises(NotImplementedError): + va.__array__() + with pytest.raises(NotImplementedError): + _ = va.shape + with pytest.raises(NotImplementedError): + _ = va.dtype + with pytest.raises(NotImplementedError): + va.to_dataset(None, None) class TestVirtualStackExtra: @@ -437,7 +440,7 @@ def test_check_dtype_mismatch(self, tmp_path): class TestVirtualLayoutExtra: def test_array_with_dtype(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() da.to_netcdf(tmp_path / "c.nc") da2 = xd.open(tmp_path / "c.nc") layout = da2.data._to_layout() @@ -445,7 +448,7 @@ def test_array_with_dtype(self, tmp_path): assert result.dtype == np.float32 def test_setitem_with_virtual_source(self, tmp_path): - da = wavelet_wavefronts() + da = xd.testing.dummy() da.to_netcdf(tmp_path / "d.nc") with h5py.File(tmp_path / "d.nc", "r") as f: src = VirtualSource(f["__values__"]) diff --git a/tests/test_xarray.py b/tests/test_xarray.py index 9e87a62a..9fc456da 100644 --- a/tests/test_xarray.py +++ b/tests/test_xarray.py @@ -2,12 +2,11 @@ import xdas as xd import xdas.core.methods as xm -from xdas.synthetics import wavelet_wavefronts class TestXarray: def test_returns_dataarray(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() for name, func in xm.HANDLED_METHODS.items(): if callable(func): if name in [ @@ -30,7 +29,7 @@ def test_returns_dataarray(self): assert isinstance(result, xd.DataArray) def test_mean(self): - da = wavelet_wavefronts() + da = xd.testing.dummy() result = xm.mean(da, "time") result_method = da.mean("time") expected = np.mean(da, 0) diff --git a/tests/test_xdas.py b/tests/test_xdas.py index 3af18b0b..d3fcf89d 100644 --- a/tests/test_xdas.py +++ b/tests/test_xdas.py @@ -1,9 +1,12 @@ +import re + import xdas as xd +# Release segment, optionally followed by a PEP 440 pre-release marker (e.g. 0.2.8rc0). +VERSION_PATTERN = re.compile(r"^\d+(\.\d+)*((a|b|rc)\d+)?$") + def test_version(): version = xd.__version__ assert isinstance(version, str) - version_parts = version.split(".") - for part in version_parts: - assert part.isdigit() + assert VERSION_PATTERN.match(version) diff --git a/xdas/__init__.py b/xdas/__init__.py index d4f1e09c..15ed61bf 100644 --- a/xdas/__init__.py +++ b/xdas/__init__.py @@ -6,9 +6,9 @@ for common DAS instrument formats. """ -__version__ = "0.2.7" +__version__ = "0.2.8rc0" -__all__ = [ +__all__ = [ # noqa: RUF022 - grouped by kind, not alphabetically # submodules "atoms", "config", @@ -24,6 +24,7 @@ "routines", "signal", "synthetics", + "testing", "virtual", # classes "Coordinate", @@ -32,7 +33,6 @@ "DataCollection", "DataMapping", "DataSequence", - "DefaultCoordinate", "DenseCoordinate", "InterpCoordinate", "SampledCoordinate", @@ -68,23 +68,23 @@ processing, signal, synthetics, + testing, virtual, ) from .coordinates import ( Coordinate, Coordinates, - DefaultCoordinate, DenseCoordinate, InterpCoordinate, SampledCoordinate, ScalarCoordinate, get_sampling_interval, ) -from .core import dataarray, datacollection, methods, numpy, routines -from .core.dataarray import DataArray -from .core.datacollection import DataCollection, DataMapping, DataSequence -from .core.methods import * # noqa: F403 -from .core.routines import ( +from .core import ( + DataArray, + DataCollection, + DataMapping, + DataSequence, align, asdataarray, broadcast_coords, @@ -94,6 +94,10 @@ concat, concat_coords, concatenate, + dataarray, + datacollection, + methods, + numpy, open, open_dataarray, open_datacollection, @@ -101,5 +105,7 @@ open_mfdatacollection, open_mfdatatree, plot_availability, + routines, split, ) +from .core.methods import * diff --git a/xdas/atoms/core.py b/xdas/atoms/core.py index e3cea04c..6bf0c88d 100644 --- a/xdas/atoms/core.py +++ b/xdas/atoms/core.py @@ -10,9 +10,7 @@ from functools import wraps from typing import Any -from ..core.dataarray import DataArray -from ..core.datacollection import DataCollection -from ..core.routines import open_datacollection +from ..core import DataArray, DataCollection, open_datacollection class State: @@ -171,7 +169,7 @@ def reset(self): """Reset all state entries to ``...`` (uninitialised sentinel).""" for key in self._state: setattr(self, key, State(...)) - for _, filter in self._atoms.items(): + for filter in self._atoms.values(): filter.reset() def save_state(self, path): @@ -311,12 +309,6 @@ def __repr__(self) -> str: s += "\n".join(f" {e}" for e in repr(value).split("\n")[:-1]) + "\n" return s - def reset(self) -> None: - """Reset the state of all stateful atoms in the sequence.""" - for atom in self: - if isinstance(atom, Partial): - atom.reset() - class Partial(Atom): """ diff --git a/xdas/atoms/ml.py b/xdas/atoms/ml.py index c953f921..8d892b1f 100644 --- a/xdas/atoms/ml.py +++ b/xdas/atoms/ml.py @@ -8,8 +8,7 @@ import numpy as np -from ..core.dataarray import DataArray -from ..core.routines import concat +from ..core import DataArray, concat from .core import Atom, State @@ -114,7 +113,7 @@ def blinding(self): def initialize(self, da, chunk_dim=None, **flags): """Allocate circular buffers sized to *da*'s batch and segment dimensions.""" self.batch_size = State( - np.prod([size for dim, size in da.sizes.items() if not dim == self.dim]) + np.prod([size for dim, size in da.sizes.items() if dim != self.dim]) ) self.circular_input = State( torch.zeros( @@ -214,7 +213,7 @@ def _attach_metadata(self, data, da, idx): coords = da.coords.copy() coords[self.dim] = coords[self.dim][idx : idx + self.step] coords["phase"] = self.phases - dims = tuple(dim for dim in da.dims if not dim == self.dim) + ( + dims = tuple(dim for dim in da.dims if dim != self.dim) + ( "phase", self.dim, ) diff --git a/xdas/atoms/signal.py b/xdas/atoms/signal.py index a077573d..36cdc4d2 100644 --- a/xdas/atoms/signal.py +++ b/xdas/atoms/signal.py @@ -10,9 +10,8 @@ import numpy as np import scipy.signal as sp -from ..coordinates.core import Coordinate, get_sampling_interval -from ..core.dataarray import DataArray -from ..core.routines import concat, split +from ..coordinates import Coordinate, get_sampling_interval +from ..core import DataArray, concat, split from ..parallel import parallelize from .core import Atom, State @@ -569,15 +568,23 @@ def call(self, da, **flags): data[slc] = da.values coords = da.coords.copy() delta = get_sampling_interval(da, self.dim, cast=False) - tie_indices = coords[self.dim].tie_indices * self.factor - tie_values = coords[self.dim].tie_values + new_delta = delta / self.factor + coord = coords[self.dim] + tie_indices = coord.tie_indices * self.factor + tie_values = coord.tie_values tie_indices[-1] += self.factor - 1 - tie_values[-1] += (self.factor - 1) / self.factor * delta - coords[self.dim] = Coordinate( - { - "tie_indices": tie_indices, - "tie_values": tie_values, - }, - self.dim, - ) + tie_values[-1] += (self.factor - 1) * new_delta + data_coord = {"tie_indices": tie_indices, "tie_values": tie_values} + if coord.isregular(): + # The derived rate may not be exactly representable (integer datetime + # resolutions truncate), so declare the representation error as jitter + # on top of the inherited one; chunk seams then stay within tolerance. + data_coord["sampling_interval"] = new_delta + data_coord["tolerance"] = coord.tolerance + np.abs( + delta - new_delta * self.factor + ) + # An irregular input gives no rate to inherit and no jitter bound to + # derive one from, so the result stays irregular rather than claiming a + # precision the source never declared. + coords[self.dim] = Coordinate(data_coord, self.dim) return DataArray(data, coords, da.dims, da.name, da.attrs) diff --git a/xdas/config.py b/xdas/config.py index 3e3a1ace..c91ea3e3 100644 --- a/xdas/config.py +++ b/xdas/config.py @@ -5,12 +5,13 @@ """ import os +from typing import ClassVar class Config: """Global configuration store backed by a plain dict.""" - config = {"n_workers": os.cpu_count()} + config: ClassVar[dict] = {"n_workers": os.cpu_count()} def get(key): diff --git a/xdas/coordinates/__init__.py b/xdas/coordinates/__init__.py index 1bcd6ae6..82ad4c7d 100644 --- a/xdas/coordinates/__init__.py +++ b/xdas/coordinates/__init__.py @@ -2,15 +2,14 @@ Coordinate types that describe how array axes map to physical values. Exports :class:`Coordinates` (container) and all concrete coordinate classes: -:class:`Coordinate` (factory/base), :class:`DefaultCoordinate`, -:class:`DenseCoordinate`, :class:`InterpCoordinate`, -:class:`SampledCoordinate`, :class:`ScalarCoordinate`. +:class:`Coordinate` (factory/base), :class:`DenseCoordinate`, +:class:`InterpCoordinate`, :class:`SampledCoordinate`, :class:`ScalarCoordinate`. """ __all__ = [ + "AxisCoordinate", "Coordinate", "Coordinates", - "DefaultCoordinate", "DenseCoordinate", "InterpCoordinate", "SampledCoordinate", @@ -18,8 +17,12 @@ "get_sampling_interval", ] -from .core import Coordinate, Coordinates, get_sampling_interval -from .default import DefaultCoordinate +from .core import ( + AxisCoordinate, + Coordinate, + Coordinates, + get_sampling_interval, +) from .dense import DenseCoordinate from .interp import InterpCoordinate from .sampled import SampledCoordinate diff --git a/xdas/coordinates/core.py b/xdas/coordinates/core.py index 5b6928dc..f76a7187 100644 --- a/xdas/coordinates/core.py +++ b/xdas/coordinates/core.py @@ -2,17 +2,33 @@ Core coordinate infrastructure. Includes the :class:`Coordinates` container, :class:`Coordinate` factory/base -class, and shared helpers used by all concrete coordinate types (parsing, -interpolation, tolerance handling). +class, :class:`AxisCoordinate` (the axis-mapping ABC), and shared helpers used by +all concrete coordinate types (parsing, interpolation, tolerance handling). """ +import warnings import weakref +from abc import ABC, abstractmethod from copy import copy, deepcopy from functools import wraps from itertools import pairwise +from typing import ClassVar import numpy as np import pandas as pd +from typing_extensions import override + +#: Mapping from numpy datetime64/timedelta64 unit codes to CF-style unit names, +#: used to serialise timedelta scalars into dataset attributes. +CODE_TO_UNITS = { + "h": "hours", + "m": "minutes", + "s": "seconds", + "ms": "milliseconds", + "us": "microseconds", + "ns": "nanoseconds", +} +UNITS_TO_CODE = {v: k for k, v in CODE_TO_UNITS.items()} def wraps_first_last(func): @@ -30,6 +46,20 @@ def wrapper(self, dim, *args, **kwargs): return wrapper +def wraps_first_last_all(func): + """Resolve ``"first"`` and ``"last"`` aliases in every positional argument.""" + + @wraps(func) + def wrapper(self, *args, **kwargs): + resolved = tuple( + self._dims[0] if a == "first" else (self._dims[-1] if a == "last" else a) + for a in args + ) + return func(self, *resolved, **kwargs) + + return wrapper + + class Coordinates(dict): """ Dictionary like container for coordinates. @@ -46,7 +76,7 @@ class Coordinates(dict): which are assumed to be a dimensional coordinate with `dim` set to the related name. - dims: squence of str, optional + dims: sequence of str, optional An ordered sequence of dimensions. It is meant to match the dimensionality of its associated data. If provided, it must at least include all dimensions found in `coords` (extras dimensions will be considered as empty coordinates). @@ -92,7 +122,7 @@ def __setitem__(self, key, value): if not isinstance(key, str): raise TypeError("dimension names must be of type str") coord = Coordinate(value) - if coord.dim is None and not coord.isscalar(): + if coord.dim is None and isinstance(coord, AxisCoordinate): coord.dim = key if self.parent is None: if coord.dim is not None and coord.dim not in self.dims: @@ -144,7 +174,7 @@ def isdim(self, name): """Return ``True`` if *name* is a dimensional coordinate (i.e. its dim equals its name).""" return self[name].dim == name - def get_query(self, item): + def _get_query(self, item): """ Format a query from one or multiple indexer. @@ -173,11 +203,11 @@ def get_query(self, item): query[self.dims[k]] = item[k] else: query[self.dims[0]] = item - for dim, item in query.items(): - if isinstance(item, tuple): - msg = f"cannot use tuple {item} to index dim '{dim}'" - if len(item) == 2: - msg += f". Did you mean: {dim}=slice({item[0]}, {item[1]})?" + for dim, indexer in query.items(): + if isinstance(indexer, tuple): + msg = f"cannot use tuple {indexer} to index dim '{dim}'" + if len(indexer) == 2: + msg += f". Did you mean: {dim}=slice({indexer[0]}, {indexer[1]})?" raise TypeError(msg) return query @@ -199,62 +229,24 @@ def to_index(self, item, method=None, endpoint=True): dict Mapping from dimension name to integer index or slice. """ - query = self.get_query(item) + query = self._get_query(item) return {dim: self[dim].to_index(query[dim], method, endpoint) for dim in query} def equals(self, other): """Return ``True`` if *other* is a :class:`Coordinates` with identical coordinate values.""" if not isinstance(other, Coordinates): return False + if set(self) != set(other): + return False for name in self: if not self[name].equals(other[name]): return False return True - def to_dict(self): - """Convert this `Coordinates` object into a pure python dictionnary. - - Examples - -------- - >>> import xdas as xd - - >>> coords = xd.Coordinates( - ... { - ... "time": {"tie_indices": [0, 999], "tie_values": [0.0, 10.0]}, - ... "distance": [0, 1, 2], - ... "channel": ("distance", ["DAS01", "DAS02", "DAS03"]), - ... "interrogator": (None, "SRN"), - ... } - ... ) - >>> coords.to_dict() - {'dims': ('time', 'distance'), - 'coords': {'time': {'dim': 'time', - 'data': {'tie_indices': [0, 999], 'tie_values': [0.0, 10.0]}, - 'dtype': 'float64'}, - 'distance': {'dim': 'distance', 'data': [0, 1, 2], 'dtype': 'int64'}, - 'channel': {'dim': 'distance', - 'data': ['DAS01', 'DAS02', 'DAS03'], - 'dtype': ' zero) + # --- queries --- def isdim(self): - """Return ``True`` if this coordinate is a dimensional coordinate in its parent container.""" + """Return ``True`` if this coordinate is a dimensional coordinate.""" if self.parent is None or self.name is None: return None else: return self.parent.isdim(self.name) + def isregular(self): + """ + Return ``True`` if this coordinate carries a nominal sampling interval. + + Scalar coordinates are never regular. Axis coordinates are regular when + :meth:`AxisCoordinate.get_sampling_interval` returns a value, i.e. when + an explicit nominal spacing is part of their data. + """ + return False + + def equals(self, other): + """Return ``True`` if *other* is the same coordinate type with identical dim and data. + + Comparison is strict on dtype. Same type implies same ``data`` structure: + either a single ``np.ndarray`` or a flat ``dict[str, np.ndarray]`` with + the same keys. + """ + if type(self) is not type(other) or self.dim != other.dim: + return False + a, b = self.data, other.data + if isinstance(a, dict): + pairs = [(a[key], b[key]) for key in a] + else: + pairs = [(a, b)] + for x, y in pairs: + x, y = np.asarray(x), np.asarray(y) + if x.dtype != y.dtype or not np.array_equal(x, y, equal_nan=False): + return False + return True + + # --- routines --- + def copy(self, deep=True): """ Return a copy of this coordinate. @@ -470,6 +506,11 @@ def copy(self, deep=True): ---------- deep : bool, optional If ``True`` (default) perform a deep copy; otherwise a shallow copy. + + Returns + ------- + Coordinate + A new coordinate of the same subclass with copied data and metadata. """ if deep: func = deepcopy @@ -477,142 +518,401 @@ def copy(self, deep=True): func = copy return self.__class__(func(self.data), func(self.dim), func(self.dtype)) - def equals(self, other): - """Return ``True`` if *other* represents the same coordinate values. Subclass must implement.""" - raise NotImplementedError + # --- IO --- - def to_index(self, item, method=None, endpoint=True): + @classmethod + def _from_dataset(cls, dataset, name): + """Read coordinates named *name* from an xarray *dataset* via each registered subclass.""" + coords = {} + for subcls in cls._registry.values(): + coords |= subcls._collect_from_dataset(dataset, name) + return coords + + # --- internals --- + + def _assign_parent(self, parent): + """Attach this coordinate to its parent :class:`Coordinates` container.""" + self._parent = weakref.ref(parent) + + +class AxisCoordinate(Coordinate, ABC): + """ + Base class for coordinates that map an array axis to physical values. + + Adds the full axis-mapping contract on top of :class:`Coordinate`: it + supports two complementary directions of lookup: + + - **Index-based selection** — ``coord[i]`` or ``coord[start:stop]``: + given integer position(s), return the corresponding physical value(s) + as a new coordinate. + - **Label-based selection** — ``coord.to_index(v)``: given a physical + value (or slice of values), return the integer index (or slice) at + that label. An optional *method* argument controls nearest/forward/ + backward matching for values that fall between samples. The returned + index can then be passed to ``coord[idx]`` to retrieve the + coordinate subset, and is also used internally to index into the + parent data array. + + Concrete subclasses are :class:`DenseCoordinate`, :class:`InterpCoordinate`, + and :class:`SampledCoordinate`. + """ + + # --- abstract contract --- + + @classmethod + @abstractmethod + def from_block(cls, start, size, step, dim=None, dtype=None): """ - Convert a label-based selector to an integer index or slice. + Construct a coordinate from a start value, element count, and step size. Parameters ---------- - item : label, slice, or array-like - Selector to resolve. - method : str, optional - Look-up method (e.g. ``"ffill"``, ``"bfill"``). - endpoint : bool, optional - Whether to include the stop of a slice. Default ``True``. + start : scalar + Value of the first element. + size : int + Number of elements. + step : scalar + Spacing between consecutive elements. + dim : str, optional + Dimension name. + dtype : dtype-like, optional + Desired dtype for the coordinate values. Returns ------- - int or slice + Coordinate + A new coordinate instance of this subclass. """ - if isinstance(item, slice): - return self.slice_indexer(item.start, item.stop, item.step, endpoint) - else: - return self.get_indexer(item, method) - def format_index(self, idx, bounds="raise"): + @abstractmethod + def __len__(self): + """Return the number of elements along this coordinate's axis.""" + + @abstractmethod + def _is_monotonic_increasing(self): + """Return ``True`` if all consecutive differences in this coordinate are positive.""" + + @abstractmethod + def _get_value(self, index): """ - Normalise integer index *idx*, handling negative indices and optional bounds checking. + Return the coordinate value(s) at integer *index*. Parameters ---------- - idx : int or array-like of int - Index or indices to normalise. - bounds : {"raise", "clip"}, optional - ``"raise"`` (default) raises :exc:`IndexError` for out-of-bounds indices; - ``"clip"`` clamps them to the valid range. + index : int or numpy.ndarray of int + Non-negative integer index or array of indices. Returns ------- - numpy.ndarray - Non-negative integer index array. + scalar or numpy.ndarray + Coordinate value(s) at the requested position(s). """ - idx = np.asarray(idx) - if not np.issubdtype(idx.dtype, np.integer): - raise IndexError("only integer are valid index") - idx = idx + (idx < 0) * len(self) - if bounds == "raise": - if np.any(idx < 0) or np.any(idx >= len(self)): - raise IndexError("index is out of bounds") - elif bounds == "clip": - idx = np.clip(idx, 0, len(self)) - return idx - def slice_indexer(self, start=None, stop=None, step=None, endpoint=True): + @abstractmethod + def _get_indexer(self, value, method=None): """ - Return an integer :class:`slice` corresponding to the label range [*start*, *stop*]. + Return the integer index for label *value* using the segment structure. Parameters ---------- - start : label, optional - First label to include (inclusive, via ``"bfill"`` look-up). - stop : label, optional - Last label to include (inclusive by default, via ``"ffill"`` look-up). - step : not supported - Reserved; raises :exc:`NotImplementedError` if provided. - endpoint : bool, optional - If ``True`` (default), include *stop* in the result. + value : scalar, str (ISO datetime), or array-like + Label(s) to locate. + method : {None, "nearest", "ffill", "bfill"}, optional + How to handle values that fall in gaps or between samples. + ``None`` (default) requires an exact match and raises ``KeyError`` + if the value is not present. ``"nearest"`` returns the index of + the closest label. ``"ffill"`` (forward-fill) returns the last + index whose label is less than or equal to *value*. ``"bfill"`` + (backward-fill) returns the first index whose label is greater + than or equal to *value*. Returns ------- - slice + int or numpy.ndarray + + Raises + ------ + KeyError + If *value* falls in an overlap region or is not found (exact mode). """ - if start is not None: - try: - start_index = self.get_indexer(start, method="bfill") - except KeyError: - start_index = len(self) + + @abstractmethod + def _slice(self, slc): + """ + Return a new coordinate covering the integer slice *slc*. + + Parameters + ---------- + slc : slice + Integer slice (already normalised by the caller). + + Returns + ------- + Coordinate + A new coordinate of the same subclass. + """ + + @abstractmethod + def _concat(self, other): + """ + Return a new coordinate formed by appending *other* after this one. + + Parameters + ---------- + other : Coordinate + Must be the same subclass and have the same ``dim`` and ``dtype``. + + Returns + ------- + Coordinate + Concatenated coordinate of the same subclass. + """ + + @abstractmethod + def get_sampling_interval(self, cast=True): + """ + Return the nominal sample spacing for this coordinate, or ``None``. + + Parameters + ---------- + cast : bool, optional + If ``True`` (default), cast timedelta64 results to seconds (float). + + Returns + ------- + float or None + ``None`` if the coordinate has fewer than two elements or has no + defined sampling interval. + """ + + @abstractmethod + def to_regular(self, sampling_interval=None, tolerance=None): + """ + Return a regular version of this coordinate, raising when impossible. + + The strict conversion entry point: the result always satisfies + :meth:`isregular` (it carries a nominal ``sampling_interval``), or a + :exc:`ValueError` is raised when the coordinate values cannot be + described by a single spacing within *tolerance*. + + Parameters + ---------- + sampling_interval : scalar, optional + Nominal sample spacing to enforce. When omitted it is taken from + the coordinate itself when available, or inferred from the values. + tolerance : scalar, optional + Tolerated jitter around *sampling_interval*. Defaults to the + coordinate's declared tolerance when present, else a zero-like + default (exact zero for datetime axes, a dtype epsilon for floats), + so a genuinely irregular axis raises. + + Returns + ------- + AxisCoordinate + A regular coordinate. The subclass may change: a + :class:`DenseCoordinate` converts to a regular + :class:`InterpCoordinate`. + """ + + @abstractmethod + def _split_candidates(self): + """ + Return the candidate segment boundaries used by :meth:`get_split_indices`. + + Returns + ------- + positions : numpy.ndarray + Integer index of each candidate boundary. Each ``positions[k]`` + marks the start of a new segment (the boundary lies between element + ``positions[k] - 1`` and ``positions[k]``). + deltas : numpy.ndarray + Signed jump at each candidate, i.e. the value step across the + boundary minus the nominal sampling interval. Positive values are + gaps, negative values are overlaps, zero means a clean continuation. + """ + + @abstractmethod + def simplify(self, tolerance=None, *, reduce=True, regularize=False): + """ + Return the simplest faithful copy of this coordinate within *tolerance*. + + A coordinate carries two independent, orthogonal properties: + + - **Monotonic** — tie values strictly increase, which is what makes + label-based selection (:meth:`to_index`) work. + - **Regular** — every continuous segment fits a single nominal + ``sampling_interval``, which signal-processing routines (FFT, + filtering, resampling) need for a clean sample rate. + + ``simplify`` spends an accuracy budget *tolerance* across two + independently toggleable stages: + + - *reduce* drops redundant points whose removal shifts the curve by no + more than *tolerance* (which also absorbs soft gaps and overlaps, + helping monotonicity). Surviving values are never moved. + - *regularize* promotes the coordinate to *regular* when the surviving + continuous segments admit a single spacing within *tolerance*. + + Parameters + ---------- + tolerance : float, timedelta, None, or ``False``, optional + Accuracy budget; maximum allowed deviation from the original + values. ``None`` (default) spends the coordinate's own declared + tolerance when it carries one, else a zero-like default (exact + zero for datetime axes, a dtype epsilon for floats). ``False`` + returns an unchanged copy regardless of the flags below. + reduce : bool, optional + Whether to drop redundant tie points. Default ``True``. + regularize : bool, optional + Whether to try to acquire a nominal ``sampling_interval``. Default + ``False`` (opt-in). A no-op for coordinates that are already regular + by construction or cannot become regular. + + Returns + ------- + Coordinate + A new coordinate of the same subclass. + """ + + # --- properties --- + + @property + def ndim(self): + """Number of dimensions (always 1 for dimensional coordinates).""" + return 1 + + @property + def shape(self): + """Shape tuple ``(len(self),)``.""" + return (len(self),) + + @property + def empty(self): + """``True`` if the coordinate has zero length.""" + return len(self) == 0 + + @property + def indices(self): + """Integer array ``[0, 1, ..., len(self) - 1]``.""" + return np.arange(len(self)) + + @property + def start(self): + """Value at index 0 (first element).""" + return self._get_value(0) + + @property + def end(self): + """Value at the last element.""" + return self._get_value(len(self) - 1) + + # --- dunders logic --- + + def __getitem__(self, item): + if isinstance(item, slice): + return self._slice(self._format_slice(item)) else: - start_index = None - if stop is not None: - try: - end_index = self.get_indexer(stop, method="ffill") - stop_index = end_index + 1 - except KeyError: - stop_index = 0 + item = self._format_index(item) + return Coordinate( + self._get_value(item), None if np.ndim(item) == 0 else self.dim + ) + + def __array__(self, dtype=None, copy=None): + if self.empty: + out = np.array([], dtype=self.dtype) else: - stop_index = None - if step is not None: - raise NotImplementedError("cannot use step yet") - if ( - (not endpoint) - and (stop is not None) - and (self[stop_index - 1].values == stop) - ): - stop_index -= 1 - return slice(start_index, stop_index) + out = self._get_value(self.indices) + if dtype is not None: + out = out.__array__(dtype) + return out - def isscalar(self): - """Return ``True`` if this is a :class:`ScalarCoordinate` (non-dimensional).""" - return False + def __repr__(self): + if self.empty: + return "empty coordinate" + elif len(self) == 1: + return f"{self.tie_values[0]}" + else: + if np.issubdtype(self.dtype, np.floating): + return f"{self.start:.3f} to {self.end:.3f}" + elif np.issubdtype(self.dtype, np.datetime64): + start_str = format_datetime(self.start) + end_str = format_datetime(self.end) + return f"{start_str} to {end_str}" + else: + return f"{self.start} to {self.end}" - def isdefault(self): - """Return ``True`` if this is a :class:`DefaultCoordinate` (integer range).""" - return False + # --- queries --- - def isdense(self): - """Return ``True`` if this is a :class:`DenseCoordinate` (explicit numpy array).""" - return False + @override + def isregular(self): + return self.get_sampling_interval() is not None - def isinterp(self): - """Return ``True`` if this is an :class:`InterpCoordinate` (piecewise-linear).""" - return False + def get_split_indices(self, kind="discontinuities", tolerance=False): + """ + Return integer indices where this coordinate should be split. - def issampled(self): - """Return ``True`` if this is a :class:`SampledCoordinate` (regularly sampled).""" - return False + Each returned index ``i`` marks the start of a new segment: the + boundary lies between element ``i - 1`` and element ``i``. The first + segment always starts at index 0, so 0 is never included in the result. - def concat(self, other): - """Concatenate *other* coordinate to this one. Subclass must implement.""" - raise NotImplementedError(f"concat is not implemented for {self.__class__}") + Parameters + ---------- + kind : {"discontinuities", "gaps", "overlaps"}, optional + Which boundary type to return. ``"gaps"`` returns only boundaries + where the axis jumps forward by more than one sampling interval; + ``"overlaps"`` returns only boundaries where the axis jumps + backward. ``"discontinuities"`` (default) returns both. + tolerance : float, timedelta, None, or ``False``, optional + Minimum absolute magnitude of the jump to report. Boundaries + smaller than *tolerance* are silently dropped. ``None`` removes + only zero-magnitude jumps (i.e. consecutive equal values). + ``False`` (default) disables magnitude filtering and returns all + boundaries of the requested kind. - def simplify(self, tolerance=None): - """Reduce tie-point count within *tolerance*. Subclass must implement.""" - raise NotImplementedError(f"simplify is not implemented for {self.__class__}") + Returns + ------- + numpy.ndarray + Integer indices of the start of each new segment (excluding the first). + """ + valid_kinds = {"discontinuities", "gaps", "overlaps"} + if kind not in valid_kinds: + raise ValueError(f"`kind` must be one of {valid_kinds}; got {kind!r}") + + positions, deltas = self._split_candidates() + + # Fast path: every candidate boundary is a discontinuity by construction + if kind == "discontinuities" and tolerance is False: + return positions + + if tolerance is False: + zero = np.timedelta64(0) if np.issubdtype(self.dtype, np.datetime64) else 0 + match kind: + case "gaps": + mask = deltas >= zero + case "overlaps": # pragma: no branch + mask = deltas < zero + else: + tolerance = parse_scalar_delta(tolerance, self.dtype, default_zero=True) + match kind: + case "discontinuities": + mask = np.abs(deltas) > tolerance + case "gaps": + mask = deltas > tolerance + case "overlaps": # pragma: no branch + mask = deltas < -tolerance - def get_split_indices(self, kind="discontinuities", tolerance=False): - """Return integer indices where this coordinate should be split. Subclass must implement.""" - raise NotImplementedError( - f"get_split_indices is not implemented for {self.__class__}" - ) + return positions[mask] def get_discontinuities(self, tolerance=None): """ Return a DataFrame containing information about the discontinuities. + Parameters + ---------- + tolerance : float, timedelta, or None, optional + Minimum magnitude of a gap or overlap to include. ``None`` + (default) reports all discontinuities regardless of size. + Returns ------- pandas.DataFrame @@ -648,8 +948,8 @@ def get_discontinuities(self, tolerance=None): for index in indices: start_index = index end_index = index + 1 - start_value = self.get_value(index) - end_value = self.get_value(index + 1) + start_value = self._get_value(index) + end_value = self._get_value(index + 1) delta = end_value - start_value if tolerance is not None and np.abs(delta) < tolerance: continue @@ -658,7 +958,7 @@ def get_discontinuities(self, tolerance=None): "end_index": end_index, "start_value": start_value, "end_value": end_value, - "delta": end_value - start_value, + "delta": delta, "type": ("gap" if end_value > start_value else "overlap"), } records.append(record) @@ -702,8 +1002,8 @@ def get_availabilities(self): records = [] for start_index, stop_index in pairwise(indices): end_index = stop_index - 1 - start_value = self.get_value(start_index) - end_value = self.get_value(end_index) + start_value = self._get_value(start_index) + end_value = self._get_value(end_index) records.append( { "start_index": start_index, @@ -716,9 +1016,131 @@ def get_availabilities(self): ) return pd.DataFrame.from_records(records) + # --- selection / indexing --- + + def to_index(self, item, method=None, endpoint=True): + """ + Convert a label-based selector to an integer index or slice. + + Parameters + ---------- + item : label, slice, or array-like + Selector to resolve. + method : {None, "nearest", "ffill", "bfill"}, optional + How to resolve *item* when it does not match a label exactly. + ``None`` (default) requires an exact match. ``"nearest"`` selects + the closest label. ``"ffill"`` selects the last label ≤ *item*; + ``"bfill"`` selects the first label ≥ *item*. Ignored when *item* + is a slice. + endpoint : bool, optional + Whether to include the stop of a slice. Default ``True``. + + Returns + ------- + int, array of ints or slice + """ + if isinstance(item, slice): + return self._slice_indexer(item.start, item.stop, item.step, endpoint) + else: + return self._get_indexer(item, method) + + def _format_index(self, idx, bounds="raise"): + """ + Normalise integer index *idx*, handling negative indices and optional bounds checking. + + Parameters + ---------- + idx : int or array-like of int + Index or indices to normalise. + bounds : {"raise", "clip"}, optional + ``"raise"`` (default) raises :exc:`IndexError` for out-of-bounds indices; + ``"clip"`` clamps them to the valid range. + + Returns + ------- + numpy.ndarray + Non-negative integer index array. + """ + idx = np.asarray(idx) + if not np.issubdtype(idx.dtype, np.integer): + raise IndexError("only integer are valid index") + idx = idx + (idx < 0) * len(self) + if bounds == "raise": + if np.any(idx < 0) or np.any(idx >= len(self)): + raise IndexError("index is out of bounds") + elif bounds == "clip": + idx = np.clip(idx, 0, len(self)) + return idx + + def _format_slice(self, slc): + """ + Normalise *slc*, resolving ``None`` bounds, negative indices, and out-of-bounds. + + Parameters + ---------- + slc : slice + Raw slice, as received from user code. + + Returns + ------- + slice + Concrete ``slice(start, stop, step)`` with non-negative integer bounds + clipped to ``[0, len(self)]``. + """ + start, stop, step = slc.indices(len(self)) + if step < 0: + raise NotImplementedError("negative slice step is not implemented") + return slice(start, stop, step) + + def _slice_indexer(self, start=None, stop=None, step=None, endpoint=True): + """ + Return an integer :class:`slice` corresponding to the label range [*start*, *stop*]. + + Parameters + ---------- + start : label, optional + First label to include (inclusive, via ``"bfill"`` look-up). + stop : label, optional + Last label to include (inclusive by default, via ``"ffill"`` look-up). + step : not supported + Reserved; raises :exc:`NotImplementedError` if provided. + endpoint : bool, optional + If ``True`` (default), include *stop* in the result. + + Returns + ------- + slice + """ + if start is not None: + try: + start_index = self._get_indexer(start, method="bfill") + except KeyError: + start_index = len(self) + else: + start_index = None + if stop is not None: + try: + end_index = self._get_indexer(stop, method="ffill") + stop_index = end_index + 1 + except KeyError: + stop_index = 0 + else: + stop_index = None + if step is not None: + raise NotImplementedError("cannot use step yet") + if ( + (not endpoint) + and (stop is not None) + and (self[stop_index - 1].values == stop) + ): + stop_index -= 1 + return slice(start_index, stop_index) + + # --- routines --- + def to_dataarray(self): """Convert this coordinate to a :class:`~xdas.DataArray` with a single dimension.""" - from ..core.dataarray import DataArray # TODO: avoid defered import? + from ..core import DataArray # TODO: avoid deferred import? if self.name is None: raise ValueError("cannot convert unnamed coordinate to DataArray") @@ -742,38 +1164,8 @@ def to_dataarray(self): name=self.name, ) - def to_dict(self): - """Serialise this coordinate to a plain-dict representation. Subclass must implement.""" - raise NotImplementedError - - @classmethod - def from_dict(cls, dct): - """Reconstruct a coordinate from the dict returned by :meth:`to_dict`.""" - return cls(**dct) - - def to_dataset(self, dataset, attrs): - """Write this coordinate into an xarray *dataset*, updating *attrs* in place.""" - dataset = dataset.assign_coords( - {self.name: (self.dim, self.values) if self.dim else self.values} - ) - return dataset, attrs - @classmethod - def from_dataset(cls, dataset, name): - """Read coordinates named *name* from an xarray *dataset* via each registered subclass.""" - coords = {} - for subcls in cls.__subclasses__(): - if hasattr(subcls, "from_dataset"): # pragma: no branch - coords |= subcls.from_dataset(dataset, name) - return coords - - @classmethod - def from_block(cls, start, size, step, dim=None, dtype=None): - """Construct a coordinate from a start value, element count, and step size. Subclass must implement.""" - raise NotImplementedError - - -def parse(data, dim=None): +def parse_data_dim(data, dim=None): """ Normalise *data* / *dim* inputs accepted by coordinate constructors. @@ -807,38 +1199,85 @@ def parse(data, dim=None): return data, dim -def parse_tolerance(tolerance, dtype): +def parse_scalar_delta(value, dtype, default_zero=False): """ - Normalise *tolerance* to the correct type for *dtype*. + Normalise a scalar *value* to the correct type for *dtype*. - Converts ``None`` to zero, and for datetime64 dtypes converts a - numeric tolerance (in seconds) to the appropriate :class:`numpy.timedelta64`. + When ``default_zero`` is ``True``, a ``None`` input is replaced by a + sensible zero-like default: ``timedelta64(0)`` for datetime64 dtypes, + a dtype-appropriate epsilon for floating-point dtypes, or plain ``0`` + otherwise. For datetime64 dtypes a numeric value is interpreted as + seconds and converted to :class:`numpy.timedelta64`. Parameters ---------- - tolerance : float or None - Raw tolerance value. + value : scalar or None + Raw scalar value to normalise. dtype : numpy.dtype - The dtype of the coordinate values the tolerance will be compared against. + Target dtype that determines the output type. + default_zero : bool, optional + If ``True``, replace ``None`` with a zero-like default for *dtype*. + Default is ``False``. Returns ------- - tolerance : int, float, or numpy.timedelta64 + value : numpy scalar + Normalised scalar cast to the appropriate numpy scalar type. + + Raises + ------ + ValueError + If *value* is not a scalar (i.e. has non-zero ndim), or if *value* is + ``None`` while *default_zero* is ``False`` (no default is available). """ + # check shape + if not np.ndim(value) == 0: + raise ValueError("`value` must be a scalar value") + + # default + if value is None: + if not default_zero: + raise ValueError("`value` cannot be None when `default_zero` is False") + if np.issubdtype(dtype, np.datetime64): + value = np.timedelta64(0) + elif dtype == np.float16: + value = 1e-2 + elif dtype == np.float32: + value = 1e-5 + elif dtype == np.float64: + value = 1e-8 + else: + value = 0 + + # ensure numpy scalar + value = np.asarray(value)[()] + + # check dtype if np.issubdtype(dtype, np.datetime64): - if tolerance is None: - tolerance = np.timedelta64(0) - elif isinstance(tolerance, (int, float)): - tolerance = np.timedelta64(round(tolerance * 1e9), "ns") + if not np.issubdtype(value.dtype, np.timedelta64): + value = np.timedelta64(round(value * 1e9), "ns") # TODO: not `dtype` else: - if tolerance is None: - tolerance = 0 - return tolerance + value = value.astype(dtype) + + return value def get_sampling_interval(da, dim, cast=True): """ - Return the sample spacing along a given dimension. + Return the nominal sample spacing along a given dimension. + + Convenience used by every signal-processing routine: the coordinate should + be regular (carry a nominal sampling interval). Convert an irregular + coordinate first, e.g. ``da[dim] = da[dim].to_regular(tolerance=...)``, or + open the files with a tolerance so gaps and jitter are absorbed upfront. + + .. deprecated:: 0.2.8 + For backward compatibility with data saved by earlier versions (whose + coordinates carry no ``sampling_interval``), an irregular coordinate + currently falls back to inferring a spacing and emits a + :exc:`FutureWarning` stating the inferred value and the tolerance it + requires. This fallback will be removed in a future release, after + which irregular coordinates will raise. Parameters ---------- @@ -851,25 +1290,95 @@ def get_sampling_interval(da, dim, cast=True): Returns ------- - float - The sample spacing. + float or None + The sample spacing. ``None`` when *dim* has no axis coordinate. + + Raises + ------ + ValueError + If the coordinate is not regular and no spacing can be inferred. """ - return da[dim].get_sampling_interval(cast=cast) + from .interp import InterpCoordinate # avoid circular import + + coord = da[dim] + if not isinstance(coord, AxisCoordinate): + return None + delta = coord.get_sampling_interval(cast=cast) + if delta is not None: + return delta + + # Deprecated fallback: data written by earlier versions carries no + # sampling_interval metadata, so infer one rather than break every + # signal-processing call on existing archives. + hint = ( + f"make the coordinate regular with `da[{dim!r}] = " + f"da[{dim!r}].to_regular(tolerance=...)`, or open the files with a " + f"tolerance" + ) + if isinstance(coord, InterpCoordinate): + sampling_interval, tolerance = coord._infer_regular() + else: + try: + regular = coord.to_regular() + except ValueError as exc: + raise ValueError( + f"coordinate {dim!r} has no nominal sampling interval and " + f"none could be inferred ({exc}); {hint}" + ) from exc + sampling_interval = regular.get_sampling_interval(cast=False) + tolerance = regular.tolerance if sampling_interval is not None else None + if sampling_interval is None: + raise ValueError( + f"coordinate {dim!r} has no nominal sampling interval and none " + f"could be inferred; {hint}" + ) + warnings.warn( + f"coordinate {dim!r} has no declared sampling interval; inferred " + f"{sampling_interval} (accepting jitter up to tolerance={tolerance}). " + f"This implicit inference is deprecated and will raise in a future " + f"release; {hint}", + FutureWarning, + stacklevel=2, + ) + if cast and np.issubdtype(np.asarray(sampling_interval).dtype, np.timedelta64): + sampling_interval = sampling_interval / np.timedelta64(1, "s") + return sampling_interval + + +def encode_delta(key, value): + """Serialise a scalar (possibly timedelta64) into a dict of dataset attributes.""" + if value is None: + return {} + if np.issubdtype(np.asarray(value).dtype, np.timedelta64): + code, count = np.datetime_data(value.dtype) + if code == "generic": # e.g. timedelta64(0); promote to nanoseconds + value = value.astype("timedelta64[ns]") + code, count = np.datetime_data(value.dtype) + return { + key: int(count * value.astype(int)), + f"{key}_dtype": "timedelta64[ns]", + f"{key}_units": CODE_TO_UNITS[code], + } + return {key: value} -def isscalar(data): - """Return ``True`` if *data* converts to a 0-d non-object numpy array.""" - data = np.asarray(data) - return (data.dtype != np.dtype(object)) and (data.ndim == 0) +def decode_delta(key, attrs): + """Inverse of :func:`encode_delta`: read a scalar back from dataset attributes.""" + if key not in attrs: + return None + value = attrs[key] + if f"{key}_units" in attrs: + value = np.timedelta64(value, UNITS_TO_CODE[attrs[f"{key}_units"]]).astype( + attrs[f"{key}_dtype"] + ) + return value def is_monotonic_increasing(x): """Return ``True`` if every element of *x* is strictly greater than the previous one.""" - if np.issubdtype(x.dtype, np.datetime64): - return np.all(np.diff(x) > np.timedelta64(0)) - else: - return np.all(np.diff(x) > 0) + zero = np.timedelta64(0) if np.issubdtype(x.dtype, np.datetime64) else 0 + return np.all(np.diff(x) > zero) def format_datetime(x): @@ -878,6 +1387,6 @@ def format_datetime(x): if "." in string: datetime, digits = string.split(".") digits = digits[:3] - return ".".join([datetime, digits]) + return f"{datetime}.{digits}" else: return string diff --git a/xdas/coordinates/default.py b/xdas/coordinates/default.py deleted file mode 100644 index 70c5e2f9..00000000 --- a/xdas/coordinates/default.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -:class:`DefaultCoordinate`: integer-range coordinate. - -Used when no coordinate is explicitly provided for an axis. -""" - -import numpy as np - -from .core import Coordinate, isscalar, parse - - -class DefaultCoordinate(Coordinate, name="default"): - """ - Integer-range coordinate, equivalent to ``np.arange(size)``. - - Used automatically when no explicit coordinate is provided for an axis. - Internally stored as ``{"size": int}`` rather than a full array to avoid - memory allocation until values are actually needed. - - Parameters - ---------- - data : {"size": int} or None, optional - Mapping with a single ``"size"`` key. ``None`` creates an empty coordinate. - dim : str, optional - Dimension name. - dtype : ignored - Not supported; raises :exc:`ValueError` if provided. - """ - - def __init__(self, data=None, dim=None, dtype=None): - # empty - if data is None: - data = {"size": 0} - - # parse data - data, dim = parse(data, dim) - if not self.isvalid(data): - raise TypeError("`data` must be a mapping {'size': }") - - # check dtype - if dtype is not None: - raise ValueError("`dtype` is not supported for DefaultCoordinate") - - # store data - self.data = data - self.dim = dim - - @property - def empty(self): - """``True`` if the coordinate has size zero.""" - return self.data["size"] == 0 - - @property - def dtype(self): - """Always ``numpy.int64``.""" - return np.int64 - - @property - def ndim(self): - """Always 1.""" - return 1 - - @property - def shape(self): - """Shape tuple ``(size,)``.""" - return (len(self),) - - @staticmethod - def isvalid(data): - """Return ``True`` if *data* is ``{"size": int}``.""" - match data: - case {"size": None | int(_)}: - return True - case _: - return False - - def __len__(self): - if self.data["size"] is None: - return 0 - else: - return self.data["size"] - - def __getitem__(self, item): - data = self.__array__()[item] - dim = None if isscalar(data) else self.dim - return Coordinate(data, dim) - - def __array__(self, dtype=None): - return np.arange(self.data["size"], dtype=dtype) - - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - raise NotImplementedError - - def __array_function__(self, func, types, args, kwargs): - raise NotImplementedError - - def isdefault(self): - """Return ``True`` (this is a :class:`DefaultCoordinate`).""" - return True - - def get_sampling_interval(self, cast=True): - """Return the sample spacing, always 1 for integer-range coordinates.""" - return 1 - - def equals(self, other): - """Return ``True`` if *other* is a :class:`DefaultCoordinate` of the same size.""" - if isinstance(other, self.__class__): - return self.data["size"] == other.data["size"] - - def get_indexer(self, value, method=None): - """Return *value* directly (integer index equals label for range coordinates).""" - return value - - def slice_indexer(self, start=None, stop=None, step=None, endpoint=True): - """Return a :class:`slice` with *start*, *stop*, *step* unchanged.""" - return slice(start, stop, step) - - def concat(self, other): - """Return a new :class:`DefaultCoordinate` whose size is the sum of both sizes.""" - if not isinstance(other, self.__class__): - raise TypeError(f"cannot concatenate {type(other)} to {self.__class__}") - if not self.dim == other.dim: - raise ValueError("cannot concatenate coordinate with different dimension") - return self.__class__({"size": len(self) + len(other)}, self.dim) - - def to_dict(self): - """Serialise to ``{"dim": ..., "data": ...}``.""" - return {"dim": self.dim, "data": self.data} diff --git a/xdas/coordinates/dense.py b/xdas/coordinates/dense.py index 39743f2a..3001f08c 100644 --- a/xdas/coordinates/dense.py +++ b/xdas/coordinates/dense.py @@ -2,11 +2,13 @@ import numpy as np import pandas as pd +from typing_extensions import override -from .core import Coordinate, parse +from .core import AxisCoordinate, parse_data_dim, parse_scalar_delta +from .interp import InterpCoordinate -class DenseCoordinate(Coordinate, name="dense"): +class DenseCoordinate(AxisCoordinate, ctype="dense"): """ Coordinate backed by an explicit numpy array. @@ -23,66 +25,61 @@ class DenseCoordinate(Coordinate, name="dense"): Cast *data* to this dtype on construction. """ + @override def __init__(self, data=None, dim=None, dtype=None): # empty if data is None: data = [] # parse data - data, dim = parse(data, dim) - if not self.isvalid(data): + data, dim = parse_data_dim(data, dim) + if not self._isvalid(data): raise TypeError("`data` must be array-like") # store data self.data = np.asarray(data, dtype=dtype) self.dim = dim + @classmethod + @override + def from_block(cls, start, size, step, dim=None, dtype=None): + data = start + step * np.arange(size) + return cls(data, dim=dim, dtype=dtype) + + @override + def __len__(self): + return self.data.__len__() + + @property + @override + def dtype(self): + return self.data.dtype + @property def index(self): """A :class:`pandas.Index` view of the underlying data array.""" return pd.Index(self.data) @staticmethod - def isvalid(data): - """Return ``True`` if *data* converts to a 1-D non-object numpy array.""" + @override + def _isvalid(data): data = np.asarray(data) return (data.dtype != np.dtype(object)) and (data.ndim == 1) - def isdense(self): - """Return ``True`` (this is a :class:`DenseCoordinate`).""" - return True - - def equals(self, other): - """Return ``True`` if *other* is a :class:`DenseCoordinate` with identical values and dtype.""" - if isinstance(other, self.__class__): - return ( - np.array_equal(self.data, other.data) - and self.dim == other.dim - and self.dtype == other.dtype - ) + @override + def _is_monotonic_increasing(self): + if np.issubdtype(self.dtype, np.datetime64): + zero = np.timedelta64(0) else: - return False + zero = 0 + return np.all(np.diff(self.values) > zero) - def get_indexer(self, value, method=None): - """ - Return the integer index (or indices) for *value*. - - Parameters - ---------- - value : scalar or array-like - Label(s) to look up. - method : str, optional - Forwarded to :meth:`pandas.Index.get_indexer` (e.g. ``"ffill"``). - - Returns - ------- - int or numpy.ndarray - - Raises - ------ - KeyError - If any requested label is not found (indexer returns -1). - """ + @override + def _get_value(self, index): + return self.data[index] + + @override + def _get_indexer(self, value, method=None): if np.isscalar(value): out = self.index.get_indexer([value], method).item() else: @@ -91,19 +88,12 @@ def get_indexer(self, value, method=None): raise KeyError("index not found") return out - def slice_indexer(self, start=None, stop=None, step=None, endpoint=True): - """Return an integer :class:`slice` for label range [*start*, *stop*] via :class:`pandas.Index`.""" - slc = self.index.slice_indexer(start, stop, step) - if ( - (not endpoint) - and (stop is not None) - and (self[slc.stop - 1].values == stop) - ): - slc = slice(slc.start, slc.stop - 1, slc.step) - return slc - - def concat(self, other): - """Concatenate *other* :class:`DenseCoordinate` values to this one.""" + @override + def _slice(self, slc): + return self.__class__(self.data[slc], self.dim) + + @override + def _concat(self, other): if not isinstance(other, self.__class__): raise TypeError(f"cannot concatenate {type(other)} to {self.__class__}") if not self.dim == other.dim: @@ -116,29 +106,18 @@ def concat(self, other): raise ValueError("cannot concatenate coordinate with different dtype") return self.__class__(np.concatenate([self.data, other.data]), self.dim) - def get_div_points(self, tolerance=None): - """Return sorted split-point indices where consecutive differences exceed *tolerance*.""" - deltas = np.diff(self.data) - if tolerance is not None: - div_points = np.nonzero(np.abs(deltas) >= tolerance)[0] + 1 - else: - raise NotImplementedError( - "get_div_points without tolerance is not implemented for DenseCoordinate" - ) - div_points = np.concatenate(([0], div_points, [len(self)])) - return div_points - - def to_dict(self): - """Serialise to ``{"dim": ..., "data": ..., "dtype": ...}``.""" - if np.issubdtype(self.dtype, np.datetime64): - data = self.data.astype(str).tolist() - else: - data = self.data.tolist() - return {"dim": self.dim, "data": data, "dtype": str(self.dtype)} + @override + def _to_dataset(self, dataset, attrs): + if self.name is None: + raise ValueError("cannot serialize a coordinate with no name") + dataset = dataset.assign_coords( + {self.name: (self.dim, self.values) if self.dim else self.values} + ) + return dataset, attrs @classmethod - def from_dataset(cls, dataset, name): - """Extract all coordinates from an xarray *dataset* variable *name* as plain arrays.""" + @override + def _collect_from_dataset(cls, dataset, name): return { name: ( ( @@ -155,8 +134,76 @@ def from_dataset(cls, dataset, name): for name, coord in dataset[name].coords.items() } - @classmethod - def from_block(cls, start, size, step, dim=None, dtype=None): - """Build a :class:`DenseCoordinate` from ``start + step * arange(size)``.""" - data = start + step * np.arange(size) - return cls(data, dim=dim, dtype=dtype) + @override + def __repr__(self): + return np.array2string(self.data, threshold=0, edgeitems=1) + + def __add__(self, other): + return self.__class__(self.data + other, self.dim) + + def __sub__(self, other): + return self.__class__(self.data - other, self.dim) + + @override + def get_sampling_interval(self, cast=True): + """ + Return ``None``: a dense coordinate never carries a nominal spacing. + + The raw values may happen to be evenly spaced, but regularity is an + explicit declaration; convert with :meth:`to_regular` to obtain a + regular :class:`InterpCoordinate`. + """ + return + + @override + def to_regular(self, sampling_interval=None, tolerance=None): + """Convert to a regular :class:`InterpCoordinate` (single continuous ramp). + + The spacing defaults to the end-to-end slope, and every value must lie + within *tolerance* of the regular grid anchored at the first value; + otherwise a :exc:`ValueError` is raised. See + :meth:`AxisCoordinate.to_regular` for the parameter contract. + """ + if len(self) < 2: + raise ValueError( + "cannot make a regular coordinate from fewer than two values" + ) + tolerance = parse_scalar_delta(tolerance, self.dtype, default_zero=True) + if sampling_interval is None: + sampling_interval = (self.data[-1] - self.data[0]) / (len(self) - 1) + else: + sampling_interval = parse_scalar_delta(sampling_interval, self.dtype) + grid = self.data[0] + sampling_interval * np.arange(len(self)) + if not np.all(np.abs(self.data - grid) <= tolerance): + raise ValueError( + "values are not evenly spaced by `sampling_interval` within `tolerance`" + ) + data = { + "tie_indices": [0, len(self) - 1], + "tie_values": [self.data[0], self.data[-1]], + "sampling_interval": sampling_interval, + "tolerance": tolerance, + } + return InterpCoordinate(data, self.dim) + + @override + def _split_candidates(self): + steps = np.diff(self.data) + positions = np.arange(1, len(self)) + if steps.size == 0: + return positions, steps + reference = np.median(steps) + deltas = np.empty(steps.shape, dtype=np.asarray(steps[0] - reference).dtype) + for i in range(steps.size): + deltas[i] = steps[i] - reference + if i + 1 < steps.size and abs(steps[i + 1] - steps[i]) < abs( + steps[i + 1] - reference + ): + reference = steps[i] + return positions, deltas + + @override + def simplify(self, tolerance=None, *, reduce=True, regularize=False): + # a dense coordinate stores every value explicitly; there is nothing to + # drop and no spacing to promote, so both stages are no-ops. + return self.copy() diff --git a/xdas/coordinates/interp.py b/xdas/coordinates/interp.py index 44401666..c50f85a2 100644 --- a/xdas/coordinates/interp.py +++ b/xdas/coordinates/interp.py @@ -2,54 +2,106 @@ :class:`InterpCoordinate`: piecewise-linear coordinate. Defined by tie points, using ``xinterp`` for forward and inverse interpolation. +Optionally carries a nominal ``sampling_interval`` (and ``tolerance``) making the +coordinate *regular* and providing a clean sample rate for signal-processing routines. """ import re import numpy as np +from numba import njit +from typing_extensions import override from xinterp import forward, inverse from .core import ( + AxisCoordinate, Coordinate, - format_datetime, + decode_delta, + encode_delta, is_monotonic_increasing, - parse, - parse_tolerance, + parse_data_dim, + parse_scalar_delta, ) -class InterpCoordinate(Coordinate, name="interpolated"): +class InterpCoordinate(AxisCoordinate, ctype="interpolated"): """ - Array-like object representing piecewise evenly spaced coordinates (CF convention). + Piecewise-linear coordinate described by tie points (CF subsampling, 8.3). - The coordinate ticks are described by tie points that are interpolated when - intermediate values are required. Coordinate objects provide label-based - selection methods. + Following the CF conventions for compression by coordinate subsampling. + Values between tie points are recovered by linear interpolation (via + ``xinterp``), which also enables label-based selection through + :meth:`~Coordinate.to_index`. The index axis is split into *continuous + areas* separated by *discontinuities*; a discontinuity is encoded as two + consecutive tie points at adjacent indices (a gap of one). + + When *data* contains a ``sampling_interval`` key the coordinate also + enforces a nominal sample spacing, making it *regular* + (:meth:`isregular` returns ``True``) and giving signal-processing routines a + clean sample rate. A ``tolerance`` key may accompany it to allow bounded + jitter around that rate. Parameters ---------- - tie_indices : sequence of integers - The indices of the tie points. Must include index 0 and be strictly increasing. - tie_values : sequence of float or datetime64 - The values of the tie points. Must be strictly increasing to enable label-based - selection. The len of `tie_indices` and `tie_values` sizes must match. + data : dict with keys ``tie_indices`` and ``tie_values`` + ``tie_indices`` : sequence of int + Positions of the tie points. Must start at 0 and be strictly + increasing. + ``tie_values`` : sequence of float or datetime64 + Values at the tie points. Must be strictly increasing to enable + label-based selection. Length must match ``tie_indices``. + ``sampling_interval`` : scalar, optional + Nominal sample spacing. When provided the coordinate is + *regular* and :meth:`get_sampling_interval` returns it directly. + ``tolerance`` : scalar, optional + Allowed jitter around ``sampling_interval``. Checked for + consistency with the tie points at construction. Ignored when + ``sampling_interval`` is absent. + dim : str, optional + Name of the dimension this coordinate is associated with. + dtype : dtype-like, optional + Desired dtype for ``tie_values``. + + Notes + ----- + Regularity is judged on the continuous areas only: a tie-point gap of one + index (``den == 1``) is a CF discontinuity and carries no sampling-rate + information. A ``sampling_interval`` is valid when, for every continuous + segment, the accumulated drift ``|sampling_interval * den - num|`` stays + within ``2 * tolerance`` (each tie value may jitter by ±``tolerance``). + A coordinate with no continuous area (e.g. ``tie_indices=[0, 1, 2]``) has no + inferable spacing, so an explicitly provided one is stored as-is. Use + :meth:`simplify` to canonicalise a coordinate and acquire a spacing from + the continuous areas within an accuracy budget. + + Examples + -------- + >>> import xdas as xd + >>> coord = xd.Coordinate( + ... {"tie_indices": [0, 9, 10, 19], "tie_values": [0.0, 90.0, 200.0, 290.0]} + ... ) + >>> coord + 0.000 to 290.000 """ + @override def __init__(self, data=None, dim=None, dtype=None): # empty if data is None: data = {"tie_indices": [], "tie_values": []} # parse data - data, dim = parse(data, dim) - if not self.__class__.isvalid(data): - raise TypeError("`data` must be dict-like") - if not set(data) == {"tie_indices", "tie_values"}: - raise ValueError( - "both `tie_indices` and `tie_values` key should be provided" + data, dim = parse_data_dim(data, dim) + if not InterpCoordinate._isvalid(data): + raise TypeError( + "`data` must be dict-like with `tie_indices` and `tie_values` " + "(and optionally `sampling_interval` / `tolerance`)" ) + tie_indices = np.asarray(data["tie_indices"]) tie_values = np.asarray(data["tie_values"], dtype=dtype) + sampling_interval = data.get("sampling_interval", None) + tolerance = data.get("tolerance", None) # check shapes if not tie_indices.ndim == 1: @@ -60,7 +112,7 @@ def __init__(self, data=None, dim=None, dtype=None): raise ValueError("`tie_indices` and `tie_values` must have the same length") # check dtypes - if not tie_indices.shape == (0,): + if tie_indices.shape != (0,): if not np.issubdtype(tie_indices.dtype, np.integer): raise ValueError("`tie_indices` must be integer-like") if not tie_indices[0] == 0: @@ -73,11 +125,14 @@ def __init__(self, data=None, dim=None, dtype=None): ): raise ValueError("`tie_values` must have either numeric or datetime dtype") - # store data + # store base data tie_indices = tie_indices.astype(int) - self.data = dict(tie_indices=tie_indices, tie_values=tie_values) + self.data = {"tie_indices": tie_indices, "tie_values": tie_values} self.dim = dim + # optional regular sampling + self._assign_sampling_interval(sampling_interval, tolerance) + @property def tie_indices(self): """Integer array of tie-point positions (starts at 0, strictly increasing).""" @@ -89,168 +144,105 @@ def tie_values(self): return self.data["tie_values"] @property - def dtype(self): - """Dtype of the tie values (and of all materialised coordinate values).""" - return self.tie_values.dtype + def sampling_interval(self): + """Nominal sample spacing, or ``None`` when the coordinate is not regular.""" + return self.data["sampling_interval"] @property - def empty(self): - """``True`` if no tie points have been set.""" - return self.tie_indices.shape == (0,) + def tolerance(self): + """Allowed jitter around :attr:`sampling_interval`, or ``None``.""" + return self.data["tolerance"] @property - def ndim(self): - """Always 1.""" - return self.tie_values.ndim - - @property - def shape(self): - """Shape tuple ``(len(self),)``.""" - return (len(self),) + @override + def dtype(self): + return self.tie_values.dtype - @property - def indices(self): - """Full integer index array from 0 to the last tie-point index (inclusive).""" - if self.empty: - return np.array([], dtype="int") + @classmethod + @override + def from_block(cls, start, size, step, dim=None, dtype=None): + start = np.asarray(start, dtype=dtype) + step = parse_scalar_delta(step, start.dtype) + if size < 2: + # A single (or zero) sample cannot span two tie points; keep the + # declared spacing as metadata. + data = { + "tie_indices": [0][:size], + "tie_values": [start][:size], + "sampling_interval": step, + } else: - return np.arange(self.tie_indices[-1] + 1) + end = start + step * (size - 1) + data = { + "tie_indices": [0, size - 1], + "tie_values": [start, end], + "sampling_interval": step, + } + return cls(data, dim=dim, dtype=dtype) - @property - def values(self): - """Materialised numpy array of all coordinate values via piecewise interpolation.""" - if self.empty: - return np.array([], dtype=self.dtype) + @override + def __len__(self): + if len(self.tie_indices) > 0: + return int(self.tie_indices[-1]) + 1 else: - return self.get_value(self.indices) + return 0 @staticmethod - def isvalid(data): - """Return ``True`` if *data* is a dict with ``tie_indices`` and ``tie_values`` keys.""" + @override + def _isvalid(data): match data: - case {"tie_indices": _, "tie_values": _}: + case {"tie_indices": _, "tie_values": _, **rest} if set(rest) <= { + "sampling_interval", + "tolerance", + }: return True case _: return False - def __len__(self): - if self.empty: - return 0 - else: - return self.tie_indices[-1] - self.tie_indices[0] + 1 + @override + def _is_monotonic_increasing(self): + return not self.get_split_indices("overlaps", tolerance=False).size - def __repr__(self): - if len(self) == 0: - return "empty coordinate" - elif len(self) == 1: - return f"{self.tie_values[0]}" - else: - if np.issubdtype(self.dtype, np.floating): - return f"{self.tie_values[0]:.3f} to {self.tie_values[-1]:.3f}" - elif np.issubdtype(self.dtype, np.datetime64): - start = format_datetime(self.tie_values[0]) - end = format_datetime(self.tie_values[-1]) - return f"{start} to {end}" - else: - return f"{self.tie_values[0]} to {self.tie_values[-1]}" + @override + def _get_value(self, index): + return forward(index, self.tie_indices, self.tie_values) - def __getitem__(self, item): - if isinstance(item, slice): - return self.slice_index(item) - elif np.isscalar(item): - return Coordinate(self.get_value(item), None) + @override + def _get_indexer(self, value, method=None): + if isinstance(value, str): + value = np.datetime64(value) else: - return Coordinate(self.get_value(item), self.dim) - - def __add__(self, other): - return self.__class__( - {"tie_indices": self.tie_indices, "tie_values": self.tie_values + other}, - self.dim, - ) - - def __sub__(self, other): - return self.__class__( - {"tie_indices": self.tie_indices, "tie_values": self.tie_values - other}, - self.dim, - ) - - def __array__(self, dtype=None): - out = self.values - if dtype is not None: - out = out.__array__(dtype) - return out - - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - raise NotImplementedError - - def __array_function__(self, func, types, args, kwargs): - raise NotImplementedError - - def isinterp(self): - """Return ``True`` (this is an :class:`InterpCoordinate`).""" - return True - - def get_sampling_interval(self, cast=True): - """ - Return the median sample spacing across all tie-point segments. - - Parameters - ---------- - cast : bool, optional - If ``True`` (default), cast timedelta64 to seconds. - - Returns - ------- - float or None - ``None`` if fewer than two elements. - """ - if len(self) < 2: - return None - num = np.diff(self.tie_values) - den = np.diff(self.tie_indices) - mask = den != 1 - num = num[mask] - den = den[mask] - delta = np.median(num / den) - if cast and np.issubdtype(delta.dtype, np.timedelta64): - delta = delta / np.timedelta64(1, "s") - return delta - - def is_monotonic_increasing(self): - """Return ``True`` if no segment starts before the end of the previous one.""" - return not self.get_split_indices("overlaps", tolerance=False).size + value = np.asarray(value) + try: + indexer = inverse(value, self.tie_indices, self.tie_values, method) + except ValueError as e: + if str(e) == "fp must be strictly increasing": + raise ValueError( + "overlaps were found in the coordinate. If this is due to some " + "jitter in the tie values, consider smoothing the coordinate by " + "including some tolerance. This can be done by " + "`da[dim] = da[dim].simplify(tolerance)`, or by specifying a " + "tolerance when opening multiple files." + ) + else: # pragma: no cover + raise + return indexer - def equals(self, other): - """Return ``True`` if *other* has identical tie points, dim, and dtype.""" - return ( - np.array_equal(self.tie_indices, other.tie_indices) - and np.array_equal(self.tie_values, other.tie_values) - and self.dim == other.dim - and self.dtype == other.dtype + @override + def _slice(self, index_slice): + start_index, stop_index, step_index = ( + index_slice.start, + index_slice.stop, + index_slice.step, ) - - def get_value(self, index): - """Interpolate coordinate values at integer position(s) *index*.""" - index = self.format_index(index) - return forward(index, self.tie_indices, self.tie_values) - - def slice_index(self, index_slice): - """Return a new :class:`InterpCoordinate` for the integer slice *index_slice*.""" - start_index, stop_index, step_index = index_slice.indices(len(self)) - if step_index < 0: - raise NotImplementedError("negative slice step is not implemented") if stop_index - start_index <= 0: - return self.__class__(dict(tie_indices=[], tie_values=[]), dim=self.dim) + data = {"tie_indices": [], "tie_values": []} elif (stop_index - start_index) <= step_index: - tie_indices = [0] - tie_values = [self.get_value(start_index)] - return self.__class__( - dict(tie_indices=tie_indices, tie_values=tie_values), dim=self.dim - ) + data = {"tie_indices": [0], "tie_values": [self._get_value(start_index)]} else: end_index = stop_index - 1 - start_value = self.get_value(start_index) - end_value = self.get_value(end_index) + start_value = self._get_value(start_index) + end_value = self._get_value(end_index) mask = (start_index < self.tie_indices) & (self.tie_indices < end_index) tie_indices = np.insert( self.tie_indices[mask], @@ -263,48 +255,26 @@ def slice_index(self, index_slice): (start_value, end_value), ) tie_indices -= tie_indices[0] - data = {"tie_indices": tie_indices, "tie_values": tie_values} - coord = self.__class__(data, self.dim) - if step_index != 1: - coord = coord.decimate(step_index) - return coord - def get_indexer(self, value, method=None): - """ - Return the integer index for a label *value* via inverse interpolation. - - Parameters - ---------- - value : scalar, str (ISO datetime), or array-like - Label(s) to locate. - method : str, optional - Forwarded to ``xinterp.inverse`` (e.g. ``"ffill"``, ``"bfill"``). + if step_index != 1: + tie_indices = (tie_indices // step_index) * step_index + for k in range(1, len(tie_indices) - 1): + if tie_indices[k] == tie_indices[k - 1]: + tie_indices[k] += step_index + tie_values = self._get_value(start_index + tie_indices) + tie_indices //= step_index - Returns - ------- - int or numpy.ndarray - """ - if isinstance(value, str): - value = np.datetime64(value) - else: - value = np.asarray(value) - try: - indexer = inverse(value, self.tie_indices, self.tie_values, method) - except ValueError as e: - if str(e) == "fp must be strictly increasing": - raise ValueError( - "overlaps were found in the coordinate. If this is due to some " - "jitter in the tie values, consider smoothing the coordinate by " - "including some tolerance. This can be done by " - "`da[dim] = da[dim].simplify(tolerance)`, or by specifying a " - "tolerance when opening multiple files." - ) - else: # pragma: no cover - raise e - return indexer + data = {"tie_indices": tie_indices, "tie_values": tie_values} + if self.sampling_interval is not None: + data = { + **data, + "sampling_interval": self.sampling_interval * step_index, + "tolerance": self.tolerance, + } + return self.__class__(data, self.dim) - def concat(self, other): - """Append *other* :class:`InterpCoordinate` after this one, shifting its tie indices.""" + @override + def _concat(self, other): if not isinstance(other, self.__class__): raise TypeError(f"cannot concatenate {type(other)} to {self.__class__}") if not self.dim == other.dim: @@ -315,125 +285,30 @@ def concat(self, other): return self if not self.dtype == other.dtype: raise ValueError("cannot concatenate coordinate with different dtype") - coord = self.__class__( - { - "tie_indices": np.append( - self.tie_indices, other.tie_indices + len(self) - ), - "tie_values": np.append(self.tie_values, other.tie_values), - }, - self.dim, - ) - return coord - - def decimate(self, q): - """Return a new coordinate keeping every *q*-th sample (integer decimation).""" - tie_indices = (self.tie_indices // q) * q - for k in range(1, len(tie_indices) - 1): - if tie_indices[k] == tie_indices[k - 1]: - tie_indices[k] += q - tie_values = [self.get_value(idx) for idx in tie_indices] - tie_indices //= q - return self.__class__( - dict(tie_indices=tie_indices, tie_values=tie_values), self.dim - ) - - def simplify(self, tolerance=None): - """ - Reduce the number of tie points using the Douglas-Peucker algorithm. - - Parameters - ---------- - tolerance : float, timedelta, or None - Maximum allowed deviation from the original piecewise-linear curve. - ``None`` uses zero tolerance (lossless). ``False`` returns ``self`` unchanged. - """ - if tolerance is False: - return self # TODO: copy - tolerance = parse_tolerance(tolerance, self.dtype) - tie_indices, tie_values = douglas_peucker( - self.tie_indices, self.tie_values, tolerance - ) - return self.__class__( - dict(tie_indices=tie_indices, tie_values=tie_values), self.dim - ) - - def get_split_indices(self, kind="discontinuities", tolerance=False): - """ - Return tie-point indices where consecutive segments are discontinuous. - - Parameters - ---------- - kind : {"discontinuities", "gaps", "overlaps"}, optional - Which type of split to detect. Default ``"discontinuities"``. - tolerance : float, timedelta, or ``False`` - Minimum magnitude of gap/overlap to report. ``False`` returns all - consecutive tie-point pairs regardless of size. - - Returns - ------- - numpy.ndarray - Integer positions (into the full coordinate array) of each split. - """ - valid_kinds = {"discontinuities", "gaps", "overlaps"} - if kind not in valid_kinds: - raise ValueError(f"`kind` must be one of {valid_kinds}; got {kind!r}") - - (indices,) = np.nonzero(np.diff(self.tie_indices) == 1) - indices += 1 - - # Fast path: no filtering requested - if kind == "discontinuities" and tolerance is False: - return self.tie_indices[indices] - - sampling_interval = self.get_sampling_interval(cast=False) - deltas = ( - self.tie_values[indices] - self.tie_values[indices - 1] - sampling_interval - ) - - if tolerance is False: - zero = np.timedelta64(0) if np.issubdtype(self.dtype, np.datetime64) else 0 - - match kind: - case "gaps": - mask = deltas >= zero - case "overlaps": # pragma: no branch - mask = deltas < zero - - else: - tolerance = parse_tolerance(tolerance, self.dtype) - - match kind: - case "discontinuities": - mask = np.abs(deltas) > tolerance - case "gaps": - mask = deltas > tolerance - case "overlaps": # pragma: no branch - mask = deltas < -tolerance - - return self.tie_indices[indices[mask]] - - @classmethod - def from_array(cls, arr, dim=None, tolerance=None): - """Build an :class:`InterpCoordinate` from a full array *arr*, optionally simplified.""" - return cls( - {"tie_indices": np.arange(len(arr)), "tie_values": arr}, dim - ).simplify(tolerance) - - def to_dict(self): - """Serialise to ``{"dim": ..., "data": {"tie_indices": ..., "tie_values": ...}, "dtype": ...}``.""" - tie_indices = self.data["tie_indices"] - tie_values = self.data["tie_values"] - if np.issubdtype(tie_values.dtype, np.datetime64): - tie_values = tie_values.astype(str) data = { - "tie_indices": tie_indices.tolist(), - "tie_values": tie_values.tolist(), + "tie_indices": np.append(self.tie_indices, other.tie_indices + len(self)), + "tie_values": np.append(self.tie_values, other.tie_values), } - return {"dim": self.dim, "data": data, "dtype": str(self.dtype)} + # Strict primitive: preserve the regular contract only when both sides + # advertise the exact same spacing; otherwise the merged coord is + # irregular by construction. The joining tie pair has ``den == 1`` (a + # CF discontinuity) so each side's segments validate independently, + # and ``max(tolerance)`` bounds the union. Reconciling slightly + # different rates is the job of user-facing routines (see + # :func:`concat_coords`, which delegates to :meth:`simplify`). + if ( + self.sampling_interval is not None + and self.sampling_interval == other.sampling_interval + ): + data = { + **data, + "sampling_interval": self.sampling_interval, + "tolerance": max(self.tolerance, other.tolerance), + } + return self.__class__(data, self.dim) - def to_dataset(self, dataset, attrs): - """Write tie points into an xarray *dataset* using CF coordinate interpolation conventions.""" + @override + def _to_dataset(self, dataset, attrs): mapping = f"{self.name}: {self.name}_indices {self.name}_values" if "coordinate_interpolation" in attrs: attrs["coordinate_interpolation"] += " " + mapping @@ -449,6 +324,12 @@ def to_dataset(self, dataset, attrs): "interpolation_name": "linear", "tie_points_mapping": f"{self.name}_points: {self.name}_indices {self.name}_values", } + if self.sampling_interval is not None: + interp_attrs.update( + encode_delta("sampling_interval", self.sampling_interval) + ) + if self.tolerance is not None: + interp_attrs.update(encode_delta("tolerance", self.tolerance)) dataset.update( { f"{self.name}_interpolation": ((), np.nan, interp_attrs), @@ -459,31 +340,339 @@ def to_dataset(self, dataset, attrs): return dataset, attrs @classmethod - def from_dataset(cls, dataset, name): - """Read interpolated coordinates from *dataset* using the ``coordinate_interpolation`` attribute.""" + @override + def _collect_from_dataset(cls, dataset, name): coords = {} mapping = dataset[name].attrs.pop("coordinate_interpolation", None) if mapping is not None: - matches = re.findall(r"(\w+): (\w+) (\w+)", mapping) - for match in matches: - dim, indices, values = match - data = {"tie_indices": dataset[indices], "tie_values": dataset[values]} + for dim, indices, values in re.findall(r"(\w+): (\w+) (\w+)", mapping): + data = { + "tie_indices": dataset[indices].values, + "tie_values": dataset[values].values, + } + interp_attrs = dataset[f"{dim}_interpolation"].attrs + if "sampling_interval" in interp_attrs: + data["sampling_interval"] = decode_delta( + "sampling_interval", interp_attrs + ) + data["tolerance"] = decode_delta("tolerance", interp_attrs) coords[dim] = Coordinate(data, dim) return coords - @classmethod - def from_block(cls, start, size, step, dim=None, dtype=None): - """Build a two-point :class:`InterpCoordinate` covering [start, start + step*(size-1)].""" - return cls( - { - "tie_indices": [0, size - 1], - "tie_values": [start, start + step * (size - 1)], - }, - dim=dim, + def __add__(self, other): + data = {"tie_indices": self.tie_indices, "tie_values": self.tie_values + other} + if self.sampling_interval is not None: + data = { + **data, + "sampling_interval": self.sampling_interval, + "tolerance": self.tolerance, + } + return self.__class__(data, self.dim) + + def __sub__(self, other): + data = {"tie_indices": self.tie_indices, "tie_values": self.tie_values - other} + if self.sampling_interval is not None: + data = { + **data, + "sampling_interval": self.sampling_interval, + "tolerance": self.tolerance, + } + return self.__class__(data, self.dim) + + @override + def get_sampling_interval(self, cast=True): + delta = self.sampling_interval + if delta is None: + return None + if cast and np.issubdtype(delta.dtype, np.timedelta64): + delta = delta / np.timedelta64(1, "s") + return delta + + def _assign_sampling_interval(self, sampling_interval, tolerance=None): + """Parse, validate and store the sampling interval and its tolerance. + + ``None`` clears both; a value is kept only if consistent with the tie + points (see :meth:`_is_valid_sampling_interval`). + """ + if sampling_interval is None: + if tolerance is not None: + raise ValueError( + "`tolerance` cannot be set without a `sampling_interval`" + ) + self.data["sampling_interval"] = None + self.data["tolerance"] = None + return + + sampling_interval = parse_scalar_delta(sampling_interval, self.dtype) + tolerance = parse_scalar_delta(tolerance, self.dtype, default_zero=True) + + # Normalise datetime deltas to the tie-value resolution so an in-memory + # coordinate matches the one read back after serialisation (which always + # encodes timedeltas at the coordinate's datetime resolution). + if np.issubdtype(self.dtype, np.datetime64): + unit = np.datetime_data(self.dtype)[0] + sampling_interval = sampling_interval.astype(f"timedelta64[{unit}]") + tolerance = tolerance.astype(f"timedelta64[{unit}]") + + if self._is_valid_sampling_interval(sampling_interval, tolerance): + self.data["sampling_interval"] = sampling_interval + self.data["tolerance"] = tolerance + else: + raise ValueError( + "`sampling_interval` and `tolerance` are not consistent with " + "the `tie_indices` and `tie_values`" + ) + + def _is_valid_sampling_interval(self, sampling_interval, tolerance): + """Whether *sampling_interval* fits every continuous area within *tolerance*.""" + num, den = self._continuous_segments() + # Bound the per-segment accumulated drift: each tie value may jitter by + # ±tolerance, so a segment span may be off by up to 2 * tolerance. With no + # continuous area `np.all([])` is vacuously True, accepting an explicit + # spacing as metadata (e.g. a two-tie-point block). Datetime bounds use + # integer division and are only accurate to the dtype resolution. + dmin = (num - 2 * tolerance) / den + dmax = (num + 2 * tolerance) / den + valid = np.all((dmin <= sampling_interval) & (sampling_interval <= dmax)) + return bool(valid) + + def _minimal_tolerance(self, sampling_interval): + """Smallest tolerance for which *sampling_interval* validates, or ``None``. + + Private helper behind :meth:`simplify`. Validity bounds the accumulated + drift of every continuous segment by ``2 * tolerance``, so the tightest + admissible value is half the worst drift, rounded up at the dtype + resolution. Returns ``None`` when even that value fails to validate, + which callers treat as "this spacing cannot be declared". + """ + num, den = self._continuous_segments() + if num.size == 0: + # No continuous area: any tolerance is vacuously valid. + return parse_scalar_delta(None, self.dtype, default_zero=True) + drift = np.abs(num - sampling_interval * den).max() + if np.issubdtype(self.dtype, np.datetime64): + # Integer resolution: round up so the halved drift is not truncated. + unit = np.datetime_data(self.dtype)[0] + counts = int(drift / np.timedelta64(1, unit)) + tolerance = np.timedelta64(-(-counts // 2), unit) + else: + # A few ULPs of slack so re-validation cannot reject the value we + # just derived from the same quantities (as in :meth:`_infer_regular`). + tolerance = drift / 2 + 4 * np.spacing(np.abs(num).max()) + if not self._is_valid_sampling_interval(sampling_interval, tolerance): + return None # pragma: no cover + return tolerance + + def _infer_regular(self): + """ + Estimate the nominal spacing and tightest tolerance for this coordinate. + + Private helper behind :meth:`simplify` and :meth:`to_regular`: returns + the spacing that minimises the worst per-segment drift and the smallest + tolerance that would still validate it, without enforcing either on the + coordinate. + + Returns + ------- + sampling_interval : scalar or None + Spacing minimising ``max_i |sampling_interval * den_i - num_i|`` + over the continuous segments. ``None`` when no continuous segment + is available (every tie-point gap is a ``den == 1`` CF + discontinuity). + tolerance : scalar or None + Half the worst residual drift at ``sampling_interval``, plus a few + ULPs so the value stays valid under re-validation. ``None`` when + ``sampling_interval`` is ``None``. + + Notes + ----- + Spacing is judged on continuous areas only, ``den == 1`` gaps being CF + discontinuities (see :meth:`_continuous_segments`). For such a segment + ``i``, ``num_i`` is the change in ``tie_values`` and ``den_i`` the + change in ``tie_indices``. The quantity ``si * den_i - num_i`` is the + drift accumulated between the regular grid and the tie values at the + end of that segment, and :meth:`_is_valid_sampling_interval` accepts + ``si`` exactly when every such drift stays within ``2 * tolerance``. + + The inferred spacing minimises the worst-case drift:: + + si* = argmin_si max_i |si * den_i - num_i| + + This convex, piecewise-linear objective is a length-weighted Chebyshev + center of the per-segment rates ``r = num / den``. Its minimum is + reached where the two most disagreeing segments balance, so over all + pairs the binding one maximises + ``den_i * den_j * |r_i - r_j| / (den_i + den_j)`` and the optimum is + the rate of that merged pair:: + + si* = (num_i + num_j) / (den_i + den_j) + + That pair is found in ``O(n log n)`` via :func:`_chebyshev_center_pair` + rather than scanning all pairs. The matching tolerance is half the + worst drift, since validity compares the drift against + ``2 * tolerance``. + """ + num, den = self._continuous_segments() + if num.size == 0: + return None, None + # Float seconds for datetime axes pick the binding pair without + # integer/timedelta overflow; the final values stay in the native dtype. + is_datetime = np.issubdtype(num.dtype, np.timedelta64) + num_seconds = num / np.timedelta64(1, "s") if is_datetime else num.astype(float) + pos_idx, neg_idx = _chebyshev_center_pair(num_seconds, den.astype(float)) + sampling_interval = (num[pos_idx] + num[neg_idx]) / ( + den[pos_idx] + den[neg_idx] + ) + si_seconds = ( + sampling_interval / np.timedelta64(1, "s") + if is_datetime + else float(sampling_interval) + ) + drift = np.abs(si_seconds * den - num_seconds).max() + # A few ULPs of slack so re-validation cannot reject the returned pair. + tolerance = drift / 2 + 4 * np.spacing(np.abs(num_seconds).max()) + if is_datetime: + tolerance = np.timedelta64(int(np.ceil(tolerance * 1e9)), "ns") + return sampling_interval, tolerance + + @override + def to_regular(self, sampling_interval=None, tolerance=None): + """Enforce a nominal sampling interval, inferring it when omitted. + + The inferred spacing is the length-weighted Chebyshev center of the + per-segment rates (see :meth:`_infer_regular`). Raises when no spacing + can be inferred (no continuous area, i.e. every tie-point gap is a + ``den == 1`` CF discontinuity) or when the spacing does not fit the tie + points within *tolerance*. See :meth:`AxisCoordinate.to_regular` for + the parameter contract. + """ + # Default each unspecified argument to the stored regular config; an + # explicit value still overrides it. + if sampling_interval is None: + sampling_interval = self.sampling_interval + if tolerance is None: + tolerance = self.tolerance + if sampling_interval is None: + sampling_interval, _ = self._infer_regular() + if sampling_interval is None: + raise ValueError( + "cannot infer a sampling interval: the coordinate has no " + "continuous area; pass `sampling_interval` explicitly" + ) + data = { + "tie_indices": self.tie_indices, + "tie_values": self.tie_values, + "sampling_interval": sampling_interval, + "tolerance": tolerance, + } + return self.__class__(data, self.dim) + + @override + def simplify(self, tolerance=None, *, reduce=True, regularize=False): + """Canonicalise within *tolerance*: drop tie points, then promote to regular. + + The *reduce* stage runs Douglas-Peucker to drop tie points whose removal + shifts the curve by no more than *tolerance*. The CF 8.3 structure is + preserved as an emergent property of that bound: real discontinuities are + kept (any spanning line crosses them by far more than *tolerance*), soft + ones are fused into a single ramp, and synchronisation tie points survive + because removing them would, by definition, drift more than *tolerance*. + Surviving values are never moved. + + The *regularize* stage promotes the result to *regular* when the + surviving continuous segments admit a single ``sampling_interval`` within + *tolerance* (the internal Chebyshev fit's worst residual stays inside the + budget). The promotion is per-continuous-segment and sign-agnostic, so + two same-rate segments joined by a CF overlap are still described by one + spacing. An already-regular coordinate keeps its spacing regardless of + *regularize* and just widens its stored tolerance to absorb any jump + fused by the reduce stage. + + See :meth:`Coordinate.simplify` for the parameter contract. + """ + if tolerance is False: + return self.copy() + if tolerance is None: + # Default the budget to the coordinate's own declared jitter. + tolerance = self.tolerance + tolerance = parse_scalar_delta(tolerance, self.dtype, default_zero=True) + if reduce: + tie_indices, tie_values = _douglas_peucker( + self.tie_indices, self.tie_values, tolerance + ) + else: + tie_indices, tie_values = self.tie_indices, self.tie_values + data = {"tie_indices": tie_indices, "tie_values": tie_values} + if self.sampling_interval is not None: + # Already regular: keep the spacing. A reduce pass may fuse a soft + # discontinuity into a ramp whose absorbed jump exceeds the declared + # jitter; keep the declared tolerance when it still validates and + # only widen by the spent budget when it does not, so a lossless or + # seam-only reduction round-trips the exact regular metadata. + reduced = self.__class__(data, self.dim) + if reduce and not reduced._is_valid_sampling_interval( + self.sampling_interval, self.tolerance + ): + # Reduction fused a discontinuity, so the surviving tie points + # now drift further from the nominal grid than the declared + # jitter allows. Widen to the least value that describes them: + # the budget bounds how far *values* may move, not how much + # drift fusing a seam exposes, so it is no bound at all here. + new_tolerance = reduced._minimal_tolerance(self.sampling_interval) + if new_tolerance is None: # pragma: no cover + # Numerical safety net: the spacing cannot be declared for + # the reduced tie points, so stay irregular rather than let + # the constructor raise. + return reduced + else: + new_tolerance = self.tolerance + data = { + **data, + "sampling_interval": self.sampling_interval, + "tolerance": new_tolerance, + } + return self.__class__(data, self.dim) + # Otherwise try to promote: infer the best spacing on the surviving + # continuous segments and keep it only if it validates within the budget. + if regularize: + reduced = self.__class__(data, self.dim) + sampling_interval, _ = reduced._infer_regular() + if sampling_interval is not None and reduced._is_valid_sampling_interval( + sampling_interval, tolerance + ): + data = { + **data, + "sampling_interval": sampling_interval, + "tolerance": tolerance, + } + return self.__class__(data, self.dim) + + @override + def _split_candidates(self): + """Discontinuity split points, each paired with its step's deviation from the neighbouring interval.""" + tie_intervals = np.diff(self.tie_values) / np.diff(self.tie_indices) + (positions,) = np.nonzero(np.diff(self.tie_indices) == 1) + references = np.where( + positions > 0, + positions - 1, + np.minimum(positions + 1, len(tie_intervals) - 1), ) + deltas = tie_intervals[positions] - tie_intervals[references] + return self.tie_indices[positions + 1], deltas + + def _continuous_segments(self): + """Per-segment value/index spans ``(num, den)`` for the continuous areas. + + A ``den == 1`` gap is a CF discontinuity (section 8.3), not a segment, so + it is excluded and carries no sampling-rate information. + """ + num = np.diff(self.tie_values) + den = np.diff(self.tie_indices) + mask = den != 1 + return num[mask], den[mask] -def douglas_peucker(x, y, epsilon): +def _douglas_peucker(x, y, epsilon): """ Reduce the piecewise-linear curve *(x, y)* using the Douglas-Peucker algorithm. @@ -523,3 +712,72 @@ def douglas_peucker(x, y, epsilon): else: mask[start + 1 : stop - 1] = False return x[mask], y[mask] + + +def _chebyshev_center_pair(num, den): + """ + Segment indices binding the length-weighted Chebyshev center, in O(n log n). + + Returns the pair maximising ``den_i den_j |r_i - r_j| / (den_i + den_j)`` with + ``r = num / den``, equivalently the lowest point of the upper envelope of the + ``2 n`` lines ``±(den_i si - num_i)``. That vertex is the meeting of the + binding negative- and positive-slope lines, found with the convex-hull trick + instead of the O(n^2) pairwise scan. + + Parameters + ---------- + num : numpy.ndarray + Per-segment numerators as floats (seconds for datetime axes). + den : numpy.ndarray + Per-segment denominators as floats, all strictly positive. + + Returns + ------- + pos_idx, neg_idx : int + Segment indices of the binding positive- and negative-slope lines. A + single segment trivially selects itself (``pos_idx == neg_idx``). + """ + seg = np.arange(len(den)) + # Positive-slope lines (den si - num) and negative-slope lines (num - den si). + slopes = np.concatenate([den, -den]) + intercepts = np.concatenate([-num, num]) + idx = np.concatenate([seg, seg]) + # Process lines by ascending slope, equal slopes ordered by descending + # intercept so the dominant one comes first. + order = np.lexsort((-intercepts, slopes)) + return _upper_envelope_min_pair(slopes, intercepts, idx, order) + + +@njit(cache=True) +def _upper_envelope_min_pair(slopes, intercepts, idx, order): # pragma: no cover + """Binding (positive, negative) line indices at the upper-envelope minimum.""" + n = order.size + hull_s = np.empty(n, dtype=slopes.dtype) + hull_b = np.empty(n, dtype=intercepts.dtype) + hull_i = np.empty(n, dtype=idx.dtype) + m = 0 # current hull size + for t in range(n): + k = order[t] + s, b, seg = slopes[k], intercepts[k], idx[k] + # Equal slopes: the dominant (larger intercept) one came first; skip rest. + if m > 0 and hull_s[m - 1] == s: + continue + # Drop any line the convex-hull trick proves can never be the maximum. + while m >= 2: + s1, b1 = hull_s[m - 2], hull_b[m - 2] + s2, b2 = hull_s[m - 1], hull_b[m - 1] + if (b - b1) / (s1 - s) <= (b2 - b1) / (s1 - s2): + m -= 1 + else: + break + hull_s[m], hull_b[m], hull_i[m] = s, b, seg + m += 1 + # The envelope is convex with slope increasing along x; its minimum sits at + # the negative-to-positive slope transition, between the binding pair. The + # scan is safely bounded because `_chebyshev_center_pair` always feeds in + # both `+den_i` and `-den_i` lines, so at least one slope of each sign + # reaches the hull. + t = 0 + while hull_s[t] < 0.0: + t += 1 + return hull_i[t], hull_i[t - 1] diff --git a/xdas/coordinates/sampled.py b/xdas/coordinates/sampled.py index d05f159f..8faff535 100644 --- a/xdas/coordinates/sampled.py +++ b/xdas/coordinates/sampled.py @@ -7,40 +7,60 @@ import re import numpy as np +from typing_extensions import override from .core import ( + CODE_TO_UNITS, + UNITS_TO_CODE, + AxisCoordinate, Coordinate, - format_datetime, is_monotonic_increasing, - parse, - parse_tolerance, + parse_data_dim, + parse_scalar_delta, ) -CODE_TO_UNITS = { - "h": "hours", - "m": "minutes", - "s": "seconds", - "ms": "milliseconds", - "us": "microseconds", - "ns": "nanoseconds", -} -UNITS_TO_CODE = {v: k for k, v in CODE_TO_UNITS.items()} - -class SampledCoordinate(Coordinate, name="sampled"): +class SampledCoordinate(AxisCoordinate, ctype="sampled"): """ - A coordinate that is sampled at regular intervals. + Coordinate sampled at a fixed interval, with optional gaps between segments. + + More compact and numerically stable than + :class:`InterpCoordinate` for strictly uniform grids. Each contiguous + block is described by its start value and element count; all blocks share + the same ``sampling_interval``. Parameters ---------- - data : dict-like - The data of the coordinate. + data : dict with keys ``tie_values``, ``tie_lengths``, and ``sampling_interval`` + ``tie_values`` : sequence of float or datetime64 + Start value of each segment. + ``tie_lengths`` : sequence of int + Number of samples in each segment. All values must be > 0. + ``sampling_interval`` : scalar + Fixed step between consecutive samples, shared across all segments. + Must be :class:`numpy.timedelta64` when ``tie_values`` are + :class:`numpy.datetime64`. dim : str, optional - The dimension name of the coordinate, by default None. - dtype : str or numpy.dtype, optional - The data type of the coordinate, by default None. + Name of the dimension this coordinate is associated with. + dtype : dtype-like, optional + Desired dtype for ``tie_values``. + + Examples + -------- + >>> import numpy as np + >>> from xdas.coordinates import SampledCoordinate + >>> coord = SampledCoordinate( + ... { + ... "tie_values": [np.datetime64("2024-01-01T00:00:00", "ms")], + ... "tie_lengths": [1000], + ... "sampling_interval": np.timedelta64(4, "ms"), + ... } + ... ) + >>> coord + 2024-01-01T00:00:00.000 to 2024-01-01T00:00:03.996 """ + @override def __init__(self, data=None, dim=None, dtype=None): # empty if data is None: @@ -50,8 +70,8 @@ def __init__(self, data=None, dim=None, dtype=None): empty = False # parse data - data, dim = parse(data, dim) - if not self.__class__.isvalid(data): + data, dim = parse_data_dim(data, dim) + if not self._isvalid(data): raise ValueError( "`data` must be dict-like and contain `tie_values`, `tie_lengths`, and " "`sampling_interval`" @@ -89,13 +109,12 @@ def __init__(self, data=None, dim=None, dtype=None): if not np.ndim(sampling_interval) == 0: raise ValueError("`sampling_interval` must be a scalar value") sampling_interval = np.asarray(sampling_interval)[()] # ensure numpy scalar - if np.issubdtype(tie_values.dtype, np.datetime64): - if not np.issubdtype( - np.asarray(sampling_interval).dtype, np.timedelta64 - ): - raise ValueError( - "`sampling_interval` must be timedelta64 for datetime64 `tie_values`" - ) + if np.issubdtype(tie_values.dtype, np.datetime64) and not np.issubdtype( + np.asarray(sampling_interval).dtype, np.timedelta64 + ): + raise ValueError( + "`sampling_interval` must be timedelta64 for datetime64 `tie_values`" + ) # store data self.data = { @@ -120,60 +139,33 @@ def sampling_interval(self): """Fixed step between consecutive samples (shared across all segments).""" return self.data["sampling_interval"] - @property - def dtype(self): - """Dtype of the tie values (and of all materialised coordinate values).""" - return self.tie_values.dtype - @property def tie_indices(self): """Start integer index of each segment within the full coordinate array.""" return np.concatenate(([0], np.cumsum(self.tie_lengths[:-1]))) @property - def empty(self): - """``True`` if no segments have been set.""" - return self.tie_values.shape == (0,) - - @property - def ndim(self): - """Always 1.""" - return self.tie_values.ndim - - @property - def shape(self): - """Shape tuple ``(len(self),)``.""" - return (len(self),) - - @property - def indices(self): - """Full integer index array from 0 to ``len(self) - 1``.""" - if self.empty: - return np.array([], dtype="int") - else: - return np.arange(len(self)) - - @property - def values(self): - """Materialised numpy array of all coordinate values.""" - if self.empty: - return np.array([], dtype=self.dtype) - else: - return self.get_value(self.indices) + @override + def dtype(self): + return self.tie_values.dtype - @property - def start(self): - """Value at index 0 (first tie value).""" - return self.tie_values[0] + @classmethod + @override + def from_block(cls, start, size, step, dim=None, dtype=None): + data = { + "tie_values": [start], + "tie_lengths": [size], + "sampling_interval": step, + } + return cls(data, dim=dim, dtype=dtype) - @property - def end(self): - """Value one step past the last sample (exclusive upper bound).""" - return self.tie_values[-1] + self.sampling_interval * self.tie_lengths[-1] + @override + def __len__(self): + return sum(self.tie_lengths) @staticmethod - def isvalid(data): - """Return ``True`` if *data* has ``tie_values``, ``tie_lengths``, and ``sampling_interval`` keys.""" + @override + def _isvalid(data): match data: case { "tie_values": _, @@ -184,163 +176,21 @@ def isvalid(data): case _: return False - def __len__(self): - if self.empty: - return 0 - else: - return sum(self.tie_lengths) - - def __repr__(self): - if self.empty: - return "empty coordinate" - elif len(self) == 1: - return f"{self.tie_values[0]}" - else: - if np.issubdtype(self.dtype, np.floating): - return f"{self.start:.3f} to {self.end:.3f}" - elif np.issubdtype(self.dtype, np.datetime64): - start_str = format_datetime(self.start) - end_str = format_datetime(self.end) - return f"{start_str} to {end_str}" - else: - return f"{self.start} to {self.end}" - - def __getitem__(self, item): - if isinstance(item, slice): - return self.slice_index(item) - else: - return Coordinate( - self.get_value(item), None if np.isscalar(item) else self.dim - ) - - def __add__(self, other): - return self.__class__( - { - "tie_values": self.tie_values + other, - "tie_lengths": self.tie_lengths, - "sampling_interval": self.sampling_interval, - }, - self.dim, - ) - - def __sub__(self, other): - return self.__class__( - { - "tie_values": self.tie_values - other, - "tie_lengths": self.tie_lengths, - "sampling_interval": self.sampling_interval, - }, - self.dim, - ) - - def __array__(self, dtype=None): - out = self.values - if dtype is not None: - out = out.__array__(dtype) - return out - - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - raise NotImplementedError - - def __array_function__(self, func, types, args, kwargs): - raise NotImplementedError - - def issampled(self): - """Return ``True`` (this is a :class:`SampledCoordinate`).""" - return True + @override + def _is_monotonic_increasing(self): + return not self.get_split_indices( + "overlaps", tolerance=False + ).size # TODO: do not clall split_indices - def get_sampling_interval(self, cast=True): - """ - Return the sampling interval. - - Parameters - ---------- - cast : bool, optional - If ``True`` (default), cast timedelta64 to seconds (float). - """ - delta = self.sampling_interval - if cast and np.issubdtype(delta.dtype, np.timedelta64): - delta = delta / np.timedelta64(1, "s") - return delta - - def is_monotonic_increasing(self): - """Return ``True`` if no segment starts before the end of the previous one.""" - return not self.get_split_indices("overlaps", tolerance=False).size - - def equals(self, other): - """Return ``True`` if *other* has identical tie values, lengths, sampling interval, dim, and dtype.""" - return ( - np.array_equal(self.tie_values, other.tie_values) - and np.array_equal(self.tie_lengths, other.tie_lengths) - and self.sampling_interval == other.sampling_interval - and self.dim == other.dim - and self.dtype == other.dtype - ) - - def get_value(self, index): - """Compute coordinate value(s) at integer position(s) *index* using the stored segments.""" - index = self.format_index(index, bounds="raise") + @override + def _get_value(self, index): reference = np.searchsorted(self.tie_indices, index, side="right") - 1 return self.tie_values[reference] + ( (index - self.tie_indices[reference]) * self.sampling_interval ) - def slice_index(self, index_slice): - """Return a new :class:`SampledCoordinate` for the integer slice *index_slice*.""" - # normalize slice - start, stop, step = index_slice.indices(len(self)) - - if step < 0: - raise NotImplementedError("negative slice step is not implemented") - - # align stop - stop += (start - stop) % step # TODO: check for negative step - - # get relative start and stop within each tie - q, r = np.divmod(start - self.tie_indices, step) - lo = np.maximum(q, 0) * step + r - - q, r = np.divmod(self.tie_indices + self.tie_lengths - stop, step) - hi = self.tie_lengths - np.maximum(q, 0) * step + r - - # filter empty segments - mask = hi > lo - lo = lo[mask] - hi = hi[mask] - - # compute new tie values, tie lengths and sampling interval - tie_values = self.tie_values[mask] + lo * self.sampling_interval - tie_lengths = (hi - lo) // step - sampling_interval = self.sampling_interval * step - - # build new coordinate - data = { - "tie_values": tie_values, - "tie_lengths": tie_lengths, - "sampling_interval": sampling_interval, - } - return self.__class__(data, self.dim) - - def get_indexer(self, value, method=None): - """ - Return the integer index for label *value* using the segment structure. - - Parameters - ---------- - value : scalar, str (ISO datetime), or array-like - Label(s) to locate. - method : {None, "nearest", "ffill", "bfill"}, optional - How to handle values that fall in gaps or between samples. - - Returns - ------- - int or numpy.ndarray - - Raises - ------ - KeyError - If *value* falls in an overlap region or is not found (exact mode). - """ + @override + def _get_indexer(self, value, method=None): if isinstance(value, str): value = np.datetime64(value) else: @@ -411,8 +261,40 @@ def get_indexer(self, value, method=None): offset = np.maximum(offset, 0) return self.tie_indices[reference] + offset - def concat(self, other): - """Append *other* :class:`SampledCoordinate` segments after this one.""" + @override + def _slice(self, slc): + start, stop, step = slc.start, slc.stop, slc.step + + # align stop + stop += (start - stop) % step # TODO: check for negative step + + # get relative start and stop within each tie + q, r = np.divmod(start - self.tie_indices, step) + lo = np.maximum(q, 0) * step + r + + q, r = np.divmod(self.tie_indices + self.tie_lengths - stop, step) + hi = self.tie_lengths - np.maximum(q, 0) * step + r + + # filter empty segments + mask = hi > lo + lo = lo[mask] + hi = hi[mask] + + # compute new tie values, tie lengths and sampling interval + tie_values = self.tie_values[mask] + lo * self.sampling_interval + tie_lengths = (hi - lo) // step + sampling_interval = self.sampling_interval * step + + # build new coordinate + data = { + "tie_values": tie_values, + "tie_lengths": tie_lengths, + "sampling_interval": sampling_interval, + } + return self.__class__(data, self.dim) + + @override + def _concat(self, other): if not isinstance(other, self.__class__): raise TypeError(f"cannot concatenate {type(other)} to {self.__class__}") if not self.dim == other.dim: @@ -438,112 +320,8 @@ def concat(self, other): self.dim, ) - def decimate(self, q): - """Return a new coordinate keeping every *q*-th sample (integer decimation).""" - return self[::q] - - def simplify(self, tolerance=None): - """ - Merge adjacent segments whose gap is within *tolerance* of the sampling interval. - - Parameters - ---------- - tolerance : float, timedelta, or None - Maximum allowed discrepancy between the expected and actual start of the - next segment. ``None`` uses zero tolerance. ``False`` returns ``self`` unchanged. - """ - if tolerance is False: - return self # TODO: copy - tolerance = parse_tolerance(tolerance, self.dtype) - tie_values = [self.tie_values[0]] - tie_lengths = [self.tie_lengths[0]] - for value, length in zip(self.tie_values[1:], self.tie_lengths[1:]): - delta = value - (tie_values[-1] + self.sampling_interval * tie_lengths[-1]) - if np.abs(delta) <= tolerance: - tie_lengths[-1] += length - else: - tie_values.append(value) - tie_lengths.append(length) - return self.__class__( - { - "tie_values": np.array(tie_values), - "tie_lengths": np.array(tie_lengths), - "sampling_interval": self.sampling_interval, - }, - self.dim, - ) - - def get_split_indices(self, kind="discontinuities", tolerance=False): - """ - Return integer indices of segment boundaries (start of each segment except the first). - - Parameters - ---------- - kind : {"discontinuities", "gaps", "overlaps"}, optional - Which boundary type to return. Default ``"discontinuities"``. - tolerance : float, timedelta, or ``False`` - Minimum magnitude of the discrepancy to report. - - Returns - ------- - numpy.ndarray - """ - valid_kinds = {"discontinuities", "gaps", "overlaps"} - if kind not in valid_kinds: - raise ValueError(f"`kind` must be one of {valid_kinds}; got {kind!r}") - - indices = self.tie_indices[1:] - - # Fast path: no filtering requested - if kind == "discontinuities" and tolerance is False: - return indices - - deltas = self.tie_values[1:] - ( - self.tie_values[:-1] + self.sampling_interval * self.tie_lengths[:-1] - ) - - if tolerance is False: - zero = np.timedelta64(0) if np.issubdtype(self.dtype, np.datetime64) else 0 - - match kind: # pragma: no branch - case "gaps": - mask = deltas >= zero - case "overlaps": # pragma: no branch - mask = deltas < zero - - else: - tolerance = parse_tolerance(tolerance, self.dtype) - - match kind: # pragma: no branch - case "discontinuities": - mask = np.abs(deltas) > tolerance - case "gaps": - mask = deltas > tolerance - case "overlaps": # pragma: no branch - mask = deltas < -tolerance - - return indices[mask] - - @classmethod - def from_array(cls, arr, dim=None, sampling_interval=None): - """Not supported — raises :exc:`NotImplementedError`.""" - raise NotImplementedError("from_array is not implemented for SampledCoordinate") - - def to_dict(self): - """Serialise to ``{"dim": ..., "data": {"tie_values": ..., "tie_lengths": ..., "sampling_interval": ...}, "dtype": ...}``.""" - tie_values = self.data["tie_values"] - tie_lengths = self.data["tie_lengths"] - if np.issubdtype(tie_values.dtype, np.datetime64): - tie_values = tie_values.astype(str) - data = { - "tie_values": tie_values.tolist(), - "tie_lengths": tie_lengths.tolist(), - "sampling_interval": self.sampling_interval, - } - return {"dim": self.dim, "data": data, "dtype": str(self.dtype)} - - def to_dataset(self, dataset, attrs): - """Write sampling metadata into an xarray *dataset* using CF tie-point conventions.""" + @override + def _to_dataset(self, dataset, attrs): mapping = f"{self.name}: {self.name}_sampling" if "coordinate_sampling" in attrs: attrs["coordinate_sampling"] += " " + mapping @@ -578,8 +356,8 @@ def to_dataset(self, dataset, attrs): return dataset, attrs @classmethod - def from_dataset(cls, dataset, name): - """Read sampled coordinates from *dataset* using the ``coordinate_sampling`` attribute.""" + @override + def _collect_from_dataset(cls, dataset, name): coords = {} mapping = dataset[name].attrs.pop("coordinate_sampling", None) if mapping is not None: @@ -608,12 +386,83 @@ def from_dataset(cls, dataset, name): coords[name] = Coordinate(data, dim) return coords - @classmethod - def from_block(cls, start, size, step, dim=None, dtype=None): - """Build a single-segment :class:`SampledCoordinate` starting at *start* with *size* samples and step *step*.""" - data = { - "tie_values": [start], - "tie_lengths": [size], - "sampling_interval": step, - } - return cls(data, dim=dim, dtype=dtype) + def __add__(self, other): + return self.__class__( + { + "tie_values": self.tie_values + other, + "tie_lengths": self.tie_lengths, + "sampling_interval": self.sampling_interval, + }, + self.dim, + ) + + def __sub__(self, other): + return self.__class__( + { + "tie_values": self.tie_values - other, + "tie_lengths": self.tie_lengths, + "sampling_interval": self.sampling_interval, + }, + self.dim, + ) + + @override + def get_sampling_interval(self, cast=True): + if len(self) < 2: + return None + delta = self.sampling_interval + if cast and np.issubdtype(delta.dtype, np.timedelta64): + delta = delta / np.timedelta64(1, "s") + return delta + + @override + def to_regular(self, sampling_interval=None, tolerance=None): + """Regular by construction: validate any explicit spacing and return a copy. + + See :meth:`AxisCoordinate.to_regular` for the parameter contract. + """ + if sampling_interval is not None: + sampling_interval = parse_scalar_delta(sampling_interval, self.dtype) + tolerance = parse_scalar_delta(tolerance, self.dtype, default_zero=True) + if np.abs(sampling_interval - self.sampling_interval) > tolerance: + raise ValueError( + "`sampling_interval` does not match the stored sampling interval" + ) + return self.copy() + + @override + def simplify(self, tolerance=None, *, reduce=True, regularize=False): + """Fuse adjacent segments whose junction drift is within *tolerance*. + + The coordinate is regular by construction (it carries a single + ``sampling_interval``), so *regularize* is a no-op; fusing happens only + when *reduce* is set. See :meth:`Coordinate.simplify` for the parameter + contract. + """ + if tolerance is False or not reduce: + return self.copy() + tolerance = parse_scalar_delta(tolerance, self.dtype, default_zero=True) + tie_values = [self.tie_values[0]] + tie_lengths = [self.tie_lengths[0]] + for value, length in zip(self.tie_values[1:], self.tie_lengths[1:]): + delta = value - (tie_values[-1] + self.sampling_interval * tie_lengths[-1]) + if np.abs(delta) <= tolerance: + tie_lengths[-1] += length + else: + tie_values.append(value) + tie_lengths.append(length) + return self.__class__( + { + "tie_values": np.array(tie_values), + "tie_lengths": np.array(tie_lengths), + "sampling_interval": self.sampling_interval, + }, + self.dim, + ) + + @override + def _split_candidates(self): + deltas = self.tie_values[1:] - ( + self.tie_values[:-1] + self.sampling_interval * self.tie_lengths[:-1] + ) + return self.tie_indices[1:], deltas diff --git a/xdas/coordinates/scalar.py b/xdas/coordinates/scalar.py index dd89cbeb..a0181211 100644 --- a/xdas/coordinates/scalar.py +++ b/xdas/coordinates/scalar.py @@ -5,17 +5,20 @@ """ import numpy as np +from typing_extensions import override -from .core import Coordinate, parse +from .core import Coordinate, parse_data_dim -class ScalarCoordinate(Coordinate, name="scalar"): +class ScalarCoordinate(Coordinate, ctype="scalar"): """ Non-dimensional coordinate that carries a single scalar value. - Unlike dimensional coordinates, a :class:`ScalarCoordinate` is not tied - to an array axis and has no length. Typical use: metadata attached to a - :class:`DataArray` (e.g. an instrument identifier or a shot time). + Unlike :class:`~xdas.coordinates.AxisCoordinate` subclasses, a + :class:`ScalarCoordinate` is not tied to an array axis and has no length. + It therefore implements only the thin :class:`Coordinate` interface. + Typical use: metadata attached to a :class:`DataArray` (e.g. an instrument + identifier or a shot time). Parameters ---------- @@ -27,16 +30,23 @@ class ScalarCoordinate(Coordinate, name="scalar"): Cast *data* to this dtype. """ + @override def __init__(self, data=None, dim=None, dtype=None): if data is None: raise TypeError("scalar coordinate cannot be empty, please provide a value") - data, dim = parse(data, dim) + data, dim = parse_data_dim(data, dim) if dim is not None: raise ValueError("a scalar coordinate cannot be a dim") - if not self.__class__.isvalid(data): + if not self._isvalid(data): raise TypeError("`data` must be scalar-like") self.data = np.asarray(data, dtype=dtype) + @staticmethod + @override + def _isvalid(data): + data = np.asarray(data) + return (data.dtype != np.dtype(object)) and (data.ndim == 0) + @property def dim(self): """Always ``None`` — scalar coordinates have no associated dimension.""" @@ -48,35 +58,44 @@ def dim(self, value): if value is not None: raise ValueError("A scalar coordinate cannot have a `dim` other that None") - @staticmethod - def isvalid(data): - """Return ``True`` if *data* converts to a 0-d non-object numpy array.""" - data = np.asarray(data) - return (data.dtype != np.dtype(object)) and (data.ndim == 0) - - def isscalar(self): - """Return ``True`` (this is a :class:`ScalarCoordinate`).""" - return True + @property + @override + def dtype(self): + return self.data.dtype - def get_sampling_interval(self, cast=True): - """Return ``None`` — scalar coordinates have no sample spacing.""" - return None + @property + def ndim(self): + """Always ``0`` — scalar coordinates have no axis.""" + return 0 - def equals(self, other): - """Return ``True`` if *other* is a :class:`ScalarCoordinate` with the same value.""" - if isinstance(other, self.__class__): - return self.data == other.data - else: - return False - - def to_index(self, item, method=None, endpoint=True): - """Not supported — raises :exc:`NotImplementedError`.""" - raise NotImplementedError("cannot get index of scalar coordinate") - - def to_dict(self): - """Serialise to ``{"dim": None, "data": ..., "dtype": ...}``.""" - if np.issubdtype(self.dtype, np.datetime64): - data = self.data.astype(str).item() - else: - data = self.data.item() - return {"dim": self.dim, "data": data, "dtype": str(self.dtype)} + @property + @override + def shape(self): + return () + + @override + def __array__(self, dtype=None, copy=None): + # TODO: drop this workaround once Python 3.10 is no longer supported + # (EOL Oct 2026). numpy < 2.3 raises when copy=False on a 0-d array; + # numpy 2.3+ (requires Python 3.11+) handles it correctly. + if copy: + return np.array(self.data, dtype=dtype) + return np.asarray(self.data, dtype=dtype) + + @override + def __repr__(self): + return np.array2string(self.data, threshold=0, edgeitems=1) + + @override + def _to_dataset(self, dataset, attrs): + if self.name is None: + raise ValueError("cannot serialize a coordinate with no name") + dataset = dataset.assign_coords( + {self.name: (self.dim, self.values) if self.dim else self.values} + ) + return dataset, attrs + + @classmethod + @override + def _collect_from_dataset(cls, dataset, name): + return {} diff --git a/xdas/core/__init__.py b/xdas/core/__init__.py index c0c68b57..cb570eb0 100644 --- a/xdas/core/__init__.py +++ b/xdas/core/__init__.py @@ -4,3 +4,49 @@ Includes :class:`DataArray`, :class:`DataCollection`, and supporting routines, methods, and NumPy dispatch. """ + +__all__ = [ + "DataArray", + "DataCollection", + "DataMapping", + "DataSequence", + "align", + "asdataarray", + "broadcast_coords", + "broadcast_to", + "combine_by_coords", + "combine_by_field", + "concat", + "concat_coords", + "concatenate", + "open", + "open_dataarray", + "open_datacollection", + "open_mfdataarray", + "open_mfdatacollection", + "open_mfdatatree", + "plot_availability", + "split", +] + +from .dataarray import DataArray +from .datacollection import DataCollection, DataMapping, DataSequence +from .routines import ( + align, + asdataarray, + broadcast_coords, + broadcast_to, + combine_by_coords, + combine_by_field, + concat, + concat_coords, + concatenate, + open, + open_dataarray, + open_datacollection, + open_mfdataarray, + open_mfdatacollection, + open_mfdatatree, + plot_availability, + split, +) diff --git a/xdas/core/dataarray.py b/xdas/core/dataarray.py index f2d8994f..c5c28fc9 100644 --- a/xdas/core/dataarray.py +++ b/xdas/core/dataarray.py @@ -14,9 +14,8 @@ from dask.array import Array as DaskArray from numpy.lib.mixins import NDArrayOperatorsMixin -from ..coordinates import Coordinates -from ..dask.core import from_dict, to_dict -from ..virtual import VirtualArray, _to_human +from ..coordinates import AxisCoordinate, Coordinates +from ..virtual import _to_human HANDLED_NUMPY_FUNCTIONS = {} HANDLED_METHODS = {} @@ -79,7 +78,7 @@ def __getitem__(self, key): if isinstance(key, str): return self.coords[key] else: - query = self.coords.get_query(key) + query = self.coords._get_query(key) data = self.data.__getitem__(tuple(query.values())) coords = { name: ( @@ -96,7 +95,7 @@ def __setitem__(self, key, value): if isinstance(key, str): self.coords[key] = value else: - query = self.coords.get_query(key) + query = self.coords._get_query(key) self.data.__setitem__(tuple(query.values()), value) def __repr__(self): @@ -126,16 +125,16 @@ def __repr__(self): def __len__(self): return self.shape[0] - def __array__(self, dtype=None): - if dtype is None: - return self.data.__array__() - else: - return self.data.__array__(dtype) + def __array__(self, dtype=None, copy=None): + out = np.asarray(self.data, dtype=dtype) + if copy: + out = out.copy() + return out def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): from .routines import broadcast_coords, broadcast_to # TODO: circular import - if not method == "__call__": + if method != "__call__": return NotImplemented coords = broadcast_coords( @@ -267,7 +266,7 @@ def nbytes(self): @property def values(self): """Materialised numpy array of all values.""" - return self.__array__() + return self.__array__(copy=False) @property def empty(self): @@ -282,19 +281,17 @@ def loc(self): def equals(self, other): """Return ``True`` if *other* has equal data, coordinates, dims, name, and attrs.""" if isinstance(other, self.__class__): - if not self.dtype == other.dtype: + if self.dtype != other.dtype: return False if not np.array_equal(self.values, other.values, equal_nan=True): return False if not self.coords.equals(other.coords): return False - if not self.dims == other.dims: - return False - if not self.name == other.name: + if self.dims != other.dims: return False - if not self.attrs == other.attrs: + if self.name != other.name: return False - return True + return self.attrs == other.attrs else: return False @@ -348,7 +345,7 @@ def isel(self, indexers=None, drop=False, **indexers_kwargs): da = self[indexers] if drop: for dim in indexers: - if da[dim].isscalar(): + if not isinstance(da[dim], AxisCoordinate): da = da.drop_coords(dim) return da @@ -389,13 +386,13 @@ def sel( # handle not monotonic increasing coordinates for dim in indexers: - if not self[dim].is_monotonic_increasing(): + if not self[dim]._is_monotonic_increasing(): if isinstance(indexers[dim], slice): warnings.warn( f"dimension {dim} is not monotonic increasing, " f"spliting on overlaps, slicing and concatenating can be slow..." ) - from ..core.routines import concat, split + from .routines import concat, split chunks = [ chunk.sel(indexers, method, endpoint, drop) @@ -412,7 +409,7 @@ def sel( da = self[key] if drop: for dim in indexers: - if da[dim].isscalar(): + if not isinstance(da[dim], AxisCoordinate): da = da.drop_coords(dim) return da @@ -648,7 +645,7 @@ def swap_dims(self, dims_dict=None, **dims_kwargs): raise KeyError( f"dimension {dim} not found in current object with dims {self.dims}" ) - dims = tuple(dims_dict[dim] if dim in dims_dict else dim for dim in self.dims) + dims = tuple(dims_dict.get(dim, dim) for dim in self.dims) coords = {} for name, coord in self.coords.copy(deep=False).items(): if coord.dim in dims_dict: @@ -778,7 +775,7 @@ def expand_dims(self, dim, axis=0): raise ValueError(f"cannot expand on existing dimension {dim}") coords = self.coords.copy() if dim in coords: - if coords[dim].isscalar(): + if not isinstance(coords[dim], AxisCoordinate): coords[dim] = [coords[dim].values] else: raise ValueError( @@ -818,7 +815,7 @@ def to_stream( station="DAS{:05}", location="00", channel="{:1}N1", - dim={"last": "first"}, + dim=None, ): """ Convert a data array into an obspy stream. @@ -956,34 +953,6 @@ def from_netcdf(cls, fname, group=None): return open_dataarray(fname, group) - def to_dict(self): - """Convert the DataArray to a dictionary.""" - if isinstance(self.data, VirtualArray): - raise NotImplementedError("cannot convert a virtual array to a dictionary") - elif isinstance(self.data, np.ndarray): - data = self.data.tolist() - elif isinstance(self.data, DaskArray): # pragma: no branch - data = to_dict(self.data) - return { - "data": data, - "coords": self.coords.to_dict()["coords"], - "dims": self.dims, - "name": self.name, - "attrs": self.attrs, - } - - @classmethod - def from_dict(cls, dct): - """Create a DataArray from a dictionary.""" - if isinstance(dct["data"], list): - data = np.array(dct["data"]) - elif isinstance(dct["data"], dict): - data = from_dict(dct["data"]) - else: - raise ValueError("data must be a list or a dictionary") - coords = Coordinates.from_dict({key: dct[key] for key in ["coords", "dims"]}) - return cls(data, coords, dct["dims"], dct["name"], dct["attrs"]) - def plot(self, *args, **kwargs): """ Plot a DataArray. @@ -1035,11 +1004,11 @@ class DimSizer(dict): """Dict-like mapping from dimension names to their sizes, returned by :attr:`DataArray.sizes`.""" def __init__(self, obj): - super().__init__({dim: size for dim, size in zip(obj.dims, obj.shape)}) + super().__init__(dict(zip(obj.dims, obj.shape))) def __getitem__(self, key): if key == "first": - key = list(self.keys())[0] + key = next(iter(self.keys())) if key == "last": key = list(self.keys())[-1] return super().__getitem__(key) diff --git a/xdas/core/datacollection.py b/xdas/core/datacollection.py index b4259367..d62bcc2b 100644 --- a/xdas/core/datacollection.py +++ b/xdas/core/datacollection.py @@ -271,13 +271,11 @@ def equals(self, other): """Return ``True`` if *other* is a :class:`DataMapping` with identical keys and values.""" if not isinstance(other, self.__class__): return False - if not self.name == other.name: + if self.name != other.name: return False - if not list(self.keys()) == list(other.keys()): + if list(self.keys()) != list(other.keys()): return False - if not all(self[key].equals(other[key]) for key in self): - return False - return True + return all(self[key].equals(other[key]) for key in self) def isel(self, indexers=None, **indexers_kwargs): """ @@ -445,7 +443,7 @@ def fields(self): def to_mapping(self): """Convert to an integer-keyed :class:`DataMapping`.""" - return DataMapping({key: value for key, value in enumerate(self)}, self.name) + return DataMapping(dict(enumerate(self)), self.name) @classmethod def from_mapping(cls, data): @@ -480,13 +478,11 @@ def equals(self, other): """Return ``True`` if *other* is a :class:`DataSequence` with identical elements.""" if not isinstance(other, self.__class__): return False - if not self.name == other.name: - return False - if not len(self) == len(other): + if self.name != other.name: return False - if not all(a.equals(b) for a, b in zip(self, other)): + if len(self) != len(other): return False - return True + return all(a.equals(b) for a, b in zip(self, other)) def isel(self, indexers=None, **indexers_kwargs): """ diff --git a/xdas/core/methods.py b/xdas/core/methods.py index 6d43ec5f..f0e2ff91 100644 --- a/xdas/core/methods.py +++ b/xdas/core/methods.py @@ -6,7 +6,7 @@ import numpy as np -from ..atoms.core import atomized +from ..atoms import atomized from .dataarray import HANDLED_METHODS diff --git a/xdas/core/numpy.py b/xdas/core/numpy.py index 43a3529a..0eaf2cd6 100644 --- a/xdas/core/numpy.py +++ b/xdas/core/numpy.py @@ -68,9 +68,9 @@ def wrapper(*args, **kwargs): coords = { name: coord for name, coord in da.coords.items() - if not coord.dim == da.dims[axis] + if coord.dim != da.dims[axis] } - dims = tuple(dim for dim in da.dims if not dim == da.dims[axis]) + dims = tuple(dim for dim in da.dims if dim != da.dims[axis]) else: coords = da.coords dims = da.dims diff --git a/xdas/core/routines.py b/xdas/core/routines.py index a058f880..cda37707 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -21,7 +21,7 @@ from loky import get_reusable_executor from tqdm import tqdm -from ..coordinates.core import Coordinates, get_sampling_interval +from ..coordinates import AxisCoordinate, Coordinates from ..parallel import get_workers_count from ..virtual import VirtualSource, VirtualStack from .dataarray import DataArray @@ -143,7 +143,7 @@ def open( elif isinstance(paths, list): method = "multi-file" else: - raise Exception( + raise ValueError( f"`paths` must be either a string or a list, found {type(paths)}" ) match method: @@ -151,7 +151,7 @@ def open( if engine is None: try: return open_datacollection(paths) - except Exception: + except Exception: # noqa: BLE001, S110 - fall back to dataarray pass return open_dataarray(paths, engine=engine, **kwargs) case "multi-file": @@ -165,7 +165,7 @@ def open( parallel=parallel, verbose=verbose, ) - except Exception: + except Exception: # noqa: BLE001, S110 - fall back to mfdataarray pass return open_mfdataarray( paths, @@ -468,7 +468,7 @@ def collect( def defaulttree(depth): """Generate a default tree of lists with given depth.""" if depth == 1: - return list() + return [] else: return defaultdict(lambda: defaulttree(depth - 1)) @@ -555,7 +555,7 @@ def open_mfdataarray( for path in iterator: try: objs.append(open_dataarray(path, engine=engine, **kwargs)) - except Exception as error: + except Exception as error: # noqa: BLE001 - collected and warned below failures.append((path, error)) warnings.warn(f"could not open {path}: {error}", RuntimeWarning) else: @@ -575,7 +575,7 @@ def open_mfdataarray( for future in iterator: try: obj = future.result() - except Exception as error: + except Exception as error: # noqa: BLE001 - collected and warned below path = futures_to_paths[future] failures.append((path, error)) warnings.warn(f"could not open {path}: {error}", RuntimeWarning) @@ -738,7 +738,7 @@ def combine_by_field( dc.name = leaves[0].name return dc elif nodes and not leaves: - (name,) = set(dc.name for dc in nodes) + (name,) = {dc.name for dc in nodes} keys = sorted(set.union(*[set(dc.keys()) for dc in nodes])) return DataCollection( { @@ -803,7 +803,11 @@ def combine_by_coords( if dim in objs[0].coords: objs = sorted( objs, - key=lambda da: da[dim].values if da[dim].isscalar() else da[dim][0].values, + key=lambda da: ( + da[dim][0].values + if isinstance(da[dim], AxisCoordinate) + else da[dim].values + ), ) # combine objs @@ -857,20 +861,24 @@ def initialize(self, da): """Set *da* as the first element and record its shape, coords, sampling interval, and dtype.""" self.objs = [da] self.dims = da.dims - self.subshape = tuple( - size for dim, size in da.sizes.items() if not dim == self.dim - ) + self.subshape = tuple(size for dim, size in da.sizes.items() if dim != self.dim) self.subcoords = ( da.coords.drop_dims(self.dim) if self.dim in self.dims else da.coords.drop_coords(self.dim) ) - if self.dim in da.coords: - self.delta = get_sampling_interval(da, self.dim) - else: - self.delta = None + self.delta = self._get_delta(da) self.dtype = da.dtype + def _get_delta(self, da): + """Nominal sampling interval of *da* along *dim*, or ``None`` (irregular or absent).""" + if self.dim not in da.coords: + return None + coord = da.coords[self.dim] + if not isinstance(coord, AxisCoordinate): + return None + return coord.get_sampling_interval() + def append(self, da): """Add *da* after running all compatibility checks; initialises on first call.""" if not self.objs: @@ -890,7 +898,7 @@ def check_dims(self, da): def check_shape(self, da): """Raise :exc:`CompatibilityError` if *da* has a different non-concat shape.""" - subshape = tuple(size for dim, size in da.sizes.items() if not dim == self.dim) + subshape = tuple(size for dim, size in da.sizes.items() if dim != self.dim) if not self.subshape == subshape: raise CompatibilityError("shapes are not compatible") @@ -914,12 +922,21 @@ def check_sampling_interval(self, da): if self.delta is None: pass else: - delta = get_sampling_interval(da, self.dim) - if not np.isclose(delta, self.delta): + delta = self._get_delta(da) + if delta is None or not np.isclose(delta, self.delta): raise CompatibilityError("sampling intervals are not compatible") -def concat(objs, dim="first", tolerance=None, virtual=None, verbose=None): +def concat( + objs, + dim="first", + tolerance=None, + virtual=None, + verbose=None, + *, + reduce=True, + regularize=False, +): """ Concatenate data arrays along a given dimension. @@ -932,12 +949,22 @@ def concat(objs, dim="first", tolerance=None, virtual=None, verbose=None): tolerance : float or timedelta64, optional The tolerance to consider that the end of a file is continuous with beginning of the following, For time coordinates, numeric values are considered as seconds. - Zero by default. + By default each coordinate spends its own declared tolerance when it + carries one, else a zero-like default. Pass ``False`` to disable + simplification entirely. virtual : bool, optional Whether to create a virtual dataset. It requires that all concatenated data arrays are virtual. By default tries to create a virtual dataset if possible. verbose: bool Whether to display a progress bar. + reduce : bool, optional + Whether to drop redundant tie points from the concatenated coordinate. + Default True. + regularize : bool, optional + Whether to promote the concatenated coordinate to a regular one when its + segments admit a single shared rate within *tolerance*. Default False: + regular inputs already stay regular through concatenation, so promotion + only matters for irregular inputs and stays opt-in. Returns ------- @@ -976,6 +1003,8 @@ def concat(objs, dim="first", tolerance=None, virtual=None, verbose=None): sort=True, return_order=True, tolerance=tolerance, + reduce=reduce, + regularize=regularize, ) objs = [objs[idx] for idx in order] coords[dim] = coord @@ -986,8 +1015,7 @@ def concat(objs, dim="first", tolerance=None, virtual=None, verbose=None): data = [] for da in iterator: if isinstance(da.data, VirtualStack): - for source in da.data.sources: - data.append(source) + data.extend(da.data.sources) else: data.append(da.data) @@ -1002,7 +1030,15 @@ def concat(objs, dim="first", tolerance=None, virtual=None, verbose=None): concatenate = concat # TODO: deprecate it -def concat_coords(objs, *, sort=False, return_order=False, tolerance=False): +def concat_coords( + objs, + *, + sort=False, + return_order=False, + tolerance=None, + reduce=True, + regularize=False, +): """ Concatenate coordinate objects. @@ -1018,7 +1054,15 @@ def concat_coords(objs, *, sort=False, return_order=False, tolerance=False): tolerance : float or timedelta64, optional The tolerance to consider that the end of a coordinate object is continuous with beginning of the following, For time coordinates, numeric values are - considered as seconds. No simplification by default. + considered as seconds. By default the coordinate spends its own declared + tolerance when it carries one, else a zero-like default. Pass ``False`` + to disable simplification entirely. + reduce : bool, optional + Whether to drop redundant tie points after concatenation. Default True. + regularize : bool, optional + Whether to promote the result to a regular coordinate when the merged + segments admit a single shared rate within *tolerance*. Default False: + regular inputs already stay regular through concatenation. Returns ------- @@ -1037,20 +1081,22 @@ def concat_coords(objs, *, sort=False, return_order=False, tolerance=False): # concat for obj in objs[1:]: - out = out.concat(obj) + out = out._concat(obj) # simplify if tolerance is not False: - try: - out = out.simplify(tolerance) - except NotImplementedError: - if ( - tolerance is not None - ): # TODO: Default to False and remove this condition here? - raise TypeError( - "`tolerance` can only be used with coordinates " - "that implements `simplify`" - ) + if isinstance(out, AxisCoordinate): + # `_concat` is strict: same-rate inputs stay regular, mismatched + # rates drop to irregular. `simplify` then drops redundant tie + # points (chunk seams within tolerance fuse away) and, with + # `regularize=True`, recovers a single shared rate when the merged + # segments admit one within *tolerance*. + out = out.simplify(tolerance, reduce=reduce, regularize=regularize) + elif tolerance is not None: + raise TypeError( + "`tolerance` can only be used with coordinates " + "that implements `simplify`" + ) if return_order: return out, order @@ -1217,7 +1263,7 @@ def broadcast_coords(*objs): else: sizes[dim] = size for name, coord in obj.coords.items(): - if coord.isscalar(): + if not isinstance(coord, AxisCoordinate): continue if name in coords: if not coord.equals(coords[name]): diff --git a/xdas/dask/core.py b/xdas/dask/core.py index b0a8e1e4..ab6800df 100644 --- a/xdas/dask/core.py +++ b/xdas/dask/core.py @@ -80,14 +80,11 @@ def fuse(graph): def iskey(obj): """Return ``True`` if *obj* looks like a dask graph key (string or ``(str, int…)`` tuple).""" - if isinstance(obj, str) and len(obj) > 0: - return True - elif ( + if isinstance(obj, str): + return len(obj) > 0 + return ( isinstance(obj, tuple) and len(obj) > 1 and isinstance(obj[0], str) and all(isinstance(index, int) for index in obj[1:]) - ): - return True - else: - return False + ) diff --git a/xdas/fft.py b/xdas/fft.py index 3c498268..47893c46 100644 --- a/xdas/fft.py +++ b/xdas/fft.py @@ -7,14 +7,14 @@ import numpy as np -from .atoms.core import atomized -from .coordinates.core import get_sampling_interval -from .core.dataarray import DataArray +from .atoms import atomized +from .coordinates import get_sampling_interval +from .core import DataArray from .parallel import parallelize @atomized -def fft(da, n=None, dim={"last": "spectrum"}, norm=None, parallel=None): +def fft(da, n=None, dim=None, norm=None, parallel=None): """ Compute the discrete Fourier Transform along a given dimension. @@ -52,21 +52,25 @@ def fft(da, n=None, dim={"last": "spectrum"}, norm=None, parallel=None): -------- >>> import xdas as xd >>> import xdas.fft as xfft - >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0, 1, 2, 3]}) + >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0., 1., 2., 3.]}) + >>> signal["time"] = signal["time"].to_regular() >>> xfft.fft(signal, dim={"time": "frequency"}) [0.+0.j 0.+2.j 0.+0.j 0.-2.j] Coordinates: - * frequency (frequency): [-0.5 ... 0.25] + * frequency (frequency): -0.500 to 0.250 """ + if dim is None: + dim = {"last": "spectrum"} ((olddim, newdim),) = dim.items() olddim = da.dims[da.get_axis_num(olddim)] if n is None: n = da.sizes[olddim] axis = da.get_axis_num(olddim) d = get_sampling_interval(da, olddim) - f = np.fft.fftshift(np.fft.fftfreq(n, d)) + start = np.fft.fftshift(np.fft.fftfreq(n, d))[0] + f = type(da.coords[olddim]).from_block(start, n, 1 / (n * d), dim=newdim) def func(x): return np.fft.fftshift(np.fft.fft(x, n, axis, norm), axis) @@ -84,7 +88,7 @@ def func(x): @atomized -def rfft(da, n=None, dim={"last": "spectrum"}, norm=None, parallel=None): +def rfft(da, n=None, dim=None, norm=None, parallel=None): """ Compute the discrete Fourier Transform for real inputs along a given dimension. @@ -123,14 +127,17 @@ def rfft(da, n=None, dim={"last": "spectrum"}, norm=None, parallel=None): -------- >>> import xdas as xd >>> import xdas.fft as xfft - >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0, 1, 2, 3]}) + >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0., 1., 2., 3.]}) + >>> signal["time"] = signal["time"].to_regular() >>> xfft.rfft(signal, dim={"time": "frequency"}) [0.+0.j 0.-2.j 0.+0.j] Coordinates: - * frequency (frequency): [0. ... 0.5] + * frequency (frequency): 0.000 to 0.500 """ + if dim is None: + dim = {"last": "spectrum"} ((olddim, newdim),) = dim.items() olddim = da.dims[da.get_axis_num(olddim)] if n is None: @@ -139,7 +146,7 @@ def rfft(da, n=None, dim={"last": "spectrum"}, norm=None, parallel=None): d = get_sampling_interval(da, olddim) across = int(axis == 0) func = parallelize(across, across, parallel)(np.fft.rfft) - f = np.fft.rfftfreq(n, d) + f = type(da.coords[olddim]).from_block(0.0, n // 2 + 1, 1 / (n * d), dim=newdim) data = func(da.values, n, axis, norm) coords = { newdim if name == olddim else name: f if name == olddim else da.coords[name] @@ -151,7 +158,7 @@ def rfft(da, n=None, dim={"last": "spectrum"}, norm=None, parallel=None): @atomized -def ifft(da, n=None, dim={"last": "signal"}, norm=None, parallel=None): +def ifft(da, n=None, dim=None, norm=None, parallel=None): """ Compute the inverse of `fft`. @@ -186,20 +193,24 @@ def ifft(da, n=None, dim={"last": "signal"}, norm=None, parallel=None): -------- >>> import xdas as xd >>> import xdas.fft as xfft - >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0, 1, 2, 3]}) + >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0., 1., 2., 3.]}) + >>> signal["time"] = signal["time"].to_regular() >>> spectrum = xfft.fft(signal, dim={"time": "frequency"}) >>> result = xfft.ifft(spectrum, dim={"frequency": "time"}) >>> result["time"] = signal["time"] # to match time coordinates >>> assert np.real(result).equals(signal) """ + if dim is None: + dim = {"last": "signal"} ((olddim, newdim),) = dim.items() olddim = da.dims[da.get_axis_num(olddim)] if n is None: n = da.sizes[olddim] axis = da.get_axis_num(olddim) d = get_sampling_interval(da, olddim) - f = np.fft.ifftshift(np.fft.fftfreq(n, d)) + start = np.fft.fftshift(np.fft.fftfreq(n, d))[0] + f = type(da.coords[olddim]).from_block(start, n, 1 / (n * d), dim=newdim) def func(x): return np.fft.ifft(np.fft.ifftshift(x, axis), n, axis, norm) @@ -217,7 +228,7 @@ def func(x): @atomized -def irfft(da, n=None, dim={"last": "signal"}, norm=None, parallel=None): +def irfft(da, n=None, dim=None, norm=None, parallel=None): """ Compute the inverse of `rfft`. @@ -257,7 +268,8 @@ def irfft(da, n=None, dim={"last": "signal"}, norm=None, parallel=None): -------- >>> import xdas as xd >>> import xdas.fft as xfft - >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0, 1, 2, 3]}) + >>> signal = xd.DataArray([0., 1., 0., -1.], coords={"time": [0., 1., 2., 3.]}) + >>> signal["time"] = signal["time"].to_regular() >>> spectrum = xfft.rfft(signal, dim={"time": "frequency"}) >>> result = xfft.irfft( ... spectrum, @@ -268,6 +280,8 @@ def irfft(da, n=None, dim={"last": "signal"}, norm=None, parallel=None): >>> assert np.real(result).equals(signal) """ + if dim is None: + dim = {"last": "signal"} ((olddim, newdim),) = dim.items() olddim = da.dims[da.get_axis_num(olddim)] if n is None: @@ -276,7 +290,8 @@ def irfft(da, n=None, dim={"last": "signal"}, norm=None, parallel=None): d = get_sampling_interval(da, olddim) across = int(axis == 0) func = parallelize(across, across, parallel)(np.fft.irfft) - f = np.fft.fftshift(np.fft.fftfreq(n, d)) + start = np.fft.fftshift(np.fft.fftfreq(n, d))[0] + f = type(da.coords[olddim]).from_block(start, n, 1 / (n * d), dim=newdim) data = func(da.values, n, axis, norm) coords = { newdim if name == olddim else name: f if name == olddim else da.coords[name] diff --git a/xdas/io/apsensing.py b/xdas/io/apsensing.py index 54630129..b6187713 100644 --- a/xdas/io/apsensing.py +++ b/xdas/io/apsensing.py @@ -1,10 +1,12 @@ """I/O engine for APSensing HDF5 files (:class:`APSensingEngine`).""" +from typing import ClassVar + import h5py import numpy as np -from ..coordinates.core import Coordinate -from ..core.dataarray import DataArray +from ..coordinates import Coordinate +from ..core import DataArray from ..virtual import VirtualSource from .core import Engine @@ -12,8 +14,8 @@ class APSensingEngine(Engine, name="apsensing"): """Engine for reading APSensing HDF5 files.""" - _supported_vtypes = ["hdf5"] - _supported_ctypes = { + _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated", "sampled", "dense"], } @@ -39,7 +41,6 @@ def open_dataarray(self, fname): time = Coordinate[self.ctype["time"]].from_block(t0, nt, dt, dim="time") # distance - dx distance = Coordinate[self.ctype["distance"]].from_block( x0, nd, dx, dim="distance" ) diff --git a/xdas/io/asn.py b/xdas/io/asn.py index 49b7ea5e..414e28f8 100644 --- a/xdas/io/asn.py +++ b/xdas/io/asn.py @@ -7,13 +7,14 @@ import json from bisect import bisect_left, bisect_right +from typing import ClassVar import h5py import numpy as np import zmq -from ..coordinates.core import Coordinate, get_sampling_interval -from ..core.dataarray import DataArray +from ..coordinates import Coordinate, get_sampling_interval +from ..core import DataArray, concat_coords from ..virtual import VirtualSource from .core import Engine @@ -21,8 +22,8 @@ class ASNEngine(Engine, name="asn"): """Engine for reading ASN HDF5 files.""" - _supported_vtypes = ["hdf5"] - _supported_ctypes = { + _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated"], } @@ -42,13 +43,17 @@ def open_dataarray(self, fname): # Note that this vector is not continuous for more than one ROI all_dists = file["cableSpec"]["sensorDistances"][...] - # Buffer for the data index at which each ROI starts/stops - dist_tie_inds = [] - # Buffer for the optical distance at which each ROI starts/stops - dist_tie_vals = [] + # One regular block per ROI, concatenated below + roi_blocks = [] + + # Channel spacing is dx times the ROI decimation. Some files omit + # roiDec; those fall back to the spacing the ROI bounds imply. + roi_decs = demod["roiDec"][...] if "roiDec" in demod else None # Loop over ROIs, get the start/stop index before downsampling - for n_start, n_end in zip(demod["roiStart"], demod["roiEnd"]): + for n_roi, (n_start, n_end) in enumerate( + zip(demod["roiStart"], demod["roiEnd"]) + ): # ASN stores ROI end as an upper boundary. Use the last sampled distance # that does not exceed that boundary instead of indexing the insertion point. i_start, i_end = self._get_roi_bound_indices( @@ -57,17 +62,32 @@ def open_dataarray(self, fname): # Get the index where the ROI starts based on the position in the # distance vector. This solves the issue of rounding during decimation - # Append the data index and optical distance to the buffers - dist_tie_inds.append(i_start) - dist_tie_vals.append(float(all_dists[i_start])) - - # Repeat the procedure for the index/distance at which the ROI ends. - dist_tie_inds.append(i_end) - dist_tie_vals.append(float(all_dists[i_end])) + start = float(all_dists[i_start]) + size = i_end - i_start + 1 + if roi_decs is not None: + # Taking the spacing from the metadata keeps it bit-identical + # across ROIs that share a decimation, which is what lets + # concatenation preserve a regular axis. + step = dx * int(roi_decs[n_roi]) + elif size > 1: + step = (float(all_dists[i_end]) - start) / (i_end - i_start) + else: + step = dx + roi_blocks.append( + Coordinate[self.ctype["distance"]].from_block( + start, size, step, dim="distance" + ) + ) nt = data.shape[0] time = Coordinate[self.ctype["time"]].from_block(t0, nt, dt, dim="time") - distance = {"tie_indices": dist_tie_inds, "tie_values": dist_tie_vals} + # Concatenation keeps the declared spacing when every ROI agrees on it + # and drops to irregular otherwise, so unevenly decimated files stay + # honest without the engine having to test for it. Regularizing recovers + # the spacing when ROI steps differ only by float rounding (which the + # fallback above can produce), while genuinely different decimations + # still fail the fit. Reducing is off so the ROI structure is preserved. + distance = concat_coords(roi_blocks, reduce=False, regularize=True) return DataArray(data, {"time": time, "distance": distance}) def _get_roi_bound_indices(self, all_dists, n_start, n_end, dx): @@ -122,7 +142,7 @@ def __init__(self, address): >>> address = f"tcp://localhost:{port}" >>> publisher = ZMQPublisher(address) - >>> da = xd.synthetics.dummy() + >>> da = xd.testing.dummy() >>> chunks = xd.split(da, 10) >>> def publish(): @@ -174,18 +194,20 @@ def _update_header(self, message): roiTable = header["roiTable"][0] di = (roiTable["roiStart"] // roiTable["roiDec"]) * header["dx"] de = (roiTable["roiEnd"] // roiTable["roiDec"]) * header["dx"] - self.distance = { # TODO: use from_block + self.distance = { "tie_indices": [0, header["nChannels"] - 1], "tie_values": [di, de], + "sampling_interval": (de - di) / (header["nChannels"] - 1), } self.delta = float_to_timedelta(header["dt"], header["dtUnit"]) def _unpack(self, message): t0 = np.frombuffer(message[:8], "datetime64[ns]").reshape(()) data = np.frombuffer(message[8:], self.dtype).reshape(self.shape) - time = { # TODO: use from_block + time = { "tie_indices": [0, self.shape[0] - 1], "tie_values": [t0, t0 + (self.shape[0] - 1) * self.delta], + "sampling_interval": self.delta, } return DataArray(data, {"time": time, "distance": self.distance}) @@ -214,7 +236,7 @@ class ZMQPublisher: >>> import xdas as xd >>> from xdas.io.asn import ZMQPublisher - >>> da = xd.synthetics.dummy() + >>> da = xd.testing.dummy() >>> port = xd.io.get_free_port() >>> address = f"tcp://localhost:{port}" @@ -277,7 +299,7 @@ def _send(self, da): header = self._get_header(da) if self.header is None: self.header = header - if not header == self.header: + if header != self.header: self.header = header self._send_header() self._send_data(da) diff --git a/xdas/io/core.py b/xdas/io/core.py index a96d39dd..f4dffb55 100644 --- a/xdas/io/core.py +++ b/xdas/io/core.py @@ -6,6 +6,7 @@ """ import socket +from typing import ClassVar class Engine: @@ -65,8 +66,8 @@ class Engine: >>> engine = Engine["nc"](ctype="dense") # Using alias """ - _registry = {} - _aliases = {} + _registry: ClassVar[dict] = {} + _aliases: ClassVar[dict] = {} _supported_vtypes = None _supported_ctypes = None @@ -131,7 +132,7 @@ def _parse_ctype(self, ctype): key: self._supported_ctypes[key][0] for key in self._supported_ctypes } elif isinstance(ctype, str): - ctype = {key: ctype for key in self._supported_ctypes} + ctype = dict.fromkeys(self._supported_ctypes, ctype) elif isinstance(ctype, dict): ctype = { key: ctype.get(key, self._supported_ctypes[key][0]) @@ -206,7 +207,7 @@ def open_dataarray(self, fname, **kwargs): ) AutoEngine._last_successful_engine = engine return out - except Exception: + except Exception: # noqa: BLE001, S112 - try the next engine continue message = f"no engine could open the file '{fname}'" if self.ctype is not None: diff --git a/xdas/io/febus.py b/xdas/io/febus.py index 6915a5f4..342f672b 100644 --- a/xdas/io/febus.py +++ b/xdas/io/febus.py @@ -1,13 +1,13 @@ """I/O engine for Febus HDF5 files (:class:`FebusEngine`).""" import warnings +from typing import ClassVar import h5py import numpy as np -from ..coordinates.core import Coordinate -from ..core.dataarray import DataArray -from ..core.routines import concat +from ..coordinates import Coordinate +from ..core import DataArray, concat from ..virtual import VirtualSource from .core import Engine @@ -15,8 +15,8 @@ class FebusEngine(Engine, name="febus"): """Engine for reading Febus HDF5 files.""" - _supported_vtypes = ["hdf5"] - _supported_ctypes = { + _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated", "sampled", "dense"], } diff --git a/xdas/io/miniseed.py b/xdas/io/miniseed.py index 4af52943..948958e3 100644 --- a/xdas/io/miniseed.py +++ b/xdas/io/miniseed.py @@ -1,20 +1,26 @@ """I/O engine for MiniSEED files via ObsPy (:class:`MiniSEEDEngine`).""" +from typing import ClassVar + import dask import numpy as np import obspy -from ..coordinates.core import Coordinate, Coordinates, get_sampling_interval -from ..core.dataarray import DataArray -from ..core.routines import concat_coords +from ..coordinates import ( + AxisCoordinate, + Coordinate, + Coordinates, + get_sampling_interval, +) +from ..core import DataArray, concat_coords from .core import Engine class MiniSEEDEngine(Engine, name="miniseed"): """Engine for reading MiniSEED files via ObsPy as lazy dask-backed DataArrays.""" - _supported_vtypes = ["dask"] - _supported_ctypes = { + _supported_vtypes: ClassVar[list] = ["dask"] + _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], } @@ -81,7 +87,9 @@ def read_header(self, path, ignore_last_sample, ctype): } ) - shape = tuple(len(coord) for coord in coords.values() if not coord.isscalar()) + shape = tuple( + len(coord) for coord in coords.values() if isinstance(coord, AxisCoordinate) + ) return shape, dtype, coords, method def read_data(self, path, method, ignore_last_sample): @@ -112,7 +120,7 @@ def to_stream( station="DAS{:05}", location="00", channel="{:1}N1", - dim={"last": "first"}, + dim=None, ): """ Convert a 2-D :class:`DataArray` to an :class:`obspy.Stream`. @@ -131,6 +139,8 @@ def to_stream( ------- obspy.Stream """ + if dim is None: + dim = {"last": "first"} dimdist, dimtime = dim.copy().popitem() if not da.ndim == 2: raise ValueError("the data array must be 2D") @@ -174,13 +184,11 @@ def from_stream(st, dims=("channel", "time")): """ data = np.stack([tr.data for tr in st]) channel = [tr.id for tr in st] - time = { - "tie_indices": [0, st[0].stats.npts - 1], - "tie_values": [ - np.datetime64(st[0].stats.starttime.datetime), - np.datetime64(st[0].stats.endtime.datetime), - ], - } + # Regular by construction from the stream's own sample rate, at ns + # resolution so a `to_stream` round trip preserves the coordinate. + t0 = np.datetime64(st[0].stats.starttime.datetime) + dt = np.rint(1e6 * st[0].stats.delta).astype("m8[us]").astype("m8[ns]") + time = Coordinate["interpolated"].from_block(t0, st[0].stats.npts, dt, dim=dims[1]) return DataArray(data, {dims[0]: channel, dims[1]: time}) @@ -195,7 +203,7 @@ def get_time_coord(tr, ignore_last_sample, ctype): def uniquifiy(seq): """Return the unique elements of *seq* in order; unwrap to scalar if only one.""" seen = set() - seq = list(x for x in seq if x not in seen and not seen.add(x)) + seq = [x for x in seq if x not in seen and not seen.add(x)] if len(seq) == 1: return seq[0] else: diff --git a/xdas/io/prodml.py b/xdas/io/prodml.py index a71a0694..349d30c6 100644 --- a/xdas/io/prodml.py +++ b/xdas/io/prodml.py @@ -4,11 +4,13 @@ Also known as OptaSense and Sintela format. """ +from typing import ClassVar + import h5py import pandas as pd -from ..coordinates.core import Coordinate -from ..core.dataarray import DataArray +from ..coordinates import Coordinate +from ..core import DataArray from ..virtual import VirtualSource from .core import Engine @@ -16,8 +18,8 @@ class ProdML(Engine, name="prodml", aliases=["optasense", "sintela"]): """Engine for reading ProdML / OptaSense / Sintela HDF5 files.""" - _supported_vtypes = ["hdf5"] - _supported_ctypes = { + _supported_vtypes: ClassVar[list] = ["hdf5"] + _supported_ctypes: ClassVar[dict] = { "time": ["interpolated"], "distance": ["interpolated", "sampled", "dense"], } @@ -48,11 +50,12 @@ def open_dataarray(self, fname, swapped_dims=False): else: nt, nd = data.shape - # time + # time (regular by declaration, rate derived from the file's own stamps) time = { "tie_indices": [0, nt - 1], "tie_values": [tstart, tend], - } # TODO: use from_block + "sampling_interval": (tend - tstart) / (nt - 1), + } # distance distance = Coordinate[self.ctype["distance"]].from_block( diff --git a/xdas/io/silixa.py b/xdas/io/silixa.py index d158d0ef..a86a5734 100644 --- a/xdas/io/silixa.py +++ b/xdas/io/silixa.py @@ -1,10 +1,12 @@ """I/O engine for Silixa TDMS files (:class:`SilixaEngine`).""" +from typing import ClassVar + import dask import numpy as np -from ..coordinates.core import Coordinate -from ..core.dataarray import DataArray +from ..coordinates import Coordinate +from ..core import DataArray from .core import Engine from .tdms import TdmsReader @@ -12,8 +14,8 @@ class SilixaEngine(Engine, name="silixa"): """Engine for reading Silixa iDAS TDMS files as lazy dask-backed DataArrays.""" - _supported_vtypes = ["dask"] - _supported_ctypes = { + _supported_vtypes: ClassVar[list] = ["dask"] + _supported_ctypes: ClassVar[dict] = { "time": ["interpolated", "sampled", "dense"], "distance": ["interpolated", "sampled", "dense"], } diff --git a/xdas/io/tdms.py b/xdas/io/tdms.py index ab16f7a5..082130dc 100644 --- a/xdas/io/tdms.py +++ b/xdas/io/tdms.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Copyright (c) 2018 Silixa Ltd. @@ -49,10 +48,9 @@ def write_property_dict(prop_dict, out_file): """Write *prop_dict* as a Python-literal assignment to *out_file*.""" from pprint import pformat - f = open(out_file, "w") - f.write("tdms_property_map=") - f.write(pformat(prop_dict)) - f.close() + with open(out_file, "w") as f: + f.write("tdms_property_map=") + f.write(pformat(prop_dict)) def type_not_supported(vargin): @@ -68,7 +66,8 @@ def parse_time_stamp(fractions, seconds): 1/1/1904). Returns datetime.datetime or None. """ if fractions is not None and seconds is not None and fractions + seconds > 0: - return datetime.timedelta(0, fractions * 2**-64 + seconds) + datetime.datetime( + # the TDMS epoch is naive by spec; callers convert to tz-naive datetime64 + return datetime.timedelta(0, fractions * 2**-64 + seconds) + datetime.datetime( # noqa: DTZ001 1904, 1, 1 ) else: @@ -77,56 +76,52 @@ def parse_time_stamp(fractions, seconds): # Enum mapping TDM data types to description string, numpy type where exists # See Ref[2] for enum values -TDS_DATA_TYPE = dict( - { - 0x00: "void", # tdsTypeVoid - 0x01: "int8", # tdsTypeI8 - 0x02: "int16", # tdsTypeI16 - 0x03: "int32", # tdsTypeI32 - 0x04: "int64", # tdsTypeI64 - 0x05: "uint8", # tdsTypeU8 - 0x06: "uint16", # tdsTypeU16 - 0x07: "uint32", # tdsTypeU32 - 0x08: "uint64", # tdsTypeU64 - 0x09: "float32", # tdsTypeSingleFloat - 0x0A: "float64", # tdsTypeDoubleFloat - 0x0B: "float128", # tdsTypeExtendedFloat - 0x19: "singleFloatWithUnit", # tdsTypeSingleFloatWithUnit - 0x1A: "doubleFloatWithUnit", # tdsTypeDoubleFloatWithUnit - 0x1B: "extendedFloatWithUnit", # tdsTypeExtendedFloatWithUnit - 0x20: "str", # tdsTypeString - 0x21: "bool", # tdsTypeBoolean - 0x44: "datetime", # tdsTypeTimeStamp - 0xFFFFFFFF: "raw", # tdsTypeDAQmxRawData - } -) +TDS_DATA_TYPE = { + 0x00: "void", # tdsTypeVoid + 0x01: "int8", # tdsTypeI8 + 0x02: "int16", # tdsTypeI16 + 0x03: "int32", # tdsTypeI32 + 0x04: "int64", # tdsTypeI64 + 0x05: "uint8", # tdsTypeU8 + 0x06: "uint16", # tdsTypeU16 + 0x07: "uint32", # tdsTypeU32 + 0x08: "uint64", # tdsTypeU64 + 0x09: "float32", # tdsTypeSingleFloat + 0x0A: "float64", # tdsTypeDoubleFloat + 0x0B: "float128", # tdsTypeExtendedFloat + 0x19: "singleFloatWithUnit", # tdsTypeSingleFloatWithUnit + 0x1A: "doubleFloatWithUnit", # tdsTypeDoubleFloatWithUnit + 0x1B: "extendedFloatWithUnit", # tdsTypeExtendedFloatWithUnit + 0x20: "str", # tdsTypeString + 0x21: "bool", # tdsTypeBoolean + 0x44: "datetime", # tdsTypeTimeStamp + 0xFFFFFFFF: "raw", # tdsTypeDAQmxRawData +} # Function mapping for reading TDMS data types -TDS_READ_VAL = dict( - { - "void": lambda f: None, # tdsTypeVoid - "int8": lambda f: struct.unpack(" self.file_size: - self.fileinfo["next_segment_offset"] = self.file_size - # raise(ValueError, "Next Segment Offset too large in TDMS header") + self.fileinfo["next_segment_offset"] = min( + self.fileinfo["next_segment_offset"], self.file_size + ) + # raise(ValueError, "Next Segment Offset too large in TDMS header") def __enter__(self): return self @@ -238,9 +235,7 @@ def _read_property(self): # Read length of object path: var = struct.unpack(">> packets = xd.split(xd.synthetics.dummy(), 10) + >>> packets = xd.split(xd.testing.dummy(), 10) We initialize the publisher at a given address @@ -646,7 +645,7 @@ def write(self, da): def result(self): """Return ``None`` — ZMQPublisher has no aggregated result.""" - return None + return class ZMQSubscriber: @@ -672,7 +671,7 @@ class ZMQSubscriber: First we generate some data and split it into packets - >>> da = xd.synthetics.dummy() + >>> da = xd.testing.dummy() >>> packets = xd.split(da, 10) We then publish the packets asynchronously diff --git a/xdas/signal.py b/xdas/signal.py index c93e2fe9..54e06c2b 100644 --- a/xdas/signal.py +++ b/xdas/signal.py @@ -5,12 +5,14 @@ all coordinate-aware and multi-threaded via :func:`~xdas.parallel.parallelize`. """ +import itertools + import numpy as np import scipy.signal as sp -from .atoms.core import atomized -from .coordinates.core import Coordinate, get_sampling_interval -from .core.dataarray import DataArray +from .atoms import atomized +from .coordinates import InterpCoordinate, get_sampling_interval +from .core import DataArray from .parallel import parallelize from .spectral import stft # noqa @@ -118,8 +120,10 @@ def filter(da, freq, btype, corners=4, zerophase=False, dim="last", parallel=Non """ axis = da.get_axis_num(dim) + dim = da.dims[axis] + d = get_sampling_interval(da, dim) across = int(axis == 0) - fs = 1.0 / get_sampling_interval(da, dim) + fs = 1.0 / d sos = sp.iirfilter(corners, freq, btype=btype, ftype="butter", output="sos", fs=fs) if zerophase: func = parallelize((None, across), across, parallel)(sp.sosfiltfilt) @@ -246,14 +250,21 @@ def resample(da, num, dim="last", window=None, domain="time", parallel=None): """ axis = da.get_axis_num(dim) dim = da.dims[axis] + get_sampling_interval(da, dim) # warn or raise on irregular axes upfront across = int(axis == 0) func = parallelize(across, across, parallel)(sp.resample) data, t = func(da.values, num, da[dim].values, axis, window, domain) - new_coord = {"tie_indices": [0, num - 1], "tie_values": [t[0], t[-1]]} + source = da.coords[dim] + new_coord = type(source).from_block(t[0], num, t[1] - t[0], dim=dim) + # Resampling derives a new rate; it does not make the sample times better + # known, so the declared jitter carries over. Only interpolated coordinates + # declare one. + if isinstance(source, InterpCoordinate) and source.tolerance is not None: + new_coord = new_coord.to_regular(new_coord.sampling_interval, source.tolerance) coords = { name: new_coord if name == dim else coord for name, coord in da.coords.items() - if not (coord.dim == dim and not name == dim) # don't handle non-dimensional + if not (coord.dim == dim and name != dim) # don't handle non-dimensional } return DataArray(data, coords, da.dims, da.name, da.attrs) @@ -342,24 +353,26 @@ def resample_poly( """ axis = da.get_axis_num(dim) dim = da.dims[axis] + d = get_sampling_interval(da, dim, cast=False) across = int(axis == 0) func = parallelize(across, across, parallel)(sp.resample_poly) data = func(da.values, up, down, axis, window, padtype, cval) start = da[dim][0].values - d = da[dim][-1].values - da[dim][-2].values - end = da[dim][-1].values + d - new_coord = Coordinate( - { - "tie_indices": [0, data.shape[axis]], - "tie_values": [start, end], - }, - dim, - ) - new_coord = new_coord[:-1] + step = d * down / up + source = da.coords[dim] + new_coord = type(source).from_block(start, data.shape[axis], step, dim=dim) + # The derived rate may not be exactly representable (integer datetime + # resolutions truncate), so declare that error as jitter on top of the + # inherited one, as UpSample does; chunk seams then stay within tolerance. + if isinstance(source, InterpCoordinate): + tolerance = np.abs(d * down - step * up) + if source.tolerance is not None: + tolerance = source.tolerance + tolerance + new_coord = new_coord.to_regular(step, tolerance) coords = { name: new_coord if name == dim else coord for name, coord in da.coords.items() - if not (coord.dim == dim and not name == dim) # don't handle non-dimensional + if not (coord.dim == dim and name != dim) # don't handle non-dimensional } return DataArray(data, coords, da.dims, da.name, da.attrs) @@ -795,6 +808,7 @@ def integrate(da, midpoints=False, dim="last", parallel=None): """ axis = da.get_axis_num(dim) + dim = da.dims[axis] d = get_sampling_interval(da, dim) def func(x): @@ -838,6 +852,7 @@ def differentiate(da, midpoints=False, dim="last", parallel=None): """ axis = da.get_axis_num(dim) + dim = da.dims[axis] d = get_sampling_interval(da, dim) def func(x): @@ -875,7 +890,7 @@ def segment_mean_removal(da, limits, window="hann", dim="last"): # TODO: parall """ out = da.copy() axis = da.get_axis_num(dim) - for sstart, send in zip(limits[:-1], limits[1:]): + for sstart, send in itertools.pairwise(limits): key = {dim: slice(sstart, np.nextafter(send, -np.inf))} data = out.loc[key].values win = sp.get_window(window, data.shape[axis]) @@ -921,6 +936,7 @@ def sliding_mean_removal( """ axis = da.get_axis_num(dim) + dim = da.dims[axis] d = get_sampling_interval(da, dim) n = round(wlen / d) if n % 2 == 0: @@ -989,8 +1005,8 @@ def medfilt(da, kernel_dim): # TODO: parallelize * distance (distance): 0.000 to 10000.000 """ - if not all(dim in da.dims for dim in kernel_dim.keys()): + if not all(dim in da.dims for dim in kernel_dim): raise ValueError("dims provided not in dataarray") - kernel_size = tuple(kernel_dim[dim] if dim in kernel_dim else 1 for dim in da.dims) + kernel_size = tuple(kernel_dim.get(dim, 1) for dim in da.dims) data = sp.medfilt(da.values, kernel_size) return da.copy(data=data) diff --git a/xdas/spectral.py b/xdas/spectral.py index 2e92d990..7aec1a4a 100644 --- a/xdas/spectral.py +++ b/xdas/spectral.py @@ -8,8 +8,8 @@ from scipy.fft import fft, fftfreq, fftshift, rfft, rfftfreq from scipy.signal import get_window -from .coordinates.core import get_sampling_interval -from .core.dataarray import DataArray +from .coordinates import get_sampling_interval +from .core import DataArray from .parallel import parallelize @@ -20,7 +20,7 @@ def stft( noverlap=None, nfft=None, return_onesided=True, - dim={"last": "sprectrum"}, + dim=None, scaling="spectrum", parallel=None, ): @@ -79,9 +79,12 @@ def stft( noverlap = nperseg // 2 if nfft is None: nfft = nperseg + if dim is None: + dim = {"last": "sprectrum"} win = get_window(window, nperseg) input_dim, output_dim = next(iter(dim.items())) axis = da.get_axis_num(input_dim) + input_dim = da.dims[axis] # resolve "first"/"last" aliases dt = get_sampling_interval(da, input_dim) if scaling == "spectrum": scale = 1.0 / win.sum() ** 2 @@ -90,11 +93,12 @@ def stft( else: raise ValueError("Scaling must be 'spectrum' or 'psd'") scale = np.sqrt(scale) + coord_cls = type(da.coords[input_dim]) if return_onesided: freqs = rfftfreq(nfft, dt) else: freqs = fftshift(fftfreq(nfft, dt)) - freqs = {"tie_indices": [0, len(freqs) - 1], "tie_values": [freqs[0], freqs[-1]]} + freqs = coord_cls.from_block(freqs[0], len(freqs), 1.0 / (nfft * dt)) def func(x): """Apply windowed FFT to produce the STFT output array.""" @@ -123,11 +127,7 @@ def func(x): dt = get_sampling_interval(da, input_dim, cast=False) t0 = da.coords[input_dim].values[0] starttime = t0 + (nperseg / 2) * dt - endtime = starttime + (data.shape[axis] - 1) * (nperseg - noverlap) * dt - time = { - "tie_indices": [0, data.shape[axis] - 1], - "tie_values": [starttime, endtime], - } + time = coord_cls.from_block(starttime, data.shape[axis], (nperseg - noverlap) * dt) coords = {} for name in da.coords: diff --git a/xdas/synthetics.py b/xdas/synthetics.py index ecf593b8..426a0a24 100644 --- a/xdas/synthetics.py +++ b/xdas/synthetics.py @@ -7,14 +7,17 @@ import numpy as np import scipy.signal as sp -from .core.dataarray import DataArray -from .core.routines import split +from .coordinates import Coordinate +from .core import DataArray, split + +# module-level singleton so the default is not built at each call (ruff B008) +DEFAULT_RESOLUTION = (np.timedelta64(20, "ms"), 25.0) def wavelet_wavefronts( *, starttime="2023-01-01T00:00:00", - resolution=(np.timedelta64(20, "ms"), 25.0), + resolution=DEFAULT_RESOLUTION, nchunk=None, ): """ @@ -84,14 +87,12 @@ def wavelet_wavefronts( da = DataArray( data=data, coords={ - "time": { - "tie_indices": [0, shape[0] - 1], - "tie_values": [starttime, starttime + resolution[0] * (shape[0] - 1)], - }, - "distance": { - "tie_indices": [0, shape[1] - 1], - "tie_values": [0.0, resolution[1] * (shape[1] - 1)], - }, + "time": Coordinate["interpolated"].from_block( + starttime, shape[0], resolution[0], dim="time" + ), + "distance": Coordinate["interpolated"].from_block( + 0.0, shape[1], resolution[1], dim="distance" + ), }, ) if nchunk is not None: @@ -145,42 +146,12 @@ def randn_wavefronts(): da = DataArray( data=data, coords={ - "time": { - "tie_indices": [0, shape[0] - 1], - "tie_values": [starttime, starttime + resolution[0] * (shape[0] - 1)], - }, - "distance": { - "tie_indices": [0, shape[1] - 1], - "tie_values": [0.0, resolution[1] * (shape[1] - 1)], - }, + "time": Coordinate["interpolated"].from_block( + starttime, shape[0], resolution[0], dim="time" + ), + "distance": Coordinate["interpolated"].from_block( + 0.0, shape[1], resolution[1], dim="distance" + ), }, ) return da - - -def dummy(shape=(1000, 100)): - """ - Return a minimal random :class:`DataArray` for quick testing. - - Parameters - ---------- - shape : tuple of int, optional - ``(n_time, n_distance)`` shape. Defaults to ``(1000, 100)``. - - Returns - ------- - DataArray - DataArray filled with Gaussian noise, sampled at 10 Hz over - ``[0, 1000]`` m with ``time`` starting at 2024-01-01. - """ - starttime = np.datetime64("2024-01-01T00:00:00.000000000") - endtime = starttime + (shape[0] - 1) * np.timedelta64(100, "ms") - time = {"tie_indices": [0, shape[0] - 1], "tie_values": [starttime, endtime]} - distance = {"tie_indices": [0, shape[1] - 1], "tie_values": [0.0, 1000.0]} - return DataArray( - data=np.random.randn(*shape), - coords={ - "time": time, - "distance": distance, - }, - ) diff --git a/xdas/testing.py b/xdas/testing.py new file mode 100644 index 00000000..f03f13db --- /dev/null +++ b/xdas/testing.py @@ -0,0 +1,80 @@ +"""Test utilities for xdas.""" + +import numpy as np + +from .coordinates import Coordinate +from .core import DataArray + + +def dummy( + dims=("time", "distance"), + shape=(100, 10), + dtype=float, + step=(0.01, 10.0), + ctype="interpolated", + datetime=True, +): + """ + Return a minimal :class:`DataArray` for quick testing. + + Parameters + ---------- + dims : tuple of str, optional + Dimension names. Length must match ``shape``. Defaults to + ``("time", "distance")``. + shape : tuple of int, optional + Size along each dimension. Defaults to ``(100, 10)``. + dtype : dtype-like, optional + Data type for the array values. Defaults to ``float``. + step : scalar or tuple, optional + Step size for each dimension. A single value is applied to all + dimensions; a tuple must have the same length as ``dims``. Defaults + to ``(0.01, 10.0)`` (100 Hz, 10 m spacing → 1 s × 100 m total). + When ``datetime=True``, a float step for the first dimension is + interpreted as seconds and converted to :class:`numpy.timedelta64`. + ctype : {"interpolated", "sampled", "dense"}, optional + Coordinate type for all dimensions. Defaults to ``"interpolated"``. + datetime : bool, optional + If ``True`` (default), the first dimension uses + :class:`numpy.datetime64` coordinates starting at 2024-05-21. + All other dimensions use float coordinates starting at 0.0. + + Returns + ------- + DataArray + Array filled with sequential integers (via :func:`numpy.arange`) + reshaped to ``shape`` and cast to ``dtype``. + + Examples + -------- + >>> import xdas as xd + >>> da = xd.testing.dummy() + >>> da.shape + (100, 10) + >>> da = xd.testing.dummy(dims=("x",), shape=(50,), datetime=False, step=1.0) + >>> da.shape + (50,) + >>> da = xd.testing.dummy(dims=("x",), shape=(10,), datetime=False, step=2.0) + >>> float(da.coords["x"].sampling_interval) + 2.0 + + """ + if len(dims) != len(shape): + raise ValueError(f"len(dims)={len(dims)} must equal len(shape)={len(shape)}") + if isinstance(step, (tuple, list)) and len(step) != len(dims): + raise ValueError(f"len(step)={len(step)} must equal len(dims)={len(dims)}") + + data = np.arange(int(np.prod(shape))).reshape(shape).astype(dtype) + + coords = {} + for i, (dim, size) in enumerate(zip(dims, shape)): + s = step[i] if isinstance(step, (tuple, list)) else step + if datetime and i == 0: + start = np.datetime64("2024-05-21T00:00:00.000000000") + if isinstance(s, (int, float)): + s = np.timedelta64(int(s * 1e9), "ns") + else: + start = 0.0 + coords[dim] = Coordinate[ctype].from_block(start, size, s, dim=dim) + + return DataArray(data=data, coords=coords) diff --git a/xdas/trigger.py b/xdas/trigger.py index c9e2465e..4f07fbac 100644 --- a/xdas/trigger.py +++ b/xdas/trigger.py @@ -10,7 +10,8 @@ from numba import njit from .atoms.core import Atom, State, atomized -from .coordinates.core import Coordinate +from .coordinates import Coordinate +from .core import concat_coords class Trigger(Atom): @@ -140,7 +141,7 @@ def call(self, cft, **flags): """ data = np.asarray(cft.values, dtype=float) values, coords = self._call_numeric(data) - self.coord = _concat([self.coord, cft.coords[self.dim]]) + self.coord = concat_coords([self.coord, cft.coords[self.dim]], tolerance=None) picks = {} for axis, dim in enumerate(cft.dims): @@ -333,7 +334,9 @@ def find_picks(cft, thresh, dim="last", state_dict=None): # TODO: state_dict => buffer=state_dict["buffer"], offset=state_dict["offset"], ) - state_dict["coord"] = _concat([state_dict["coord"], cft.coords[dim]]) + state_dict["coord"] = concat_coords( + [state_dict["coord"], cft.coords[dim]], tolerance=None + ) else: indices, values = _find_picks_numeric(data, thresh, axis) state_dict["coord"] = cft.coords[dim] @@ -355,59 +358,6 @@ def find_picks(cft, thresh, dim="last", state_dict=None): # TODO: state_dict => return picks -def _concat(list_of_coord): # TODO: make it a public function/method - """ - Concatenates a list of interpolated coordinates. - - Parameters - ---------- - list_of_coord : list - A list of InterpCoordinate objects to be concatenated. - - Returns - ------- - InterpCoordinate - The concatenated interpolated coordinate. - - Examples - -------- - >>> import xdas as xd - - >>> coord1 = xd.Coordinate( - ... {"tie_indices": [0, 2], "tie_values": [10, 30]}, - ... dim="dim", - ... ) - >>> coord2 = xd.Coordinate( - ... {"tie_indices": [0, 3], "tie_values": [40, 70]}, - ... dim="dim", - ... ) - - >>> concatenated = _concat([coord1, coord2]) - - >>> concatenated.tie_indices - array([0, 6]) - >>> concatenated.tie_values - array([10, 70]) - >>> concatenated.dim - 'dim' - - """ - tie_indices = [] - tie_values = [] - idx = 0 - dim = list_of_coord[0].dim - for coord in list_of_coord: - if not coord.isinterp(): - raise ValueError("Only interpolated coordinates can be concatenated.") - if not coord.dim == dim: - raise ValueError("All coordinates must have the same dimension.") - tie_indices.extend(idx + coord.tie_indices) - tie_values.extend(coord.tie_values) - idx += len(coord) - coord = Coordinate({"tie_indices": tie_indices, "tie_values": tie_values}, dim) - return coord.simplify() - - def _find_picks_numeric(cft, thresh, axis=-1, buffer=None, offset=None): """ Find picks in a N-dimensional array along a given axis based on a given threshold. diff --git a/xdas/virtual.py b/xdas/virtual.py index a94d1033..17748539 100644 --- a/xdas/virtual.py +++ b/xdas/virtual.py @@ -26,24 +26,24 @@ def __repr__(self): return f"{self.__class__.__name__}: {_to_human(self.nbytes)} ({self.dtype})" def __getitem__(self, key): - NotImplemented + raise NotImplementedError - def __array__(self, dtype=None): - NotImplemented + def __array__(self, dtype=None, copy=None): + raise NotImplementedError @property def shape(self): """Tuple of array dimensions (abstract — must be overridden).""" - NotImplemented + raise NotImplementedError @property def dtype(self): """NumPy dtype of the array elements (abstract — must be overridden).""" - NotImplemented + raise NotImplementedError def to_dataset(self, file_or_group, name): """Write this virtual array as an HDF5 dataset (abstract — must be overridden).""" - NotImplemented + raise NotImplementedError @property def ndim(self): @@ -113,11 +113,11 @@ class VirtualStack(VirtualArray): Concatenation axis. Defaults to ``0``. """ - def __init__(self, sources=[], axis=0): - self._sources = list() + def __init__(self, sources=None, axis=0): + self._sources = [] self._axis = axis self._shape = () - self.extend(sources) + self.extend([] if sources is None else sources) def __getitem__(self, key): if not isinstance(key, tuple): @@ -175,10 +175,10 @@ def __getitem__(self, key): sources = [source[tuple(indexers)] for source in self._sources] return VirtualStack(sources, self._axis) - def __array__(self, dtype=None): + def __array__(self, dtype=None, copy=None): if not self._sources: raise ValueError("no sources in stack") - return self._to_layout().__array__(dtype) + return self._to_layout().__array__(dtype, copy=copy) @property def sources(self): @@ -331,7 +331,7 @@ def __init__(self, shape, dtype, maxshape=None, filename=None): self._layout = h5py.VirtualLayout(shape, dtype, maxshape, filename) self._sel = Selection(self._layout.shape) - def __array__(self, dtype=None): + def __array__(self, dtype=None, copy=None): with TemporaryDirectory() as tmpdirname: fname = os.path.join(tmpdirname, "vds.h5") with h5py.File(fname, "w") as file: @@ -344,9 +344,9 @@ def __array__(self, dtype=None): return out def __getitem__(self, key): - self = copy(self) - self._sel = self._sel.__getitem__(key) - return self + out = copy(self) + out._sel = out._sel.__getitem__(key) + return out def __setitem__(self, key, value): if not self._sel._whole: @@ -461,11 +461,11 @@ def __init__( self._sel = Selection(self._vsource.shape) def __getitem__(self, key): - self = copy(self) - self._sel = self._sel.__getitem__(key) - return self + out = copy(self) + out._sel = out._sel.__getitem__(key) + return out - def __array__(self, dtype=None): + def __array__(self, dtype=None, copy=None): with h5py.File(self.vsource.path) as file: dataset = file[self.vsource.name] return np.asarray(dataset[self._sel.get_indexer()], dtype=dtype)