|
| 1 | +"""Read Heidelberg spiking-audio splits (SHD, SSC) from their HDF5 directly. |
| 2 | +
|
| 3 | +Tonic downloads these datasets correctly and then decodes them wrongly. Its |
| 4 | +reader converts the file's timestamps from seconds to microseconds with |
| 5 | +``times * 1e6``, but the Heidelberg files store them as ``float16`` (maximum |
| 6 | +65504). Under NumPy 2's NEP 50 promotion a ``float16`` array times a Python |
| 7 | +float stays ``float16``, so the multiply overflows to ``inf``, becomes ``NaN``, |
| 8 | +and casts to ``INT64_MIN`` -- for every timestamp in every sample. Binning then |
| 9 | +sees a zero-width span and collapses the whole recording into one time step, so |
| 10 | +a 25-step spiking network trains on a single static frame and still reports a |
| 11 | +plausible accuracy. |
| 12 | +
|
| 13 | +So this module replaces that one conversion. It does **not** replace tonic: |
| 14 | +the download, extraction, cache layout, and sensor geometry all still come |
| 15 | +from the tonic dataset object, which is passed in. Only ``__getitem__`` is |
| 16 | +ours, and it scales in ``float64`` where the arithmetic is exact. |
| 17 | +
|
| 18 | +Two incidental wins. Tonic reopens the HDF5 file on every ``__getitem__``; |
| 19 | +this opens it once, which is what makes |
| 20 | +:class:`~spikeforge.events.event_source.EventSampleSource`'s open-once caching |
| 21 | +actually pay off here. And the emitted stream keeps tonic's own |
| 22 | +``(t, x, p)`` structured layout, so nothing downstream can tell the |
| 23 | +difference -- a cochlea has channels rather than pixel rows, and |
| 24 | +:func:`~spikeforge.data.event_loader.events_to_sample` already places a |
| 25 | +one-row sensor's events on row 0. |
| 26 | +
|
| 27 | +``h5py`` arrives with ``tonic`` itself, so the ``events`` extra covers both, |
| 28 | +but the import is defensive here exactly as it is in |
| 29 | +:mod:`spikeforge.events.tonic_api`: this is the only module in the project |
| 30 | +that imports ``h5py``. |
| 31 | +""" |
| 32 | + |
| 33 | +import os |
| 34 | +from importlib import import_module |
| 35 | +from typing import Any, Optional, Tuple |
| 36 | + |
| 37 | +import numpy as np |
| 38 | + |
| 39 | +#: Name a registry entry sets in ``DatasetSpec.native_reader`` to route here. |
| 40 | +HSD = "hsd" |
| 41 | +#: The structured layout tonic's own HSD reader emits, kept verbatim so |
| 42 | +#: nothing downstream has to special-case this path. |
| 43 | +DTYPE = np.dtype([("t", int), ("x", int), ("p", int)]) |
| 44 | +#: Seconds-to-microseconds scale, applied in float64 where it is exact. |
| 45 | +MICROSECONDS = 1e6 |
| 46 | + |
| 47 | + |
| 48 | +def _h5py() -> Optional[Any]: |
| 49 | + """Return the ``h5py`` module, or ``None`` when it is unavailable.""" |
| 50 | + try: |
| 51 | + return import_module("h5py") |
| 52 | + except ImportError: |
| 53 | + return None |
| 54 | + |
| 55 | + |
| 56 | +def available() -> bool: |
| 57 | + """Return True when the HDF5 reader can run in this environment.""" |
| 58 | + return _h5py() is not None |
| 59 | + |
| 60 | + |
| 61 | +class HsdSplit: |
| 62 | + """One SHD/SSC split, read from its HDF5 file. |
| 63 | +
|
| 64 | + Presents the same surface the rest of the event path uses of a tonic |
| 65 | + dataset -- ``len()``, ``[index]``, and ``sensor_size`` -- so it drops in |
| 66 | + wherever the tonic object went. |
| 67 | + """ |
| 68 | + |
| 69 | + def __init__(self, dataset: Any) -> None: |
| 70 | + """Open the split ``dataset`` points at, reading its layout from it. |
| 71 | +
|
| 72 | + ``dataset`` is the constructed tonic dataset: it has already done the |
| 73 | + download and extraction, and it knows the cache layout and the sensor |
| 74 | + geometry. Nothing here second-guesses any of that. |
| 75 | + """ |
| 76 | + module = _h5py() |
| 77 | + if module is None: |
| 78 | + raise RuntimeError( |
| 79 | + "h5py is required to read SHD/SSC and is normally installed " |
| 80 | + 'with tonic; reinstall the `events` extra: pip install -e ' |
| 81 | + '".[events]"' |
| 82 | + ) |
| 83 | + self._path = os.path.join( |
| 84 | + dataset.location_on_system, dataset.data_filename |
| 85 | + ) |
| 86 | + self.sensor_size: Tuple[int, int, int] = dataset.sensor_size |
| 87 | + self._file = module.File(self._path, "r") |
| 88 | + self._times = self._file["spikes/times"] |
| 89 | + self._units = self._file["spikes/units"] |
| 90 | + self._labels = self._file["labels"] |
| 91 | + |
| 92 | + def __len__(self) -> int: |
| 93 | + """Return the split's sample count.""" |
| 94 | + return int(len(self._labels)) |
| 95 | + |
| 96 | + def __getitem__(self, index: int) -> Tuple[np.ndarray, int]: |
| 97 | + """Return one ``((t, x, p) array, label)`` pair. |
| 98 | +
|
| 99 | + The timestamps are widened to ``float64`` *before* being scaled, which |
| 100 | + is the whole point of this module: doing it in the file's own |
| 101 | + ``float16`` overflows and loses every timestamp. |
| 102 | + """ |
| 103 | + position = int(index) |
| 104 | + seconds = np.asarray(self._times[position], dtype=np.float64) |
| 105 | + units = np.asarray(self._units[position], dtype=np.int64) |
| 106 | + events = np.empty(seconds.shape[0], dtype=DTYPE) |
| 107 | + events["t"] = np.rint(seconds * MICROSECONDS).astype(np.int64) |
| 108 | + events["x"] = units |
| 109 | + # The Heidelberg recordings carry no polarity; tonic supplies a |
| 110 | + # constant 1 and we match it rather than inventing a second channel. |
| 111 | + events["p"] = 1 |
| 112 | + return events, int(np.asarray(self._labels[position])) |
| 113 | + |
| 114 | + def close(self) -> None: |
| 115 | + """Close the underlying HDF5 file.""" |
| 116 | + self._file.close() |
| 117 | + |
| 118 | + @property |
| 119 | + def path(self) -> str: |
| 120 | + """Return the HDF5 file this split reads.""" |
| 121 | + return self._path |
| 122 | + |
| 123 | + |
| 124 | +def open_split(dataset: Any) -> HsdSplit: |
| 125 | + """Return an :class:`HsdSplit` reading the split ``dataset`` located.""" |
| 126 | + return HsdSplit(dataset) |
0 commit comments