From 36225b5b1ed75e70efec7fd3fc418bcd26319c9b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 25 Sep 2025 07:18:46 +0100 Subject: [PATCH 01/12] integrate codspeed --- .github/test_code.sh | 5 +- .github/workflows/codspeed.yml | 35 +++++ .gitignore | 1 + asv.conf.json | 184 -------------------------- benchmarks/__init__.py | 5 - benchmarks/generic_io_benchmarks.py | 49 ------- benchmarks/memory_spool_benchmarks.py | 84 ------------ benchmarks/patch_benchmarks.py | 110 --------------- benchmarks/readme.md | 41 +++++- benchmarks/test_io_benchmarks.py | 58 ++++++++ benchmarks/test_patch_benchmarks.py | 119 +++++++++++++++++ benchmarks/test_spool_benchmarks.py | 107 +++++++++++++++ pyproject.toml | 2 +- readme.md | 1 + scripts/_benchmark_uilts.py | 94 ------------- scripts/run_benchmarks.py | 34 ----- scripts/visualize_benchmarks.py | 11 -- 17 files changed, 366 insertions(+), 574 deletions(-) create mode 100644 .github/workflows/codspeed.yml delete mode 100644 asv.conf.json delete mode 100644 benchmarks/__init__.py delete mode 100644 benchmarks/generic_io_benchmarks.py delete mode 100644 benchmarks/memory_spool_benchmarks.py delete mode 100644 benchmarks/patch_benchmarks.py create mode 100644 benchmarks/test_io_benchmarks.py create mode 100644 benchmarks/test_patch_benchmarks.py create mode 100644 benchmarks/test_spool_benchmarks.py delete mode 100644 scripts/_benchmark_uilts.py delete mode 100644 scripts/run_benchmarks.py delete mode 100644 scripts/visualize_benchmarks.py diff --git a/.github/test_code.sh b/.github/test_code.sh index 7fa95f694..f2589f913 100755 --- a/.github/test_code.sh +++ b/.github/test_code.sh @@ -1,10 +1,13 @@ #!/bin/bash # Script to run tests to account for wonkiness of periodic mac failures. -args=" -s --cov dascore --cov-append --cov-report=xml" +args="tests -s --cov dascore --cov-append --cov-report=xml" if [[ "$1" == "doctest" ]]; then args="dascore --doctest-modules" fi +if [[ "$1" == "benchmarks" ]]; then + args="benchmarks --codspeed" +fi exit_code=0 diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..e19bffd6d --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,35 @@ +name: CodSpeed Benchmarks + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + +env: + python_version: "3.13" + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-tags: "true" + fetch-depth: '0' + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.python_version }} + + - name: Install dependencies + run: pip install ".[test]" + + - name: Run benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: instrumentation + run: pytest benchmarks/ --codspeed + token: ${{ secrets.CODSPEED_TOKEN }} # Optional for public repos diff --git a/.gitignore b/.gitignore index 1ae472ecf..d6902cf66 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ docs/**/*.ipynb # profile stuff from asv .asv +.codspeed # misc scratch/** diff --git a/asv.conf.json b/asv.conf.json deleted file mode 100644 index c9795bde5..000000000 --- a/asv.conf.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - // The version of the config file format. Do not change, unless - // you know what you are doing. - "version": 1, - - // The name of the project being benchmarked - "project": "dascore", - - // The project's homepage - "project_url": "https://github.com/dasdae/dascore", - - // The URL or local path of the source code repository for the - // project being benchmarked - "repo": ".", - - // The Python project's subdirectory in your repo. If missing or - // the empty string, the project is assumed to be located at the root - // of the repository. - // "repo_subdir": "", - - // Customizable commands for building, installing, and - // uninstalling the project. See asv.conf.json documentation. - // - // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], - // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], - // "build_command": [ - // "python setup.py build", - // "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" - // ], - - // List of branches to benchmark. If not provided, defaults to "master" - // (for git) or "default" (for mercurial). - // "branches": ["master"], // for git - // "branches": ["default"], // for mercurial - - // The DVCS being used. If not set, it will be automatically - // determined from "repo" by looking at the protocol in the URL - // (if remote), or by looking for special directories, such as - // ".git" (if local). - // "dvcs": "git", - - // The tool to use to create environments. May be "conda", - // "virtualenv" or other value depending on the plugins in use. - // If missing or the empty string, the tool will be automatically - // determined by looking for tools on the PATH environment - // variable. - "environment_type": "conda", - - // timeout in seconds for installing any dependencies in environment - // defaults to 10 min - //"install_timeout": 600, - - // the base URL to show a commit for the project. - // "show_commit_url": "http://github.com/owner/project/commit/", - - // The Pythons you'd like to test against. If not provided, defaults - // to the current version of Python used to run `asv`. - // "pythons": ["2.7", "3.6"], - - // The list of conda channel names to be searched for benchmark - // dependency packages in the specified order - // "conda_channels": ["conda-forge", "defaults"], - - // A conda environment file that is used for environment creation. - // "conda_environment_file": "environment.yml", - - // The matrix of dependencies to test. Each key of the "req" - // requirements dictionary is the name of a package (in PyPI) and - // the values are version numbers. An empty list or empty string - // indicates to just test against the default (latest) - // version. null indicates that the package is to not be - // installed. If the package to be tested is only available from - // PyPi, and the 'environment_type' is conda, then you can preface - // the package name by 'pip+', and the package will be installed - // via pip (with all the conda available packages installed first, - // followed by the pip installed packages). - // - // The ``@env`` and ``@env_nobuild`` keys contain the matrix of - // environment variables to pass to build and benchmark commands. - // An environment will be created for every combination of the - // cartesian product of the "@env" variables in this matrix. - // Variables in "@env_nobuild" will be passed to every environment - // during the benchmark phase, but will not trigger creation of - // new environments. A value of ``null`` means that the variable - // will not be set for the current combination. - // - // "matrix": { - // "req": { - // "numpy": ["1.6", "1.7"], - // "six": ["", null], // test with and without six installed - // "pip+emcee": [""] // emcee is only available for install with pip. - // }, - // "env": {"ENV_VAR_1": ["val1", "val2"]}, - // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, - // }, - - // Combinations of libraries/python versions can be excluded/included - // from the set to test. Each entry is a dictionary containing additional - // key-value pairs to include/exclude. - // - // An exclude entry excludes entries where all values match. The - // values are regexps that should match the whole string. - // - // An include entry adds an environment. Only the packages listed - // are installed. The 'python' key is required. The exclude rules - // do not apply to includes. - // - // In addition to package names, the following keys are available: - // - // - python - // Python version, as in the *pythons* variable above. - // - environment_type - // Environment type, as above. - // - sys_platform - // Platform, as in sys.platform. Possible values for the common - // cases: 'linux2', 'win32', 'cygwin', 'darwin'. - // - req - // Required packages - // - env - // Environment variables - // - env_nobuild - // Non-build environment variables - // - // "exclude": [ - // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows - // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda - // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 - // ], - // - // "include": [ - // // additional env for python2.7 - // {"python": "2.7", "req": {"numpy": "1.8"}, "env_nobuild": {"FOO": "123"}}, - // // additional env if run on windows+conda - // {"platform": "win32", "environment_type": "conda", "python": "2.7", "req": {"libpython": ""}}, - // ], - - // The directory (relative to the current directory) that benchmarks are - // stored in. If not provided, defaults to "benchmarks" - // "benchmark_dir": "benchmarks", - - // The directory (relative to the current directory) to cache the Python - // environments in. If not provided, defaults to "env" - "env_dir": ".asv/env", - - // The directory (relative to the current directory) that raw benchmark - // results are stored in. If not provided, defaults to "results". - "results_dir": ".asv/results", - - // The directory (relative to the current directory) that the html tree - // should be written to. If not provided, defaults to "html". - "html_dir": ".asv/html", - - // The number of characters to retain in the commit hashes. - // "hash_length": 8, - - // `asv` will cache results of the recent builds in each - // environment, making them faster to install next time. This is - // the number of builds to keep, per environment. - // "build_cache_size": 2, - - // The commits after which the regression search in `asv publish` - // should start looking for regressions. Dictionary whose keys are - // regexps matching to benchmark names, and values corresponding to - // the commit (exclusive) after which to start looking for - // regressions. The default is to start from the first commit - // with results. If the commit is `null`, regression detection is - // skipped for the matching benchmark. - // - // "regressions_first_commits": { - // "some_benchmark": "352cdf", // Consider regressions only after this commit - // "another_benchmark": null, // Skip regression detection altogether - // }, - - // The thresholds for relative change in results, after which `asv - // publish` starts reporting regressions. Dictionary of the same - // form as in ``regressions_first_commits``, with values - // indicating the thresholds. If multiple entries match, the - // maximum is taken. If no entry matches, the default is 5%. - // - // "regressions_thresholds": { - // "some_benchmark": 0.01, // Threshold of 1% - // "another_benchmark": 0.5, // Threshold of 50% - // }, -} diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py deleted file mode 100644 index 9ad605673..000000000 --- a/benchmarks/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -A suite of benchmarks for DASCore. This __init__.py file is apparently -required. -""" -from __future__ import annotations diff --git a/benchmarks/generic_io_benchmarks.py b/benchmarks/generic_io_benchmarks.py deleted file mode 100644 index 54b64eeec..000000000 --- a/benchmarks/generic_io_benchmarks.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Benchmark for generic memory spool operations.""" - -from __future__ import annotations - -from functools import cache - -import dascore as dc -from dascore.utils.downloader import fetch, get_registry_df - - -@cache -def test_file_paths(): - """Get a dict of name: path for all files in data registry.""" - df = get_registry_df().loc[lambda x: ~x["name"].str.endswith(".csv")] - out = {row["name"]: fetch(row["name"]) for _, row in df.iterrows()} - return out - - -class IoSuite: - """Basic io functions.""" - - def setup(self): - """Get paths of test files.""" - self.path_dict = test_file_paths() - - def time_scan(self): - """Time for basic scanning of all datafiles.""" - for path in self.path_dict.values(): - dc.scan(path) - - def time_scan_df(self): - """Time for basic scanning of all datafiles.""" - for path in self.path_dict.values(): - dc.scan_to_df(path) - - def time_get_format(self): - """Time for basic scanning of all datafiles.""" - for path in self.path_dict.values(): - dc.get_format(path) - - def time_read(self): - """Time for basic scanning of all datafiles.""" - for path in self.path_dict.values(): - dc.read(path)[0] - - def time_spool(self): - """Time for basic scanning of all datafiles.""" - for path in self.path_dict.values(): - dc.spool(path)[0] diff --git a/benchmarks/memory_spool_benchmarks.py b/benchmarks/memory_spool_benchmarks.py deleted file mode 100644 index 75db3e875..000000000 --- a/benchmarks/memory_spool_benchmarks.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Benchmark for generic memory spool operations.""" - -from __future__ import annotations - -import dascore as dc - - -class ChunkSuite: - """Benchmark for chunking patches inside spools.""" - - def setup(self): - """Get test spools.""" - self._spool_no_gap = dc.get_example_spool("random_das", length=10) - self._spool_no_overlap = dc.get_example_spool( - "random_das", length=10, time_gap=10 - ) - self._diverse_spool = dc.get_example_spool("diverse_das") - - def _chunk_n_check(self, spool, length: int | None = 1, time=None): - """Helper function to merge and check the spool.""" - new = spool.chunk(time=time) - if length is not None: - assert len(new) == length - # just in case we make spools more lazy, force iteration - for patch in new: - assert isinstance(patch, dc.Patch) - - def time_contiguous_merge(self): - """Time merging contiguous patches from in-memory spool.""" - self._chunk_n_check(self._spool_no_gap) - - def time_no_overlap_merge(self): - """Timing for trying to chunk patches that have no overlap.""" - self._chunk_n_check(self._spool_no_gap) - - def time_diverse_merge(self): - """Time trying to merge the diverse spool.""" - self._chunk_n_check(self._diverse_spool, length=None) - - def time_1second_chunk(self): - """Time chunking for one second along no gap spool.""" - self._chunk_n_check(self._spool_no_gap, time=1, length=None) - - def time_half_second_chunk(self): - """Time chunking for 0.5 along no gap spool.""" - self._chunk_n_check(self._spool_no_gap, time=0.5, length=None) - - -class SelectSuite: - """Suite of selection timing.""" - - def setup(self): - """Get test spools.""" - self._spool_no_gap = dc.get_example_spool("random_das", length=10) - self._spool_no_gap_df = self._spool_no_gap.get_contents() - self._diverse_spool = dc.get_example_spool("diverse_das") - - def time_select_full_range(self): - """Timing selecting the full time range.""" - df, spool = self._spool_no_gap_df, self._spool_no_gap - t1, t2 = df["time_min"].min(), df["time_max"].max() - spool.select(time=(t1, t2)) - spool.select(time=(None, t2)) - spool.select(time=(t1, None)) - - def time_select_half_range(self): - """Time selecting and trimming.""" - df, spool = self._spool_no_gap_df, self._spool_no_gap - t1, t2 = df["time_min"].min(), df["time_max"].max() - duration = (t2 - t1) / 2 - spool.select(time=(t1, t2 - duration)) - spool.select(time=(t1 + duration, t2)) - - def time_select_strings(self): - """Time select non-dimensional selects.""" - spool = self._diverse_spool - spool.select(tag="some_tag") - spool.select(station="wayout") - - def time_select_string_match(self): - """Time select non-dimensional selects.""" - spool = self._diverse_spool - spool.select(tag="some_*") - spool.select(station="wayou?") diff --git a/benchmarks/patch_benchmarks.py b/benchmarks/patch_benchmarks.py deleted file mode 100644 index c1def980d..000000000 --- a/benchmarks/patch_benchmarks.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Benchmarks for patch functions.""" - -from __future__ import annotations - -import numpy as np - -import dascore as dc -from dascore.utils.patch import get_start_stop_step - - -class ProcessingSuite: - """Suite for processing functions.""" - - patch: dc.Patch - - def setup(self): - """Just load the default patch.""" - self.patch = dc.get_example_patch() - - def time_pass_filter(self): - """Time the pass filter.""" - self.patch.pass_filter(distance=(0.1, 0.2)) - self.patch.pass_filter(time=(10.2, None)) - self.patch.pass_filter(time=(None, 100.22)) - self.patch.pass_filter(time=(10, 100)) - - def time_median_filter(self): - """Time the median filter.""" - self.patch.median_filter(distance=5, time=5, samples=True) - self.patch.median_filter(time=5, samples=True) - - def time_resample(self): - """Time resample operations.""" - # upsample time - start, stop, step = get_start_stop_step(self.patch, "time") - self.patch.interpolate(time=np.arange(start, stop, step / 2)) - # up sample distance - start, stop, step = get_start_stop_step(self.patch, "distance") - new_coord = np.arange(start, stop, step / 2.2) - self.patch.interpolate(distance=new_coord) - - def time_decimate(self): - """Timing decimate.""" - self.patch.decimate(time=2) - self.patch.decimate(time=10, filter_type="iir") - self.patch.decimate(time=10, filter_type="fir") - self.patch.decimate(time=10, filter_type=None) - - def time_select(self): - """Timing select.""" - self.patch.select(distance=(100, 200)) - t1 = self.patch.attrs["time_min"] + np.timedelta64(1, "s") - t2 = t1 + np.timedelta64(3, "s") - self.patch.select(time=(None, t1)) - self.patch.select(time=(t1, None)) - self.patch.select(time=(t1, t2)) - - -class TransformSuite: - """Timing for various transformations.""" - - def setup(self): - """Just load the default patch.""" - self.patch = dc.get_example_patch() - self.dft_patch = self.patch.tran.dft("time") - - def time_indefinite_integrate(self): - """Integrate along time axis.""" - self.patch.integrate(dim="time", definite=False) - - def time_definite_integrate(self): - """Integrate along time axis.""" - self.patch.integrate(dim="time", definite=True) - - def time_differentiate(self): - """Differentiate along time axis.""" - self.patch.differentiate(dim="time") - - def time_dft(self): - """The discrete fourier transform.""" - self.patch.dft(dim="time") - - def time_idft(self): - """The inverse of the fourier transform.""" - self.dft_patch.idft() - - -class VizSuite: - """Timing for visualizations.""" - - patch: dc.Patch - - def setup(self): - """Just load the default patch.""" - self.patch = dc.get_example_patch() - - def teardown(self): - """Just load the default patch.""" - import matplotlib.pyplot as plt - - plt.close("all") - - def time_waterfall(self): - """Timing for waterfall patch.""" - patch = self.patch - patch.viz.waterfall() - - def time_str(self): - """Timing for getting str rep.""" - str(self.patch) diff --git a/benchmarks/readme.md b/benchmarks/readme.md index 1a17fe31f..df439b850 100644 --- a/benchmarks/readme.md +++ b/benchmarks/readme.md @@ -1,3 +1,42 @@ # Benchmarks -These are DASCore's benchmark suite. They are run using the python package [Airspeed Velocity](https://github.com/airspeed-velocity/asv), but a simple script has been setup for convenience (scripts/run_benchmarks.py). +DASCore's benchmark suite uses [CodSpeed](https://codspeed.io/) for continuous performance monitoring. + +## Running Benchmarks Locally + +To run benchmarks locally: + +```bash +# Install test dependencies (includes pytest-codspeed) +pip install -e ".[test]" + +# Run all benchmarks +pytest benchmarks/ --codspeed + +# Run specific benchmark files +pytest benchmarks/test_patch_benchmarks.py --codspeed +pytest benchmarks/test_io_benchmarks.py --codspeed +pytest benchmarks/test_spool_benchmarks.py --codspeed +``` + +## Benchmark Structure + +Benchmarks are now organized as pytest tests in the `benchmarks/` directory: + +- `test_patch_benchmarks.py` - Core Patch processing, transform, and visualization benchmarks +- `test_io_benchmarks.py` - File I/O operations benchmarks +- `test_spool_benchmarks.py` - Spool chunking and selection benchmarks + +Each benchmark uses the `@pytest.mark.benchmark` decorator to automatically measure performance. + +## Continuous Performance Monitoring + +Benchmarks automatically run on: +- Push to main/master branch +- Pull requests + +Performance results are tracked by CodSpeed and reported in pull requests, helping identify performance regressions before they're merged. + +## Migration Notes + +The legacy ASV benchmarks in the `benchmarks/` directory have been converted to pytest format. The new benchmarks maintain the same functionality while providing better integration with the existing test suite. diff --git a/benchmarks/test_io_benchmarks.py b/benchmarks/test_io_benchmarks.py new file mode 100644 index 000000000..f22ae0c88 --- /dev/null +++ b/benchmarks/test_io_benchmarks.py @@ -0,0 +1,58 @@ +"""Benchmarks for generic IO operations using pytest-codspeed.""" + +from __future__ import annotations + +from functools import cache + +import pytest + +import dascore as dc +from dascore.utils.downloader import fetch, get_registry_df + + +@cache +def get_test_file_paths(): + """Get a dict of name: path for all files in data registry.""" + df = get_registry_df().loc[lambda x: ~x["name"].str.endswith(".csv")] + out = {row["name"]: fetch(row["name"]) for _, row in df.iterrows()} + return out + + +@pytest.fixture(scope="session") +def test_file_paths(): + """Get paths of test files.""" + return get_test_file_paths() + + +class TestIOBenchmarks: + """Benchmarks for IO operations.""" + + @pytest.mark.benchmark + def test_scan_performance(self, test_file_paths): + """Time for basic scanning of all datafiles.""" + for path in test_file_paths.values(): + dc.scan(path) + + @pytest.mark.benchmark + def test_scan_df_performance(self, test_file_paths): + """Time for basic scanning of all datafiles to DataFrame.""" + for path in test_file_paths.values(): + dc.scan_to_df(path) + + @pytest.mark.benchmark + def test_get_format_performance(self, test_file_paths): + """Time for format detection of all datafiles.""" + for path in test_file_paths.values(): + dc.get_format(path) + + @pytest.mark.benchmark + def test_read_performance(self, test_file_paths): + """Time for basic reading of all datafiles.""" + for path in test_file_paths.values(): + dc.read(path)[0] + + @pytest.mark.benchmark + def test_spool_performance(self, test_file_paths): + """Time for creating spools from all datafiles.""" + for path in test_file_paths.values(): + dc.spool(path)[0] diff --git a/benchmarks/test_patch_benchmarks.py b/benchmarks/test_patch_benchmarks.py new file mode 100644 index 000000000..9933863dd --- /dev/null +++ b/benchmarks/test_patch_benchmarks.py @@ -0,0 +1,119 @@ +"""Benchmarks for patch functions using pytest-codspeed.""" + +from __future__ import annotations + +import numpy as np +import pytest + +import dascore as dc +from dascore.utils.patch import get_start_stop_step + + +@pytest.fixture +def example_patch(): + """Get the example patch for benchmarks.""" + return dc.get_example_patch() + + +class TestProcessingBenchmarks: + """Benchmarks for patch processing operations.""" + + @pytest.mark.benchmark + def test_pass_filter_performance(self, example_patch): + """Time the pass filter.""" + patch = example_patch + patch.pass_filter(distance=(0.1, 0.2)) + patch.pass_filter(time=(10.2, None)) + patch.pass_filter(time=(None, 100.22)) + patch.pass_filter(time=(10, 100)) + + @pytest.mark.benchmark + def test_median_filter_performance(self, example_patch): + """Time the median filter.""" + patch = example_patch + patch.median_filter(distance=5, time=5, samples=True) + patch.median_filter(time=5, samples=True) + + @pytest.mark.benchmark + def test_resample_performance(self, example_patch): + """Time resample operations.""" + patch = example_patch + # upsample time + start, stop, step = get_start_stop_step(patch, "time") + patch.interpolate(time=np.arange(start, stop, step / 2)) + # up sample distance + start, stop, step = get_start_stop_step(patch, "distance") + new_coord = np.arange(start, stop, step / 2.2) + patch.interpolate(distance=new_coord) + + @pytest.mark.benchmark + def test_decimate_performance(self, example_patch): + """Timing decimate.""" + patch = example_patch + patch.decimate(time=2) + patch.decimate(time=10, filter_type="iir") + patch.decimate(time=10, filter_type="fir") + patch.decimate(time=10, filter_type=None) + + @pytest.mark.benchmark + def test_select_performance(self, example_patch): + """Timing select.""" + patch = example_patch + patch.select(distance=(100, 200)) + t1 = patch.attrs["time_min"] + np.timedelta64(1, "s") + t2 = t1 + np.timedelta64(3, "s") + patch.select(time=(None, t1)) + patch.select(time=(t1, None)) + patch.select(time=(t1, t2)) + + +class TestTransformBenchmarks: + """Benchmarks for patch transform operations.""" + + @pytest.fixture + def dft_patch(self, example_patch): + """Get DFT patch for benchmarks.""" + return example_patch.tran.dft("time") + + @pytest.mark.benchmark + def test_indefinite_integrate_performance(self, example_patch): + """Integrate along time axis.""" + example_patch.integrate(dim="time", definite=False) + + @pytest.mark.benchmark + def test_definite_integrate_performance(self, example_patch): + """Integrate along time axis.""" + example_patch.integrate(dim="time", definite=True) + + @pytest.mark.benchmark + def test_differentiate_performance(self, example_patch): + """Differentiate along time axis.""" + example_patch.differentiate(dim="time") + + @pytest.mark.benchmark + def test_dft_performance(self, example_patch): + """The discrete fourier transform.""" + example_patch.dft(dim="time") + + @pytest.mark.benchmark + def test_idft_performance(self, dft_patch): + """The inverse of the fourier transform.""" + dft_patch.idft() + + +class TestVisualizationBenchmarks: + """Benchmarks for patch visualization operations.""" + + @pytest.mark.benchmark + def test_waterfall_performance(self, example_patch): + """Timing for waterfall patch.""" + example_patch.viz.waterfall() + # Clean up matplotlib figures + import matplotlib.pyplot as plt + + plt.close("all") + + @pytest.mark.benchmark + def test_str_performance(self, example_patch): + """Timing for getting str rep.""" + str(example_patch) diff --git a/benchmarks/test_spool_benchmarks.py b/benchmarks/test_spool_benchmarks.py new file mode 100644 index 000000000..7b1bfc119 --- /dev/null +++ b/benchmarks/test_spool_benchmarks.py @@ -0,0 +1,107 @@ +"""Benchmarks for spool operations using pytest-codspeed.""" + +from __future__ import annotations + +import pytest + +import dascore as dc + + +@pytest.fixture +def spool_no_gap(): + """Get test spool with no gaps.""" + return dc.get_example_spool("random_das", length=10) + + +@pytest.fixture +def gapped_spool_no_overlap(): + """Get test spool with no overlap.""" + return dc.get_example_spool("random_das", length=10, time_gap=10) + + +@pytest.fixture +def diverse_spool(): + """Get diverse test spool.""" + return dc.get_example_spool("diverse_das") + + +def _chunk_and_check(spool, length: int | None = 1, time=None): + """Helper function to merge and check the spool.""" + new = spool.chunk(time=time) + if length is not None: + assert len(new) == length + # Check contents + for patch in new: + assert isinstance(patch, dc.Patch) + + +class TestChunkBenchmarks: + """Benchmarks for spool chunking operations.""" + + @pytest.mark.benchmark + def test_contiguous_merge_performance(self, spool_no_gap): + """Time merging contiguous patches from in-memory spool.""" + _chunk_and_check(spool_no_gap) + + @pytest.mark.benchmark + def test_no_overlap_merge_performance(self, gapped_spool_no_overlap): + """Timing for trying to chunk patches that have no overlap.""" + chunked = gapped_spool_no_overlap.chunk(time=None) + # In this case the spool should not be merged. + assert len(chunked) == len(gapped_spool_no_overlap) + + @pytest.mark.benchmark + def test_diverse_merge_performance(self, diverse_spool): + """Time trying to merge the diverse spool.""" + _chunk_and_check(diverse_spool, length=None) + + @pytest.mark.benchmark + def test_1second_chunk_performance(self, spool_no_gap): + """Time chunking for one second along no gap spool.""" + _chunk_and_check(spool_no_gap, time=1, length=None) + + @pytest.mark.benchmark + def test_half_second_chunk_performance(self, spool_no_gap): + """Time chunking for 0.5 along no gap spool.""" + _chunk_and_check(spool_no_gap, time=0.5, length=None) + + +class TestSelectionBenchmarks: + """Benchmarks for spool selection operations.""" + + @pytest.fixture + def spool_no_gap_df(self, spool_no_gap): + """Get contents dataframe for no gap spool.""" + return spool_no_gap.get_contents() + + @pytest.mark.benchmark + def test_select_full_range_performance(self, spool_no_gap, spool_no_gap_df): + """Timing selecting the full time range.""" + df, spool = spool_no_gap_df, spool_no_gap + t1, t2 = df["time_min"].min(), df["time_max"].max() + spool.select(time=(t1, t2)) + spool.select(time=(None, t2)) + spool.select(time=(t1, None)) + + @pytest.mark.benchmark + def test_select_half_range_performance(self, spool_no_gap, spool_no_gap_df): + """Time selecting and trimming.""" + df, spool = spool_no_gap_df, spool_no_gap + t1, t2 = df["time_min"].min(), df["time_max"].max() + duration = (t2 - t1) / 2 + spool.select(time=(t1, t2 - duration)) + spool.select(time=(t1 + duration, t2)) + + @pytest.mark.benchmark + def test_select_strings_performance(self, diverse_spool): + """Time select non-dimensional selects.""" + spool = diverse_spool + spool.select(tag="some_tag") + spool.select(station="wayout") + + @pytest.mark.benchmark + def test_select_string_match_performance(self, diverse_spool): + """Time select non-dimensional selects with wildcards.""" + spool = diverse_spool + spool.select(tag="some_*") + spool.select(station="wayou?") diff --git a/pyproject.toml b/pyproject.toml index 361c74ecd..4a102d7e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,11 +80,11 @@ test = [ "pytest", "pytest-codeblocks", "pytest-cov", + "pytest-codspeed", "twine", ] profile = [ - "asv", "typer", ] diff --git a/readme.md b/readme.md index d14228fb2..5e64ac533 100644 --- a/readme.md +++ b/readme.md @@ -3,6 +3,7 @@ A python library for distributed fiber optic sensing. [![coverage](https://codecov.io/gh/dasdae/dascore/branch/master/graph/badge.svg)](https://codecov.io/gh/dasdae/dascore) +[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/DASDAE/dascore) [![PyPI Version](https://img.shields.io/pypi/v/dascore.svg)](https://pypi.python.org/pypi/dascore) [![supported versions](https://img.shields.io/pypi/pyversions/dascore.svg?label=python_versions)](https://pypi.python.org/pypi/dascore) [![PyPI Downloads](https://img.shields.io/pypi/dm/dascore.svg?label=pypi)](https://pypi.org/project/dascore/) diff --git a/scripts/_benchmark_uilts.py b/scripts/_benchmark_uilts.py deleted file mode 100644 index f07537844..000000000 --- a/scripts/_benchmark_uilts.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Utils for running benchmarks.""" - -from __future__ import annotations - -import os -from contextlib import contextmanager -from pathlib import Path -from subprocess import run - -from rich.console import Console - -console = Console() - - -BASE_PATH = Path(__file__).absolute().parent.parent -REFERENCE_BRANCH = "master" - - -@contextmanager -def cd(path): - """Change directory temporarily.""" - current = os.getcwd() - os.chdir(path) - try: - yield - except Exception: - os.chdir(current) - raise - else: - os.chdir(current) - - -def run_asv(name): - """Run asv, rename output.""" - expected_output_file = BASE_PATH / ".asv" / "results" / "benchmarks.json" - if expected_output_file.exists(): - expected_output_file.unlink() - # check for machine file, if it doesnt exist just use defaults. - machine_file_path = Path.home() / ".asv-machine.json" - if not machine_file_path.exists(): - run("asv machine --yes", check=True, shell=True) - assert machine_file_path.exists() - # tell the user what is going on - console.print() - console.rule(f"[bold red]Running benchmarks for {name}") - console.print() - # run benchmarks - hash = git_hash() - cmd = f"asv run -E existing --set-commit-hash {hash}" - run(cmd, check=True, shell=True) - assert expected_output_file.exists() - return - - -def compare_asv(hash1, hash2): - """Compare two commit hash in something vaguely approximating a table.""" - kwargs = dict(check=True, shell=True, capture_output=True, text=True) - out = run(f"asv compare {hash1} {hash2}", **kwargs) - return out.stdout.strip() - - -def git_branch_name(): - """Return the current branch name.""" - kwargs = dict(check=True, shell=True, capture_output=True, text=True) - out = run("git branch --show-current", **kwargs) - return out.stdout.strip() - - -def git_hash(branch_name=None): - """Return the current git hash name.""" - branch = branch_name or git_branch_name() - kwargs = dict(check=True, shell=True, capture_output=True, text=True) - out = run(f"git rev-parse --short --verify {branch}", **kwargs) - return out.stdout.strip() - - -@contextmanager -def git_checkout(new_branch): - """Checkout branch/commit within context manager.""" - - def _checkout(name): - """Checkout something in git.""" - run(f"git checkout {name}", check=True, shell=True) - return - - current = git_branch_name() - _checkout(new_branch) - try: - yield - except Exception: - _checkout(current) - raise - else: - _checkout(current) diff --git a/scripts/run_benchmarks.py b/scripts/run_benchmarks.py deleted file mode 100644 index 91d36280c..000000000 --- a/scripts/run_benchmarks.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Python script to run benchmarks for dascore. - -It works by using airspeed velocity to run code on current commit, -collecting the results, then running code on reference branch (eg master) -the resetting the repo to current commit. - -A few notes: -- Before running this script make sure all your changes are committed. The - script will raise an exception otherwise. -- asv can usually be run in a more intelligent way, but it doesn't yet - support projects without setup.py files, so this is a bit of a hack - until that gets fixed. -""" - -from __future__ import annotations - -from _benchmark_uilts import ( - BASE_PATH, - REFERENCE_BRANCH, - cd, - git_branch_name, - git_checkout, - run_asv, -) - -if __name__ == "__main__": - with cd(BASE_PATH): - current_name = git_branch_name() - # first run benchmarks of reference. - with git_checkout(REFERENCE_BRANCH): - run_asv(REFERENCE_BRANCH) - # now run benchmarks on current branch. - run_asv(current_name) diff --git a/scripts/visualize_benchmarks.py b/scripts/visualize_benchmarks.py deleted file mode 100644 index 2a48c622f..000000000 --- a/scripts/visualize_benchmarks.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Load the benchmark files and create simple table.""" - -from __future__ import annotations - -from _benchmark_uilts import REFERENCE_BRANCH, compare_asv, console, git_hash - -if __name__ == "__main__": - commit1 = git_hash(REFERENCE_BRANCH) - commit2 = git_hash() - out = compare_asv(commit1, commit2) - console.print(out) From 4d232b9cd4b881a67c51638da900ed94740012fa Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 25 Sep 2025 08:40:48 +0100 Subject: [PATCH 02/12] try fix profile job --- .github/test_code.sh | 2 +- .github/workflows/{codspeed.yml => profile.yml} | 3 ++- benchmarks/test_io_benchmarks.py | 17 ++++++++++++----- pyproject.toml | 3 +-- 4 files changed, 16 insertions(+), 9 deletions(-) rename .github/workflows/{codspeed.yml => profile.yml} (86%) diff --git a/.github/test_code.sh b/.github/test_code.sh index f2589f913..52a452260 100755 --- a/.github/test_code.sh +++ b/.github/test_code.sh @@ -5,7 +5,7 @@ args="tests -s --cov dascore --cov-append --cov-report=xml" if [[ "$1" == "doctest" ]]; then args="dascore --doctest-modules" fi -if [[ "$1" == "benchmarks" ]]; then +if [[ "$1" == "profile" ]]; then args="benchmarks --codspeed" fi diff --git a/.github/workflows/codspeed.yml b/.github/workflows/profile.yml similarity index 86% rename from .github/workflows/codspeed.yml rename to .github/workflows/profile.yml index e19bffd6d..fbedf32bb 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/profile.yml @@ -1,3 +1,4 @@ +# An action for running DASCore's performance benchmarks. name: CodSpeed Benchmarks on: @@ -31,5 +32,5 @@ jobs: uses: CodSpeedHQ/action@v4 with: mode: instrumentation - run: pytest benchmarks/ --codspeed + run: ./.github/test_code.sh profile token: ${{ secrets.CODSPEED_TOKEN }} # Optional for public repos diff --git a/benchmarks/test_io_benchmarks.py b/benchmarks/test_io_benchmarks.py index f22ae0c88..f9dda83ff 100644 --- a/benchmarks/test_io_benchmarks.py +++ b/benchmarks/test_io_benchmarks.py @@ -2,11 +2,13 @@ from __future__ import annotations +from contextlib import suppress from functools import cache import pytest import dascore as dc +from dascore.exceptions import MissingOptionalDependencyError from dascore.utils.downloader import fetch, get_registry_df @@ -31,28 +33,33 @@ class TestIOBenchmarks: def test_scan_performance(self, test_file_paths): """Time for basic scanning of all datafiles.""" for path in test_file_paths.values(): - dc.scan(path) + with suppress(MissingOptionalDependencyError): + dc.scan(path) @pytest.mark.benchmark def test_scan_df_performance(self, test_file_paths): """Time for basic scanning of all datafiles to DataFrame.""" for path in test_file_paths.values(): - dc.scan_to_df(path) + with suppress(MissingOptionalDependencyError): + dc.scan_to_df(path) @pytest.mark.benchmark def test_get_format_performance(self, test_file_paths): """Time for format detection of all datafiles.""" for path in test_file_paths.values(): - dc.get_format(path) + with suppress(MissingOptionalDependencyError): + dc.get_format(path) @pytest.mark.benchmark def test_read_performance(self, test_file_paths): """Time for basic reading of all datafiles.""" for path in test_file_paths.values(): - dc.read(path)[0] + with suppress(MissingOptionalDependencyError): + dc.read(path)[0] @pytest.mark.benchmark def test_spool_performance(self, test_file_paths): """Time for creating spools from all datafiles.""" for path in test_file_paths.values(): - dc.spool(path)[0] + with suppress(MissingOptionalDependencyError): + dc.spool(path)[0] diff --git a/pyproject.toml b/pyproject.toml index 4a102d7e4..32366171e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,12 +80,11 @@ test = [ "pytest", "pytest-codeblocks", "pytest-cov", - "pytest-codspeed", "twine", ] profile = [ - "typer", + "pytest-codspeed", ] all = ["dascore[extras]"] From e461ac03f018ac416eebec1fe16b2fa7ced57847 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 25 Sep 2025 08:46:13 +0100 Subject: [PATCH 03/12] try add timeout skip --- tests/test_io/test_common_io.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index 8dd3bf30c..9521a3ac2 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -107,7 +107,7 @@ def _cached_read(path, io=None): read = dc.read else: read = io.read - with skip_missing_dependency(): + with skip_missing_or_timeout(): out = read(path) return out @@ -122,12 +122,12 @@ def _get_flat_io_test(): @contextmanager -def skip_missing_dependency(): +def skip_missing_or_timeout(): """Skip if missing dependencies found.""" try: yield - except MissingOptionalDependencyError: - pytest.skip("Missing optional dep to read file.") + except (MissingOptionalDependencyError, TimeoutError): + pytest.skip("Missing optional dep to read file or unable to fetch.") @pytest.fixture(scope="session", params=list(COMMON_IO_READ_TESTS)) @@ -159,7 +159,7 @@ def data_file_path(request): @pytest.fixture(scope="session") def read_spool(data_file_path): """Read each file into a spool.""" - with skip_missing_dependency(): + with skip_missing_or_timeout(): out = dc.read(data_file_path) return out @@ -167,7 +167,7 @@ def read_spool(data_file_path): @pytest.fixture(scope="session") def scanned_attrs(data_file_path): """Read each file into a spool.""" - with skip_missing_dependency(): + with skip_missing_or_timeout(): out = dc.scan(data_file_path) return out @@ -293,7 +293,7 @@ def test_slice_single_dim_both_ends(self, io_path_tuple): a patch containing the requested data is returned. """ io, path = io_path_tuple - with skip_missing_dependency(): + with skip_missing_or_timeout(): attrs_from_file = dc.scan(path) assert len(attrs_from_file) # skip files that have more than one patch for now @@ -341,7 +341,7 @@ class TestScan: def test_scan_basics(self, data_file_path): """Ensure each file can be scanned.""" - with skip_missing_dependency(): + with skip_missing_or_timeout(): attrs_list = dc.scan(data_file_path) assert len(attrs_list) @@ -352,7 +352,7 @@ def test_scan_basics(self, data_file_path): def test_scan_has_version_and_format(self, io_path_tuple): """Scan output should contain version and format.""" io, path = io_path_tuple - with skip_missing_dependency(): + with skip_missing_or_timeout(): attr_list = io.scan(path) for attrs in attr_list: assert attrs.file_format == io.name @@ -422,7 +422,7 @@ def test_scan_attrs_match_patch_attrs(self, data_file_path): "tag", "network", ) - with skip_missing_dependency(): + with skip_missing_or_timeout(): scan_attrs_list = dc.scan(data_file_path) patch_attrs_list = [x.attrs for x in _cached_read(data_file_path)] assert len(scan_attrs_list) == len(patch_attrs_list) From cd6b95587246d5c693cd16e9aec21de8da5727d7 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 25 Sep 2025 08:54:35 +0100 Subject: [PATCH 04/12] try fix profile job --- .github/workflows/profile.yml | 2 +- pyproject.toml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index fbedf32bb..85b6132d6 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -26,7 +26,7 @@ jobs: python-version: ${{ env.python_version }} - name: Install dependencies - run: pip install ".[test]" + run: pip install ".[profile]" - name: Run benchmarks uses: CodSpeedHQ/action@v4 diff --git a/pyproject.toml b/pyproject.toml index 32366171e..5655237d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ test = [ "twine", ] -profile = [ +profile_base = [ "pytest-codspeed", ] @@ -91,6 +91,9 @@ all = ["dascore[extras]"] dev = ["dascore[test]", "dascore[docs]", "dascore[profile]", "dascore[extras]"] +profile = ["dascore[test]", "dascore[profile_base]"] + + # --- URLs for project [project.urls] From 792e24e6875634e562f48731b026669627514412 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 09:01:56 +0100 Subject: [PATCH 05/12] update profiling --- .gitignore | 3 + benchmarks/test_patch_benchmarks.py | 185 ++++++++++++++++++++- docs/contributing/profiling_benchmarks.qmd | 24 +-- docs/contributors.qmd | 2 - pyproject.toml | 5 +- readme.md | 2 + 6 files changed, 203 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index d6902cf66..f35307add 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,6 @@ docs/index.quarto_ipynb # Claude stuff .claude CLAUDE.md + +# profile stuff +prof/ diff --git a/benchmarks/test_patch_benchmarks.py b/benchmarks/test_patch_benchmarks.py index 9933863dd..b25363c5f 100644 --- a/benchmarks/test_patch_benchmarks.py +++ b/benchmarks/test_patch_benchmarks.py @@ -2,6 +2,7 @@ from __future__ import annotations +import matplotlib.pyplot as plt import numpy as np import pytest @@ -9,12 +10,22 @@ from dascore.utils.patch import get_start_stop_step -@pytest.fixture +@pytest.fixture(scope="module") def example_patch(): """Get the example patch for benchmarks.""" return dc.get_example_patch() +@pytest.fixture(scope="module") +def patch_uneven_time(): + """Get a patch with uneven time coord.""" + patch = dc.get_example_patch() + time = patch.get_coord("time") + rand = np.random.RandomState(39) + new_vales = rand.random(len(time)) + return patch.update_coords(time=new_vales) + + class TestProcessingBenchmarks: """Benchmarks for patch processing operations.""" @@ -48,7 +59,7 @@ def test_resample_performance(self, example_patch): @pytest.mark.benchmark def test_decimate_performance(self, example_patch): - """Timing decimate.""" + """Time decimation.""" patch = example_patch patch.decimate(time=2) patch.decimate(time=10, filter_type="iir") @@ -57,7 +68,7 @@ def test_decimate_performance(self, example_patch): @pytest.mark.benchmark def test_select_performance(self, example_patch): - """Timing select.""" + """Selecting on time/distance dimension""" patch = example_patch patch.select(distance=(100, 200)) t1 = patch.attrs["time_min"] + np.timedelta64(1, "s") @@ -66,6 +77,58 @@ def test_select_performance(self, example_patch): patch.select(time=(t1, None)) patch.select(time=(t1, t2)) + @pytest.mark.benchmark + def test_sobel_filter_performance(self, example_patch): + """Time the Sobel filter.""" + patch = example_patch + patch.sobel_filter(dim="time") + + @pytest.mark.benchmark + def test_standardize_performance(self, example_patch): + """Time standardization operation.""" + patch = example_patch + patch.standardize(dim="time") + + @pytest.mark.benchmark + def test_taper_performance(self, example_patch): + """Time tapering operations.""" + patch = example_patch + patch.taper(time=0.1) + + @pytest.mark.benchmark + def test_transpose_performance(self, example_patch): + """Time transpose operations.""" + patch = example_patch + dims = patch.dims[::-1] + patch.transpose(*dims) + + @pytest.mark.benchmark + def test_roll_performance(self, example_patch): + """Time roll/shift operations.""" + patch = example_patch + patch.roll(time=10, samples=True) + + @pytest.mark.benchmark + def test_snap_coords_performance(self, patch_uneven_time): + """Time coordinate snapping.""" + patch = patch_uneven_time + patch.snap_coords("time") + + @pytest.mark.benchmark + def test_hampel_filter_performance(self, example_patch): + """Time the Hampel filter.""" + patch = example_patch + patch.hampel_filter(threshold=3.0, time=5, samples=True) + patch.hampel_filter( + threshold=2.5, distance=3, time=5, samples=True, separable=True + ) + + @pytest.mark.benchmark + def test_wiener_filter_performance(self, example_patch): + """Time the Wiener filter.""" + patch = example_patch + patch.wiener_filter(time=3, samples=True) + class TestTransformBenchmarks: """Benchmarks for patch transform operations.""" @@ -100,6 +163,24 @@ def test_idft_performance(self, dft_patch): """The inverse of the fourier transform.""" dft_patch.idft() + @pytest.mark.benchmark + def test_stft(self, example_patch): + """Time Hilbert transform.""" + patch = example_patch + patch.stft(time=1, overlap=0.25) + + @pytest.mark.benchmark + def test_hilbert_performance(self, example_patch): + """Time Hilbert transform.""" + patch = example_patch + patch.hilbert(dim="time") + + @pytest.mark.benchmark + def test_envelope_performance(self, example_patch): + """Time envelope calculation.""" + patch = example_patch + patch.envelope(dim="time") + class TestVisualizationBenchmarks: """Benchmarks for patch visualization operations.""" @@ -117,3 +198,101 @@ def test_waterfall_performance(self, example_patch): def test_str_performance(self, example_patch): """Timing for getting str rep.""" str(example_patch) + + @pytest.mark.benchmark + def test_repr_performance(self, example_patch): + """Time representation generation.""" + repr(example_patch) + + @pytest.mark.benchmark + def test_wiggle_performance(self, example_patch): + """Time wiggle plot visualization.""" + patch = example_patch.select(distance=(0, 100)) # Subset for performance + patch.viz.wiggle() + # Clean up matplotlib figures + plt.close("all") + + +class TestAggregationBenchmarks: + """Benchmarks for patch aggregation operations.""" + + @pytest.mark.benchmark + def test_mean_performance(self, example_patch): + """Time mean aggregation.""" + patch = example_patch + patch.mean(dim="time") + patch.mean(dim="distance") + + @pytest.mark.benchmark + def test_max_performance(self, example_patch): + """Time max aggregation.""" + patch = example_patch + patch.max(dim="time") + patch.max(dim="distance") + + @pytest.mark.benchmark + def test_min_performance(self, example_patch): + """Time min aggregation.""" + patch = example_patch + patch.min(dim="time") + patch.min(dim="distance") + + @pytest.mark.benchmark + def test_std_performance(self, example_patch): + """Time standard deviation aggregation.""" + patch = example_patch + patch.std(dim="time") + patch.std(dim="distance") + + @pytest.mark.benchmark + def test_sum_performance(self, example_patch): + """Time sum aggregation.""" + patch = example_patch + patch.sum(dim="time") + patch.sum(dim="distance") + + @pytest.mark.benchmark + def test_median_performance(self, example_patch): + """Time median aggregation.""" + patch = example_patch + patch.median(dim="time") + patch.median(dim="distance") + + @pytest.mark.benchmark + def test_first_performance(self, example_patch): + """Time first aggregation.""" + patch = example_patch + patch.first(dim="time") + + @pytest.mark.benchmark + def test_last_performance(self, example_patch): + """Time last aggregation.""" + patch = example_patch + patch.last(dim="distance") + + +class TestRollingBenchmarks: + """Benchmarks for rolling window operations.""" + + @pytest.fixture(scope="module") + def small_roller(self, example_patch): + """Get a rolling object""" + return example_patch.rolling(time=5, samples=True) + + @pytest.fixture(scope="module") + def big_roller(self, example_patch): + """Get a large rolling object""" + patch = example_patch + time = patch.get_coord("time") + roll_time = dc.to_float(time.coord_range()) / 4 + return example_patch.rolling(time=roll_time) + + @pytest.mark.benchmark + def test_rolling_small_roller_mean_performance(self, small_roller): + """Time rolling mean calculation for small roller.""" + small_roller.mean() + + @pytest.mark.benchmark + def test_rolling_large_roller_mean_performance(self, big_roller): + """Time rolling mean calculation.""" + big_roller.mean() diff --git a/docs/contributing/profiling_benchmarks.qmd b/docs/contributing/profiling_benchmarks.qmd index ef6417c90..0d3fa5e80 100644 --- a/docs/contributing/profiling_benchmarks.qmd +++ b/docs/contributing/profiling_benchmarks.qmd @@ -3,30 +3,30 @@ title: Profiling and Benchmarks --- # Benchmarks -DASCore uses [Airspeed Velocity (asv)](https://github.com/airspeed-velocity/) to create and run a simple benchmark suite. The benchmarks are found in the benchmark folder at the top level of the repository. - -The [asv documentation](https://asv.readthedocs.io/en/stable/) is great, and you can do all sorts of things with the benchmarks. For the most common use case, that is to compare changes in performance on a development branch to the main branch, you can simply use two scripts in the script directory. +DASCore uses [codspeed](https://codspeed.io/) to create and run a simple benchmark suite. The benchmarks are found in the benchmarks folder at the top level of the repository. To run the benchmarks: ```bash -python scripts/run_benchmarks.py +python -m pytest benchmarks --codspeed ``` -And to get a simple (text-based) visualization of the differences: -```bash -python scripts/visualize_benchmarks.py -``` +However, when you create a pull request, the benchmarks will be run in the CI/CD and a report displayed. If there are significant performance regressions, more discussion is needed before merging the pull request. -If you notice any significant degradations, it is probably worth looking into via profiling. +If you add significant new functionality, you should probably add a benchmark. # Profiling -You can profile the problematic benchmark(s) to see *why* their performance degraded. This is done with the [asv profile](https://asv.readthedocs.io/en/stable/commands.html#id13) command. +If you find a significant issue, you can profile the problematic benchmark(s) to see *why* their performance degraded. This can be done with the [pytest profile plugin](https://pypi.org/project/pytest-profiling/). -For example, suppose you notice a large increase in runtime for the time_pass_filter benchmark in the patch_benchmarks' ProcessingSuite. You can load a nice interactive [snakeviz](https://jiffyclub.github.io/snakeviz/) gui (assuming you installed snakeviz `pip install snakeviz`) like so: +For example, suppose you notice a large increase in runtime for the pass_filter benchmark in the patch_benchmarks' TestProcessingBenchmarks class. Run the benchmark again under profiling: ```bash -asv profile patch_benchmarks.ProcessingSuite.time_pass_filter -E existing --gui snakeviz +pytest benchmarks/test_patch_benchmarks.py::TestProcessingBenchmarks::test_pass_filter_performance --profile +``` +This will create a new `prof` folder with `test_pass_filter_performance.prof` as one of the files. You can view these with a variety of tools, such as [snakeviz](https://jiffyclub.github.io/snakeviz/) (assuming you installed snakeviz with `pip install snakeviz`). + +```bash +snakeviz prof/test_pass_filter_performance.prof ``` You can then click through the call stack and see which functions can be improved. After tweaking them, re-run the profiling and see if the overall time improves. diff --git a/docs/contributors.qmd b/docs/contributors.qmd index c4c1aeff5..9f117d85c 100644 --- a/docs/contributors.qmd +++ b/docs/contributors.qmd @@ -1,5 +1,3 @@ # Contributors A huge thanks to [all the DASCore contributors](https://github.com/DASDAE/dascore/graphs/contributors)! - -[![Contributors](https://contrib.rocks/image?repo=DASDAE/dascore)](https://github.com/DASDAE/dascore/graphs/contributors) diff --git a/pyproject.toml b/pyproject.toml index 5655237d9..4bc0b38f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,13 +85,16 @@ test = [ profile_base = [ "pytest-codspeed", + "pytest-profiling", ] all = ["dascore[extras]"] +profile = ["dascore[test]", "dascore[profile_base]"] + dev = ["dascore[test]", "dascore[docs]", "dascore[profile]", "dascore[extras]"] -profile = ["dascore[test]", "dascore[profile_base]"] + # --- URLs for project diff --git a/readme.md b/readme.md index 5e64ac533..2791d1bc8 100644 --- a/readme.md +++ b/readme.md @@ -18,3 +18,5 @@ Documentation [[stable](https://dascore.org), [development](https://dascore.netl [Citation](https://seismica.library.mcgill.ca/article/view/1184) > Chambers, D., Jin, G., Tourei, A., Issah, A. H. S., Lellouch, A., Martin, E., Zhu, D., Girard, A., Yuan, S., Cullison, T., Snyder, T., Kim, S., Danes, N., Pnithan, N., Boltz, M. S. & Mendoza, M. M. (2024). DASCore: a Python Library for Distributed Fiber Optic Sensing. Seismica, 3(2). + +[![Contributors](https://contrib.rocks/image?repo=DASDAE/dascore)](https://github.com/DASDAE/dascore/graphs/contributors) From 77913602eed8178cccd20f00cc4279968f4a1b09 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 10:12:22 +0100 Subject: [PATCH 06/12] rabbit --- benchmarks/test_patch_benchmarks.py | 23 +++++++++++----------- docs/contributing/profiling_benchmarks.qmd | 2 +- tests/test_io/test_common_io.py | 10 +++++++--- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/benchmarks/test_patch_benchmarks.py b/benchmarks/test_patch_benchmarks.py index b25363c5f..85cb2baa3 100644 --- a/benchmarks/test_patch_benchmarks.py +++ b/benchmarks/test_patch_benchmarks.py @@ -26,6 +26,13 @@ def patch_uneven_time(): return patch.update_coords(time=new_vales) +@pytest.fixture() +def cleanup_mpl(): + """Close all open matplotlib figures after test.""" + yield + plt.close("all") + + class TestProcessingBenchmarks: """Benchmarks for patch processing operations.""" @@ -136,7 +143,7 @@ class TestTransformBenchmarks: @pytest.fixture def dft_patch(self, example_patch): """Get DFT patch for benchmarks.""" - return example_patch.tran.dft("time") + return example_patch.dft("time") @pytest.mark.benchmark def test_indefinite_integrate_performance(self, example_patch): @@ -165,7 +172,7 @@ def test_idft_performance(self, dft_patch): @pytest.mark.benchmark def test_stft(self, example_patch): - """Time Hilbert transform.""" + """Time short time fourier transform transform.""" patch = example_patch patch.stft(time=1, overlap=0.25) @@ -183,16 +190,12 @@ def test_envelope_performance(self, example_patch): class TestVisualizationBenchmarks: - """Benchmarks for patch visualization operations.""" + """Benchmarks for patch visualization operations (or str repr).""" @pytest.mark.benchmark - def test_waterfall_performance(self, example_patch): + def test_waterfall_performance(self, example_patch, cleanup_mpl): """Timing for waterfall patch.""" example_patch.viz.waterfall() - # Clean up matplotlib figures - import matplotlib.pyplot as plt - - plt.close("all") @pytest.mark.benchmark def test_str_performance(self, example_patch): @@ -205,12 +208,10 @@ def test_repr_performance(self, example_patch): repr(example_patch) @pytest.mark.benchmark - def test_wiggle_performance(self, example_patch): + def test_wiggle_performance(self, example_patch, cleanup_mpl): """Time wiggle plot visualization.""" patch = example_patch.select(distance=(0, 100)) # Subset for performance patch.viz.wiggle() - # Clean up matplotlib figures - plt.close("all") class TestAggregationBenchmarks: diff --git a/docs/contributing/profiling_benchmarks.qmd b/docs/contributing/profiling_benchmarks.qmd index 0d3fa5e80..7cfd8aa14 100644 --- a/docs/contributing/profiling_benchmarks.qmd +++ b/docs/contributing/profiling_benchmarks.qmd @@ -7,7 +7,7 @@ DASCore uses [codspeed](https://codspeed.io/) to create and run a simple benchma To run the benchmarks: ```bash -python -m pytest benchmarks --codspeed +python -m pytest benchmarks/ --codspeed ``` However, when you create a pull request, the benchmarks will be run in the CI/CD and a report displayed. If there are significant performance regressions, more discussion is needed before merging the pull request. diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index 9521a3ac2..cd3f971ed 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -122,12 +122,16 @@ def _get_flat_io_test(): @contextmanager -def skip_missing_or_timeout(): +def skip_missing_dependency(): """Skip if missing dependencies found.""" try: yield - except (MissingOptionalDependencyError, TimeoutError): - pytest.skip("Missing optional dep to read file or unable to fetch.") + except MissingOptionalDependencyError as exc: + pytest.skip( + f"Missing optional dependency required to read file: {exc}" + ) + except TimeoutError as exc: + pytest.skip(f"Unable to fetch data due to timeout: {exc}") @pytest.fixture(scope="session", params=list(COMMON_IO_READ_TESTS)) From 4bb2b29084170d2cabd579a3bbd84cd1ef7a752b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 11:15:13 +0100 Subject: [PATCH 07/12] fix tests and lint --- tests/test_io/test_common_io.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index cd3f971ed..0480b3da1 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -97,6 +97,17 @@ SKIP_DATA_FILES = {"whale_1.hdf5", "brady_hs_DAS_DTS_coords.csv"} +@contextmanager +def skip_missing_or_timeout(): + """Skip if missing dependencies found.""" + try: + yield + except MissingOptionalDependencyError as exc: + pytest.skip(f"Missing optional dependency required to read file: {exc}") + except TimeoutError as exc: + pytest.skip(f"Unable to fetch data due to timeout: {exc}") + + @cache def _cached_read(path, io=None): """ @@ -121,19 +132,6 @@ def _get_flat_io_test(): return flat_io -@contextmanager -def skip_missing_dependency(): - """Skip if missing dependencies found.""" - try: - yield - except MissingOptionalDependencyError as exc: - pytest.skip( - f"Missing optional dependency required to read file: {exc}" - ) - except TimeoutError as exc: - pytest.skip(f"Unable to fetch data due to timeout: {exc}") - - @pytest.fixture(scope="session", params=list(COMMON_IO_READ_TESTS)) def io_instance(request): """Fixture for returning fiber io instances.""" From b4106751ed50428014b29929dd3f1950380b8c7b Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 12:39:55 +0100 Subject: [PATCH 08/12] rabbit --- benchmarks/test_patch_benchmarks.py | 22 +++++++++++-------- tests/test_io/test_common_io.py | 34 +++++++++++++++++++---------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/benchmarks/test_patch_benchmarks.py b/benchmarks/test_patch_benchmarks.py index 85cb2baa3..16c5ed151 100644 --- a/benchmarks/test_patch_benchmarks.py +++ b/benchmarks/test_patch_benchmarks.py @@ -36,6 +36,16 @@ def cleanup_mpl(): class TestProcessingBenchmarks: """Benchmarks for patch processing operations.""" + @pytest.fixture(scope="module") + def interp_time(self, example_patch): + """Get an array for interpolation.""" + # This is a fixture as to not affect the timing. + patch = example_patch + # upsample time + start, stop, step = get_start_stop_step(patch, "time") + step = dc.to_timedelta64(dc.to_float(step) / 2) + return np.arange(start, stop, step) + @pytest.mark.benchmark def test_pass_filter_performance(self, example_patch): """Time the pass filter.""" @@ -53,16 +63,10 @@ def test_median_filter_performance(self, example_patch): patch.median_filter(time=5, samples=True) @pytest.mark.benchmark - def test_resample_performance(self, example_patch): - """Time resample operations.""" + def test_interpolate_performance(self, example_patch, interp_time): + """Time interpolate operations.""" patch = example_patch - # upsample time - start, stop, step = get_start_stop_step(patch, "time") - patch.interpolate(time=np.arange(start, stop, step / 2)) - # up sample distance - start, stop, step = get_start_stop_step(patch, "distance") - new_coord = np.arange(start, stop, step / 2.2) - patch.interpolate(distance=new_coord) + patch.interpolate(time=interp_time) @pytest.mark.benchmark def test_decimate_performance(self, example_patch): diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index 0480b3da1..b007a358a 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -98,7 +98,7 @@ @contextmanager -def skip_missing_or_timeout(): +def skip_missing(): """Skip if missing dependencies found.""" try: yield @@ -108,6 +108,15 @@ def skip_missing_or_timeout(): pytest.skip(f"Unable to fetch data due to timeout: {exc}") +@contextmanager +def skip_timeout(): + """Skip if downloading file times out.""" + try: + yield + except TimeoutError as exc: + pytest.skip(f"Unable to fetch data due to timeout: {exc}") + + @cache def _cached_read(path, io=None): """ @@ -118,7 +127,7 @@ def _cached_read(path, io=None): read = dc.read else: read = io.read - with skip_missing_or_timeout(): + with skip_missing(): out = read(path) return out @@ -145,7 +154,8 @@ def io_path_tuple(request): This is used for common testing. """ io, fetch_name = request.param - return io, fetch(fetch_name) + with skip_timeout(): + return io, fetch(fetch_name) @pytest.fixture(scope="session", params=get_registry_df()["name"]) @@ -155,13 +165,14 @@ def data_file_path(request): # Some files should be skipped if not DAS or too big. if str(param) in SKIP_DATA_FILES: pytest.skip(f"Skipping {param}") - return fetch(request.param) + with skip_timeout(): + return fetch(request.param) @pytest.fixture(scope="session") def read_spool(data_file_path): """Read each file into a spool.""" - with skip_missing_or_timeout(): + with skip_missing(): out = dc.read(data_file_path) return out @@ -169,7 +180,7 @@ def read_spool(data_file_path): @pytest.fixture(scope="session") def scanned_attrs(data_file_path): """Read each file into a spool.""" - with skip_missing_or_timeout(): + with skip_missing(): out = dc.scan(data_file_path) return out @@ -244,7 +255,8 @@ def test_all_other_files_arent_format(self, io_instance): if isinstance(other_io, type(io_instance)): continue for key in data_files: - path = fetch(key) + with skip_timeout(): + path = fetch(key) out = io_instance.get_format(path) if out: format_name, version = out @@ -295,7 +307,7 @@ def test_slice_single_dim_both_ends(self, io_path_tuple): a patch containing the requested data is returned. """ io, path = io_path_tuple - with skip_missing_or_timeout(): + with skip_missing(): attrs_from_file = dc.scan(path) assert len(attrs_from_file) # skip files that have more than one patch for now @@ -343,7 +355,7 @@ class TestScan: def test_scan_basics(self, data_file_path): """Ensure each file can be scanned.""" - with skip_missing_or_timeout(): + with skip_missing(): attrs_list = dc.scan(data_file_path) assert len(attrs_list) @@ -354,7 +366,7 @@ def test_scan_basics(self, data_file_path): def test_scan_has_version_and_format(self, io_path_tuple): """Scan output should contain version and format.""" io, path = io_path_tuple - with skip_missing_or_timeout(): + with skip_missing(): attr_list = io.scan(path) for attrs in attr_list: assert attrs.file_format == io.name @@ -424,7 +436,7 @@ def test_scan_attrs_match_patch_attrs(self, data_file_path): "tag", "network", ) - with skip_missing_or_timeout(): + with skip_missing(): scan_attrs_list = dc.scan(data_file_path) patch_attrs_list = [x.attrs for x in _cached_read(data_file_path)] assert len(scan_attrs_list) == len(patch_attrs_list) From e4e34575cc39149e11be5c0a0a45b4f6ab0c378f Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 14:08:10 +0100 Subject: [PATCH 09/12] rabit --- tests/test_io/test_common_io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index b007a358a..39369b610 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -15,6 +15,7 @@ from io import BytesIO from operator import eq, ge, le from pathlib import Path +from urllib import error as urllib_error import numpy as np import pandas as pd @@ -113,7 +114,7 @@ def skip_timeout(): """Skip if downloading file times out.""" try: yield - except TimeoutError as exc: + except (TimeoutError, urllib_error.URLError) as exc: pytest.skip(f"Unable to fetch data due to timeout: {exc}") From eaa4412477cac9c3aaf1a4486cfb5371c1939cc1 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 14:34:41 +0100 Subject: [PATCH 10/12] remove duplicate io tests --- tests/test_io/test_common_io.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index 39369b610..d9f277007 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -68,7 +68,6 @@ "iDAS005_hdf5_example.626.h5", ), H5Simple(): ("h5_simple_2.h5", "h5_simple_1.h5"), - DASDAEV1(): ("example_dasdae_event_1.h5",), APSensingV10(): ("ap_sensing_1.hdf5",), Febus2(): ("febus_1.h5",), OptoDASV8(): ("opto_das_1.hdf5",), @@ -80,7 +79,6 @@ ), Terra15FormatterV5(): ("terra15_v5_test_file.hdf5",), Terra15FormatterV6(): ("terra15_v6_test_file.hdf5",), - Terra15FormatterV6(): ("terra15_v6_test_file.hdf5",), SegyV1_0(): ("conoco_segy_1.sgy",), DASHDF5(): ("PoroTomo_iDAS_1.h5",), SentekV5(): ("DASDMSShot00_20230328155653619.das",), From bafb2e713eaba0e46e61583e3a28a6b5a17bd591 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 14:42:16 +0100 Subject: [PATCH 11/12] address rabit review --- tests/test_io/test_common_io.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index d9f277007..ef336e04e 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -288,13 +288,13 @@ def test_read_stream(self, io_path_tuple): """If the format supports reading from a stream, test it out.""" io, path = io_path_tuple req_type = getattr(io.read, "_required_type", None) - if not isinstance(req_type, BinaryReader): - msg = f"{io} doesn't require a read type." - pytest.skip(msg) + if req_type is not BinaryReader: + pytest.skip(f"{io} doesn't support BinaryReader streams.") + spool1 = _cached_read(path) # write file contents to bytes io and ensure it can be read. bio = BytesIO() - bio.write(Path(io_path_tuple).read_bytes()) + bio.write(Path(path).read_bytes()) bio.seek(0) spool2 = io.read(bio) for patch1, patch2 in zip(spool1, spool2): @@ -444,12 +444,12 @@ def test_scan_attrs_match_patch_attrs(self, data_file_path): # first compare dimensions are related attributes for dim in pat_attrs1.dim_tuple: assert getattr(pat_attrs1, f"{dim}_min") == getattr( - pat_attrs1, f"{dim}_min" + scan_attrs2, f"{dim}_min" ) for dim_attr in DIM_RELATED_ATTRS: attr_name = dim_attr.format(dim=dim) attr1 = getattr(pat_attrs1, attr_name) - attr2 = getattr(pat_attrs1, attr_name) + attr2 = getattr(scan_attrs2, attr_name) assert attr1 == attr2 # then other expected attributes. for attr_name in comp_attrs: From 07df64b7c99aabfa420231bc7dfd286ce4f338df Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 26 Sep 2025 15:00:44 +0100 Subject: [PATCH 12/12] rabit fix, skip failing streams for now --- tests/test_io/test_common_io.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index ef336e04e..2061ebc95 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -12,7 +12,7 @@ from contextlib import contextmanager, suppress from functools import cache -from io import BytesIO +from io import BytesIO, UnsupportedOperation from operator import eq, ge, le from pathlib import Path from urllib import error as urllib_error @@ -296,7 +296,13 @@ def test_read_stream(self, io_path_tuple): bio = BytesIO() bio.write(Path(path).read_bytes()) bio.seek(0) - spool2 = io.read(bio) + try: + spool2 = io.read(bio) + except (AttributeError, OSError, UnsupportedOperation) as e: + # Skip if the format doesn't support BytesIO (e.g., missing + # 'name' attribute, fileno() not supported, or other BytesIO + # incompatibilities) + pytest.skip(f"{io} doesn't support BytesIO streams: {e}") for patch1, patch2 in zip(spool1, spool2): assert patch1.equals(patch2) @@ -450,7 +456,13 @@ def test_scan_attrs_match_patch_attrs(self, data_file_path): attr_name = dim_attr.format(dim=dim) attr1 = getattr(pat_attrs1, attr_name) attr2 = getattr(scan_attrs2, attr_name) - assert attr1 == attr2 + # Use close comparison for floating point values + if isinstance(attr1, float | np.floating) and isinstance( + attr2, float | np.floating + ): + np.testing.assert_allclose(attr1, attr2, rtol=1e-12) + else: + assert attr1 == attr2 # then other expected attributes. for attr_name in comp_attrs: patch_attr = getattr(pat_attrs1, attr_name)