diff --git a/.github/actions/mamba-install-dascore/action.yml b/.github/actions/mamba-install-dascore/action.yml index a0904f9af..3110f7646 100644 --- a/.github/actions/mamba-install-dascore/action.yml +++ b/.github/actions/mamba-install-dascore/action.yml @@ -20,11 +20,6 @@ inputs: default: "true" required: false - cache-number: - description: "Cache number. Use != 1 to reset data cache" - required: false - default: "1" - prepare-test-data: description: "If true, restore, prime, and save the test-data cache after installation" required: false @@ -77,34 +72,12 @@ runs: run: | pip install -e ".${INSTALL_GROUP_STR}" - - name: export test data cache info - id: data-cache - shell: bash -el {0} - env: - INPUT_CACHE_NUMBER: ${{ inputs.cache-number }} - RUNNER_OS: ${{ runner.os }} - run: | - python .github/scripts/export_test_data_cache_env.py >> "$GITHUB_OUTPUT" - - - name: restore test data cache + # Normally just restores the cache primed by the calling workflow's setup + # job, but also self-heals (prime + save) on a miss, e.g. for single-job + # workflows or when a re-run can't see an evicted cache. + - name: prepare test data cache if: "${{ inputs.prepare-test-data == 'true' }}" - id: restore-test-data - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ steps.data-cache.outputs.DATA_CACHE_PATH }} - key: ${{ steps.data-cache.outputs.DATA_CACHE_KEY }} - - - name: prime test data cache - if: "${{ inputs.prepare-test-data == 'true' && steps.restore-test-data.outputs.cache-hit != 'true' }}" - shell: bash -el {0} - run: python .github/scripts/cache_test_data.py - - - name: save test data cache - if: "${{ inputs.prepare-test-data == 'true' && steps.restore-test-data.outputs.cache-hit != 'true' }}" - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ steps.data-cache.outputs.DATA_CACHE_PATH }} - key: ${{ steps.data-cache.outputs.DATA_CACHE_KEY }} + uses: ./.github/actions/prime-test-data-cache # Print out the package info for current environment - name: print package info diff --git a/.github/actions/prime-test-data-cache/action.yml b/.github/actions/prime-test-data-cache/action.yml new file mode 100644 index 000000000..90d38be0f --- /dev/null +++ b/.github/actions/prime-test-data-cache/action.yml @@ -0,0 +1,74 @@ +name: "Prime Test Data Cache" +description: > + Restore the shared cross-OS test-data cache, or build it from a checkout of + DASDAE/test_data on a miss. Requires dascore to be checked out at the + workspace root first. Jobs that run tests must also set the DFS_DATA_DIR + env var to .test_data_cache under the workspace so pooch reads the restored + files (composite actions can't set caller env vars without GITHUB_ENV, + which zizmor forbids). +inputs: + lookup-only: + description: "If true, check cache existence (and prime on miss) without downloading it" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: compute test data cache key + id: cache-key + shell: bash + env: + REGISTRY_HASH: ${{ hashFiles('dascore/data_registry.txt') }} + run: | + if [ -z "$REGISTRY_HASH" ]; then + echo "dascore/data_registry.txt not found; checkout dascore first." >&2 + exit 1 + fi + # DATA_VERSION is part of the key because pooch reads files from a + # subdir of the cache; a version bump must invalidate. + data_version="$(sed -n 's/^DATA_VERSION[^=]*=[[:space:]]*"\([^"]*\)".*/\1/p' dascore/constants.py)" + if [ -z "$data_version" ]; then + echo "DATA_VERSION not found in dascore/constants.py." >&2 + exit 1 + fi + # Bump this number to manually reset the cache. + cache_number=1 + echo "key=test-data-${data_version}-${REGISTRY_HASH}-${cache_number}" >> "$GITHUB_OUTPUT" + + # The path/key literals must be identical in restore and save, and + # enableCrossOsArchive must be set on both, so one ubuntu-primed cache + # serves all operating systems. + - name: restore test data cache + id: restore + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .test_data_cache + key: ${{ steps.cache-key.outputs.key }} + enableCrossOsArchive: true + lookup-only: ${{ inputs.lookup-only }} + + - name: checkout test data repo + if: "${{ steps.restore.outputs.cache-hit != 'true' }}" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: DASDAE/test_data + ref: master + path: .test_data_repo + persist-credentials: false + + - name: prime test data cache + if: "${{ steps.restore.outputs.cache-hit != 'true' }}" + shell: bash + run: | + # Stdlib-only script; use whichever system interpreter this OS provides. + if command -v python3 >/dev/null 2>&1; then PY=python3; else PY=python; fi + "$PY" .github/scripts/prime_test_data.py + + - name: save test data cache + if: "${{ steps.restore.outputs.cache-hit != 'true' }}" + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .test_data_cache + key: ${{ steps.cache-key.outputs.key }} + enableCrossOsArchive: true diff --git a/.github/scripts/cache_test_data.py b/.github/scripts/cache_test_data.py deleted file mode 100644 index 6b4c23d19..000000000 --- a/.github/scripts/cache_test_data.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Populate the pooch cache with every file in dascore's data registry.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from dascore.utils.downloader import fetch, get_registry_df # noqa: E402 - - -def main() -> None: - """Fetch every registered test-data file into the local pooch cache.""" - registry = get_registry_df() - total = len(registry) - print(f"Priming DASCore test-data cache with {total} files") # noqa - for index, name in enumerate(registry["name"], start=1): - path = Path(fetch(name)) - print(f"[{index}/{total}] {name} -> {path}") # noqa - print("Finished priming DASCore test-data cache") # noqa - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/export_test_data_cache_env.py b/.github/scripts/export_test_data_cache_env.py deleted file mode 100644 index 451b78668..000000000 --- a/.github/scripts/export_test_data_cache_env.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Export DASCore test-data cache metadata for GitHub Actions.""" - -from __future__ import annotations - -import os -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from dascore.utils.downloader import get_test_data_cache_info # noqa: E402 - - -def main() -> None: - """Print cache metadata as KEY=VALUE lines for GitHub Actions output files.""" - runner_os = os.environ["RUNNER_OS"] - cache_number = os.environ["INPUT_CACHE_NUMBER"] - info = get_test_data_cache_info() - - print(f"DATA_REGISTRY_HASH={info.registry_hash}") # noqa: T201 - print(f"DATA_CACHE_PATH={info.cache_path}") # noqa: T201 - print(f"DATA_VERSION={info.data_version}") # noqa: T201 - print( # noqa: T201 - f"DATA_CACHE_KEY={info.get_key(runner_os=runner_os, cache_number=cache_number)}" - ) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/prime_test_data.py b/.github/scripts/prime_test_data.py new file mode 100644 index 000000000..2f8baafc8 --- /dev/null +++ b/.github/scripts/prime_test_data.py @@ -0,0 +1,101 @@ +"""Populate the CI test-data cache from a checkout of DASDAE/test_data. + +Copies every file listed in dascore's data registry from a local checkout of +https://github.com/DASDAE/test_data into the layout pooch expects +(.test_data_cache//), verifying each sha256 against the +registry. Registry entries hosted elsewhere are allowed but skipped with a +warning — they aren't cached, so the test suite shouldn't depend on them; +pooch lazy-fetches them on demand. + +Uses only the standard library so it can run on the runners' system python. +""" + +from __future__ import annotations + +import hashlib +import re +import shutil +import sys +from pathlib import Path +from urllib.parse import unquote + +ROOT = Path(__file__).resolve().parents[2] +REGISTRY_PATH = ROOT / "dascore" / "data_registry.txt" +CONSTANTS_PATH = ROOT / "dascore" / "constants.py" +REPO_PATH = ROOT / ".test_data_repo" +CACHE_PATH = ROOT / ".test_data_cache" +URL_REGEX = re.compile( + r"^https?://github\.com/dasdae/test_data/raw/master/(?P.+)$", + re.IGNORECASE, +) + + +def get_data_version() -> str: + """Parse DATA_VERSION from dascore/constants.py without importing dascore.""" + match = re.search( + r'^DATA_VERSION\s*=\s*"([^"]+)"', CONSTANTS_PATH.read_text(), re.MULTILINE + ) + if match is None: + sys.exit(f"DATA_VERSION not found in {CONSTANTS_PATH}") + return match.group(1) + + +def sha256sum(path: Path) -> str: + """Return the sha256 hex digest of a file, reading in chunks.""" + digest = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(2**20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + """Copy registry files from the repo checkout into the cache layout.""" + dest_dir = CACHE_PATH / get_data_version() + dest_dir.mkdir(parents=True, exist_ok=True) + errors = [] + copied = 0 + skipped = 0 + total_bytes = 0 + for line in REGISTRY_PATH.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + name, expected_hash, url = line.split() + match = URL_REGEX.match(url) + if match is None: + print(f"Skipping {name}: not hosted in the DASDAE/test_data repo") # noqa: T201 + skipped += 1 + continue + # Registry URLs may percent-encode characters (e.g. + as %2B); the + # files in the repo use the decoded names. + source = (REPO_PATH / unquote(match["subpath"])).resolve() + dest = (dest_dir / name).resolve() + if not source.is_relative_to(REPO_PATH) or dest.parent != dest_dir.resolve(): + errors.append(f"{name}: escapes its intended directory") + continue + if not source.exists(): + errors.append(f"{name}: {source} missing from the test_data checkout") + continue + if (digest := sha256sum(source)) != expected_hash: + errors.append( + f"{name}: sha256 mismatch (registry {expected_hash}, repo {digest})" + ) + continue + shutil.copy2(source, dest) + copied += 1 + total_bytes += source.stat().st_size + print( # noqa: T201 + f"Copied {copied} files ({total_bytes / 1_000_000:.0f} MB) to {dest_dir}" + f" ({skipped} skipped)" + ) + if errors: + print("Failed to prime test data cache:", file=sys.stderr) # noqa: T201 + for error in errors: + print(f" {error}", file=sys.stderr) # noqa: T201 + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build_deploy_master_docs.yaml b/.github/workflows/build_deploy_master_docs.yaml index 8de2dc2ce..d516d4b83 100644 --- a/.github/workflows/build_deploy_master_docs.yaml +++ b/.github/workflows/build_deploy_master_docs.yaml @@ -16,6 +16,11 @@ on: permissions: contents: read +env: + # Where the test-data cache is restored; pooch reads files from here. + # Must match the path used in .github/actions/prime-test-data-cache. + DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache + # Allow one concurrent deployment concurrency: group: "pages" diff --git a/.github/workflows/build_deploy_stable_docs.yaml b/.github/workflows/build_deploy_stable_docs.yaml index e12434a64..2072eeabe 100644 --- a/.github/workflows/build_deploy_stable_docs.yaml +++ b/.github/workflows/build_deploy_stable_docs.yaml @@ -13,6 +13,11 @@ on: permissions: contents: read +env: + # Where the test-data cache is restored; pooch reads files from here. + # Must match the path used in .github/actions/prime-test-data-cache. + DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache + # Allow one concurrent deployment. Only runs which can deploy need to # serialize, so they share the "pages" group with the master docs workflow. # A pre-release never deploys; it is grouped per tag so that publishing one diff --git a/.github/workflows/get_coverage.yml b/.github/workflows/get_coverage.yml index 4dcfcf204..4e28c8b22 100644 --- a/.github/workflows/get_coverage.yml +++ b/.github/workflows/get_coverage.yml @@ -8,6 +8,11 @@ on: permissions: contents: read +env: + # Where the test-data cache is restored; pooch reads files from here. + # Must match the path used in .github/actions/prime-test-data-cache. + DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache + jobs: calc_coverage: runs-on: ubuntu-latest @@ -26,7 +31,6 @@ jobs: with: # Defined in .github/actions/load-shared-vars/action.yml python-version: ${{ steps.shared-vars.outputs.python-default }} - cache-number: 1 prepare-test-data: "true" - name: run test suite diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index ea24bc36b..854963994 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -19,6 +19,11 @@ permissions: contents: read # required for actions/checkout id-token: write # required for OIDC authentication with CodSpeed +env: + # Where the test-data cache is restored; pooch reads files from here. + # Must match the path used in .github/actions/prime-test-data-cache. + DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache + jobs: benchmarks: name: Run benchmarks @@ -41,7 +46,6 @@ jobs: # Defined in .github/actions/load-shared-vars/action.yml python-version: ${{ steps.shared-vars.outputs.python-default }} install-group-str: "[profile]" - cache-number: 1 prepare-test-data: "true" - name: Run benchmarks diff --git a/.github/workflows/run_min_dep_tests.yml b/.github/workflows/run_min_dep_tests.yml index 3fcfdfc4d..493a988eb 100644 --- a/.github/workflows/run_min_dep_tests.yml +++ b/.github/workflows/run_min_dep_tests.yml @@ -14,6 +14,9 @@ on: - '**.py' - '.github/workflows/run_min_dep_tests.yml' - '.github/actions/**/*.yml' + # Registry edits change the test-data cache key; run the suite so the + # prime script validates new entries before merge. + - 'dascore/data_registry.txt' permissions: contents: read @@ -22,6 +25,9 @@ env: # Ensure matplotlib doesn't try to show figures in CI MPLBACKEND: Agg QT_QPA_PLATFORM: offscreen + # Where the test-data cache is restored; pooch reads files from here. + # Must match the path used in .github/actions/prime-test-data-cache. + DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache # Cancel previous runs when this one starts. concurrency: @@ -31,6 +37,8 @@ concurrency: jobs: # Load Python version matrix from shared vars setup: + # only run if CI isn't turned off + if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') runs-on: ubuntu-latest outputs: # Shared values live in .github/actions/load-shared-vars/action.yml @@ -41,6 +49,11 @@ jobs: persist-credentials: false - uses: ./.github/actions/load-shared-vars id: load-vars + # Make sure the shared cross-OS test-data cache exists before the matrix + # jobs start; see the same step in runtests.yml. + - uses: ./.github/actions/prime-test-data-cache + with: + lookup-only: "true" test_code_min_deps: needs: setup @@ -67,7 +80,6 @@ jobs: python-version: ${{ matrix.python-version }} environment-file: './.github/min_deps_environment.yml' install-group-str: "[test]" - cache-number: 1 prepare-test-data: "true" # Runs test suite and calculates coverage diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml index 61b3ea9d5..f8ff13877 100644 --- a/.github/workflows/runtests.yml +++ b/.github/workflows/runtests.yml @@ -14,13 +14,17 @@ on: - '**.py' - '.github/workflows/*.yml' - '.github/actions/**/*.yml' + # Registry edits change the test-data cache key; run the suite so the + # prime script validates new entries before merge. + - 'dascore/data_registry.txt' permissions: contents: read env: - # used to manually trigger cache reset. Just increment if needed. - CACHE_NUMBER: 1 + # Where the test-data cache is restored; pooch reads files from here. + # Must match the path used in .github/actions/prime-test-data-cache. + DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache # Ensure matplotlib doesn't try to show figures in CI MPLBACKEND: Agg QT_QPA_PLATFORM: offscreen @@ -33,6 +37,8 @@ concurrency: jobs: # Load Python version matrix from shared vars setup: + # only run if CI isn't turned off + if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') runs-on: ubuntu-latest outputs: # Shared values live in .github/actions/load-shared-vars/action.yml @@ -43,6 +49,13 @@ jobs: persist-credentials: false - uses: ./.github/actions/load-shared-vars id: load-vars + # Make sure the shared cross-OS test-data cache exists before the matrix + # jobs start, so at most one job builds it. On a simultaneous cold miss + # the other test workflow's setup job may benignly race-prime the same + # key; the losing save just logs a warning. + - uses: ./.github/actions/prime-test-data-cache + with: + lookup-only: "true" test_code: needs: setup @@ -71,7 +84,6 @@ jobs: - uses: ./.github/actions/mamba-install-dascore with: python-version: ${{ matrix.python-version }} - cache-number: ${{ env.CACHE_NUMBER }} prepare-test-data: "true" # Runs test suite and calculates coverage diff --git a/.github/workflows/test_doc_build.yml b/.github/workflows/test_doc_build.yml index 0867699be..30322b0dc 100644 --- a/.github/workflows/test_doc_build.yml +++ b/.github/workflows/test_doc_build.yml @@ -10,6 +10,11 @@ on: permissions: contents: read +env: + # Where the test-data cache is restored; pooch reads files from here. + # Must match the path used in .github/actions/prime-test-data-cache. + DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache + jobs: test_build_docs: if: | @@ -38,7 +43,6 @@ jobs: # Defined in .github/actions/load-shared-vars/action.yml python-version: ${{ steps.shared-vars.outputs.python-default }} environment-file: './.github/doc_environment.yml' - cache-number: 1 prepare-test-data: "true" - uses: ./.github/actions/build-docs diff --git a/.gitignore b/.gitignore index acf55e407..377afdf9e 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,10 @@ worktrees/ # profile stuff prof/ +# CI test-data cache workspace dirs (see .github/actions/prime-test-data-cache) +.test_data_cache/ +.test_data_repo/ + ## Common IDE project files #Spyder *.spyproject diff --git a/dascore/data_registry.txt b/dascore/data_registry.txt index a17874b32..402d4a261 100644 --- a/dascore/data_registry.txt +++ b/dascore/data_registry.txt @@ -18,7 +18,6 @@ dispersion_event.h5 598c8baa2a5610c930e1c003f2ba02da13f8d8686e3ccf2a034e94bfc5e1 PoroTomo_iDAS_1.h5 967a2885e79937ac0426b2022a9c03d5f24790ecf3abbaa9a16eb28055566fc6 https://github.com/dasdae/test_data/raw/master/das/PoroTomo_iDAS_1.h5 DASDMSShot00_20230328155653619.das 12ac53f78b32d8b0e32cc674c43ff5b4c79a6c8b19de2ad577fd481679b2b7b3 https://github.com/dasdae/test_data/raw/master/das/DASDMSShot00_20230328155653619.das opto_das_1.hdf5 0437d1f02d93c9f00d31133388efaf6a28c21883bcfac457b97f1224464c7dca https://github.com/dasdae/test_data/raw/master/das/opto_das_1.hdf5 -whale_1.hdf5 a09922969e740307bf26dc6ffa7fb9fbb834dc7cd7d4ced02c66b159fb1ce0cd http://piweb.ooirsn.uw.edu/das/data/Optasense/NorthCable/TransmitFiber/North-C1-LR-P1kHz-GL50m-Sp2m-FS200Hz_2021-11-03T15_06_51-0700/North-C1-LR-P1kHz-GL50m-Sp2m-FS200Hz_2021-11-04T020002Z.h5 febus_1.h5 73eba2b6e183b3bca51f8a1448c3b423c979d05ce6c18bfd7fb76b4f9bda5c0b https://github.com/dasdae/test_data/raw/master/das/febus_1.h5 ap_sensing_1.hdf5 322429f2c44bed5dc72fb9a02f79bb0d3cb71048e93d906d3d24b0605b431b12 https://github.com/dasdae/test_data/raw/master/das/ap_sensing_1.hdf5 silixa_h5_1.hdf5 d3f1b92b17ae2d00f900426e80d48964fb5a33b9480ef9805721ac756acd4a21 https://github.com/dasdae/test_data/raw/master/das/silixa_h5_1.hdf5 diff --git a/dascore/utils/downloader.py b/dascore/utils/downloader.py index c289a4803..814944e10 100644 --- a/dascore/utils/downloader.py +++ b/dascore/utils/downloader.py @@ -2,9 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass from functools import cache -from hashlib import sha256 from importlib.resources import files from pathlib import Path @@ -26,34 +24,6 @@ fetcher.load_registry(REGISTRY_PATH) -@dataclass(frozen=True) -class TestDataCacheInfo: - """Metadata needed to restore or prime the CI test-data cache.""" - - registry_path: Path - cache_path: Path - data_version: str - registry_hash: str - - def get_key(self, runner_os: str, cache_number: str | int) -> str: - """Return the GitHub Actions cache key for the given OS and cache number.""" - return ( - f"data-{runner_os}-{self.data_version}-{self.registry_hash}-{cache_number}" - ) - - -@cache -def get_test_data_cache_info() -> TestDataCacheInfo: - """Return the metadata needed to populate the CI test-data cache.""" - registry_path = Path(REGISTRY_PATH) - return TestDataCacheInfo( - registry_path=registry_path, - cache_path=Path(fetcher.path).parent, - data_version=DATA_VERSION, - registry_hash=sha256(registry_path.read_bytes()).hexdigest(), - ) - - @cache def get_registry_df() -> pd.DataFrame: """Returns a dataframe of all files in the data registry.""" diff --git a/docs/contributing/adding_test_data.qmd b/docs/contributing/adding_test_data.qmd index 3bb3300c1..deb19438b 100644 --- a/docs/contributing/adding_test_data.qmd +++ b/docs/contributing/adding_test_data.qmd @@ -45,11 +45,15 @@ If, in the test code, the example patch or spool is used only once, just call th Of course, not all data can easily be generated in python. For example, testing [support for new file formats](./new_format.qmd) typically requires a test file. -If you have a small file that isn't already hosted on a permanent site, you can put it into [dasdae's data repo](https://github.com/DASDAE/test_data). Simply clone the repo, add you file format, and push back to master or open a PR on a separate branch and someone will merge it. +All test-suite data files must be hosted in [dasdae's data repo](https://github.com/DASDAE/test_data). Simply clone the repo, add your file, and push back to master or open a PR on a separate branch and someone will merge it. Files for other purposes (e.g. documentation examples) may be hosted elsewhere and still be listed in the data registry, but CI won't cache them, so the test suite shouldn't depend on them (add them to `SKIP_DATA_FILES` in `tests/test_io/test_common_io.py`). Next, add your file to dascore's data registry (dascore/data_registry.txt). You will have to get the sha256 hash of your test file, for that you can simply use [Pooch's hash_file function](https://www.fatiando.org/pooch/latest/api/generated/pooch.file_hash.html), and you can create the proper download url using the other entries as examples. +:::{.callout-note} +CI doesn't download registry files one by one; it checks out the whole test_data repo once and builds a single test-data cache shared by all operating systems (see `.github/actions/prime-test-data-cache`). The cache invalidates automatically whenever data_registry.txt or `DATA_VERSION` changes. If it ever needs a manual reset, increment the cache number in that action. +::: + The name, hash, and url might look something like this: ``` jingle_test_file.jgl diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index 9502a6f1d..42a88450a 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -101,7 +101,7 @@ # Specifies data registry entries which should not be tested. # DASVader is covered in its own test module to isolate its compatibility-path # testing from the shared common-IO matrix. -SKIP_DATA_FILES = {"whale_1.hdf5", "brady_hs_DAS_DTS_coords.csv", "das_vader_1.jld2"} +SKIP_DATA_FILES = {"brady_hs_DAS_DTS_coords.csv", "das_vader_1.jld2"} @contextmanager diff --git a/tests/test_utils/test_downloader.py b/tests/test_utils/test_downloader.py index 97988e344..5b4cf5053 100644 --- a/tests/test_utils/test_downloader.py +++ b/tests/test_utils/test_downloader.py @@ -5,14 +5,7 @@ import pandas as pd import pytest -from dascore.constants import DATA_VERSION -from dascore.utils.downloader import ( - REGISTRY_PATH, - fetch, - fetcher, - get_registry_df, - get_test_data_cache_info, -) +from dascore.utils.downloader import fetch, get_registry_df @pytest.fixture() @@ -43,24 +36,3 @@ def test_existing_file(self, registry_df): """Ensure an existing file just returns.""" path = fetch(registry_df["name"].iloc[0]) assert fetch(path) == path - - -class TestTestDataCacheInfo: - """Tests for CI cache metadata derived from downloader state.""" - - def test_cache_info_matches_downloader_configuration(self): - """Ensure cache metadata stays aligned with downloader config.""" - info = get_test_data_cache_info() - - assert info.registry_path == REGISTRY_PATH - assert info.cache_path == fetcher.path.parent - assert info.data_version == DATA_VERSION - assert len(info.registry_hash) == 64 - - def test_cache_key_includes_expected_parts(self): - """Ensure the generated cache key matches the CI convention.""" - info = get_test_data_cache_info() - - out = info.get_key(runner_os="Linux", cache_number=7) - - assert out == f"data-Linux-{DATA_VERSION}-{info.registry_hash}-7"