Prime CI test data from one cross-OS clone of DASDAE/test_data

.github/actions/mamba-install-dascore/action.yml

@@ -20,11 +20,6 @@ inputs:
     default: "true"
     required: false
 
-  cache-number:
-    description: "Cache number. Use != 1 to reset data cache"
-    required: false
-    default: "1"
-
   prepare-test-data:
     description: "If true, restore, prime, and save the test-data cache after installation"
     required: false
@@ -77,34 +72,12 @@ runs:
       run: |
         pip install -e ".${INSTALL_GROUP_STR}"
 
-    - name: export test data cache info
-      id: data-cache
-      shell: bash -el {0}
-      env:
-        INPUT_CACHE_NUMBER: ${{ inputs.cache-number }}
-        RUNNER_OS: ${{ runner.os }}
-      run: |
-        python .github/scripts/export_test_data_cache_env.py >> "$GITHUB_OUTPUT"
-
-    - name: restore test data cache
+    # Normally just restores the cache primed by the calling workflow's setup
+    # job, but also self-heals (prime + save) on a miss, e.g. for single-job
+    # workflows or when a re-run can't see an evicted cache.
+    - name: prepare test data cache
       if: "${{ inputs.prepare-test-data == 'true' }}"
-      id: restore-test-data
-      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
-      with:
-        path: ${{ steps.data-cache.outputs.DATA_CACHE_PATH }}
-        key: ${{ steps.data-cache.outputs.DATA_CACHE_KEY }}
-
-    - name: prime test data cache
-      if: "${{ inputs.prepare-test-data == 'true' && steps.restore-test-data.outputs.cache-hit != 'true' }}"
-      shell: bash -el {0}
-      run: python .github/scripts/cache_test_data.py
-
-    - name: save test data cache
-      if: "${{ inputs.prepare-test-data == 'true' && steps.restore-test-data.outputs.cache-hit != 'true' }}"
-      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
-      with:
-        path: ${{ steps.data-cache.outputs.DATA_CACHE_PATH }}
-        key: ${{ steps.data-cache.outputs.DATA_CACHE_KEY }}
+      uses: ./.github/actions/prime-test-data-cache
 
     # Print out the package info for current environment
     - name: print package info

.github/actions/prime-test-data-cache/action.yml

@@ -0,0 +1,74 @@
+name: "Prime Test Data Cache"
+description: >
+  Restore the shared cross-OS test-data cache, or build it from a checkout of
+  DASDAE/test_data on a miss. Requires dascore to be checked out at the
+  workspace root first. Jobs that run tests must also set the DFS_DATA_DIR
+  env var to .test_data_cache under the workspace so pooch reads the restored
+  files (composite actions can't set caller env vars without GITHUB_ENV,
+  which zizmor forbids).
+inputs:
+  lookup-only:
+    description: "If true, check cache existence (and prime on miss) without downloading it"
+    required: false
+    default: "false"
+
+runs:
+  using: "composite"
+  steps:
+    - name: compute test data cache key
+      id: cache-key
+      shell: bash
+      env:
+        REGISTRY_HASH: ${{ hashFiles('dascore/data_registry.txt') }}
+      run: |
+        if [ -z "$REGISTRY_HASH" ]; then
+          echo "dascore/data_registry.txt not found; checkout dascore first." >&2
+          exit 1
+        fi
+        # DATA_VERSION is part of the key because pooch reads files from a
+        # <DATA_VERSION> subdir of the cache; a version bump must invalidate.
+        data_version="$(sed -n 's/^DATA_VERSION[^=]*=[[:space:]]*"\([^"]*\)".*/\1/p' dascore/constants.py)"
+        if [ -z "$data_version" ]; then
+          echo "DATA_VERSION not found in dascore/constants.py." >&2
+          exit 1
+        fi
+        # Bump this number to manually reset the cache.
+        cache_number=1
+        echo "key=test-data-${data_version}-${REGISTRY_HASH}-${cache_number}" >> "$GITHUB_OUTPUT"
+
+    # The path/key literals must be identical in restore and save, and
+    # enableCrossOsArchive must be set on both, so one ubuntu-primed cache
+    # serves all operating systems.
+    - name: restore test data cache
+      id: restore
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+      with:
+        path: .test_data_cache
+        key: ${{ steps.cache-key.outputs.key }}
+        enableCrossOsArchive: true
+        lookup-only: ${{ inputs.lookup-only }}
+
+    - name: checkout test data repo
+      if: "${{ steps.restore.outputs.cache-hit != 'true' }}"
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      with:
+        repository: DASDAE/test_data
+        ref: master
+        path: .test_data_repo
+        persist-credentials: false
+
+    - name: prime test data cache
+      if: "${{ steps.restore.outputs.cache-hit != 'true' }}"
+      shell: bash
+      run: |
+        # Stdlib-only script; use whichever system interpreter this OS provides.
+        if command -v python3 >/dev/null 2>&1; then PY=python3; else PY=python; fi
+        "$PY" .github/scripts/prime_test_data.py
+
+    - name: save test data cache
+      if: "${{ steps.restore.outputs.cache-hit != 'true' }}"
+      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+      with:
+        path: .test_data_cache
+        key: ${{ steps.cache-key.outputs.key }}
+        enableCrossOsArchive: true

.github/scripts/prime_test_data.py

@@ -0,0 +1,101 @@
+"""Populate the CI test-data cache from a checkout of DASDAE/test_data.
+
+Copies every file listed in dascore's data registry from a local checkout of
+https://github.com/DASDAE/test_data into the layout pooch expects
+(.test_data_cache/<DATA_VERSION>/<name>), verifying each sha256 against the
+registry. Registry entries hosted elsewhere are allowed but skipped with a
+warning — they aren't cached, so the test suite shouldn't depend on them;
+pooch lazy-fetches them on demand.
+
+Uses only the standard library so it can run on the runners' system python.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import shutil
+import sys
+from pathlib import Path
+from urllib.parse import unquote
+
+ROOT = Path(__file__).resolve().parents[2]
+REGISTRY_PATH = ROOT / "dascore" / "data_registry.txt"
+CONSTANTS_PATH = ROOT / "dascore" / "constants.py"
+REPO_PATH = ROOT / ".test_data_repo"
+CACHE_PATH = ROOT / ".test_data_cache"
+URL_REGEX = re.compile(
+    r"^https?://github\.com/dasdae/test_data/raw/master/(?P<subpath>.+)$",
+    re.IGNORECASE,
+)
+
+
+def get_data_version() -> str:
+    """Parse DATA_VERSION from dascore/constants.py without importing dascore."""
+    match = re.search(
+        r'^DATA_VERSION\s*=\s*"([^"]+)"', CONSTANTS_PATH.read_text(), re.MULTILINE
+    )
+    if match is None:
+        sys.exit(f"DATA_VERSION not found in {CONSTANTS_PATH}")
+    return match.group(1)
+
+
+def sha256sum(path: Path) -> str:
+    """Return the sha256 hex digest of a file, reading in chunks."""
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(2**20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def main() -> int:
+    """Copy registry files from the repo checkout into the cache layout."""
+    dest_dir = CACHE_PATH / get_data_version()
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    errors = []
+    copied = 0
+    skipped = 0
+    total_bytes = 0
+    for line in REGISTRY_PATH.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        name, expected_hash, url = line.split()
+        match = URL_REGEX.match(url)
+        if match is None:
+            print(f"Skipping {name}: not hosted in the DASDAE/test_data repo")  # noqa: T201
+            skipped += 1
+            continue
+        # Registry URLs may percent-encode characters (e.g. + as %2B); the
+        # files in the repo use the decoded names.
+        source = (REPO_PATH / unquote(match["subpath"])).resolve()
+        dest = (dest_dir / name).resolve()
+        if not source.is_relative_to(REPO_PATH) or dest.parent != dest_dir.resolve():
+            errors.append(f"{name}: escapes its intended directory")
+            continue
+        if not source.exists():
+            errors.append(f"{name}: {source} missing from the test_data checkout")
+            continue
+        if (digest := sha256sum(source)) != expected_hash:
+            errors.append(
+                f"{name}: sha256 mismatch (registry {expected_hash}, repo {digest})"
+            )
+            continue
+        shutil.copy2(source, dest)
+        copied += 1
+        total_bytes += source.stat().st_size
+    print(  # noqa: T201
+        f"Copied {copied} files ({total_bytes / 1_000_000:.0f} MB) to {dest_dir}"
+        f" ({skipped} skipped)"
+    )
+    if errors:
+        print("Failed to prime test data cache:", file=sys.stderr)  # noqa: T201
+        for error in errors:
+            print(f"  {error}", file=sys.stderr)  # noqa: T201
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/build_deploy_master_docs.yaml

@@ -16,6 +16,11 @@ on:
 permissions:
   contents: read
 
+env:
+  # Where the test-data cache is restored; pooch reads files from here.
+  # Must match the path used in .github/actions/prime-test-data-cache.
+  DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache
+
 # Allow one concurrent deployment
 concurrency:
   group: "pages"

.github/workflows/build_deploy_stable_docs.yaml

@@ -13,6 +13,11 @@ on:
 permissions:
   contents: read
 
+env:
+  # Where the test-data cache is restored; pooch reads files from here.
+  # Must match the path used in .github/actions/prime-test-data-cache.
+  DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache
+
 # Allow one concurrent deployment. Only runs which can deploy need to
 # serialize, so they share the "pages" group with the master docs workflow.
 # A pre-release never deploys; it is grouped per tag so that publishing one

.github/workflows/get_coverage.yml

@@ -8,6 +8,11 @@ on:
 permissions:
   contents: read
 
+env:
+  # Where the test-data cache is restored; pooch reads files from here.
+  # Must match the path used in .github/actions/prime-test-data-cache.
+  DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache
+
 jobs:
   calc_coverage:
     runs-on: ubuntu-latest
@@ -26,7 +31,6 @@ jobs:
         with:
           # Defined in .github/actions/load-shared-vars/action.yml
           python-version: ${{ steps.shared-vars.outputs.python-default }}
-          cache-number: 1
           prepare-test-data: "true"
 
       - name: run test suite

.github/workflows/profile.yml

@@ -19,6 +19,11 @@ permissions:
   contents: read # required for actions/checkout
   id-token: write # required for OIDC authentication with CodSpeed
 
+env:
+  # Where the test-data cache is restored; pooch reads files from here.
+  # Must match the path used in .github/actions/prime-test-data-cache.
+  DFS_DATA_DIR: ${{ github.workspace }}/.test_data_cache
+
 jobs:
   benchmarks:
     name: Run benchmarks
@@ -41,7 +46,6 @@ jobs:
           # Defined in .github/actions/load-shared-vars/action.yml
           python-version: ${{ steps.shared-vars.outputs.python-default }}
           install-group-str: "[profile]"
-          cache-number: 1
           prepare-test-data: "true"
 
       - name: Run benchmarks