diff --git a/.github/actions/cache-test-data/action.yml b/.github/actions/cache-test-data/action.yml index 35fa4ca1e..f63f821eb 100644 --- a/.github/actions/cache-test-data/action.yml +++ b/.github/actions/cache-test-data/action.yml @@ -5,6 +5,18 @@ inputs: description: "Cache number. Use != 1 to reset data cache" required: false default: "1" + mode: + description: "One of restore, save, or restore-save" + required: false + default: "restore" + +outputs: + cache-hit: + description: "True if an exact cache key was restored" + value: ${{ steps.restore-test-data.outputs.cache-hit }} + cache-key: + description: "The cache key derived from the registry hash" + value: ${{ env.DATA_CACHE_KEY }} runs: using: "composite" @@ -17,29 +29,59 @@ runs: echo "This action requires Python to be installed first." exit 1 fi - if ! python -c "import pooch" 2>/dev/null; then - echo "❌ Error: Pooch is not installed" - echo "This action requires Pooch to be installed (pip install pooch or include in dependencies)." - exit 1 - fi - echo "✅ Prerequisites verified: Python and Pooch are available" + echo "✅ Prerequisites verified: Python is available" - name: get data registry hash shell: bash -el {0} run: | - echo "DATA_REGISTRY_HASH=$(python -c "import pooch; print(pooch.file_hash('dascore/data_registry.txt'))")" >> $GITHUB_ENV + python - <<'PY' >> "$GITHUB_ENV" + from hashlib import sha256 + from pathlib import Path + + registry_path = Path("dascore/data_registry.txt") + print(f"DATA_REGISTRY_HASH={sha256(registry_path.read_bytes()).hexdigest()}") + PY - name: get data cache path shell: bash -el {0} run: | - echo "DATA_CACHE_PATH=$(python -c "import pooch; print(pooch.os_cache('dascore'))")" >> $GITHUB_ENV + python - <<'PY' >> "$GITHUB_ENV" + import os + import sys + from pathlib import Path - - name: cache test data - uses: actions/cache@v4 - id: cache-test-data + home = Path.home() + if sys.platform == "win32": + cache_root = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local")) + elif sys.platform == "darwin": + cache_root = home / "Library" / "Caches" + else: + cache_root = Path(os.environ.get("XDG_CACHE_HOME", home / ".cache")) + + print(f"DATA_CACHE_PATH={cache_root / 'dascore'}") + PY + + - name: get data cache key + shell: bash -el {0} + run: | + echo "DATA_CACHE_KEY=data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}" >> $GITHUB_ENV + + - name: restore test data cache + if: ${{ inputs.mode == 'restore' || inputs.mode == 'restore-save' }} + uses: actions/cache/restore@v4 + id: restore-test-data with: enableCrossOsArchive: true path: ${{ env.DATA_CACHE_PATH }} - key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }} + key: ${{ env.DATA_CACHE_KEY }} restore-keys: | data-${{ env.DATA_REGISTRY_HASH }}- + + - name: save test data cache + if: ${{ inputs.mode == 'save' || inputs.mode == 'restore-save' }} + uses: actions/cache/save@v4 + id: cache-test-data + with: + enableCrossOsArchive: true + path: ${{ env.DATA_CACHE_PATH }} + key: ${{ env.DATA_CACHE_KEY }} diff --git a/.github/actions/mamba-install-dascore/action.yml b/.github/actions/mamba-install-dascore/action.yml index dd3019e1a..3dcdfce72 100644 --- a/.github/actions/mamba-install-dascore/action.yml +++ b/.github/actions/mamba-install-dascore/action.yml @@ -17,14 +17,13 @@ inputs: install-package: description: "If true, install dascore" - default: true + default: "true" required: false - type: boolean cache-number: description: "Cache number. Use != 1 to reset data cache" required: false - default: 1 + default: "1" runs: using: "composite" diff --git a/.github/scripts/cache_test_data.py b/.github/scripts/cache_test_data.py new file mode 100644 index 000000000..3c2d7cf38 --- /dev/null +++ b/.github/scripts/cache_test_data.py @@ -0,0 +1,22 @@ +"""Populate the pooch cache with every file in dascore's data registry.""" + +from __future__ import annotations + +from pathlib import Path + +from dascore.utils.downloader import fetch, get_registry_df + + +def main() -> None: + """Fetch every registered test-data file into the local pooch cache.""" + registry = get_registry_df() + total = len(registry) + print(f"Priming DASCore test-data cache with {total} files") # noqa + for index, name in enumerate(registry["name"], start=1): + path = Path(fetch(name)) + print(f"[{index}/{total}] {name} -> {path}") # noqa + print("Finished priming DASCore test-data cache") # noqa + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/get_coverage.yml b/.github/workflows/get_coverage.yml index 9db4eae76..df8f8ec85 100644 --- a/.github/workflows/get_coverage.yml +++ b/.github/workflows/get_coverage.yml @@ -6,7 +6,44 @@ on: - master jobs: + prime_test_data_cache: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/load-shared-vars + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_DEFAULT }} + + - name: Install dascore for cache priming + shell: bash + run: | + python -m pip install --upgrade pip + pip install -e . + + - uses: ./.github/actions/cache-test-data + id: cache-test-data + with: + mode: restore + cache-number: 1 + + - name: Download all registered test data + if: steps.cache-test-data.outputs.cache-hit != 'true' + shell: bash + run: python .github/scripts/cache_test_data.py + + - name: Save primed test data cache + if: steps.cache-test-data.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-test-data + with: + mode: save + cache-number: 1 + calc_coverage: + needs: prime_test_data_cache runs-on: ubuntu-latest steps: diff --git a/.github/workflows/prime_test_data_cache.yml b/.github/workflows/prime_test_data_cache.yml new file mode 100644 index 000000000..6f34032d9 --- /dev/null +++ b/.github/workflows/prime_test_data_cache.yml @@ -0,0 +1,60 @@ +name: PrimeTestDataCache + +on: + workflow_dispatch: + schedule: + - cron: "0 4 * * 1" + push: + branches: + - master + paths: + - "dascore/data_registry.txt" + - ".github/actions/cache-test-data/action.yml" + - ".github/scripts/cache_test_data.py" + - ".github/workflows/prime_test_data_cache.yml" + pull_request: + branches: + - master + paths: + - "dascore/data_registry.txt" + - ".github/actions/cache-test-data/action.yml" + - ".github/scripts/cache_test_data.py" + - ".github/workflows/prime_test_data_cache.yml" + +jobs: + prime_test_data_cache: + name: Prime shared test data cache + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/load-shared-vars + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_DEFAULT }} + + - name: Install dascore + shell: bash + run: | + python -m pip install --upgrade pip + pip install -e . + + # The cache key is shared across OSes. Each consuming job restores this + # archive into the platform-specific cache path that pooch uses locally. + - uses: ./.github/actions/cache-test-data + id: cache-test-data + with: + mode: restore + + - name: Download all registered test data + if: steps.cache-test-data.outputs.cache-hit != 'true' + shell: bash + run: python .github/scripts/cache_test_data.py + + - name: Save primed test data cache + if: steps.cache-test-data.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-test-data + with: + mode: save diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index f91b870c7..4d3bb479a 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -6,6 +6,12 @@ on: branches: - master pull_request: + types: + - opened + - synchronize + - reopened + - labeled + - unlabeled workflow_dispatch: permissions: @@ -13,8 +19,11 @@ permissions: id-token: write # required for OIDC authentication with CodSpeed jobs: - benchmarks: - name: Run benchmarks + prime_test_data_cache: + name: Prime test data cache + if: > + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'benchmark') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -26,18 +35,57 @@ jobs: - uses: actions/setup-python@v6 with: - # Defined in .github/actions/load-shared-vars/action.yml python-version: ${{ env.PYTHON_DEFAULT }} - - name: Install dependencies - run: pip install ".[profile]" + - name: Install dascore for cache priming + run: | + python -m pip install --upgrade pip + pip install -e . - uses: ./.github/actions/cache-test-data + id: cache-test-data + with: + mode: restore + cache-number: 1 + + - name: Download all registered test data + if: steps.cache-test-data.outputs.cache-hit != 'true' + run: python .github/scripts/cache_test_data.py + + - name: Save primed test data cache + if: steps.cache-test-data.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-test-data with: + mode: save + cache-number: 1 + + benchmarks: + name: Run benchmarks + if: > + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'benchmark') + needs: prime_test_data_cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-tags: "true" + fetch-depth: '0' + + - uses: ./.github/actions/load-shared-vars + + - uses: ./.github/actions/mamba-install-dascore + with: + # Defined in .github/actions/load-shared-vars/action.yml + python-version: ${{ env.PYTHON_DEFAULT }} + install-group-str: "[profile]" cache-number: 1 - name: Run benchmarks uses: CodSpeedHQ/action@v4 with: - mode: instrumentation - run: ./.github/test_code.sh profile + mode: simulation + # CodSpeed does not inherit the shell activation done by the + # micromamba setup step, so run benchmarks explicitly inside the + # dascore environment to ensure pytest and pytest-codspeed resolve. + run: micromamba run -n dascore ./.github/test_code.sh profile diff --git a/.github/workflows/run_min_dep_tests.yml b/.github/workflows/run_min_dep_tests.yml index 8cee0f5f0..229110c2a 100644 --- a/.github/workflows/run_min_dep_tests.yml +++ b/.github/workflows/run_min_dep_tests.yml @@ -11,6 +11,7 @@ on: - 'pyproject.toml' - '**.py' - '.github/workflows/run_min_dep_tests.yml' + - '.github/actions/**/*.yml' env: # Ensure matplotlib doesn't try to show figures in CI @@ -35,8 +36,47 @@ jobs: id: load-vars # Runs the tests on combinations of the supported python/os matrix. - test_code_min_deps: + prime_test_data_cache: needs: setup + timeout-minutes: 20 + if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/load-shared-vars + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_DEFAULT }} + + - name: Install dascore for cache priming + shell: bash + run: | + python -m pip install --upgrade pip + pip install -e . + + - uses: ./.github/actions/cache-test-data + id: cache-test-data + with: + mode: restore + cache-number: 1 + + - name: Download all registered test data + if: steps.cache-test-data.outputs.cache-hit != 'true' + shell: bash + run: python .github/scripts/cache_test_data.py + + - name: Save primed test data cache + if: steps.cache-test-data.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-test-data + with: + mode: save + cache-number: 1 + + test_code_min_deps: + needs: [setup, prime_test_data_cache] timeout-minutes: 25 runs-on: ${{ matrix.os }} strategy: diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml index 68e105ad7..8e561d711 100644 --- a/.github/workflows/runtests.yml +++ b/.github/workflows/runtests.yml @@ -11,6 +11,7 @@ on: - 'pyproject.toml' - '**.py' - '.github/workflows/*.yml' + - '.github/actions/**/*.yml' env: # used to manually trigger cache reset. Just increment if needed. @@ -37,8 +38,49 @@ jobs: id: load-vars # Runs the tests on combinations of the supported python/os matrix. - test_code: + prime_test_data_cache: needs: setup + timeout-minutes: 20 + if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/load-shared-vars + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_DEFAULT }} + + - name: Install dascore for cache priming + shell: bash + run: | + python -m pip install --upgrade pip + pip install -e . + + # Prime the shared test-data cache once. Each downstream OS restores this + # archive into the platform-specific cache path that pooch uses locally. + - uses: ./.github/actions/cache-test-data + id: cache-test-data + with: + mode: restore + cache-number: ${{ env.CACHE_NUMBER }} + + - name: Download all registered test data + if: steps.cache-test-data.outputs.cache-hit != 'true' + shell: bash + run: python .github/scripts/cache_test_data.py + + - name: Save primed test data cache + if: steps.cache-test-data.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-test-data + with: + mode: save + cache-number: ${{ env.CACHE_NUMBER }} + + test_code: + needs: [setup, prime_test_data_cache] timeout-minutes: 25 runs-on: ${{ matrix.os }} strategy: diff --git a/.github/workflows/test_doc_build.yml b/.github/workflows/test_doc_build.yml index 480e7c741..7abd52e28 100644 --- a/.github/workflows/test_doc_build.yml +++ b/.github/workflows/test_doc_build.yml @@ -5,10 +5,52 @@ on: types: [labeled, synchronize] jobs: + prime_test_data_cache: + if: | + (github.event.action == 'labeled' && github.event.label.name == 'documentation') + || (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'documentation')) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-tags: "true" + fetch-depth: '0' + + - uses: ./.github/actions/load-shared-vars + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_DEFAULT }} + + - name: Install dascore for cache priming + shell: bash + run: | + python -m pip install --upgrade pip + pip install -e . + + - uses: ./.github/actions/cache-test-data + id: cache-test-data + with: + mode: restore + cache-number: 1 + + - name: Download all registered test data + if: steps.cache-test-data.outputs.cache-hit != 'true' + shell: bash + run: python .github/scripts/cache_test_data.py + + - name: Save primed test data cache + if: steps.cache-test-data.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-test-data + with: + mode: save + cache-number: 1 + test_build_docs: if: | (github.event.action == 'labeled' && github.event.label.name == 'documentation') || (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'documentation')) + needs: prime_test_data_cache runs-on: ubuntu-latest steps: - uses: actions/checkout@v4