diff --git a/.github/actions/cache-test-data/action.yml b/.github/actions/cache-test-data/action.yml new file mode 100644 index 000000000..35fa4ca1e --- /dev/null +++ b/.github/actions/cache-test-data/action.yml @@ -0,0 +1,45 @@ +name: "Cache Test Data" +description: "Caches DASCore test data using Pooch cache directory" +inputs: + cache-number: + description: "Cache number. Use != 1 to reset data cache" + required: false + default: "1" + +runs: + using: "composite" + steps: + - name: verify prerequisites + shell: bash -el {0} + run: | + if ! command -v python &> /dev/null; then + echo "❌ Error: Python is not installed or not in PATH" + echo "This action requires Python to be installed first." + exit 1 + fi + if ! python -c "import pooch" 2>/dev/null; then + echo "❌ Error: Pooch is not installed" + echo "This action requires Pooch to be installed (pip install pooch or include in dependencies)." + exit 1 + fi + echo "✅ Prerequisites verified: Python and Pooch are available" + + - name: get data registry hash + shell: bash -el {0} + run: | + echo "DATA_REGISTRY_HASH=$(python -c "import pooch; print(pooch.file_hash('dascore/data_registry.txt'))")" >> $GITHUB_ENV + + - name: get data cache path + shell: bash -el {0} + run: | + echo "DATA_CACHE_PATH=$(python -c "import pooch; print(pooch.os_cache('dascore'))")" >> $GITHUB_ENV + + - name: cache test data + uses: actions/cache@v4 + id: cache-test-data + with: + enableCrossOsArchive: true + path: ${{ env.DATA_CACHE_PATH }} + key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }} + restore-keys: | + data-${{ env.DATA_REGISTRY_HASH }}- diff --git a/.github/actions/load-shared-vars/action.yml b/.github/actions/load-shared-vars/action.yml new file mode 100644 index 000000000..221fe7732 --- /dev/null +++ b/.github/actions/load-shared-vars/action.yml @@ -0,0 +1,68 @@ +name: "Load Shared Variables" +description: "Loads shared variables from .github/shared-vars.yml and sets them as environment variables and outputs" + +outputs: + test-matrix: + description: "Python version matrix for full tests" + value: ${{ steps.load.outputs.test-matrix }} + min-deps-matrix: + description: "Python version matrix for minimum dependency tests" + value: ${{ steps.load.outputs.min-deps-matrix }} + +runs: + using: "composite" + steps: + - name: install pyyaml + shell: bash + run: python -m pip install --quiet pyyaml + + - name: load shared variables + id: load + shell: bash + run: | + # Read shared-vars.yml and set environment variables + python << 'EOF' + import yaml + import os + import json + + def flatten_dict(d, parent_key='', sep='_'): + """Recursively flatten nested dictionary into env var format.""" + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}".upper() if parent_key else k.upper() + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + elif isinstance(v, list): + # Convert lists to JSON strings for use in workflows + items.append((new_key, json.dumps(v))) + else: + items.append((new_key, str(v))) + return dict(items) + + # Load shared variables file + with open('.github/shared-vars.yml', 'r') as f: + config = yaml.safe_load(f) + + # Flatten all variables + env_vars = flatten_dict(config) + + # Write to GITHUB_ENV + with open(os.environ['GITHUB_ENV'], 'a') as env_file: + for key, value in sorted(env_vars.items()): + env_file.write(f"{key}={value}\n") + print(f"✅ Set {key}={value}") + + # Also output specific matrices for job outputs + python_config = config.get('python', {}) + test_matrix = python_config.get('test_matrix', []) + min_deps_matrix = python_config.get('min_deps_matrix', []) + + # Write to GITHUB_OUTPUT for action outputs + with open(os.environ['GITHUB_OUTPUT'], 'a') as output_file: + output_file.write(f"test-matrix={json.dumps(test_matrix)}\n") + output_file.write(f"min-deps-matrix={json.dumps(min_deps_matrix)}\n") + + print(f"📋 Output test-matrix: {test_matrix}") + print(f"📋 Output min-deps-matrix: {min_deps_matrix}") + EOF diff --git a/.github/actions/mamba-install-dascore/action.yml b/.github/actions/mamba-install-dascore/action.yml index 4700d8dab..dd3019e1a 100644 --- a/.github/actions/mamba-install-dascore/action.yml +++ b/.github/actions/mamba-install-dascore/action.yml @@ -64,43 +64,15 @@ runs: run: | git fetch --tags --force - - name: get data registry hash - shell: bash - run: | - echo "DATA_REGISTRY_HASH=$(sha256sum dascore/data_registry.txt | cut -d' ' -f1)" >> $GITHUB_ENV - - name: install dascore if: "${{ inputs.install-package == 'true' }}" shell: bash -l {0} run: | pip install -e .${{ inputs.install-group-str }} - - name: set data cache path - shell: bash -el {0} - run: | - echo "DATA_CACHE_PATH=$(python -c "import pooch; print(pooch.os_cache('dascore'))")" >> $GITHUB_ENV - - - name: cache test data - uses: actions/cache@v4 - id: cache-test-data + - uses: ./.github/actions/cache-test-data with: - enableCrossOsArchive: true - path: ${{ env.DATA_CACHE_PATH }} - key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }} - restore-keys: | - data-${{ env.DATA_REGISTRY_HASH }}- - - - name: verify cache status - shell: bash -el {0} - run: | - if [[ "${{ steps.cache-test-data.outputs.cache-hit }}" == "true" ]]; then - echo "✅ Cache hit! Using cached test data from: ${{ env.DATA_CACHE_PATH }}" - echo "Cache key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}" - else - echo "❌ Cache miss. Test data will be downloaded fresh." - echo "Cache key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}" - fi - ls -la "${{ env.DATA_CACHE_PATH }}" || echo "Cache directory does not exist yet" + cache-number: ${{ inputs.cache-number }} # Print out the package info for current environment - name: print package info diff --git a/.github/doc_environment.yml b/.github/doc_environment.yml index 8bde2d6cc..946709951 100644 --- a/.github/doc_environment.yml +++ b/.github/doc_environment.yml @@ -2,7 +2,6 @@ name: dascore channels: - conda-forge dependencies: - - python=3.12 - pytest - pydantic>2.0 - pip diff --git a/.github/shared-vars.yml b/.github/shared-vars.yml new file mode 100644 index 000000000..7c7bdeeef --- /dev/null +++ b/.github/shared-vars.yml @@ -0,0 +1,20 @@ +# Shared variables used across all GitHub Actions workflows +# Update these values to propagate changes to all workflows + +python: + # Default Python version for most workflows (docs, linting, coverage, etc.) + default: "3.13" + + # Full test matrix versions (runtests.yml) + test_matrix: + - "3.11" + - "3.12" + - "3.13" + - "3.14" + + # Minimum dependency test matrix versions (run_min_dep_tests.yml) + min_deps_matrix: + - "3.13" + - "3.14" + +# Add other shared variables here as needed diff --git a/.github/workflows/build_deploy_master_docs.yaml b/.github/workflows/build_deploy_master_docs.yaml index fb4d8cd1d..6191c23ef 100644 --- a/.github/workflows/build_deploy_master_docs.yaml +++ b/.github/workflows/build_deploy_master_docs.yaml @@ -37,9 +37,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} environment-file: './.github/doc_environment.yml' - uses: ./.github/actions/prep_doc_build diff --git a/.github/workflows/build_deploy_stable_docs.yaml b/.github/workflows/build_deploy_stable_docs.yaml index 4fda3aa89..8b1534cf3 100644 --- a/.github/workflows/build_deploy_stable_docs.yaml +++ b/.github/workflows/build_deploy_stable_docs.yaml @@ -31,10 +31,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} environment-file: './.github/doc_environment.yml' - uses: ./.github/actions/build-docs diff --git a/.github/workflows/get_coverage.yml b/.github/workflows/get_coverage.yml index 774ff6889..141ffc5c9 100644 --- a/.github/workflows/get_coverage.yml +++ b/.github/workflows/get_coverage.yml @@ -15,10 +15,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} cache-number: 1 - name: run test suite diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 14503b9a5..f16a0265f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,12 +15,14 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - name: Install uv uses: astral-sh/setup-uv@v3 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: ${{ env.PYTHON_DEFAULT }} - name: install linting packages run: uv tool install pre-commit diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 4a4ca9a8e..89460ff2c 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -8,9 +8,6 @@ on: pull_request: workflow_dispatch: -env: - python_version: "3.13" - jobs: benchmarks: name: Run benchmarks @@ -21,13 +18,19 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - uses: actions/setup-python@v6 with: - python-version: ${{ env.python_version }} + python-version: ${{ env.PYTHON_DEFAULT }} - name: Install dependencies run: pip install ".[profile]" + - uses: ./.github/actions/cache-test-data + with: + cache-number: 1 + - name: Run benchmarks uses: CodSpeedHQ/action@v4 with: diff --git a/.github/workflows/run_min_dep_tests.yml b/.github/workflows/run_min_dep_tests.yml index 68939ea96..28ac3eecf 100644 --- a/.github/workflows/run_min_dep_tests.yml +++ b/.github/workflows/run_min_dep_tests.yml @@ -23,16 +23,25 @@ concurrency: cancel-in-progress: true jobs: + # Load Python version matrix from shared-vars.yml + setup: + runs-on: ubuntu-latest + outputs: + python-matrix: ${{ steps.load-vars.outputs.min-deps-matrix }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/load-shared-vars + id: load-vars + # Runs the tests on combinations of the supported python/os matrix. test_code_min_deps: - + needs: setup timeout-minutes: 25 runs-on: ${{ matrix.os }} strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ '3.12', '3.13' ] - + python-version: ${{ fromJson(needs.setup.outputs.python-matrix) }} # only run if CI isn't turned off if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml index 4f934eea3..70d75df85 100644 --- a/.github/workflows/runtests.yml +++ b/.github/workflows/runtests.yml @@ -25,27 +25,37 @@ concurrency: cancel-in-progress: true jobs: + # Load Python version matrix from shared-vars.yml + setup: + runs-on: ubuntu-latest + outputs: + python-matrix: ${{ steps.load-vars.outputs.test-matrix }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/load-shared-vars + id: load-vars + # Runs the tests on combinations of the supported python/os matrix. test_code: - + needs: setup timeout-minutes: 25 runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.10', '3.11', "3.12", "3.13"] + python-version: ${{ fromJson(needs.setup.outputs.python-matrix) }} # only run if CI isn't turned off if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') env: # set conda environment file with dependencies - env_file: "environment.yml" + env_file: 'environment.yml' steps: - uses: actions/checkout@v4 with: - fetch-tags: "true" + fetch-tags: 'true' fetch-depth: '0' - uses: ./.github/actions/mamba-install-dascore diff --git a/.github/workflows/test_doc_build.yml b/.github/workflows/test_doc_build.yml index e4cc4ffd6..ebcc7de82 100644 --- a/.github/workflows/test_doc_build.yml +++ b/.github/workflows/test_doc_build.yml @@ -16,10 +16,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} environment-file: './.github/doc_environment.yml' cache-number: 1 diff --git a/.github/workflows/upload_pypi.yml b/.github/workflows/upload_pypi.yml index 383be8aa7..3f9d819fe 100644 --- a/.github/workflows/upload_pypi.yml +++ b/.github/workflows/upload_pypi.yml @@ -14,6 +14,8 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - name: setup conda uses: conda-incubator/setup-miniconda@v3 with: @@ -21,7 +23,7 @@ jobs: channels: conda-forge,defaults channel-priority: true environment-file: environment.yml - python-version: 3.11 + python-version: ${{ env.PYTHON_DEFAULT }} - name: create dists shell: bash -l {0} diff --git a/pyproject.toml b/pyproject.toml index 25eaef8ab..a82342d55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + ] keywords = ["geophysics", "distributed-acoustic-sensing"]