From 451d8ac4804ba78715dca23a539cfadffb83b227 Mon Sep 17 00:00:00 2001 From: derrick chambers Date: Mon, 22 Dec 2025 09:19:58 -0700 Subject: [PATCH 1/6] update cache --- .github/actions/mamba-install-dascore/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/mamba-install-dascore/action.yml b/.github/actions/mamba-install-dascore/action.yml index 4700d8dab..22612fc4e 100644 --- a/.github/actions/mamba-install-dascore/action.yml +++ b/.github/actions/mamba-install-dascore/action.yml @@ -65,9 +65,9 @@ runs: git fetch --tags --force - name: get data registry hash - shell: bash + shell: bash -el {0} run: | - echo "DATA_REGISTRY_HASH=$(sha256sum dascore/data_registry.txt | cut -d' ' -f1)" >> $GITHUB_ENV + echo "DATA_REGISTRY_HASH=$(python -c "import hashlib; print(hashlib.sha256(open('dascore/data_registry.txt', 'rb').read()).hexdigest())")" >> $GITHUB_ENV - name: install dascore if: "${{ inputs.install-package == 'true' }}" From 14e723767019832feb8b7bce38ff25e5408cb1fb Mon Sep 17 00:00:00 2001 From: derrick chambers Date: Mon, 22 Dec 2025 09:49:49 -0700 Subject: [PATCH 2/6] update caching action --- .github/actions/cache-test-data/action.yml | 45 +++++++++++++++++++ .../actions/mamba-install-dascore/action.yml | 32 +------------ .github/workflows/profile.yml | 4 ++ 3 files changed, 51 insertions(+), 30 deletions(-) create mode 100644 .github/actions/cache-test-data/action.yml diff --git a/.github/actions/cache-test-data/action.yml b/.github/actions/cache-test-data/action.yml new file mode 100644 index 000000000..35fa4ca1e --- /dev/null +++ b/.github/actions/cache-test-data/action.yml @@ -0,0 +1,45 @@ +name: "Cache Test Data" +description: "Caches DASCore test data using Pooch cache directory" +inputs: + cache-number: + description: "Cache number. Use != 1 to reset data cache" + required: false + default: "1" + +runs: + using: "composite" + steps: + - name: verify prerequisites + shell: bash -el {0} + run: | + if ! command -v python &> /dev/null; then + echo "❌ Error: Python is not installed or not in PATH" + echo "This action requires Python to be installed first." + exit 1 + fi + if ! python -c "import pooch" 2>/dev/null; then + echo "❌ Error: Pooch is not installed" + echo "This action requires Pooch to be installed (pip install pooch or include in dependencies)." + exit 1 + fi + echo "✅ Prerequisites verified: Python and Pooch are available" + + - name: get data registry hash + shell: bash -el {0} + run: | + echo "DATA_REGISTRY_HASH=$(python -c "import pooch; print(pooch.file_hash('dascore/data_registry.txt'))")" >> $GITHUB_ENV + + - name: get data cache path + shell: bash -el {0} + run: | + echo "DATA_CACHE_PATH=$(python -c "import pooch; print(pooch.os_cache('dascore'))")" >> $GITHUB_ENV + + - name: cache test data + uses: actions/cache@v4 + id: cache-test-data + with: + enableCrossOsArchive: true + path: ${{ env.DATA_CACHE_PATH }} + key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }} + restore-keys: | + data-${{ env.DATA_REGISTRY_HASH }}- diff --git a/.github/actions/mamba-install-dascore/action.yml b/.github/actions/mamba-install-dascore/action.yml index 22612fc4e..dd3019e1a 100644 --- a/.github/actions/mamba-install-dascore/action.yml +++ b/.github/actions/mamba-install-dascore/action.yml @@ -64,43 +64,15 @@ runs: run: | git fetch --tags --force - - name: get data registry hash - shell: bash -el {0} - run: | - echo "DATA_REGISTRY_HASH=$(python -c "import hashlib; print(hashlib.sha256(open('dascore/data_registry.txt', 'rb').read()).hexdigest())")" >> $GITHUB_ENV - - name: install dascore if: "${{ inputs.install-package == 'true' }}" shell: bash -l {0} run: | pip install -e .${{ inputs.install-group-str }} - - name: set data cache path - shell: bash -el {0} - run: | - echo "DATA_CACHE_PATH=$(python -c "import pooch; print(pooch.os_cache('dascore'))")" >> $GITHUB_ENV - - - name: cache test data - uses: actions/cache@v4 - id: cache-test-data + - uses: ./.github/actions/cache-test-data with: - enableCrossOsArchive: true - path: ${{ env.DATA_CACHE_PATH }} - key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }} - restore-keys: | - data-${{ env.DATA_REGISTRY_HASH }}- - - - name: verify cache status - shell: bash -el {0} - run: | - if [[ "${{ steps.cache-test-data.outputs.cache-hit }}" == "true" ]]; then - echo "✅ Cache hit! Using cached test data from: ${{ env.DATA_CACHE_PATH }}" - echo "Cache key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}" - else - echo "❌ Cache miss. Test data will be downloaded fresh." - echo "Cache key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}" - fi - ls -la "${{ env.DATA_CACHE_PATH }}" || echo "Cache directory does not exist yet" + cache-number: ${{ inputs.cache-number }} # Print out the package info for current environment - name: print package info diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 85b6132d6..9635e43a1 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -28,6 +28,10 @@ jobs: - name: Install dependencies run: pip install ".[profile]" + - uses: ./.github/actions/cache-test-data + with: + cache-number: 1 + - name: Run benchmarks uses: CodSpeedHQ/action@v4 with: From 1ab7e6374d4811c18830b0fe4974abf3ed8b7e6a Mon Sep 17 00:00:00 2001 From: derrick chambers Date: Mon, 22 Dec 2025 10:17:16 -0700 Subject: [PATCH 3/6] make actions global var --- .github/actions/load-shared-vars/action.yml | 40 +++++++++++++++++++ .github/shared-vars.yml | 8 ++++ .../workflows/build_deploy_master_docs.yaml | 4 +- .../workflows/build_deploy_stable_docs.yaml | 3 +- .github/workflows/get_coverage.yml | 3 +- .github/workflows/lint.yml | 4 +- .github/workflows/profile.yml | 7 ++-- .github/workflows/run_min_dep_tests.yml | 3 +- .github/workflows/runtests.yml | 6 +-- .github/workflows/test_doc_build.yml | 3 +- .github/workflows/upload_pypi.yml | 4 +- 11 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 .github/actions/load-shared-vars/action.yml create mode 100644 .github/shared-vars.yml diff --git a/.github/actions/load-shared-vars/action.yml b/.github/actions/load-shared-vars/action.yml new file mode 100644 index 000000000..1f653ac8d --- /dev/null +++ b/.github/actions/load-shared-vars/action.yml @@ -0,0 +1,40 @@ +name: "Load Shared Variables" +description: "Loads shared variables from .github/shared-vars.yml and sets them as environment variables" + +runs: + using: "composite" + steps: + - name: load shared variables + shell: bash + run: | + # Read shared-vars.yml and set environment variables + python << 'EOF' + import yaml + import os + + # Load shared variables file + with open('.github/shared-vars.yml', 'r') as f: + config = yaml.safe_load(f) + + # Set Python version environment variables + python_config = config.get('python', {}) + + env_vars = { + 'PYTHON_DEFAULT': python_config.get('default', '3.12'), + 'PYTHON_LATEST': python_config.get('latest', '3.13'), + 'PYTHON_RELEASE': python_config.get('release', '3.11'), + } + + # Write to GITHUB_ENV + with open(os.environ['GITHUB_ENV'], 'a') as env_file: + for key, value in env_vars.items(): + env_file.write(f"{key}={value}\n") + print(f"✅ Set {key}={value}") + + # For matrix values, we can't set them directly in env + # Workflows will need to read the file directly for matrix strategy + test_matrix = python_config.get('test_matrix', []) + min_deps_matrix = python_config.get('min_deps_matrix', []) + print(f"📋 Test matrix available: {test_matrix}") + print(f"📋 Min deps matrix available: {min_deps_matrix}") + EOF diff --git a/.github/shared-vars.yml b/.github/shared-vars.yml new file mode 100644 index 000000000..3c8287bab --- /dev/null +++ b/.github/shared-vars.yml @@ -0,0 +1,8 @@ +# Shared variables used across all GitHub Actions workflows +# Update these values to propagate changes to all workflows + +python: + # Default Python version for most workflows (docs, linting, coverage, etc.) + default: "3.13" + +# Add other shared variables here as needed diff --git a/.github/workflows/build_deploy_master_docs.yaml b/.github/workflows/build_deploy_master_docs.yaml index fb4d8cd1d..6191c23ef 100644 --- a/.github/workflows/build_deploy_master_docs.yaml +++ b/.github/workflows/build_deploy_master_docs.yaml @@ -37,9 +37,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} environment-file: './.github/doc_environment.yml' - uses: ./.github/actions/prep_doc_build diff --git a/.github/workflows/build_deploy_stable_docs.yaml b/.github/workflows/build_deploy_stable_docs.yaml index 4fda3aa89..8b1534cf3 100644 --- a/.github/workflows/build_deploy_stable_docs.yaml +++ b/.github/workflows/build_deploy_stable_docs.yaml @@ -31,10 +31,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} environment-file: './.github/doc_environment.yml' - uses: ./.github/actions/build-docs diff --git a/.github/workflows/get_coverage.yml b/.github/workflows/get_coverage.yml index 774ff6889..141ffc5c9 100644 --- a/.github/workflows/get_coverage.yml +++ b/.github/workflows/get_coverage.yml @@ -15,10 +15,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} cache-number: 1 - name: run test suite diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 14503b9a5..f16a0265f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,12 +15,14 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - name: Install uv uses: astral-sh/setup-uv@v3 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: ${{ env.PYTHON_DEFAULT }} - name: install linting packages run: uv tool install pre-commit diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 9635e43a1..1c6e02a14 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -8,9 +8,6 @@ on: pull_request: workflow_dispatch: -env: - python_version: "3.13" - jobs: benchmarks: name: Run benchmarks @@ -21,9 +18,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - uses: actions/setup-python@v6 with: - python-version: ${{ env.python_version }} + python-version: ${{ env.PYTHON_DEFAULT }} - name: Install dependencies run: pip install ".[profile]" diff --git a/.github/workflows/run_min_dep_tests.yml b/.github/workflows/run_min_dep_tests.yml index 68939ea96..ff7d90206 100644 --- a/.github/workflows/run_min_dep_tests.yml +++ b/.github/workflows/run_min_dep_tests.yml @@ -31,8 +31,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ '3.12', '3.13' ] - + python-version: [ '3.13', '3.14' ] # only run if CI isn't turned off if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml index 4f934eea3..8e9862366 100644 --- a/.github/workflows/runtests.yml +++ b/.github/workflows/runtests.yml @@ -33,19 +33,19 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.10', '3.11', "3.12", "3.13"] + python-version: ['3.11', '3.12', '3.13', '3.14' ] # only run if CI isn't turned off if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') env: # set conda environment file with dependencies - env_file: "environment.yml" + env_file: 'environment.yml' steps: - uses: actions/checkout@v4 with: - fetch-tags: "true" + fetch-tags: 'true' fetch-depth: '0' - uses: ./.github/actions/mamba-install-dascore diff --git a/.github/workflows/test_doc_build.yml b/.github/workflows/test_doc_build.yml index e4cc4ffd6..ebcc7de82 100644 --- a/.github/workflows/test_doc_build.yml +++ b/.github/workflows/test_doc_build.yml @@ -16,10 +16,11 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars - uses: ./.github/actions/mamba-install-dascore with: - python-version: "3.12" + python-version: ${{ env.PYTHON_DEFAULT }} environment-file: './.github/doc_environment.yml' cache-number: 1 diff --git a/.github/workflows/upload_pypi.yml b/.github/workflows/upload_pypi.yml index 383be8aa7..3f9d819fe 100644 --- a/.github/workflows/upload_pypi.yml +++ b/.github/workflows/upload_pypi.yml @@ -14,6 +14,8 @@ jobs: fetch-tags: "true" fetch-depth: '0' + - uses: ./.github/actions/load-shared-vars + - name: setup conda uses: conda-incubator/setup-miniconda@v3 with: @@ -21,7 +23,7 @@ jobs: channels: conda-forge,defaults channel-priority: true environment-file: environment.yml - python-version: 3.11 + python-version: ${{ env.PYTHON_DEFAULT }} - name: create dists shell: bash -l {0} From d39bd0e045dd97a03f82e352a072baaa21cb6597 Mon Sep 17 00:00:00 2001 From: derrick chambers Date: Mon, 22 Dec 2025 10:55:42 -0700 Subject: [PATCH 4/6] try to fix doc build, define matricies in one place --- .github/actions/load-shared-vars/action.yml | 52 +++++++++++++++------ .github/doc_environment.yml | 1 - .github/shared-vars.yml | 12 +++++ .github/workflows/run_min_dep_tests.yml | 14 +++++- .github/workflows/runtests.yml | 14 +++++- 5 files changed, 74 insertions(+), 19 deletions(-) diff --git a/.github/actions/load-shared-vars/action.yml b/.github/actions/load-shared-vars/action.yml index 1f653ac8d..1fb20dd5f 100644 --- a/.github/actions/load-shared-vars/action.yml +++ b/.github/actions/load-shared-vars/action.yml @@ -1,40 +1,64 @@ name: "Load Shared Variables" -description: "Loads shared variables from .github/shared-vars.yml and sets them as environment variables" +description: "Loads shared variables from .github/shared-vars.yml and sets them as environment variables and outputs" + +outputs: + test-matrix: + description: "Python version matrix for full tests" + value: ${{ steps.load.outputs.test-matrix }} + min-deps-matrix: + description: "Python version matrix for minimum dependency tests" + value: ${{ steps.load.outputs.min-deps-matrix }} runs: using: "composite" steps: - name: load shared variables + id: load shell: bash run: | # Read shared-vars.yml and set environment variables python << 'EOF' import yaml import os + import json + + def flatten_dict(d, parent_key='', sep='_'): + """Recursively flatten nested dictionary into env var format.""" + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}".upper() if parent_key else k.upper() + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + elif isinstance(v, list): + # Convert lists to JSON strings for use in workflows + items.append((new_key, json.dumps(v))) + else: + items.append((new_key, str(v))) + return dict(items) # Load shared variables file with open('.github/shared-vars.yml', 'r') as f: config = yaml.safe_load(f) - # Set Python version environment variables - python_config = config.get('python', {}) - - env_vars = { - 'PYTHON_DEFAULT': python_config.get('default', '3.12'), - 'PYTHON_LATEST': python_config.get('latest', '3.13'), - 'PYTHON_RELEASE': python_config.get('release', '3.11'), - } + # Flatten all variables + env_vars = flatten_dict(config) # Write to GITHUB_ENV with open(os.environ['GITHUB_ENV'], 'a') as env_file: - for key, value in env_vars.items(): + for key, value in sorted(env_vars.items()): env_file.write(f"{key}={value}\n") print(f"✅ Set {key}={value}") - # For matrix values, we can't set them directly in env - # Workflows will need to read the file directly for matrix strategy + # Also output specific matrices for job outputs + python_config = config.get('python', {}) test_matrix = python_config.get('test_matrix', []) min_deps_matrix = python_config.get('min_deps_matrix', []) - print(f"📋 Test matrix available: {test_matrix}") - print(f"📋 Min deps matrix available: {min_deps_matrix}") + + # Write to GITHUB_OUTPUT for action outputs + with open(os.environ['GITHUB_OUTPUT'], 'a') as output_file: + output_file.write(f"test-matrix={json.dumps(test_matrix)}\n") + output_file.write(f"min-deps-matrix={json.dumps(min_deps_matrix)}\n") + + print(f"📋 Output test-matrix: {test_matrix}") + print(f"📋 Output min-deps-matrix: {min_deps_matrix}") EOF diff --git a/.github/doc_environment.yml b/.github/doc_environment.yml index 8bde2d6cc..946709951 100644 --- a/.github/doc_environment.yml +++ b/.github/doc_environment.yml @@ -2,7 +2,6 @@ name: dascore channels: - conda-forge dependencies: - - python=3.12 - pytest - pydantic>2.0 - pip diff --git a/.github/shared-vars.yml b/.github/shared-vars.yml index 3c8287bab..7c7bdeeef 100644 --- a/.github/shared-vars.yml +++ b/.github/shared-vars.yml @@ -5,4 +5,16 @@ python: # Default Python version for most workflows (docs, linting, coverage, etc.) default: "3.13" + # Full test matrix versions (runtests.yml) + test_matrix: + - "3.11" + - "3.12" + - "3.13" + - "3.14" + + # Minimum dependency test matrix versions (run_min_dep_tests.yml) + min_deps_matrix: + - "3.13" + - "3.14" + # Add other shared variables here as needed diff --git a/.github/workflows/run_min_dep_tests.yml b/.github/workflows/run_min_dep_tests.yml index ff7d90206..28ac3eecf 100644 --- a/.github/workflows/run_min_dep_tests.yml +++ b/.github/workflows/run_min_dep_tests.yml @@ -23,15 +23,25 @@ concurrency: cancel-in-progress: true jobs: + # Load Python version matrix from shared-vars.yml + setup: + runs-on: ubuntu-latest + outputs: + python-matrix: ${{ steps.load-vars.outputs.min-deps-matrix }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/load-shared-vars + id: load-vars + # Runs the tests on combinations of the supported python/os matrix. test_code_min_deps: - + needs: setup timeout-minutes: 25 runs-on: ${{ matrix.os }} strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ '3.13', '3.14' ] + python-version: ${{ fromJson(needs.setup.outputs.python-matrix) }} # only run if CI isn't turned off if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml index 8e9862366..70d75df85 100644 --- a/.github/workflows/runtests.yml +++ b/.github/workflows/runtests.yml @@ -25,15 +25,25 @@ concurrency: cancel-in-progress: true jobs: + # Load Python version matrix from shared-vars.yml + setup: + runs-on: ubuntu-latest + outputs: + python-matrix: ${{ steps.load-vars.outputs.test-matrix }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/load-shared-vars + id: load-vars + # Runs the tests on combinations of the supported python/os matrix. test_code: - + needs: setup timeout-minutes: 25 runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.11', '3.12', '3.13', '3.14' ] + python-version: ${{ fromJson(needs.setup.outputs.python-matrix) }} # only run if CI isn't turned off if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci') From b08568825b8c4c3429475b33f03f1aaa74f303f3 Mon Sep 17 00:00:00 2001 From: derrick chambers Date: Mon, 22 Dec 2025 11:03:08 -0700 Subject: [PATCH 5/6] ensure pyyaml is installed --- .github/actions/load-shared-vars/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/load-shared-vars/action.yml b/.github/actions/load-shared-vars/action.yml index 1fb20dd5f..221fe7732 100644 --- a/.github/actions/load-shared-vars/action.yml +++ b/.github/actions/load-shared-vars/action.yml @@ -12,6 +12,10 @@ outputs: runs: using: "composite" steps: + - name: install pyyaml + shell: bash + run: python -m pip install --quiet pyyaml + - name: load shared variables id: load shell: bash From d2e19fd64d80e51767ed868140eb2caf022c5c52 Mon Sep 17 00:00:00 2001 From: derrick chambers Date: Mon, 22 Dec 2025 11:04:07 -0700 Subject: [PATCH 6/6] add py3.14 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 25eaef8ab..a82342d55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + ] keywords = ["geophysics", "distributed-acoustic-sensing"]