Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/actions/cache-test-data/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
name: "Cache Test Data"
description: "Caches DASCore test data using Pooch cache directory"
inputs:
cache-number:
description: "Cache number. Use != 1 to reset data cache"
required: false
default: "1"

runs:
using: "composite"
steps:
- name: verify prerequisites
shell: bash -el {0}
run: |
if ! command -v python &> /dev/null; then
echo "❌ Error: Python is not installed or not in PATH"
echo "This action requires Python to be installed first."
exit 1
fi
if ! python -c "import pooch" 2>/dev/null; then
echo "❌ Error: Pooch is not installed"
echo "This action requires Pooch to be installed (pip install pooch or include in dependencies)."
exit 1
fi
echo "✅ Prerequisites verified: Python and Pooch are available"

- name: get data registry hash
shell: bash -el {0}
run: |
echo "DATA_REGISTRY_HASH=$(python -c "import pooch; print(pooch.file_hash('dascore/data_registry.txt'))")" >> $GITHUB_ENV

- name: get data cache path
shell: bash -el {0}
run: |
echo "DATA_CACHE_PATH=$(python -c "import pooch; print(pooch.os_cache('dascore'))")" >> $GITHUB_ENV

- name: cache test data
uses: actions/cache@v4
id: cache-test-data
with:
enableCrossOsArchive: true
path: ${{ env.DATA_CACHE_PATH }}
key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}
restore-keys: |
data-${{ env.DATA_REGISTRY_HASH }}-
68 changes: 68 additions & 0 deletions .github/actions/load-shared-vars/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: "Load Shared Variables"
description: "Loads shared variables from .github/shared-vars.yml and sets them as environment variables and outputs"

outputs:
test-matrix:
description: "Python version matrix for full tests"
value: ${{ steps.load.outputs.test-matrix }}
min-deps-matrix:
description: "Python version matrix for minimum dependency tests"
value: ${{ steps.load.outputs.min-deps-matrix }}

runs:
using: "composite"
steps:
- name: install pyyaml
shell: bash
run: python -m pip install --quiet pyyaml

- name: load shared variables
id: load
shell: bash
run: |
# Read shared-vars.yml and set environment variables
python << 'EOF'
import yaml
import os
import json

Comment thread
coderabbitai[bot] marked this conversation as resolved.
def flatten_dict(d, parent_key='', sep='_'):
"""Recursively flatten nested dictionary into env var format."""
items = []
for k, v in d.items():
new_key = f"{parent_key}{sep}{k}".upper() if parent_key else k.upper()
if isinstance(v, dict):
items.extend(flatten_dict(v, new_key, sep=sep).items())
elif isinstance(v, list):
# Convert lists to JSON strings for use in workflows
items.append((new_key, json.dumps(v)))
else:
items.append((new_key, str(v)))
return dict(items)

# Load shared variables file
with open('.github/shared-vars.yml', 'r') as f:
config = yaml.safe_load(f)

# Flatten all variables
env_vars = flatten_dict(config)

# Write to GITHUB_ENV
with open(os.environ['GITHUB_ENV'], 'a') as env_file:
for key, value in sorted(env_vars.items()):
env_file.write(f"{key}={value}\n")
print(f"✅ Set {key}={value}")

# Also output specific matrices for job outputs
python_config = config.get('python', {})
test_matrix = python_config.get('test_matrix', [])
min_deps_matrix = python_config.get('min_deps_matrix', [])

# Write to GITHUB_OUTPUT for action outputs
with open(os.environ['GITHUB_OUTPUT'], 'a') as output_file:
output_file.write(f"test-matrix={json.dumps(test_matrix)}\n")
output_file.write(f"min-deps-matrix={json.dumps(min_deps_matrix)}\n")

print(f"📋 Output test-matrix: {test_matrix}")
print(f"📋 Output min-deps-matrix: {min_deps_matrix}")
EOF
32 changes: 2 additions & 30 deletions .github/actions/mamba-install-dascore/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,43 +64,15 @@ runs:
run: |
git fetch --tags --force

- name: get data registry hash
shell: bash
run: |
echo "DATA_REGISTRY_HASH=$(sha256sum dascore/data_registry.txt | cut -d' ' -f1)" >> $GITHUB_ENV

- name: install dascore
if: "${{ inputs.install-package == 'true' }}"
shell: bash -l {0}
run: |
pip install -e .${{ inputs.install-group-str }}

- name: set data cache path
shell: bash -el {0}
run: |
echo "DATA_CACHE_PATH=$(python -c "import pooch; print(pooch.os_cache('dascore'))")" >> $GITHUB_ENV

- name: cache test data
uses: actions/cache@v4
id: cache-test-data
- uses: ./.github/actions/cache-test-data
with:
enableCrossOsArchive: true
path: ${{ env.DATA_CACHE_PATH }}
key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}
restore-keys: |
data-${{ env.DATA_REGISTRY_HASH }}-

- name: verify cache status
shell: bash -el {0}
run: |
if [[ "${{ steps.cache-test-data.outputs.cache-hit }}" == "true" ]]; then
echo "✅ Cache hit! Using cached test data from: ${{ env.DATA_CACHE_PATH }}"
echo "Cache key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}"
else
echo "❌ Cache miss. Test data will be downloaded fresh."
echo "Cache key: data-${{ env.DATA_REGISTRY_HASH }}-${{ inputs.cache-number }}"
fi
ls -la "${{ env.DATA_CACHE_PATH }}" || echo "Cache directory does not exist yet"
cache-number: ${{ inputs.cache-number }}

# Print out the package info for current environment
- name: print package info
Expand Down
1 change: 0 additions & 1 deletion .github/doc_environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ name: dascore
channels:
- conda-forge
dependencies:
- python=3.12
- pytest
- pydantic>2.0
- pip
Expand Down
20 changes: 20 additions & 0 deletions .github/shared-vars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Shared variables used across all GitHub Actions workflows
# Update these values to propagate changes to all workflows

python:
# Default Python version for most workflows (docs, linting, coverage, etc.)
default: "3.13"

# Full test matrix versions (runtests.yml)
test_matrix:
- "3.11"
- "3.12"
- "3.13"
- "3.14"

# Minimum dependency test matrix versions (run_min_dep_tests.yml)
min_deps_matrix:
- "3.13"
- "3.14"

# Add other shared variables here as needed
Comment thread
coderabbitai[bot] marked this conversation as resolved.
4 changes: 3 additions & 1 deletion .github/workflows/build_deploy_master_docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,11 @@ jobs:
fetch-tags: "true"
fetch-depth: '0'

- uses: ./.github/actions/load-shared-vars

- uses: ./.github/actions/mamba-install-dascore
with:
python-version: "3.12"
python-version: ${{ env.PYTHON_DEFAULT }}
environment-file: './.github/doc_environment.yml'

- uses: ./.github/actions/prep_doc_build
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/build_deploy_stable_docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@ jobs:
fetch-tags: "true"
fetch-depth: '0'

- uses: ./.github/actions/load-shared-vars

- uses: ./.github/actions/mamba-install-dascore
with:
python-version: "3.12"
python-version: ${{ env.PYTHON_DEFAULT }}
environment-file: './.github/doc_environment.yml'

- uses: ./.github/actions/build-docs
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/get_coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@ jobs:
fetch-tags: "true"
fetch-depth: '0'

- uses: ./.github/actions/load-shared-vars

- uses: ./.github/actions/mamba-install-dascore
with:
python-version: "3.12"
python-version: ${{ env.PYTHON_DEFAULT }}
cache-number: 1

- name: run test suite
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@ jobs:
fetch-tags: "true"
fetch-depth: '0'

- uses: ./.github/actions/load-shared-vars

- name: Install uv
uses: astral-sh/setup-uv@v3

- uses: actions/setup-python@v5
with:
python-version: '3.12'
python-version: ${{ env.PYTHON_DEFAULT }}

- name: install linting packages
run: uv tool install pre-commit
Expand Down
11 changes: 7 additions & 4 deletions .github/workflows/profile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@ on:
pull_request:
workflow_dispatch:

env:
python_version: "3.13"

jobs:
benchmarks:
name: Run benchmarks
Expand All @@ -21,13 +18,19 @@ jobs:
fetch-tags: "true"
fetch-depth: '0'

- uses: ./.github/actions/load-shared-vars

- uses: actions/setup-python@v6
with:
python-version: ${{ env.python_version }}
python-version: ${{ env.PYTHON_DEFAULT }}

- name: Install dependencies
run: pip install ".[profile]"

- uses: ./.github/actions/cache-test-data
with:
cache-number: 1

- name: Run benchmarks
uses: CodSpeedHQ/action@v4
with:
Expand Down
15 changes: 12 additions & 3 deletions .github/workflows/run_min_dep_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,25 @@ concurrency:
cancel-in-progress: true

jobs:
# Load Python version matrix from shared-vars.yml
setup:
runs-on: ubuntu-latest
outputs:
python-matrix: ${{ steps.load-vars.outputs.min-deps-matrix }}
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/load-shared-vars
id: load-vars

# Runs the tests on combinations of the supported python/os matrix.
test_code_min_deps:

needs: setup
timeout-minutes: 25
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-latest, macos-latest, windows-latest ]
python-version: [ '3.12', '3.13' ]

python-version: ${{ fromJson(needs.setup.outputs.python-matrix) }}

# only run if CI isn't turned off
if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci')
Expand Down
18 changes: 14 additions & 4 deletions .github/workflows/runtests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,27 +25,37 @@ concurrency:
cancel-in-progress: true

jobs:
# Load Python version matrix from shared-vars.yml
setup:
runs-on: ubuntu-latest
outputs:
python-matrix: ${{ steps.load-vars.outputs.test-matrix }}
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/load-shared-vars
id: load-vars

# Runs the tests on combinations of the supported python/os matrix.
test_code:

needs: setup
timeout-minutes: 25
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ['3.10', '3.11', "3.12", "3.13"]
python-version: ${{ fromJson(needs.setup.outputs.python-matrix) }}

# only run if CI isn't turned off
if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci')

env:
# set conda environment file with dependencies
env_file: "environment.yml"
env_file: 'environment.yml'

steps:
- uses: actions/checkout@v4
with:
fetch-tags: "true"
fetch-tags: 'true'
fetch-depth: '0'

- uses: ./.github/actions/mamba-install-dascore
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/test_doc_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@ jobs:
fetch-tags: "true"
fetch-depth: '0'

- uses: ./.github/actions/load-shared-vars

- uses: ./.github/actions/mamba-install-dascore
with:
python-version: "3.12"
python-version: ${{ env.PYTHON_DEFAULT }}
environment-file: './.github/doc_environment.yml'
cache-number: 1

Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/upload_pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,16 @@ jobs:
fetch-tags: "true"
fetch-depth: '0'

- uses: ./.github/actions/load-shared-vars

- name: setup conda
uses: conda-incubator/setup-miniconda@v3
with:
mamba-version: "*"
channels: conda-forge,defaults
channel-priority: true
environment-file: environment.yml
python-version: 3.11
python-version: ${{ env.PYTHON_DEFAULT }}

- name: create dists
shell: bash -l {0}
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ classifiers = [
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",

]

keywords = ["geophysics", "distributed-acoustic-sensing"]
Expand Down
Loading