Add HPC compatible build #475
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "CD: PyPI package" | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| pull_request: | |
| branches: [ "main" ] | |
| env: | |
| BUILD_TYPE: Release | |
| DSF_OPTIMIZE_ARCH: "OFF" | |
| jobs: | |
| check-version: | |
| name: Check if version exists on PyPI/TestPyPI | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_build: ${{ steps.check.outputs.should_build }} | |
| version: ${{ steps.extract.outputs.version }} | |
| publish_version: ${{ steps.plan.outputs.publish_version }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Extract version from dsf.hpp | |
| id: extract | |
| run: | | |
| VERSION_MAJOR=$(grep -oP 'DSF_VERSION_MAJOR = \K\d+' src/dsf/dsf.hpp) | |
| VERSION_MINOR=$(grep -oP 'DSF_VERSION_MINOR = \K\d+' src/dsf/dsf.hpp) | |
| VERSION_PATCH=$(grep -oP 'DSF_VERSION_PATCH = \K\d+' src/dsf/dsf.hpp) | |
| VERSION="${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}" | |
| echo "version=${VERSION}" >> $GITHUB_OUTPUT | |
| echo "Extracted version: ${VERSION}" | |
| - name: Plan publish version | |
| id: plan | |
| run: | | |
| BASE_VERSION="${{ steps.extract.outputs.version }}" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| # TestPyPI only: use a numeric dev suffix so the version is PEP 440 compliant. | |
| # Concatenating run id and attempt keeps it unique across PRs and re-runs. | |
| PUBLISH_VERSION="${BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}" | |
| else | |
| # PyPI: publish the exact release version with no run-id suffix. | |
| PUBLISH_VERSION="${BASE_VERSION}" | |
| fi | |
| echo "publish_version=${PUBLISH_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo "Planned publish version: ${PUBLISH_VERSION}" | |
| - name: Check publish policy for PyPI | |
| id: check | |
| run: | | |
| VERSION="${{ steps.plan.outputs.publish_version }}" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo "PR build uses unique TestPyPI version ${VERSION}; skipping existence check." | |
| echo "should_build=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| PYPI_NAME="PyPI" | |
| PACKAGES=("dsf-mobility" "dsf-mobility-hpc") | |
| for PACKAGE in "${PACKAGES[@]}"; do | |
| PYPI_URL="https://pypi.org/pypi/${PACKAGE}/${VERSION}/json" | |
| echo "Checking if ${PACKAGE} version ${VERSION} exists on ${PYPI_NAME}..." | |
| # Check PyPI API for the specific version | |
| HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "${PYPI_URL}") | |
| if [ "$HTTP_STATUS" = "200" ]; then | |
| echo "Version ${VERSION} already exists for ${PACKAGE} on ${PYPI_NAME}." | |
| echo "Version already published for at least one package; skipping build and publish jobs." | |
| echo "should_build=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "Version ${VERSION} does not exist for ${PACKAGE} on ${PYPI_NAME} (HTTP ${HTTP_STATUS})" | |
| done | |
| echo "should_build=true" >> "$GITHUB_OUTPUT" | |
| uv-validate: | |
| name: Validate uv workflows | |
| needs: [check-version] | |
| runs-on: ubuntu-latest | |
| if: needs.check-version.outputs.should_build == 'true' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y cmake build-essential doxygen libtbb-dev | |
| - name: Validate uv build | |
| env: | |
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | |
| DSF_OPTIMIZE_ARCH: "OFF" | |
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | |
| run: uv build | |
| - name: Validate uv source install | |
| env: | |
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | |
| run: | | |
| uv venv .venv-uv-install | |
| source .venv-uv-install/bin/activate | |
| uv pip install . | |
| python -c "import dsf; print(dsf.__version__)" | |
| - name: Validate uv editable install | |
| env: | |
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | |
| run: | | |
| uv venv .venv-uv-editable | |
| source .venv-uv-editable/bin/activate | |
| uv pip install -e . | |
| python -c "import dsf; print(dsf.__version__)" | |
| build-wheels-linux: | |
| name: Build wheels on Linux (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) | |
| needs: [check-version, uv-validate] | |
| runs-on: ubuntu-latest | |
| if: needs.check-version.outputs.should_build == 'true' | |
| strategy: | |
| matrix: | |
| python-version: ['3.10', '3.12'] | |
| build_variant: ['standard', 'hpc'] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y cmake build-essential doxygen libtbb-dev | |
| - name: Install build dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install build wheel setuptools pybind11-stubgen auditwheel | |
| - name: Switch package name for HPC distribution | |
| if: matrix.build_variant == 'hpc' | |
| run: | | |
| python - <<'PY' | |
| from pathlib import Path | |
| pyproject_path = Path("pyproject.toml") | |
| content = pyproject_path.read_text(encoding="utf-8") | |
| old = 'name = "dsf-mobility"' | |
| new = 'name = "dsf-mobility-hpc"' | |
| if old not in content: | |
| raise RuntimeError("Unable to locate project name in pyproject.toml") | |
| pyproject_path.write_text(content.replace(old, new, 1), encoding="utf-8") | |
| print("Switched project name to dsf-mobility-hpc for HPC build") | |
| PY | |
| - name: Clean previous build artifacts | |
| run: | | |
| rm -rf dist wheelhouse | |
| rm -rf build/temp.* build/lib.* | |
| - name: Build wheel | |
| env: | |
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | |
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | |
| DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} | |
| run: | | |
| python -m build --wheel | |
| - name: Repair wheel (auditwheel) | |
| run: | | |
| mkdir -p wheelhouse | |
| auditwheel repair dist/*.whl -w wheelhouse | |
| - name: Test wheel installation | |
| run: | | |
| python -m pip install wheelhouse/*.whl | |
| python -c "import dsf; print('DSF wheel installation successful')" | |
| - name: Upload wheels as artifacts | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: wheel-linux-${{ matrix.python-version }}-${{ matrix.build_variant }} | |
| path: wheelhouse/*.whl | |
| build-wheels-macos: | |
| name: Build wheels on macOS (Python ${{ matrix.python-version }}) | |
| needs: [check-version, uv-validate] | |
| runs-on: macos-latest | |
| if: needs.check-version.outputs.should_build == 'true' | |
| strategy: | |
| matrix: | |
| python-version: ['3.10', '3.12'] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install system dependencies | |
| run: | | |
| brew install cmake doxygen tbb | |
| - name: Install build dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install build wheel setuptools pybind11-stubgen delocate | |
| - name: Set macOS environment variables | |
| run: | | |
| echo "ARCHFLAGS=-arch $(uname -m)" >> $GITHUB_ENV | |
| echo "_PYTHON_HOST_PLATFORM=macosx-$(sw_vers -productVersion | cut -d. -f1)-$(uname -m)" >> $GITHUB_ENV | |
| - name: Clean previous build artifacts | |
| run: | | |
| rm -rf dist wheelhouse | |
| rm -rf build/temp.* build/lib.* | |
| - name: Build wheel | |
| env: | |
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | |
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | |
| DSF_HPC_BUILD: "0" | |
| run: python -m build --wheel | |
| - name: Repair wheel (bundle libraries) | |
| run: | | |
| mkdir -p wheelhouse | |
| delocate-wheel -w wheelhouse -v --require-archs $(uname -m) dist/*.whl | |
| - name: Upload wheels as artifacts | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: wheel-macos-${{ matrix.python-version }} | |
| path: wheelhouse/*.whl | |
| build-wheels-windows: | |
| name: Build wheels on Windows (Python ${{ matrix.python-version }}) | |
| needs: [check-version, uv-validate] | |
| runs-on: windows-latest | |
| if: needs.check-version.outputs.should_build == 'true' | |
| strategy: | |
| matrix: | |
| python-version: ['3.10', '3.12'] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install system dependencies | |
| run: | | |
| choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' | |
| choco install doxygen.install | |
| git clone https://github.com/microsoft/vcpkg.git vcpkg | |
| cd vcpkg | |
| .\bootstrap-vcpkg.bat | |
| .\vcpkg install tbb:x64-windows | |
| echo "CMAKE_TOOLCHAIN_FILE=$env:GITHUB_WORKSPACE\vcpkg\scripts\buildsystems\vcpkg.cmake" >> $env:GITHUB_ENV | |
| echo "VCPKG_TARGET_TRIPLET=x64-windows" >> $env:GITHUB_ENV | |
| - name: Install build dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install build wheel setuptools pybind11-stubgen | |
| - name: Clean previous build artifacts | |
| shell: bash | |
| run: | | |
| rm -rf dist wheelhouse | |
| rm -rf build/temp.* build/lib.* | |
| - name: Build wheel | |
| env: | |
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | |
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | |
| DSF_HPC_BUILD: "0" | |
| run: python -m build --wheel | |
| - name: Upload wheels as artifacts | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: wheel-windows-${{ matrix.python-version }} | |
| path: dist/*.whl | |
| build-sdist: | |
| name: Build source distribution | |
| needs: [check-version, uv-validate] | |
| runs-on: ubuntu-latest | |
| if: needs.check-version.outputs.should_build == 'true' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y cmake build-essential doxygen libtbb-dev | |
| - name: Install build dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install build | |
| - name: Build source distribution | |
| env: | |
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | |
| run: python -m build --sdist | |
| - name: Upload sdist as artifact | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: sdist | |
| path: dist/*.tar.gz | |
| publish: | |
| name: Publish to PyPI | |
| needs: [check-version, uv-validate, build-wheels-linux, build-wheels-macos, build-wheels-windows, build-sdist] | |
| runs-on: ubuntu-latest | |
| if: needs.check-version.outputs.should_build == 'true' | |
| steps: | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: dist-artifacts | |
| - name: Collect all distributions | |
| run: | | |
| mkdir -p dist | |
| find dist-artifacts -name '*.whl' -exec cp {} dist/ \; | |
| find dist-artifacts -name '*.tar.gz' -exec cp {} dist/ \; | |
| echo "Contents of dist/:" | |
| ls -la dist/ | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install twine | |
| run: python -m pip install twine | |
| - name: Publish to TestPyPI (on PR) | |
| if: github.event_name == 'pull_request' | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.TEST_PYPI }} | |
| run: | | |
| python -m twine upload --repository testpypi dist/* --verbose | |
| - name: Publish to PyPI (on push to main) | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI }} | |
| run: | | |
| python -m twine upload dist/* --verbose |