Add HPC compatible build #471
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "CD: PyPI package" | ||
| on: | ||
| push: | ||
| branches: [ "main" ] | ||
| pull_request: | ||
| branches: [ "main" ] | ||
| env: | ||
| BUILD_TYPE: Release | ||
| DSF_OPTIMIZE_ARCH: "OFF" | ||
| jobs: | ||
| check-version: | ||
| name: Check if version exists on PyPI/TestPyPI | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| should_build: ${{ steps.check.outputs.should_build }} | ||
| version: ${{ steps.extract.outputs.version }} | ||
| publish_version: ${{ steps.plan.outputs.publish_version }} | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v6 | ||
| - name: Extract version from dsf.hpp | ||
| id: extract | ||
| run: | | ||
| VERSION_MAJOR=$(grep -oP 'DSF_VERSION_MAJOR = \K\d+' src/dsf/dsf.hpp) | ||
| VERSION_MINOR=$(grep -oP 'DSF_VERSION_MINOR = \K\d+' src/dsf/dsf.hpp) | ||
| VERSION_PATCH=$(grep -oP 'DSF_VERSION_PATCH = \K\d+' src/dsf/dsf.hpp) | ||
| VERSION="${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}" | ||
| echo "version=${VERSION}" >> $GITHUB_OUTPUT | ||
| echo "Extracted version: ${VERSION}" | ||
| - name: Plan publish version | ||
| id: plan | ||
| run: | | ||
| BASE_VERSION="${{ steps.extract.outputs.version }}" | ||
| if [ "${{ github.event_name }}" = "pull_request" ]; then | ||
| # TestPyPI only: use a numeric dev suffix so the version is PEP 440 compliant. | ||
| # Concatenating run id and attempt keeps it unique across PRs and re-runs. | ||
| PUBLISH_VERSION="${BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}" | ||
| else | ||
| # PyPI: publish the exact release version with no run-id suffix. | ||
| PUBLISH_VERSION="${BASE_VERSION}" | ||
| fi | ||
| echo "publish_version=${PUBLISH_VERSION}" >> "$GITHUB_OUTPUT" | ||
| echo "Planned publish version: ${PUBLISH_VERSION}" | ||
| - name: Check publish policy for PyPI | ||
| id: check | ||
| run: | | ||
| VERSION="${{ steps.plan.outputs.publish_version }}" | ||
| if [ "${{ github.event_name }}" = "pull_request" ]; then | ||
| echo "PR build uses unique TestPyPI version ${VERSION}; skipping existence check." | ||
| echo "should_build=true" >> "$GITHUB_OUTPUT" | ||
| exit 0 | ||
| fi | ||
| PYPI_URL="https://pypi.org/pypi/dsf-mobility/${VERSION}/json" | ||
| PYPI_NAME="PyPI" | ||
| echo "Checking if dsf-mobility version ${VERSION} exists on ${PYPI_NAME}..." | ||
| # Check PyPI/TestPyPI API for the specific version | ||
| HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "${PYPI_URL}") | ||
| if [ "$HTTP_STATUS" = "200" ]; then | ||
| echo "Version ${VERSION} already exists on ${PYPI_NAME}" | ||
| echo "ERROR: Refusing to overwrite an existing version on PyPI." | ||
| exit 1 | ||
| else | ||
| echo "Version ${VERSION} does not exist on ${PYPI_NAME} (HTTP ${HTTP_STATUS})" | ||
| echo "should_build=true" >> $GITHUB_OUTPUT | ||
| fi | ||
| uv-validate: | ||
| name: Validate uv workflows | ||
| needs: [check-version] | ||
| runs-on: ubuntu-latest | ||
| if: needs.check-version.outputs.should_build == 'true' | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v6 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: "3.12" | ||
| - name: Set up uv | ||
| uses: astral-sh/setup-uv@v5 | ||
| - name: Install system dependencies | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install -y cmake build-essential doxygen libtbb-dev | ||
| - name: Validate uv build | ||
| env: | ||
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | ||
| DSF_OPTIMIZE_ARCH: "OFF" | ||
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | ||
| run: uv build | ||
| - name: Validate uv source install | ||
| env: | ||
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | ||
| run: | | ||
| uv venv .venv-uv-install | ||
| source .venv-uv-install/bin/activate | ||
| uv pip install . | ||
| python -c "import dsf; print(dsf.__version__)" | ||
| - name: Validate uv editable install | ||
| env: | ||
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | ||
| run: | | ||
| uv venv .venv-uv-editable | ||
| source .venv-uv-editable/bin/activate | ||
| uv pip install -e . | ||
| python -c "import dsf; print(dsf.__version__)" | ||
| build-wheels-linux: | ||
| name: Build wheels on Linux (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) | ||
| needs: [check-version, uv-validate] | ||
| runs-on: ubuntu-latest | ||
| if: needs.check-version.outputs.should_build == 'true' | ||
| strategy: | ||
| matrix: | ||
| python-version: ['3.10', '3.12'] | ||
| build_variant: ['standard', 'hpc'] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v6 | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Install system dependencies | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install -y cmake build-essential doxygen libtbb-dev | ||
| - name: Install build dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| python -m pip install build wheel setuptools pybind11-stubgen auditwheel | ||
| - name: Build wheel | ||
| env: | ||
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | ||
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | ||
| DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} | ||
| run: | | ||
| rm -rf dist wheelhouse | ||
| python -m build --wheel | ||
| - name: Repair wheel (auditwheel) | ||
| run: | | ||
| mkdir -p wheelhouse | ||
| auditwheel repair dist/*.whl -w wheelhouse | ||
| - name: Rename HPC wheels with _hpc suffix | ||
| if: matrix.build_variant == 'hpc' | ||
| run: | | ||
| cd wheelhouse | ||
| for whl in *.whl; do | ||
| # Insert _hpc before .whl suffix | ||
| new_name="${whl%.whl}_hpc.whl" | ||
| mv "$whl" "$new_name" | ||
| done | ||
| ls -la | ||
| - name: Test wheel installation | ||
| run: | | ||
| python -m pip install wheelhouse/*.whl | ||
| python -c "import dsf; print('DSF wheel installation successful')" | ||
| - name: Upload wheels as artifacts | ||
| uses: actions/upload-artifact@v6 | ||
| with: | ||
| name: wheel-linux-${{ matrix.python-version }}-${{ matrix.build_variant }} | ||
| path: wheelhouse/*.whl | ||
| build-wheels-macos: | ||
| name: Build wheels on macOS (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) | ||
| needs: [check-version, uv-validate] | ||
| runs-on: macos-latest | ||
| if: needs.check-version.outputs.should_build == 'true' | ||
| strategy: | ||
| matrix: | ||
| python-version: ['3.10', '3.12'] | ||
| build_variant: ['standard', 'hpc'] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v6 | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Install system dependencies | ||
| run: | | ||
| brew install cmake doxygen tbb | ||
| - name: Install build dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| python -m pip install build wheel setuptools pybind11-stubgen delocate | ||
| - name: Set macOS environment variables | ||
| run: | | ||
| echo "ARCHFLAGS=-arch $(uname -m)" >> $GITHUB_ENV | ||
| echo "_PYTHON_HOST_PLATFORM=macosx-$(sw_vers -productVersion | cut -d. -f1)-$(uname -m)" >> $GITHUB_ENV | ||
| - name: Build wheel | ||
| env: | ||
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | ||
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | ||
| DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} | ||
| run: python -m build --wheel | ||
| - name: Repair wheel (bundle libraries) | ||
| run: | | ||
| mkdir -p wheelhouse | ||
| delocate-wheel -w wheelhouse -v --require-archs $(uname -m) dist/*.whl | ||
| - name: Rename HPC wheels with _hpc suffix | ||
| if: matrix.build_variant == 'hpc' | ||
| run: | | ||
| cd wheelhouse | ||
| for whl in *.whl; do | ||
| # Insert _hpc before .whl suffix | ||
| new_name="${whl%.whl}_hpc.whl" | ||
| mv "$whl" "$new_name" | ||
| done | ||
| ls -la | ||
| - name: Upload wheels as artifacts | ||
| uses: actions/upload-artifact@v6 | ||
| with: | ||
| name: wheel-macos-${{ matrix.python-version }}-${{ matrix.build_variant }} | ||
| path: wheelhouse/*.whl | ||
| build-wheels-windows: | ||
| name: Build wheels on Windows (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) | ||
| needs: [check-version, uv-validate] | ||
| runs-on: windows-latest | ||
| if: needs.check-version.outputs.should_build == 'true' | ||
| strategy: | ||
| matrix: | ||
| python-version: ['3.10', '3.12'] | ||
| build_variant: ['standard', 'hpc'] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v6 | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Install system dependencies | ||
| run: | | ||
| choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' | ||
| choco install doxygen.install | ||
| git clone https://github.com/microsoft/vcpkg.git vcpkg | ||
| cd vcpkg | ||
| .\bootstrap-vcpkg.bat | ||
| .\vcpkg install tbb:x64-windows | ||
| echo "CMAKE_TOOLCHAIN_FILE=$env:GITHUB_WORKSPACE\vcpkg\scripts\buildsystems\vcpkg.cmake" >> $env:GITHUB_ENV | ||
| echo "VCPKG_TARGET_TRIPLET=x64-windows" >> $env:GITHUB_ENV | ||
| - name: Install build dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| python -m pip install build wheel setuptools pybind11-stubgen | ||
| - name: Build wheel | ||
| env: | ||
| CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" | ||
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | ||
| DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} | ||
| run: python -m build --wheel | ||
| - name: Rename HPC wheels with _hpc suffix | ||
| if: matrix.build_variant == 'hpc' | ||
| shell: bash | ||
| run: | | ||
| cd dist | ||
| for whl in *.whl; do | ||
| # Insert _hpc before .whl suffix | ||
| new_name="${whl%.whl}_hpc.whl" | ||
| mv "$whl" "$new_name" | ||
| done | ||
| ls -la | ||
| - name: Upload wheels as artifacts | ||
| uses: actions/upload-artifact@v6 | ||
| with: | ||
| name: wheel-windows-${{ matrix.python-version }}-${{ matrix.build_variant }} | ||
| path: dist/*.whl | ||
| build-sdist: | ||
| name: Build source distribution | ||
| needs: [check-version, uv-validate] | ||
| runs-on: ubuntu-latest | ||
| if: needs.check-version.outputs.should_build == 'true' | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v6 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: "3.12" | ||
| - name: Install system dependencies | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install -y cmake build-essential doxygen libtbb-dev | ||
| - name: Install build dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| python -m pip install build | ||
| - name: Build source distribution | ||
| env: | ||
| DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} | ||
| run: python -m build --sdist | ||
| - name: Upload sdist as artifact | ||
| uses: actions/upload-artifact@v6 | ||
| with: | ||
| name: sdist | ||
| path: dist/*.tar.gz | ||
| publish: | ||
| name: Publish to PyPI | ||
| needs: [check-version, uv-validate, build-wheels-linux, build-wheels-macos, build-wheels-windows, build-sdist] | ||
| runs-on: ubuntu-latest | ||
| if: needs.check-version.outputs.should_build == 'true' | ||
| steps: | ||
| - name: Download all artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| path: dist-artifacts | ||
| - name: Collect all distributions | ||
| run: | | ||
| mkdir -p dist | ||
| find dist-artifacts -name '*.whl' -exec cp {} dist/ \; | ||
| find dist-artifacts -name '*.tar.gz' -exec cp {} dist/ \; | ||
| echo "Contents of dist/:" | ||
| ls -la dist/ | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: "3.12" | ||
| - name: Install twine | ||
| run: python -m pip install twine | ||
| - name: Publish to TestPyPI (on PR) | ||
| if: github.event_name == 'pull_request' | ||
| env: | ||
| TWINE_USERNAME: __token__ | ||
| TWINE_PASSWORD: ${{ secrets.TEST_PYPI }} | ||
| run: | | ||
| python -m twine upload --repository testpypi dist/* --verbose | ||
| - name: Publish to PyPI (on push to main) | ||
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | ||
| env: | ||
| TWINE_USERNAME: __token__ | ||
| TWINE_PASSWORD: ${{ secrets.PYPI }} | ||
| run: | | ||
| python -m twine upload dist/* --verbose | ||
