release_python_nightly.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: "Nightly PyPI Build"

on:
  schedule:
    - cron: "0 0 * * *" # Runs at midnight UTC every day
  workflow_dispatch: # Allows manual triggering

permissions:
  contents: read

jobs:
  set-version:
    if: github.repository == 'apache/iceberg-rust' || github.event_name == 'workflow_dispatch' # Run on schedule for apache repo, or on manual dispatch from any repo
    runs-on: ubuntu-latest
    outputs:
      timestamp: ${{ steps.set-ts.outputs.TIMESTAMP }}
    steps:
      - name: Generate version timestamp
        id: set-ts
        run: echo "TIMESTAMP=$(date +'%Y%m%d%H%M%S')" >> "$GITHUB_OUTPUT"

  sdist:
    needs: set-version
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Compute dev version
        id: version
        shell: bash
        env:
          TIMESTAMP: ${{ needs.set-version.outputs.timestamp }}
        run: |
          CURRENT=$(cargo metadata --format-version 1 --no-deps -q | jq -r '.packages[0].version')
          echo "pep440=${CURRENT}.dev${TIMESTAMP}" >> $GITHUB_OUTPUT

      - uses: ./.github/actions/overwrite-package-version
        with:
          version: ${{ steps.version.outputs.pep440 }}

      - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
        with:
          working-directory: "bindings/python"
          command: sdist
          args: -o dist

      - name: Upload sdist
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: wheels-sdist
          path: bindings/python/dist

  wheels:
    needs: set-version
    runs-on: "${{ matrix.os }}"
    strategy:
      max-parallel: 15
      matrix:
        include:
          - { os: windows-latest }
          - { os: macos-latest, target: "universal2-apple-darwin" }
          - { os: ubuntu-latest, target: "x86_64" }
          - {
              os: ubuntu-latest,
              target: "aarch64",
              manylinux: "manylinux_2_28",
            }
          - { os: ubuntu-latest, target: "armv7l" }
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Compute dev version
        id: version
        shell: bash
        env:
          TIMESTAMP: ${{ needs.set-version.outputs.timestamp }}
        run: |
          CURRENT=$(cargo metadata --format-version 1 --no-deps -q | jq -r '.packages[0].version')
          echo "pep440=${CURRENT}.dev${TIMESTAMP}" >> $GITHUB_OUTPUT

      - uses: ./.github/actions/overwrite-package-version
        with:
          version: ${{ steps.version.outputs.pep440 }}

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: 3.12

      - name: Get MSRV
        id: get-msrv
        uses: ./.github/actions/get-msrv

      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ steps.get-msrv.outputs.msrv }}

      - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
        with:
          target: ${{ matrix.target }}
          manylinux: ${{ matrix.manylinux || 'auto' }}
          working-directory: "bindings/python"
          command: build
          args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one

      - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
        with:
          version: "0.9.3"
          enable-cache: true
      # Verify the wheel is abi3-compatible and installable without building from source.
      # Skipped for cross-compiled targets since they share the same maturin config;
      # if abi3 is broken, the native target build will catch it.
      - name: Verify wheel installs with no build fallback
        if: matrix.target != 'aarch64' && matrix.target != 'armv7l'
        working-directory: "bindings/python"
        shell: bash
        run: |
          uv pip install --no-build --reinstall --system --no-index --find-links dist/ pyiceberg-core

      - name: Upload wheels
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: wheels-${{ matrix.os }}-${{ matrix.target }}
          path: bindings/python/dist

  testpypi-publish:
    if: github.repository == 'apache/iceberg-rust' # Only run for apache repo
    needs: [sdist, wheels]
    runs-on: ubuntu-latest

    environment:
      name: testpypi
      url: https://test.pypi.org/p/pyiceberg-core

    permissions:
      id-token: write # IMPORTANT: mandatory for trusted publishing

    steps:
      - name: Download all the dists
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: wheels-*
          merge-multiple: true
          path: bindings/python/dist
      - name: List downloaded artifacts
        run: ls -R bindings/python/dist
      - name: Publish to TestPyPI
        id: publish-testpypi
        continue-on-error: true
        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
        with:
          repository-url: https://test.pypi.org/legacy/
          skip-existing: true
          packages-dir: bindings/python/dist
          verbose: true
      - name: Display error message on publish failure
        if: steps.publish-testpypi.outcome == 'failure'
        run: |
          echo "::error::Failed to publish to TestPyPI"
          echo ""
          echo "⚠️  TestPyPI Publish Failed"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo ""
          echo "This may be due to TestPyPI storage limits."
          echo "See: https://docs.pypi.org/project-management/storage-limits"
          echo ""
          echo "To resolve this issue, use the pypi-cleanup utility to clean up old TestPyPI artifacts:"
          echo "https://pypi.org/project/pypi-cleanup/"
          echo ""
          echo "  uvx pypi-cleanup --package pyiceberg-core --host https://test.pypi.org/ \\"
          echo "    --verbose -d 10 --do-it --username <username>"
          echo ""
          echo "Requirements:"
          echo "  • Must be a maintainer for pyiceberg-core on TestPyPI"
          echo "    (https://test.pypi.org/project/pyiceberg-core)"
          echo "  • Requires TestPyPI password and 2FA"
          echo "  • ⚠️  ONLY do this for TestPyPI, NOT for production PyPI!"
          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          exit 1