diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 7f66302cd1..f17bf8cdc9 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -82,31 +82,34 @@ jobs: - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: 3.12 - # Linux maturin builds may run inside manylinux Docker, so host Rust caches only help Windows/macOS. + # This job passes no manylinux/target inputs, so maturin-action builds + # on the host on every OS (a manylinux container is only used when + # `manylinux` is set, as the release workflows do) - host Rust caching + # helps all three platforms. - name: Setup Rust toolchain - if: runner.os != 'Linux' uses: ./.github/actions/setup-builder # Image + build-definition files in the key so changed steps or a new # runner image rotate the cache and a stale cache can heal itself; the # rust-cache environment hash is dropped because its preinstalled-stable # component varies across concurrent image builds and breaks restores. - name: Get runner image id - if: runner.os != 'Linux' id: image shell: bash run: echo "id=${ImageOS:?}" >> "$GITHUB_OUTPUT" - name: Cache Rust artifacts - if: runner.os != 'Linux' uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: key: bindings-python-${{ steps.image.outputs.id }}-${{ hashFiles('.github/workflows/bindings_python_ci.yml', 'bindings/python/pyproject.toml', 'Cargo.toml') }} save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} add-rust-environment-hash-key: 'false' + # `--profile ci` overrides the size-optimized release profile from + # pyproject.toml: PR validation only needs a wheel that builds fast and + # runs the test suite; the publish workflows pass their own profile. - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: working-directory: "bindings/python" command: build - args: --out dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one + args: --out dist -i python3.12 --profile ci # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: version: "0.9.3" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10661dbf22..9c2cb9351d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,7 +119,6 @@ jobs: max-parallel: 15 matrix: os: - - ubuntu-latest - macos-latest - windows-latest steps: @@ -161,9 +160,12 @@ jobs: # Checks each crate individually to catch missing feature declarations that # are otherwise masked by Cargo's feature unification across the workspace. - check_standalone: + check-standalone: needs: lint runs-on: ubuntu-latest + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -190,34 +192,35 @@ jobs: run: | for pkg in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name'); do echo "Checking $pkg..." - cargo check -p "$pkg" --all-targets || exit 1 + cargo check --locked -p "$pkg" --all-targets || exit 1 done - tests: + # The ubuntu build and tests share one runner so the dependency graph + # compiles once instead of once per job. Docker containers start in the + # background while the workspace builds so their startup time overlaps the + # compile. macOS and Windows keep their own build job above; clippy and the + # standalone crate checks run as separate ubuntu jobs. + build-and-test: needs: lint runs-on: ubuntu-latest env: CARGO_INCREMENTAL: "0" CARGO_PROFILE_DEV_DEBUG: "0" - strategy: - max-parallel: 15 - matrix: - test-suite: - - { name: "default", args: "--all-targets --all-features --workspace" } - - { name: "doc", args: "--doc --all-features --workspace" } - name: Tests (${{ matrix.test-suite.name }}) steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + # Container startup overlaps the workspace build; the wait step below + # joins on the result before the tests need the containers. + - name: Start Docker containers in the background + run: make docker-up-background + - name: Setup Rust toolchain uses: ./.github/actions/setup-builder # Image + build-definition files in the key so changed steps or a new # runner image rotate the cache; see the build job for the rationale. - # The test-suite name stays in front so the default and doc suites keep - # separate cache lineages. - name: Get runner image id id: image shell: bash @@ -226,19 +229,23 @@ jobs: - name: Cache Rust artifacts uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: - key: ${{ matrix.test-suite.name }}-${{ steps.image.outputs.id }}-${{ hashFiles('.github/workflows/ci.yml', 'Makefile', 'Cargo.toml') }} + key: ${{ steps.image.outputs.id }}-${{ hashFiles('.github/workflows/ci.yml', 'Makefile', 'Cargo.toml') }} save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} add-rust-environment-hash-key: 'false' + - name: Build without default features + run: cargo build --locked -p iceberg --no-default-features + + - name: Build + run: cargo build --locked --all-targets --all-features --workspace + - name: Install cargo-nextest - if: matrix.test-suite.name == 'default' uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 with: tool: cargo-nextest - - name: Start Docker containers - if: matrix.test-suite.name == 'default' - run: make docker-up + - name: Wait for Docker containers + run: make docker-wait - name: Run tests shell: bash @@ -248,15 +255,13 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_BUCKET: ${{ secrets.HF_BUCKET }} HF_DATASET: ${{ secrets.HF_DATASET }} - run: | - if [ "${{ matrix.test-suite.name }}" = "default" ]; then - cargo nextest run ${{ matrix.test-suite.args }} - else - cargo test --no-fail-fast ${{ matrix.test-suite.args }} - fi + run: cargo nextest run --all-targets --all-features --workspace + + - name: Run doc tests + run: cargo test --no-fail-fast --doc --all-features --workspace - name: Stop Docker containers - if: always() && matrix.test-suite.name == 'default' + if: always() run: make docker-down msrv: diff --git a/Cargo.toml b/Cargo.toml index f14a94af93..1257b8bc61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,21 @@ # specific language governing permissions and limitations # under the License. +# Fast-compiling optimized profile for CI validation builds (e.g. the Python +# bindings test wheel). It explicitly sets every key that the pyproject.toml +# maturin config injects into the release profile it inherits from, so the +# size-optimized publish settings never leak into CI builds unnoticed. +# opt-level "z" is deliberate: it compiles the dependency graph measurably +# faster than opt-level 1 or 2, and disabling LTO removes the serialized +# multi-minute final link that dominates the publish profile. +[profile.ci] +codegen-units = 16 +debug = false +inherits = "release" +lto = false +opt-level = "z" +strip = "none" + [workspace] members = [ "bindings/python", diff --git a/Makefile b/Makefile index 003efbfd15..e005091c68 100644 --- a/Makefile +++ b/Makefile @@ -111,6 +111,18 @@ site: install-mdbook docker-up: docker compose -f dev/docker-compose.yaml up -d --build --wait +# Starts the containers without blocking so their startup can overlap other +# work (e.g. the CI build); docker-wait joins on the result. +# The subshell's stdio is fully detached so the caller (e.g. the CI runner) +# isn't left holding open pipes until the containers finish starting. +docker-up-background: + (docker compose -f dev/docker-compose.yaml up -d --build --wait > /tmp/docker-up.log 2>&1; echo $$? > /tmp/docker-up.exit) < /dev/null > /dev/null 2>&1 & + +docker-wait: + @while [ ! -f /tmp/docker-up.exit ]; do sleep 2; done + @cat /tmp/docker-up.log + @exit $$(cat /tmp/docker-up.exit) + docker-down: docker compose -f dev/docker-compose.yaml down -v --remove-orphans --timeout 0