From ac25e1dd79695878d0cdf7c2b13133499b9fe7e6 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Mon, 13 Jul 2026 19:14:04 -0500 Subject: [PATCH 1/4] ci: run the workspace tests inside the build job --- .github/workflows/ci.yml | 98 +++++++++++++--------------------------- 1 file changed, 32 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10661dbf22..6d176eda8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,38 @@ jobs: - name: Build run: make build + # The integration tests need the docker-compose services, which are only + # available on the Linux runner, so the test steps are gated to ubuntu. + # macOS and Windows still compile the full workspace above. + - name: Install cargo-nextest + if: runner.os == 'Linux' + uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 + with: + tool: cargo-nextest + + - name: Start Docker containers + if: runner.os == 'Linux' + run: make docker-up + + - name: Run tests + if: runner.os == 'Linux' + shell: bash + env: + # HuggingFace Hub integration tests self-skip when these are unset + # (e.g. PRs from forks); see crates/storage/opendal/tests/file_io_hf_test.rs + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_BUCKET: ${{ secrets.HF_BUCKET }} + HF_DATASET: ${{ secrets.HF_DATASET }} + run: cargo nextest run --all-targets --all-features --workspace + + - name: Run doc tests + if: runner.os == 'Linux' + run: cargo test --no-fail-fast --doc --all-features --workspace + + - name: Stop Docker containers + if: always() && runner.os == 'Linux' + run: make docker-down + # Checks each crate individually to catch missing feature declarations that # are otherwise masked by Cargo's feature unification across the workspace. check_standalone: @@ -193,72 +225,6 @@ jobs: cargo check -p "$pkg" --all-targets || exit 1 done - tests: - needs: lint - runs-on: ubuntu-latest - env: - CARGO_INCREMENTAL: "0" - CARGO_PROFILE_DEV_DEBUG: "0" - strategy: - max-parallel: 15 - matrix: - test-suite: - - { name: "default", args: "--all-targets --all-features --workspace" } - - { name: "doc", args: "--doc --all-features --workspace" } - name: Tests (${{ matrix.test-suite.name }}) - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - # Image + build-definition files in the key so changed steps or a new - # runner image rotate the cache; see the build job for the rationale. - # The test-suite name stays in front so the default and doc suites keep - # separate cache lineages. - - name: Get runner image id - id: image - shell: bash - run: echo "id=${ImageOS:?}" >> "$GITHUB_OUTPUT" - - - name: Cache Rust artifacts - uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - with: - key: ${{ matrix.test-suite.name }}-${{ steps.image.outputs.id }}-${{ hashFiles('.github/workflows/ci.yml', 'Makefile', 'Cargo.toml') }} - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - add-rust-environment-hash-key: 'false' - - - name: Install cargo-nextest - if: matrix.test-suite.name == 'default' - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 - with: - tool: cargo-nextest - - - name: Start Docker containers - if: matrix.test-suite.name == 'default' - run: make docker-up - - - name: Run tests - shell: bash - env: - # HuggingFace Hub integration tests self-skip when these are unset - # (e.g. PRs from forks); see crates/storage/opendal/tests/file_io_hf_test.rs - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_BUCKET: ${{ secrets.HF_BUCKET }} - HF_DATASET: ${{ secrets.HF_DATASET }} - run: | - if [ "${{ matrix.test-suite.name }}" = "default" ]; then - cargo nextest run ${{ matrix.test-suite.args }} - else - cargo test --no-fail-fast ${{ matrix.test-suite.args }} - fi - - - name: Stop Docker containers - if: always() && matrix.test-suite.name == 'default' - run: make docker-down - msrv: name: Verify MSRV needs: lint From a505e1f6b1f92b2a2d4b06c8e39f2a78ed68f4ce Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Mon, 13 Jul 2026 19:15:45 -0500 Subject: [PATCH 2/4] ci: give the ubuntu build and tests their own job with background docker startup --- .github/workflows/ci.yml | 102 ++++++++++++++++++++++++++------------- Makefile | 12 +++++ 2 files changed, 81 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d176eda8c..20081f2175 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,7 +119,6 @@ jobs: max-parallel: 15 matrix: os: - - ubuntu-latest - macos-latest - windows-latest steps: @@ -159,38 +158,6 @@ jobs: - name: Build run: make build - # The integration tests need the docker-compose services, which are only - # available on the Linux runner, so the test steps are gated to ubuntu. - # macOS and Windows still compile the full workspace above. - - name: Install cargo-nextest - if: runner.os == 'Linux' - uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 - with: - tool: cargo-nextest - - - name: Start Docker containers - if: runner.os == 'Linux' - run: make docker-up - - - name: Run tests - if: runner.os == 'Linux' - shell: bash - env: - # HuggingFace Hub integration tests self-skip when these are unset - # (e.g. PRs from forks); see crates/storage/opendal/tests/file_io_hf_test.rs - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_BUCKET: ${{ secrets.HF_BUCKET }} - HF_DATASET: ${{ secrets.HF_DATASET }} - run: cargo nextest run --all-targets --all-features --workspace - - - name: Run doc tests - if: runner.os == 'Linux' - run: cargo test --no-fail-fast --doc --all-features --workspace - - - name: Stop Docker containers - if: always() && runner.os == 'Linux' - run: make docker-down - # Checks each crate individually to catch missing feature declarations that # are otherwise masked by Cargo's feature unification across the workspace. check_standalone: @@ -225,6 +192,75 @@ jobs: cargo check -p "$pkg" --all-targets || exit 1 done + # The ubuntu build and tests share one runner so the dependency graph + # compiles once instead of once per job. Docker containers start in the + # background while the workspace builds so their startup time overlaps the + # compile. macOS and Windows keep their own build job above; clippy and the + # standalone crate checks run as separate ubuntu jobs. + build-and-test: + needs: lint + runs-on: ubuntu-latest + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + # Container startup overlaps the workspace build; the wait step below + # joins on the result before the tests need the containers. + - name: Start Docker containers in the background + run: make docker-up-background + + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + + # Image + build-definition files in the key so changed steps or a new + # runner image rotate the cache; see the build job for the rationale. + - name: Get runner image id + id: image + shell: bash + run: echo "id=${ImageOS:?}" >> "$GITHUB_OUTPUT" + + - name: Cache Rust artifacts + uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + key: ${{ steps.image.outputs.id }}-${{ hashFiles('.github/workflows/ci.yml', 'Makefile', 'Cargo.toml') }} + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + add-rust-environment-hash-key: 'false' + + - name: Build without default features + run: cargo build --locked -p iceberg --no-default-features + + - name: Build + run: cargo build --locked --all-targets --all-features --workspace + + - name: Install cargo-nextest + uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9 + with: + tool: cargo-nextest + + - name: Wait for Docker containers + run: make docker-wait + + - name: Run tests + shell: bash + env: + # HuggingFace Hub integration tests self-skip when these are unset + # (e.g. PRs from forks); see crates/storage/opendal/tests/file_io_hf_test.rs + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_BUCKET: ${{ secrets.HF_BUCKET }} + HF_DATASET: ${{ secrets.HF_DATASET }} + run: cargo nextest run --all-targets --all-features --workspace + + - name: Run doc tests + run: cargo test --no-fail-fast --doc --all-features --workspace + + - name: Stop Docker containers + if: always() + run: make docker-down + msrv: name: Verify MSRV needs: lint diff --git a/Makefile b/Makefile index 003efbfd15..e005091c68 100644 --- a/Makefile +++ b/Makefile @@ -111,6 +111,18 @@ site: install-mdbook docker-up: docker compose -f dev/docker-compose.yaml up -d --build --wait +# Starts the containers without blocking so their startup can overlap other +# work (e.g. the CI build); docker-wait joins on the result. +# The subshell's stdio is fully detached so the caller (e.g. the CI runner) +# isn't left holding open pipes until the containers finish starting. +docker-up-background: + (docker compose -f dev/docker-compose.yaml up -d --build --wait > /tmp/docker-up.log 2>&1; echo $$? > /tmp/docker-up.exit) < /dev/null > /dev/null 2>&1 & + +docker-wait: + @while [ ! -f /tmp/docker-up.exit ]; do sleep 2; done + @cat /tmp/docker-up.log + @exit $$(cat /tmp/docker-up.exit) + docker-down: docker compose -f dev/docker-compose.yaml down -v --remove-orphans --timeout 0 From eb15dc4e80f684bf509e631f09d9168b3df0823c Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Mon, 13 Jul 2026 19:16:12 -0500 Subject: [PATCH 3/4] ci: add debuginfo and locked flags to the standalone crate check --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20081f2175..9c2cb9351d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,9 +160,12 @@ jobs: # Checks each crate individually to catch missing feature declarations that # are otherwise masked by Cargo's feature unification across the workspace. - check_standalone: + check-standalone: needs: lint runs-on: ubuntu-latest + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -189,7 +192,7 @@ jobs: run: | for pkg in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name'); do echo "Checking $pkg..." - cargo check -p "$pkg" --all-targets || exit 1 + cargo check --locked -p "$pkg" --all-targets || exit 1 done # The ubuntu build and tests share one runner so the dependency graph From a26cbf6bcc1a5492eaa0e7c03bb9712c4e4f7b2b Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Mon, 13 Jul 2026 18:48:34 -0500 Subject: [PATCH 4/4] ci: build the bindings validation wheel with a fast profile and cache it on Linux --- .github/workflows/bindings_python_ci.yml | 13 ++++++++----- Cargo.toml | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 7f66302cd1..f17bf8cdc9 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -82,31 +82,34 @@ jobs: - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: 3.12 - # Linux maturin builds may run inside manylinux Docker, so host Rust caches only help Windows/macOS. + # This job passes no manylinux/target inputs, so maturin-action builds + # on the host on every OS (a manylinux container is only used when + # `manylinux` is set, as the release workflows do) - host Rust caching + # helps all three platforms. - name: Setup Rust toolchain - if: runner.os != 'Linux' uses: ./.github/actions/setup-builder # Image + build-definition files in the key so changed steps or a new # runner image rotate the cache and a stale cache can heal itself; the # rust-cache environment hash is dropped because its preinstalled-stable # component varies across concurrent image builds and breaks restores. - name: Get runner image id - if: runner.os != 'Linux' id: image shell: bash run: echo "id=${ImageOS:?}" >> "$GITHUB_OUTPUT" - name: Cache Rust artifacts - if: runner.os != 'Linux' uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: key: bindings-python-${{ steps.image.outputs.id }}-${{ hashFiles('.github/workflows/bindings_python_ci.yml', 'bindings/python/pyproject.toml', 'Cargo.toml') }} save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} add-rust-environment-hash-key: 'false' + # `--profile ci` overrides the size-optimized release profile from + # pyproject.toml: PR validation only needs a wheel that builds fast and + # runs the test suite; the publish workflows pass their own profile. - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: working-directory: "bindings/python" command: build - args: --out dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one + args: --out dist -i python3.12 --profile ci # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: version: "0.9.3" diff --git a/Cargo.toml b/Cargo.toml index f14a94af93..1257b8bc61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,21 @@ # specific language governing permissions and limitations # under the License. +# Fast-compiling optimized profile for CI validation builds (e.g. the Python +# bindings test wheel). It explicitly sets every key that the pyproject.toml +# maturin config injects into the release profile it inherits from, so the +# size-optimized publish settings never leak into CI builds unnoticed. +# opt-level "z" is deliberate: it compiles the dependency graph measurably +# faster than opt-level 1 or 2, and disabling LTO removes the serialized +# multi-minute final link that dominates the publish profile. +[profile.ci] +codegen-units = 16 +debug = false +inherits = "release" +lto = false +opt-level = "z" +strip = "none" + [workspace] members = [ "bindings/python",