From a505e1f6b1f92b2a2d4b06c8e39f2a78ed68f4ce Mon Sep 17 00:00:00 2001
From: Abanoub Doss <abanoub.doss@gmail.com>
Date: Mon, 13 Jul 2026 19:15:45 -0500
Subject: [PATCH] ci: give the ubuntu build and tests their own job with
 background docker startup

---
 .github/workflows/ci.yml | 102 ++++++++++++++++++++++++++-------------
 Makefile                 |  12 +++++
 2 files changed, 81 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6d176eda8c..20081f2175 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -119,7 +119,6 @@ jobs:
       max-parallel: 15
       matrix:
         os:
-          - ubuntu-latest
           - macos-latest
           - windows-latest
     steps:
@@ -159,38 +158,6 @@ jobs:
       - name: Build
         run: make build
 
-      # The integration tests need the docker-compose services, which are only
-      # available on the Linux runner, so the test steps are gated to ubuntu.
-      # macOS and Windows still compile the full workspace above.
-      - name: Install cargo-nextest
-        if: runner.os == 'Linux'
-        uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9
-        with:
-          tool: cargo-nextest
-
-      - name: Start Docker containers
-        if: runner.os == 'Linux'
-        run: make docker-up
-
-      - name: Run tests
-        if: runner.os == 'Linux'
-        shell: bash
-        env:
-          # HuggingFace Hub integration tests self-skip when these are unset
-          # (e.g. PRs from forks); see crates/storage/opendal/tests/file_io_hf_test.rs
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_BUCKET: ${{ secrets.HF_BUCKET }}
-          HF_DATASET: ${{ secrets.HF_DATASET }}
-        run: cargo nextest run --all-targets --all-features --workspace
-
-      - name: Run doc tests
-        if: runner.os == 'Linux'
-        run: cargo test --no-fail-fast --doc --all-features --workspace
-
-      - name: Stop Docker containers
-        if: always() && runner.os == 'Linux'
-        run: make docker-down
-
   # Checks each crate individually to catch missing feature declarations that
   # are otherwise masked by Cargo's feature unification across the workspace.
   check_standalone:
@@ -225,6 +192,75 @@ jobs:
             cargo check -p "$pkg" --all-targets || exit 1
           done
 
+  # The ubuntu build and tests share one runner so the dependency graph
+  # compiles once instead of once per job. Docker containers start in the
+  # background while the workspace builds so their startup time overlaps the
+  # compile. macOS and Windows keep their own build job above; clippy and the
+  # standalone crate checks run as separate ubuntu jobs.
+  build-and-test:
+    needs: lint
+    runs-on: ubuntu-latest
+    env:
+      CARGO_INCREMENTAL: "0"
+      CARGO_PROFILE_DEV_DEBUG: "0"
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      # Container startup overlaps the workspace build; the wait step below
+      # joins on the result before the tests need the containers.
+      - name: Start Docker containers in the background
+        run: make docker-up-background
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+
+      # Image + build-definition files in the key so changed steps or a new
+      # runner image rotate the cache; see the build job for the rationale.
+      - name: Get runner image id
+        id: image
+        shell: bash
+        run: echo "id=${ImageOS:?}" >> "$GITHUB_OUTPUT"
+
+      - name: Cache Rust artifacts
+        uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          key: ${{ steps.image.outputs.id }}-${{ hashFiles('.github/workflows/ci.yml', 'Makefile', 'Cargo.toml') }}
+          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+          add-rust-environment-hash-key: 'false'
+
+      - name: Build without default features
+        run: cargo build --locked -p iceberg --no-default-features
+
+      - name: Build
+        run: cargo build --locked --all-targets --all-features --workspace
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@4684b8405694ae9dd42c9f39ba901a70ae83f4a3 # v2.82.9
+        with:
+          tool: cargo-nextest
+
+      - name: Wait for Docker containers
+        run: make docker-wait
+
+      - name: Run tests
+        shell: bash
+        env:
+          # HuggingFace Hub integration tests self-skip when these are unset
+          # (e.g. PRs from forks); see crates/storage/opendal/tests/file_io_hf_test.rs
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_BUCKET: ${{ secrets.HF_BUCKET }}
+          HF_DATASET: ${{ secrets.HF_DATASET }}
+        run: cargo nextest run --all-targets --all-features --workspace
+
+      - name: Run doc tests
+        run: cargo test --no-fail-fast --doc --all-features --workspace
+
+      - name: Stop Docker containers
+        if: always()
+        run: make docker-down
+
   msrv:
     name: Verify MSRV
     needs: lint
diff --git a/Makefile b/Makefile
index 003efbfd15..e005091c68 100644
--- a/Makefile
+++ b/Makefile
@@ -111,6 +111,18 @@ site: install-mdbook
 docker-up:
 	docker compose -f dev/docker-compose.yaml up -d --build --wait
 
+# Starts the containers without blocking so their startup can overlap other
+# work (e.g. the CI build); docker-wait joins on the result.
+# The subshell's stdio is fully detached so the caller (e.g. the CI runner)
+# isn't left holding open pipes until the containers finish starting.
+docker-up-background:
+	(docker compose -f dev/docker-compose.yaml up -d --build --wait > /tmp/docker-up.log 2>&1; echo $$? > /tmp/docker-up.exit) < /dev/null > /dev/null 2>&1 &
+
+docker-wait:
+	@while [ ! -f /tmp/docker-up.exit ]; do sleep 2; done
+	@cat /tmp/docker-up.log
+	@exit $$(cat /tmp/docker-up.exit)
+
 docker-down:
 	docker compose -f dev/docker-compose.yaml down -v --remove-orphans --timeout 0