feat: Add cross-compilation support for sdist packages with native extensions

.github/workflows/ci-workflows.yaml

@@ -56,8 +56,8 @@ jobs:
                   py/tools/site_merge/site_merge.py
                   py/tools/unpack/unpack.py
                   uv/private/gazelle_manifest/generate.py
-                  uv/private/pep517_whl/build_helper.py
-                  uv/private/pep517_whl/memory_monitor.py
+                  uv/private/pep517_whl/tools/build_helper.py
+                  uv/private/pep517_whl/tools/memory_monitor.py
                   uv/private/py_entrypoint_binary/search.py
                 )
                 runtime311=(
@@ -107,6 +107,11 @@ jobs:
             - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2
               with:
                 aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }}
+            # e2e/crossbuild runs an arm64 container from this amd64 runner
+            - if: matrix.workspace.slug == 'e2e-crossbuild'
+              uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+              with:
+                platforms: arm64
             # Run `test //...`, then the workspace's test.sh if it has one
             - name: Test
               working-directory: ${{ matrix.workspace.path }}
@@ -115,9 +120,194 @@ jobs:
                 if [ -f test.sh ]; then
                     bash test.sh
                 fi
+
     # macOS minutes are billed at 10X, so it only covers the root and e2e consumer
     # workspaces, on main plus branches whose name contains 'macos'.
     # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#included-storage-and-minutes
+
+    crossbuild-images-darwin:
+        name: build on darwin
+        if: >-
+            ${{ github.ref_name == 'main'
+                || contains(github.head_ref || github.ref_name, 'macos') }}
+        runs-on: macos-latest
+        steps:
+            - uses: actions/checkout@v6
+            - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0
+              with:
+                  bazelisk-cache: true
+                  disk-cache: macos-crossbuild-images
+                  repository-cache: true
+            - name: Cross-build linux image tarballs from macOS
+              working-directory: e2e/crossbuild
+              shell: bash
+              run: |
+                mkdir -p "${RUNNER_TEMP}/images"
+                for case in psutil contourpy rpds_py; do
+                    for arch in amd64 arm64; do
+                        bazel build --config=ci --remote_download_outputs=toplevel --action_env=MESON_FORCE_BACKTRACE=1 "//${case}:${arch}_tarball"
+                        cp "bazel-bin/${case}/${arch}_load/tarball.tar" "${RUNNER_TEMP}/images/${case}-${arch}.tar"
+                    done
+                done
+            - uses: actions/upload-artifact@v4
+              with:
+                name: crossbuild-images-darwin
+                path: ${{ runner.temp }}/images/*.tar
+                retention-days: 1
+    crossbuild-run-darwin-built:
+        name: run on linux ${{ matrix.arch }})
+        needs: crossbuild-images-darwin
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                - { runner: "ubuntu-latest", arch: "amd64" }
+                - { runner: "ubuntu-24.04-arm", arch: "arm64" }
+        runs-on: ${{ matrix.runner }}
+        steps:
+            - uses: actions/download-artifact@v4
+              with:
+                name: crossbuild-images-darwin
+                path: images
+            - name: Run darwin-built images on native ${{ matrix.arch }}
+              run: |
+                status=0
+                for tar in images/*-${{ matrix.arch }}.tar; do
+                    tag=$(docker load -i "${tar}" | sed -n 's/^Loaded image: //p')
+                    echo "::group::${tag}"
+                    docker run --rm "${tag}" || status=1
+                    echo "::endgroup::"
+                done
+                exit "${status}"
+
+
+    crossbuild-images-amd64:
+        name: cross-build arm64 images on linux amd64
+        runs-on: ubuntu-22.04-32core
+        steps:
+            - uses: actions/checkout@v6
+            - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2
+              with:
+                aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }}
+            # binfmt for meson's build-time probes only (sanity checks and
+            # cc.run() execute target-arch binaries mid-build); the actual
+            # execution verdict comes from the native arm64 runner below.
+            - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+              with:
+                platforms: arm64
+            - name: Cross-build arm64 image tarballs
+              working-directory: e2e/crossbuild
+              run: |
+                mkdir -p "${RUNNER_TEMP}/images"
+                targets=$(bazel query 'filter(".*:arm64_tarball$", //...)')
+                bazel build --remote_download_outputs=toplevel ${targets}
+                for target in ${targets}; do
+                    case="${target#//}"
+                    case="${case%:arm64_tarball}"
+                    cp "bazel-bin/${case}/arm64_load/tarball.tar" "${RUNNER_TEMP}/images/${case}-arm64.tar"
+                done
+            - uses: actions/upload-artifact@v4
+              with:
+                name: crossbuild-images-amd64-built
+                path: ${{ runner.temp }}/images/*.tar
+                retention-days: 1
+    crossbuild-run-amd64-built:
+        name: run amd64-built images on native arm64
+        needs: crossbuild-images-amd64
+        runs-on: ubuntu-24.04-arm
+        steps:
+            - uses: actions/download-artifact@v4
+              with:
+                name: crossbuild-images-amd64-built
+                path: images
+            - name: Run amd64-built images on native arm64
+              run: |
+                status=0
+                for tar in images/*-arm64.tar; do
+                    tag=$(docker load -i "${tar}" | sed -n 's/^Loaded image: //p')
+                    echo "::group::${tag}"
+                    docker run --rm "${tag}" || status=1
+                    echo "::endgroup::"
+                done
+                exit "${status}"
+
+    crossbuild-images-arm64:
+        name: build on linux arm64
+        runs-on: ubuntu-24.04-arm
+        steps:
+            - uses: actions/checkout@v6
+            - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2
+              with:
+                aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }}
+            - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+              with:
+                platforms: amd64
+            - name: Cross-build amd64 image tarballs from arm64
+              working-directory: e2e/crossbuild
+              run: |
+                mkdir -p "${RUNNER_TEMP}/images"
+                for case in psutil contourpy rpds_py; do
+                    # toplevel: build-without-the-bytes must still materialize
+                    # the tarball this job exists to upload.
+                    bazel build --config=ci --remote_download_outputs=toplevel "//${case}:amd64_tarball"
+                    cp "bazel-bin/${case}/amd64_load/tarball.tar" "${RUNNER_TEMP}/images/${case}-amd64.tar"
+                done
+            - uses: actions/upload-artifact@v4
+              with:
+                name: crossbuild-images-arm-built
+                path: ${{ runner.temp }}/images/*.tar
+                retention-days: 1
+    crossbuild-run-arm64-built:
+        name: run on linux amd64
+        needs: crossbuild-images-arm64
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/download-artifact@v4
+              with:
+                name: crossbuild-images-arm-built
+                path: images
+            - name: Run arm-built images on native amd64
+              run: |
+                status=0
+                for tar in images/*-amd64.tar; do
+                    tag=$(docker load -i "${tar}" | sed -n 's/^Loaded image: //p')
+                    echo "::group::${tag}"
+                    docker run --rm "${tag}" || status=1
+                    echo "::endgroup::"
+                done
+                exit "${status}"
+
+
+    test-all:
+        name: test-all
+        runs-on: ubuntu-latest
+        needs: [buildifier, gazelle, typecheck, crossbuild-run-amd64-built, crossbuild-run-arm64-built, crossbuild-run-darwin-built]
+        if: always()
+        steps:
+            - name: All required jobs succeeded
+              run: |
+                for result in \
+                    "test=${{ needs.test.result }}" \
+                    "typecheck=${{ needs.typecheck.result }}" \
+                    "buildifier=${{ needs.buildifier.result }}" \
+                    "gazelle=${{ needs.gazelle.result }}" \
+                    "crossbuild-run-amd64-built=${{ needs['crossbuild-run-amd64-built'].result }}" \
+                    "crossbuild-run-arm64-built=${{ needs['crossbuild-run-arm64-built'].result }}"; do
+                    if [ "${result#*=}" != "success" ]; then
+                        echo "Required job did not succeed: ${result}"
+                        exit 1
+                    fi
+                done
+                # The darwin pipeline is gated to main and '*macos*' branches:
+                # skipped is expected elsewhere; anything that ran must pass.
+                case "${{ needs['crossbuild-run-darwin-built'].result }}" in
+                    success|skipped) ;;
+                    *)
+                        echo "crossbuild-run-darwin-built did not succeed: ${{ needs['crossbuild-run-darwin-built'].result }}"
+                        exit 1
+                        ;;
+                esac
+
     smoke:
         name: smoke (${{ matrix.workspace.path }}, macos)
         if: >-
@@ -154,19 +344,3 @@ jobs:
                   if [ -f test.sh ]; then
                       bash test.sh
                   fi
-    test-all:
-        name: test-all
-        runs-on: ubuntu-latest
-        needs: [test, typecheck]
-        if: always()
-        steps:
-            - name: All required jobs succeeded
-              run: |
-                if [ "${{ needs.test.result }}" != "success" ]; then
-                    echo "One or more test matrix jobs did not succeed: ${{ needs.test.result }}"
-                    exit 1
-                fi
-                if [ "${{ needs.typecheck.result }}" != "success" ]; then
-                    echo "Typecheck did not succeed: ${{ needs.typecheck.result }}"
-                    exit 1
-                fi

docs/uv.md

@@ -271,6 +271,70 @@ platform_transition_filegroup(
 )
 ```
 
+## Example: Cross-compiling sdists
+
+Wheels are selected for the target platform, but when a package only ships a
+source distribution — or no published wheel matches the target — uv builds the
+sdist as part of the build. Two modes exist:
+
+- **Pure-Python sdists** produce a `-none-any` wheel and build identically in
+  any configuration, cross or not.
+- **Native sdists** (C/C++ extensions) build in *native mode* when the exec and
+  target platforms coincide. When they differ, the build enters *cross mode*.
+
+Cross mode requires a **cross-capable C++ toolchain** registered with
+`register_toolchains` — one whose compiler can target the destination platform
+from the exec platform (for example `toolchains_llvm`, whose clang is
+multi-target). If none resolves, analysis fails with an explicit error naming
+the missing toolchain type rather than an obscure toolchain-resolution
+failure. No per-package opt-in is needed: cross mode activates automatically
+from the platform configuration.
+
+Under cross mode the build action:
+
+1. Extracts the C++ toolchain selected for the target platform (compiler,
+   compile/link flags, sysroot) and re-materializes it as compiler wrapper
+   scripts, so flags such as `-target` / `--sysroot` survive the PEP 517
+   backend's own command construction. Per backend it also generates the
+   cross artifact the backend requires: a meson cross file plus exe_wrapper
+   (meson-python only auto-synthesizes cross files on macOS), a CMake
+   toolchain file (scikit-build-core's cross detection likewise only covers
+   macOS), and a `rustc --sysroot` wrapper plus sandboxed `CARGO_HOME` for
+   cargo-based backends (maturin, setuptools-rust).
+2. Overrides the target interpreter's sysconfig for the build
+   (`_PYTHON_SYSCONFIGDATA_NAME` pointing at the target runtime's
+   `_sysconfigdata_*.py`, `_PYTHON_HOST_PLATFORM`, and the target's
+   `EXT_SUFFIX`/`SOABI`), so the produced `.so` and wheel are tagged for the
+   destination platform.
+3. Resolves the sdist's *build* dependencies (`uv.project`'s
+   `default_build_dependencies`, e.g. `build`, `setuptools`, `cffi`) for the
+   **exec platform** but the **target Python version** — the interpreter that
+   runs the build is the host one; the wheels it imports match the host.
+4. Validates the produced wheel's platform tag against the target OS/CPU and
+   fails the action if the backend leaked the exec platform into the tag.
+
+A working end-to-end suite lives in `e2e/crossbuild/`: real packages forced
+to build from sdist (`[tool.uv] no-binary-package`) and cross-compiled for
+linux/amd64 and linux/arm64, covering setuptools (`python-geohash`,
+`msgpack`, `psutil`), setuptools+CFFI (`zstandard`), setuptools-rust
+(`tiktoken`, `bcrypt`), maturin/PyO3 (`pydantic_core`, `rpds_py`),
+meson-python (`contourpy`, `numpy`), and scikit-build-core/CMake
+(`awkward_cpp`, `jpype1`). Each case asserts the ELF architecture of the
+produced `.so` (and its `EXT_SUFFIX`, where the filename carries one) and
+then actually runs the resulting container image (QEMU for arm64). `//geohash` additionally covers a
+macOS arm64 → macOS amd64 cross target when run from a macOS host (see
+`e2e/crossbuild/test.sh`).
+
+Current limitations:
+
+- CPU-feature detection that runs compiled binaries cannot work in cross
+  mode and needs per-package baselines.
+- No shared-library "repair" (auditwheel/delocate style bundling) is performed
+  yet; wheels linking against Bazel-provided native libraries need care.
+- Windows targets and MSVC are not supported.
+- Remote execution with an exec platform different from the host is untested;
+  the build-dependency resolution assumes exec == host.
+
 ## Example: Constraining library compatibility
 
 By default uv hubs let you write `py_library` and other targets which are

e2e/README.md

@@ -35,15 +35,36 @@ workspace above must not carry: its `.bazelrc` turns on the rules_python provide
 compatibility layer, so rules_python `py_*` targets can depend on a rules_py `py_library`.
 Its `test.sh` asserts the same dependency is rejected with the flag off.
 
-`crossbuild` hosts the rules_pycross ports (`pycross-*`): suites that force
-packages to build from their sdists — pure-Python backends, setuptools C
-extensions, pre/post-install patch phases, the distutils probe — and assert
-properties of the resulting wheels, including rebuilding them under non-host
-target platforms via `collect_wheels`. They live apart from `cases/` because
-this workspace is where the cross-compilation test matrix grows, and their
-sdist hubs carry package-specific overrides (`default_build_dependencies`,
-patches, `resource_set`) that shouldn't leak onto unrelated cases sharing a
-module — see its `MODULE.bazel` docstring.
+`crossbuild` covers `pep517_native_whl`'s cross-compilation path across five
+PEP 517 backends, each with more than one real, popular package so no
+backend's cross support rests on a single lucky case: two plain
+setuptools/distutils C extensions (`geohash`, `psutil`), `contourpy`
+(meson-python), two scikit-build-core/CMake packages (`awkward_cpp`,
+`jpype1` — the latter also needing a real Eclipse Temurin JDK and a
+hermetically vendored Apache Ant, both fetched directly rather than relying
+on rules_java's default remotejdk, which is actually Azul Zulu, or a system
+`ant`), two maturin/PyO3 Rust packages (`rpds_py`, `pydantic_core`), and
+`bcrypt` (setuptools-rust — a different real-world Rust-in-Python
+integration than maturin, with no build-backend value of its own to detect
+it by). Every case is built and packaged for linux/amd64 and linux/arm64.
+In-suite verification is structural (ELF arch, ABI tags, byte-diffs);
+execution happens in CI's crossbuild-verify pipelines, which upload each
+case's OCI tarball and `docker run` it on NATIVE amd64 and arm64 runners —
+no emulation in the verdict. The same workspace hosts the rules_pycross
+ports (`pycross-*`), which force packages to build from their sdists and
+assert properties of the resulting wheels, including rebuilding them under
+non-host target platforms via `collect_wheels`. One more package
+(`zstandard`) also ships official prebuilt wheels; its case diffs our
+cross-compiled output against theirs byte-for-byte instead of checking
+against a hardcoded expected value. These suites are isolated rather than
+packages under `e2e/cases` because their pip hubs need package-specific
+configuration (`default_build_dependencies`, pre-build patches, a larger
+`resource_set`) that would otherwise leak onto unrelated packages sharing
+the hub — see the module docstring in its `MODULE.bazel`. On a macOS runner
+(the `smoke` job, see below) it additionally builds and runs for
+arm64/amd64 macOS via the Xcode SDK, executing the amd64 one under
+Rosetta 2 — see its `test.sh` for why that half can't just be more
+`platform_transition_filegroup` targets under `//...`.
 
 Each isolated workspace points back at repo-root rules_py with
 `local_path_override(path = "../..")`.

e2e/cases/pep517-frontend-exec-group/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "pep517_native_whl")
+load("@aspect_rules_py//uv/private/pep517_whl:defs.bzl", "pep517_native_whl")
 load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup")
 load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
@@ -479,16 +479,7 @@ write_file(
     content = [
         "#!/usr/bin/env bash",
         "set -euo pipefail",
-    ] + select({
-        ":linux_aarch64_config": ["actual=aarch64"],
-        ":linux_x86_64_config": ["actual=x86_64"],
-        "//conditions:default": ["actual=default"],
-    }) + [
         "expected=\"$1\"",
-        "if [[ \"${actual}\" != \"${expected}\" ]]; then",
-        "  echo \"frontend used ${actual}; expected ${expected}\" >&2",
-        "  exit 1",
-        "fi",
         "compiler_actual=\"$(\"${CC}\")\"",
         "if [[ \"${compiler_actual}\" != \"${expected}\" ]]; then",
         "  echo \"compiler ${CC} reported ${compiler_actual}; expected ${expected}\" >&2",
@@ -735,6 +726,11 @@ build_test(
     ],
 )
 
+# NOTE: the frontend tool itself is configured for the host platform (see
+# exec_transition.bzl); this test verifies the CC toolchain of the "target"
+# exec group is exposed to the build, not that the tool runs in the exec
+# group's platform configuration.
+
 build_test(
     name = "configured_compiler_tools_test",
     targets = [":wheel_linux_configured"],