diff --git a/.github/workflows/ci-workflows.yaml b/.github/workflows/ci-workflows.yaml index 24b5d4538..d2e003edb 100644 --- a/.github/workflows/ci-workflows.yaml +++ b/.github/workflows/ci-workflows.yaml @@ -54,7 +54,7 @@ jobs: py/tools/pex/main.py py/tools/site_merge/site_merge.py py/tools/unpack/unpack.py - uv/private/gazelle_manifest/generate.py + uv/private/gazelle_manifest/tools/generate.py uv/private/pep517_whl/tools/build_helper.py uv/private/pep517_whl/tools/memory_monitor.py uv/private/py_entrypoint_binary/search.py diff --git a/docs/uv.md b/docs/uv.md index ac8b14f27..c93981783 100644 --- a/docs/uv.md +++ b/docs/uv.md @@ -421,6 +421,15 @@ gazelle_python_manifest( - `include_stub_packages` — Whether to index conventional stub distributions such as `types-requests` and `asyncpg-stubs`. The Gazelle Python extension then adds matching stub dependencies automatically. Defaults to `False`. +- `platform_parent` — Parent platform for the synthetic platforms this rule uses to pin + each venv's dependency group. Defaults to `@platforms//host`, which carries only bare + OS and CPU constraints. Set it when that isn't enough: pass your custom `--host_platform` + when hermetic C++ toolchains gate on extra constraints, so sdist builds in the hub can + still resolve a cc toolchain; or pass the target platform when cross-compiling, so wheel + selection follows it instead of snapping back to the host. In the cross-compilation case, + `bazel run .update` builds any sdist fallbacks *for that platform* — that needs an + execution platform able to run those builds (e.g. remote execution) unless every indexed + package resolves to a wheel. - `venvs` — List of dependency group names whose wheels should be indexed. Module mappings from all listed dependency groups are merged into a single manifest. diff --git a/e2e/cases/BUILD.bazel b/e2e/cases/BUILD.bazel index f5ebd0dd9..cac9b199d 100644 --- a/e2e/cases/BUILD.bazel +++ b/e2e/cases/BUILD.bazel @@ -241,5 +241,18 @@ write_source_files( # modules_mapping so generator drift (module additions, stub # synthesis, merge ordering) surfaces as a diff. "snapshots/uv_gazelle_778.gazelle_python.yaml": "//uv-gazelle-778:gazelle_python_manifest", + + # Probes for `gazelle_python_manifest(platform_parent = ...)` + # (issue #1416); the manifest build itself is covered by the case's + # build_test. First, a genrule transitioned onto the macro-generated + # platform, selecting on a constraint only the custom parent carries. + # Pins "custom-parent": if the macro ignores `platform_parent` the + # constraint is dropped and the diff test fails, rather than the + # regression silently building. + "snapshots/gazelle_platform_parents_1416.probe_out.txt": "//gazelle-platform-parents-1416:probe", + # Second, the wheel-selection probe: pins that each dependency in the + # transitioned index resolves to a linux_x86_64 (or pure) wheel, + # matching the case's cross-compilation parent regardless of host. + "snapshots/gazelle_platform_parents_1416.wheel_selection.txt": "//gazelle-platform-parents-1416:wheel_selection", }, ) diff --git a/e2e/cases/gazelle-platform-parents-1416/BUILD.bazel b/e2e/cases/gazelle-platform-parents-1416/BUILD.bazel new file mode 100644 index 000000000..30fc21b40 --- /dev/null +++ b/e2e/cases/gazelle-platform-parents-1416/BUILD.bazel @@ -0,0 +1,109 @@ +load("@aspect_rules_py//uv:defs.bzl", "gazelle_python_manifest") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load(":wheel_selection_probe.bzl", "wheel_selection_probe") + +# Regression test for `gazelle_python_manifest(platform_parent = ...)` +# (issue #1416: the macro's synthetic per-venv platforms were pinned to +# `@platforms//host`, breaking cross-compilation and custom host platforms). +# Reads from the //uv-deps-650 hub, which already declares an `extras` venv. +# +# Documented exception to per-case isolation: this test applies Gazelle to a +# venv from another case (//uv-deps-650), so the cross-case reference is by +# design. +package(default_visibility = ["//:__pkg__"]) + +# A case-local constraint that only `:parent_platform` carries: observing it +# downstream proves the macro-generated platform inherited from +# `:parent_platform`. +constraint_setting(name = "parentage") + +constraint_value( + name = "custom_parent", + constraint_setting = ":parentage", +) + +# Stands in for a cross-compilation target platform (issue #1416's scenario): +# fully specified rather than parented on `@platforms//host`, so wheel +# selection under the macro's synthetic platform resolves identically on +# every host and the goldens below are host-invariant. The libc flags ride on +# the platform because their defaults are detected from the host: without +# them, manylinux config_settings never match on a non-linux host and every +# native package silently degrades to its sdist fallback — the same config +# chimera the issue describes. This doubles as the documented recipe for +# real cross-compilation parents. +platform( + name = "parent_platform", + constraint_values = [ + ":custom_parent", + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + flags = [ + "--@aspect_rules_py//uv/private/constraints/platform:platform_libc=glibc", + "--@aspect_rules_py//uv/private/constraints/platform:platform_version=2.28", + ], +) + +gazelle_python_manifest( + name = "gazelle_python_manifest", + hub = "pypi_uv_deps_650", + platform_parent = ":parent_platform", + venvs = ["extras"], +) + +# The manifest YAML content itself is platform-independent (same module +# mapping regardless of which wheel variant was indexed), so a golden on it +# would only re-pin the hub's dependency set. Building it end-to-end is the +# signal: the generator action consumes wheels selected for the linux parent +# on whatever host runs the test. +build_test( + name = "manifest_build_test", + targets = [":gazelle_python_manifest"], +) + +# The macro generates a synthetic platform named `_{name}_{hub}_{venv}` in +# this package; `:probe` depends on that naming scheme, so a rename inside the +# macro breaks this target with a missing-target analysis error. +# +# The snapshot pins "custom-parent": if the macro ignores `platform_parent`, +# the generated platform loses `:custom_parent`, the select falls through to +# "default-parent", and the snapshot diff test fails. +config_setting( + name = "under_custom_parent", + constraint_values = [":custom_parent"], +) + +genrule( + name = "probe_src", + outs = ["probe_out.txt"], + cmd = select({ + ":under_custom_parent": "echo custom-parent > $@", + "//conditions:default": "echo default-parent > $@", + }), +) + +platform_transition_filegroup( + name = "probe", + srcs = [":probe_src"], + target_platform = ":_gazelle_python_manifest_pypi_uv_deps_650_extras", +) + +# Second probe, at the level the bug actually manifested (#1416): pins that +# wheel *selection* follows the parent. Transitions the hub's index filegroup +# through the same macro-generated platform and snapshots which wheel variant +# each dependency resolved to — native packages must come out `linux_x86_64`, +# never the host's, and never an sdist fallback. The constraint probe above +# discriminates on any host; this one additionally fails if selection snaps +# back to the host on non-linux-x86_64 hosts, or if a lock change quietly +# drops a dependency to an sdist fallback for the cross-compiled platform. +platform_transition_filegroup( + name = "selected_whls", + srcs = ["@pypi_uv_deps_650//:gazelle_index_whls"], + target_platform = ":_gazelle_python_manifest_pypi_uv_deps_650_extras", +) + +wheel_selection_probe( + name = "wheel_selection", + src = ":selected_whls", +) diff --git a/e2e/cases/gazelle-platform-parents-1416/wheel_selection_probe.bzl b/e2e/cases/gazelle-platform-parents-1416/wheel_selection_probe.bzl new file mode 100644 index 000000000..708db937a --- /dev/null +++ b/e2e/cases/gazelle-platform-parents-1416/wheel_selection_probe.bzl @@ -0,0 +1,48 @@ +"""Analysis-time probe over a hub's `gazelle_index_whls` filegroup. + +Writes one `distribution: platform-class` line per resolved wheel, so a +snapshot can pin *which kind* of wheel the configuration selected (e.g. +`cffi: linux_x86_64`) without materializing the wheels and without coupling +the golden to exact versions or manylinux tag spellings, both of which churn +on lock updates. Classification happens at analysis time from basenames, so +the probe never triggers sdist builds for the transitioned platform. +""" + +def _classify_platform_tag(basename): + tag = basename[:-len(".whl")].split("-")[-1] + if tag == "any": + return "any" + os = "unknown-os" + if "linux" in tag: + os = "linux" + elif "macosx" in tag: + os = "macosx" + elif "win" in tag: + os = "windows" + cpu = "unknown-cpu" + if "universal2" in tag: + cpu = "universal2" + elif "x86_64" in tag or "amd64" in tag: + cpu = "x86_64" + elif "aarch64" in tag or "arm64" in tag: + cpu = "aarch64" + return "{}_{}".format(os, cpu) + +def _wheel_selection_probe_impl(ctx): + out = ctx.actions.declare_file(ctx.label.name + ".txt") + lines = [] + for f in ctx.attr.src[DefaultInfo].files.to_list(): + if f.basename.endswith(".whl"): + dist = f.basename.split("-")[0].lower() + lines.append("{}: {}".format(dist, _classify_platform_tag(f.basename))) + elif f.basename == "whl": + lines.append("{}: sdist-fallback".format(f.owner.repo_name)) + ctx.actions.write(out, "\n".join(sorted(lines)) + "\n") + return [DefaultInfo(files = depset([out]))] + +wheel_selection_probe = rule( + implementation = _wheel_selection_probe_impl, + attrs = { + "src": attr.label(providers = [DefaultInfo]), + }, +) diff --git a/e2e/cases/snapshots/gazelle_platform_parents_1416.probe_out.txt b/e2e/cases/snapshots/gazelle_platform_parents_1416.probe_out.txt new file mode 100644 index 000000000..a3838b087 --- /dev/null +++ b/e2e/cases/snapshots/gazelle_platform_parents_1416.probe_out.txt @@ -0,0 +1 @@ +custom-parent diff --git a/e2e/cases/snapshots/gazelle_platform_parents_1416.wheel_selection.txt b/e2e/cases/snapshots/gazelle_platform_parents_1416.wheel_selection.txt new file mode 100644 index 000000000..af6ecaaff --- /dev/null +++ b/e2e/cases/snapshots/gazelle_platform_parents_1416.wheel_selection.txt @@ -0,0 +1,14 @@ +brotli: linux_x86_64 +brotlicffi: linux_x86_64 +build: any +certifi: any +cffi: linux_x86_64 +charset_normalizer: linux_x86_64 +colorama: any +idna: any +packaging: any +pycparser: any +pyproject_hooks: any +requests: any +setuptools: any +urllib3: any diff --git a/uv/private/gazelle_manifest/BUILD.bazel b/uv/private/gazelle_manifest/BUILD.bazel index 60909d932..2a1b37a0c 100644 --- a/uv/private/gazelle_manifest/BUILD.bazel +++ b/uv/private/gazelle_manifest/BUILD.bazel @@ -1,20 +1,4 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") -load("//py:defs.bzl", "py_binary") - -exports_files(["update.sh"]) - -exports_files( - ["generate.py"], - visibility = ["//uv/private/gazelle_manifest/tests:__pkg__"], -) - -py_binary( - name = "generator", - srcs = [ - "generate.py", - ], - main = "generate.py", -) bzl_library( name = "defs", diff --git a/uv/private/gazelle_manifest/defs.bzl b/uv/private/gazelle_manifest/defs.bzl index 25d0727b9..3a2c7224d 100644 --- a/uv/private/gazelle_manifest/defs.bzl +++ b/uv/private/gazelle_manifest/defs.bzl @@ -61,16 +61,21 @@ _modules_mapping = rule( "hub": attr.string(), "include_stub_packages": attr.bool(), "_generator": attr.label( - default = Label(":generator"), + default = Label("//uv/private/gazelle_manifest/tools:generator"), executable = True, cfg = "exec", ), }, ) -update = Label(":update.sh") +update = Label("//uv/private/gazelle_manifest/tools:update.sh") -def gazelle_python_manifest(name, hub, venvs = [], include_stub_packages = False): +def gazelle_python_manifest( + name, + hub, + venvs = [], + include_stub_packages = False, + platform_parent = None): """Generates a Gazelle Python manifest from uv-managed wheels. Args: @@ -79,7 +84,29 @@ def gazelle_python_manifest(name, hub, venvs = [], include_stub_packages = False venvs: Dependency groups whose wheels should be indexed. include_stub_packages: Whether conventional stub distributions should be indexed for Gazelle's automatic stub dependency resolution. + platform_parent: Parent platform for the synthetic platforms this macro + uses to select each venv's wheels. Defaults to + `Label("@platforms//host")`, resolved in rules_py's own repository — + you do not need a `bazel_dep` on `platforms` to use the default. The + host platform carries only OS and CPU constraints; point this at the + platform the wheels should be resolved for when that is not enough: + + - If the build sets a custom `--host_platform` (for example to carry + the constraints hermetic C++ toolchains require), pass that + platform here so sdist builds inside the hub can still resolve a + cc toolchain. + - When cross-compiling, pass the target platform so wheel selection + follows it instead of snapping back to the host. Note this makes + `bazel run .update` build any sdist fallbacks *for that + platform*, which requires an execution platform able to run the + build (e.g. remote execution) unless every indexed package + resolves to a wheel. """ + if platform_parent == None: + platform_parent = Label("@platforms//host") + if type(platform_parent) not in ("string", "Label"): + fail("gazelle_python_manifest: platform_parent takes a single platform label (Bazel's platform() rule accepts at most one parent); got {} of type {}".format(platform_parent, type(platform_parent))) + file = "gazelle_python.yaml" hub = hub.lstrip("@") @@ -88,9 +115,7 @@ def gazelle_python_manifest(name, hub, venvs = [], include_stub_packages = False platform_name = "_{}_{}_{}".format(name, hub, venv) native.platform( name = platform_name, - parents = [ - "@platforms//host", - ], + parents = [platform_parent], flags = [ "--@{}//dep_group={}".format(hub, venv), ], diff --git a/uv/private/gazelle_manifest/tests/BUILD.bazel b/uv/private/gazelle_manifest/tests/BUILD.bazel index 8ad960c48..446a28a60 100644 --- a/uv/private/gazelle_manifest/tests/BUILD.bazel +++ b/uv/private/gazelle_manifest/tests/BUILD.bazel @@ -7,11 +7,11 @@ py_test( srcs = [ "__test__.py", "test.py", - "//uv/private/gazelle_manifest:generate.py", + "//uv/private/gazelle_manifest/tools:generate.py", ], args = ["$(location :test.py)"], data = ["test.py"], - imports = [".."], + imports = ["../tools"], main = "__test__.py", deps = [ "@pypi//pytest", diff --git a/uv/private/gazelle_manifest/tools/BUILD.bazel b/uv/private/gazelle_manifest/tools/BUILD.bazel new file mode 100644 index 000000000..968e38eca --- /dev/null +++ b/uv/private/gazelle_manifest/tools/BUILD.bazel @@ -0,0 +1,17 @@ +load("//py:defs.bzl", "py_binary") + +exports_files(["update.sh"]) + +exports_files( + ["generate.py"], + visibility = ["//uv/private/gazelle_manifest/tests:__pkg__"], +) + +py_binary( + name = "generator", + srcs = [ + "generate.py", + ], + main = "generate.py", + visibility = ["//uv/private/gazelle_manifest:__pkg__"], +) diff --git a/uv/private/gazelle_manifest/generate.py b/uv/private/gazelle_manifest/tools/generate.py similarity index 100% rename from uv/private/gazelle_manifest/generate.py rename to uv/private/gazelle_manifest/tools/generate.py diff --git a/uv/private/gazelle_manifest/update.sh b/uv/private/gazelle_manifest/tools/update.sh similarity index 100% rename from uv/private/gazelle_manifest/update.sh rename to uv/private/gazelle_manifest/tools/update.sh