diff --git a/docs/interpreter.md b/docs/interpreter.md index c3c9e15e1..75989eb20 100644 --- a/docs/interpreter.md +++ b/docs/interpreter.md @@ -318,6 +318,10 @@ This interpreter provisioning is designed to coexist with `rules_python`: interpreters only through `rules_python`'s `python.toolchain()`. rules_py registers nothing under `rules_python`'s exec-tools type, leaving it — including precompiling — entirely to `rules_python`. +- The exec-tools toolchain also carries the wheel-unpack tool + (`unpack_tool`: executable + argument prefix + inputs) that + wheel-installing build actions run — see + [Custom wheel-unpack tool](#custom-wheel-unpack-tool). Note that runtimes provisioned by `interpreters.toolchain()` carry `rules_python`'s public `PyRuntimeInfo` (re-exported from @@ -329,3 +333,45 @@ these runtimes is unavailable. You can migrate incrementally: replace `python.toolchain()` calls with `interpreters.toolchain()` and remove the `rules_python` interpreter configuration while keeping everything else. + +## Custom wheel-unpack tool + +The `WhlInstall` action (uv-generated `whl_install` targets; full flag set) +and the `PyUnpackedWheel` action (hand-written `py_unpacked_wheel` targets; +always-passed flags only) install each wheel by running the exec-tools +toolchain's `unpack_tool`: its argument prefix, then the flags below. A +toolchain registered with a custom `unpack_tool` (e.g. a prebuilt binary) +replaces the default script. `@aspect_rules_py//py/tools/unpack` is +the reference implementation — match its observable behavior, including the +failure guards on patching and exclusion. + +| Flag | Repeated | Passed | Purpose | +|---|---|---|---| +| `--into ` | | always | output tree artifact; install the wheel here | +| `--wheel ` | | always | the `.whl` to install | +| `--python-version ` | | always | target interpreter major.minor version | +| `--exclude-glob ` | yes | on feature | remove matching site-packages files post-install | +| `--patch ` | yes | on feature | patch the installed tree, in order, cwd `` | +| `--patch-strip ` | | with `--patch` | `-p` strip count | +| `--preserve-path ` | yes | with `--patch` | fail if patching changes these paths' layout | +| `--compile-pyc ` | | on feature | pre-compile `.pyc` bytecode with this exec-config interpreter (a declared input) | +| `--pyc-invalidation-mode ` | | with `--compile-pyc` | PEP 552 mode | + +Requirements: + +- Install into `/lib/python./site-packages/` per the wheel spec's + install operation: PEP 427 `.data/` routing, entry-point launchers under + `bin/` and rewritten `#!python` shebangs (both relocatable, resolving the + venv-sibling `python3`), executable bits, regenerated `RECORD` plus + `INSTALLER`/`REQUESTED`. +- `.` is the *target* version and only names that directory — never run + target Python; under cross-compilation it may not run on the build host. +- Order: unpack, patch, exclude, compile. Exit non-zero on any failure. +- When patching, preserve the layout of every `--preserve-path` and reject + additions or removals outside site-packages; analysis-time wheel metadata + cannot reflect either change. +- Deterministic, path-mapping-safe output (`supports-path-mapping`): no + absolute or configuration-dependent paths in installed files; the action is + sandboxed to its declared inputs. +- The tool must not resolve a Python toolchain: the exec-tools toolchain + depends on it, so that resolution would cycle. diff --git a/e2e/cases/MODULE.bazel b/e2e/cases/MODULE.bazel index 513aab6f7..77c7c6c3d 100644 --- a/e2e/cases/MODULE.bazel +++ b/e2e/cases/MODULE.bazel @@ -5,6 +5,7 @@ bazel_dep(name = "bazel_features", version = "1.38.0") bazel_dep(name = "bazel_skylib", version = "1.4.2") bazel_dep(name = "bazel_lib", version = "3.0.0") bazel_dep(name = "rules_cc", version = "0.2.16") +bazel_dep(name = "zlib", version = "1.3.2") bazel_dep(name = "tar.bzl", version = "0.10.1") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "llvm", version = "0.8.3") @@ -51,6 +52,11 @@ interpreters.toolchain( ) use_repo(interpreters, "python_interpreters") +# Flag-gated custom wheel-unpack toolchain (a C binary). Registered ahead of +# the default interpreter toolchains so it wins exec-tools resolution when its +# target_settings flag is set; inert otherwise. +register_toolchains("//custom-unpack-tool:c_unpack_toolchain") + register_toolchains("@python_interpreters//:all") # rules_py tools — provides the native_build_toolchain entries for sdist builds. diff --git a/e2e/cases/MODULE.bazel.lock b/e2e/cases/MODULE.bazel.lock index a8d469d77..baa3a1862 100644 --- a/e2e/cases/MODULE.bazel.lock +++ b/e2e/cases/MODULE.bazel.lock @@ -210,8 +210,9 @@ "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", + "https://bcr.bazel.build/modules/zlib/1.3.2/MODULE.bazel": "7303cb87d80e438f313194fc640e38475a8d337b8c70ff6a0725f55236566a33", + "https://bcr.bazel.build/modules/zlib/1.3.2/source.json": "8f35ed65174ee2ad9630baeccd243cadf482b72d2ab32b57f9d84dc30ab076e0" }, "selectedYankedVersions": {}, "moduleExtensions": { diff --git a/e2e/cases/custom-unpack-tool/BUILD.bazel b/e2e/cases/custom-unpack-tool/BUILD.bazel new file mode 100644 index 000000000..517a174c5 --- /dev/null +++ b/e2e/cases/custom-unpack-tool/BUILD.bazel @@ -0,0 +1,86 @@ +load("@aspect_rules_py//py:defs.bzl", "py_test") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_test") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load(":toolchain.bzl", "custom_unpack_toolchain") + +# End-to-end check for the exec-tools toolchain's swappable `unpack_tool` +# (docs/interpreter.md, "Custom wheel-unpack tool"): a self-contained C binary +# replaces the default unpack.py for the WhlInstall action installing a real +# uv-locked wheel (iniconfig, reused from uv-whl-install-output-group's hub). +# +# The toolchain is registered module-wide (see MODULE.bazel) but gated by +# `target_settings` on the flag below, flipped only inside +# platform_transition_test — sibling cases keep the default tool. Each tool +# writes a distinctive dist-info INSTALLER (the C tool its own marker, the +# reference unpack.py `aspect_rules_py`); the tests read it to prove which +# tool installed the wheel in each configuration. + +# POSIX-only C tool; mirror the unpack_test guard. +_NOT_WINDOWS = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], +}) + +bool_flag( + name = "use_custom_unpack", + build_setting_default = False, +) + +config_setting( + name = "custom_unpack_enabled", + flag_values = {":use_custom_unpack": "true"}, +) + +cc_binary( + name = "unpack_tool", + srcs = ["unpack_tool.c"], + target_compatible_with = _NOT_WINDOWS, + deps = ["@zlib"], +) + +custom_unpack_toolchain( + name = "c_unpack", + unpack_tool = ":unpack_tool", +) + +toolchain( + name = "c_unpack_toolchain", + target_settings = [":custom_unpack_enabled"], + toolchain = ":c_unpack", + toolchain_type = "@aspect_rules_py//py/private/toolchain:exec_tools_toolchain_type", +) + +platform( + name = "custom_unpack_platform", + flags = ["--//custom-unpack-tool:use_custom_unpack=true"], + parents = ["@platforms//host"], +) + +py_test( + name = "custom_test_bin", + srcs = ["custom_test.py"], + dep_group = "uv-whl-install-output-group", + main = "custom_test.py", + tags = ["manual"], + target_compatible_with = _NOT_WINDOWS, + deps = ["@pypi_uv_whl_install_output_group//iniconfig"], +) + +platform_transition_test( + name = "custom_test", + binary = ":custom_test_bin", + target_compatible_with = _NOT_WINDOWS, + target_platform = ":custom_unpack_platform", +) + +# Control: without the flag the gated toolchain must not match and the +# default unpack.py installs the wheel. +py_test( + name = "default_test", + srcs = ["default_test.py"], + dep_group = "uv-whl-install-output-group", + main = "default_test.py", + target_compatible_with = _NOT_WINDOWS, + deps = ["@pypi_uv_whl_install_output_group//iniconfig"], +) diff --git a/e2e/cases/custom-unpack-tool/custom_test.py b/e2e/cases/custom-unpack-tool/custom_test.py new file mode 100644 index 000000000..502effe08 --- /dev/null +++ b/e2e/cases/custom-unpack-tool/custom_test.py @@ -0,0 +1,25 @@ +"""Runs with --//custom-unpack-tool:use_custom_unpack=true: the wheel must +have been installed by the C unpack tool, not the default unpack.py.""" + +import pathlib + +import iniconfig + +site_packages = pathlib.Path(iniconfig.__file__).resolve().parent.parent +dist_infos = sorted(site_packages.glob("iniconfig-*.dist-info")) +assert len(dist_infos) == 1, "expected one iniconfig dist-info, found %s" % dist_infos + +installer = dist_infos[0] / "INSTALLER" +assert installer.is_file(), ( + "INSTALLER missing: the custom C unpack tool did not run (%s)" % installer +) +content = installer.read_text(encoding="utf-8") +assert content == "rules_py-e2e-c-unpack-tool\n", ( + "unexpected INSTALLER content %r: wheel was not installed by the C tool" % content +) +assert (dist_infos[0] / "REQUESTED").is_file() + +# whl_install passes --compile-pyc by default; the C tool must have run +# compileall under the exec interpreter. +pycs = list((site_packages / "iniconfig" / "__pycache__").glob("__init__.*.pyc")) +assert pycs, "no compiled bytecode: the C tool skipped --compile-pyc" diff --git a/e2e/cases/custom-unpack-tool/default_test.py b/e2e/cases/custom-unpack-tool/default_test.py new file mode 100644 index 000000000..63d63bcbe --- /dev/null +++ b/e2e/cases/custom-unpack-tool/default_test.py @@ -0,0 +1,21 @@ +"""Runs without the flag: the default unpack.py must have installed the wheel. + +The reference tool stamps dist-info INSTALLER with `aspect_rules_py`; the C +tool's marker here would mean the flag-gated custom toolchain leaked into the +default configuration.""" + +import pathlib + +import iniconfig + +site_packages = pathlib.Path(iniconfig.__file__).resolve().parent.parent +dist_infos = sorted(site_packages.glob("iniconfig-*.dist-info")) +assert len(dist_infos) == 1, "expected one iniconfig dist-info, found %s" % dist_infos + +installer = dist_infos[0] / "INSTALLER" +assert installer.is_file(), "INSTALLER missing from %s" % dist_infos[0] +content = installer.read_text(encoding="utf-8") +assert content == "aspect_rules_py", ( + "unexpected INSTALLER content %r: custom unpack toolchain matched without its flag" + % content +) diff --git a/e2e/cases/custom-unpack-tool/toolchain.bzl b/e2e/cases/custom-unpack-tool/toolchain.bzl new file mode 100644 index 000000000..cd978fa4b --- /dev/null +++ b/e2e/cases/custom-unpack-tool/toolchain.bzl @@ -0,0 +1,33 @@ +"""Test-only exec-tools toolchain wrapping a self-contained unpack binary. + +Mirrors what a user registering a custom `unpack_tool` writes today: the +toolchain resolves the standard Python toolchain type for its runtime payloads +(safe — this target is registered only under the exec-tools type, so that +resolution cannot cycle back into it) and exposes the binary as the opaque +`unpack_tool` struct consumed by PyUnpackedWheel/WhlInstall actions. +""" + +PY_TOOLCHAIN = "@bazel_tools//tools/python:toolchain_type" + +def _custom_unpack_toolchain_impl(ctx): + return [platform_common.ToolchainInfo( + exec_runtime = ctx.toolchains[PY_TOOLCHAIN].py3_runtime, + unpack_tool = struct( + executable = ctx.attr.unpack_tool[DefaultInfo].files_to_run, + arguments = [], + inputs = depset(), + ), + )] + +custom_unpack_toolchain = rule( + implementation = _custom_unpack_toolchain_impl, + attrs = { + "unpack_tool": attr.label( + doc = "Self-contained executable implementing the unpack CLI contract.", + executable = True, + cfg = "target", + mandatory = True, + ), + }, + toolchains = [PY_TOOLCHAIN], +) diff --git a/e2e/cases/custom-unpack-tool/unpack_tool.c b/e2e/cases/custom-unpack-tool/unpack_tool.c new file mode 100644 index 000000000..db59c243b --- /dev/null +++ b/e2e/cases/custom-unpack-tool/unpack_tool.c @@ -0,0 +1,268 @@ +/* Minimal custom wheel-unpack tool, exercising the exec-tools toolchain's + * `unpack_tool` contract (docs/interpreter.md, "Custom wheel-unpack tool") + * from a self-contained non-Python binary. + * + * Implements only the always-passed subset of the contract (--into, --wheel, + * --python-version) for a simple pure wheel: stored or deflated zip entries, + * no `.data/` tree, no entry points, no patching/exclusion/pyc. It writes a + * distinctive dist-info INSTALLER so the test can prove this tool ran (the + * reference unpack.py writes `aspect_rules_py` there instead). Also handles + * --compile-pyc/--pyc-invalidation-mode, which whl_install passes by default + * (uv/private/pyc:precompile), by running compileall under the given + * exec-configuration interpreter. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EOCD_SIG 0x06054b50u +#define CENTRAL_SIG 0x02014b50u +#define LOCAL_SIG 0x04034b50u + +#define INSTALLER_MARKER "rules_py-e2e-c-unpack-tool\n" + +static void die(const char *msg, const char *detail) { + fprintf(stderr, "unpack_tool: %s%s%s\n", msg, detail ? ": " : "", detail ? detail : ""); + exit(1); +} + +static uint16_t read_u16(const unsigned char *p) { + return (uint16_t)(p[0] | (p[1] << 8)); +} + +static uint32_t read_u32(const unsigned char *p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static void mkdir_p(char *path) { + for (char *p = path + 1; *p; p++) { + if (*p != '/') { + continue; + } + *p = '\0'; + if (mkdir(path, 0755) != 0 && errno != EEXIST) { + die("mkdir failed", path); + } + *p = '/'; + } + if (mkdir(path, 0755) != 0 && errno != EEXIST) { + die("mkdir failed", path); + } +} + +static void write_file(const char *dir, const char *name, const unsigned char *data, size_t size) { + char path[4096]; + if ((size_t)snprintf(path, sizeof(path), "%s/%s", dir, name) >= sizeof(path)) { + die("path too long", name); + } + char *slash = strrchr(path, '/'); + *slash = '\0'; + mkdir_p(path); + *slash = '/'; + FILE *out = fopen(path, "wb"); + if (!out) { + die("cannot create", path); + } + if (size > 0 && fwrite(data, 1, size, out) != size) { + die("short write", path); + } + fclose(out); +} + +int main(int argc, char **argv) { + const char *into = NULL, *wheel = NULL, *python_version = NULL; + const char *compile_pyc = NULL, *pyc_invalidation_mode = "unchecked-hash"; + for (int i = 1; i + 1 < argc; i += 2) { + if (strcmp(argv[i], "--into") == 0) { + into = argv[i + 1]; + } else if (strcmp(argv[i], "--wheel") == 0) { + wheel = argv[i + 1]; + } else if (strcmp(argv[i], "--python-version") == 0) { + python_version = argv[i + 1]; + } else if (strcmp(argv[i], "--compile-pyc") == 0) { + compile_pyc = argv[i + 1]; + } else if (strcmp(argv[i], "--pyc-invalidation-mode") == 0) { + pyc_invalidation_mode = argv[i + 1]; + } else { + die("unknown flag", argv[i]); + } + } + if (!into || !wheel || !python_version) { + die("--into, --wheel and --python-version are required", NULL); + } + + FILE *in = fopen(wheel, "rb"); + if (!in) { + die("cannot open wheel", wheel); + } + if (fseek(in, 0, SEEK_END) != 0) { + die("seek failed", wheel); + } + long file_size = ftell(in); + if (file_size <= 0) { + die("empty wheel", wheel); + } + unsigned char *buf = malloc((size_t)file_size); + if (!buf) { + die("out of memory", NULL); + } + rewind(in); + if (fread(buf, 1, (size_t)file_size, in) != (size_t)file_size) { + die("short read", wheel); + } + fclose(in); + + /* End-of-central-directory record: scan backwards over the trailing + * comment space for its signature. */ + long eocd = -1; + long scan_floor = file_size - 22 - 65535; + if (scan_floor < 0) { + scan_floor = 0; + } + for (long i = file_size - 22; i >= scan_floor; i--) { + if (read_u32(buf + i) == EOCD_SIG) { + eocd = i; + break; + } + } + if (eocd < 0) { + die("no zip end-of-central-directory in", wheel); + } + uint16_t entries = read_u16(buf + eocd + 10); + uint32_t central_offset = read_u32(buf + eocd + 16); + + char site_packages[4096]; + if ((size_t)snprintf(site_packages, sizeof(site_packages), "%s/lib/python%s/site-packages", + into, python_version) >= sizeof(site_packages)) { + die("path too long", into); + } + mkdir_p(site_packages); + + char dist_info[512] = ""; + long offset = central_offset; + for (uint16_t e = 0; e < entries; e++) { + if (offset + 46 > file_size || read_u32(buf + offset) != CENTRAL_SIG) { + die("bad central directory entry in", wheel); + } + uint16_t method = read_u16(buf + offset + 10); + uint32_t comp_size = read_u32(buf + offset + 20); + uint32_t uncomp_size = read_u32(buf + offset + 24); + uint16_t name_len = read_u16(buf + offset + 28); + uint16_t extra_len = read_u16(buf + offset + 30); + uint16_t comment_len = read_u16(buf + offset + 32); + uint32_t local_offset = read_u32(buf + offset + 42); + + char name[2048]; + if (name_len >= sizeof(name)) { + die("entry name too long in", wheel); + } + memcpy(name, buf + offset + 46, name_len); + name[name_len] = '\0'; + offset += 46 + name_len + extra_len + comment_len; + + if (name[0] == '/' || strstr(name, "..")) { + die("unsafe entry path", name); + } + if (method != 0 && method != 8) { + die("unsupported compression method", name); + } + + /* Remember the dist-info directory for the INSTALLER marker. */ + const char *slash = strchr(name, '/'); + if (slash) { + size_t top_len = (size_t)(slash - name); + if (top_len > 10 && top_len < sizeof(dist_info) && + strncmp(name + top_len - 10, ".dist-info", 10) == 0) { + memcpy(dist_info, name, top_len); + dist_info[top_len] = '\0'; + } + } + + if (name_len > 0 && name[name_len - 1] == '/') { + char dir[4096]; + if ((size_t)snprintf(dir, sizeof(dir), "%s/%s", site_packages, name) >= sizeof(dir)) { + die("path too long", name); + } + dir[strlen(dir) - 1] = '\0'; + mkdir_p(dir); + continue; + } + + if (local_offset + 30 > (uint32_t)file_size || read_u32(buf + local_offset) != LOCAL_SIG) { + die("bad local header for", name); + } + uint16_t local_name_len = read_u16(buf + local_offset + 26); + uint16_t local_extra_len = read_u16(buf + local_offset + 28); + uint32_t data_offset = local_offset + 30 + local_name_len + local_extra_len; + if (data_offset + comp_size > (uint32_t)file_size) { + die("truncated entry", name); + } + if (method == 0) { + if (comp_size != uncomp_size) { + die("stored entry size mismatch", name); + } + write_file(site_packages, name, buf + data_offset, comp_size); + } else { + unsigned char *plain = malloc(uncomp_size ? uncomp_size : 1); + if (!plain) { + die("out of memory", name); + } + z_stream zs; + memset(&zs, 0, sizeof(zs)); + /* Negative windowBits: raw deflate, as stored in zip entries. */ + if (inflateInit2(&zs, -MAX_WBITS) != Z_OK) { + die("inflateInit2 failed", name); + } + zs.next_in = (Bytef *)(buf + data_offset); + zs.avail_in = comp_size; + zs.next_out = plain; + zs.avail_out = uncomp_size; + int zr = inflate(&zs, Z_FINISH); + if (zr != Z_STREAM_END || zs.total_out != uncomp_size) { + die("inflate failed", name); + } + inflateEnd(&zs); + write_file(site_packages, name, plain, uncomp_size); + free(plain); + } + } + + if (dist_info[0] != '\0') { + char meta[2048]; + if ((size_t)snprintf(meta, sizeof(meta), "%s/INSTALLER", dist_info) >= sizeof(meta)) { + die("path too long", dist_info); + } + write_file(site_packages, meta, (const unsigned char *)INSTALLER_MARKER, + strlen(INSTALLER_MARKER)); + snprintf(meta, sizeof(meta), "%s/REQUESTED", dist_info); + write_file(site_packages, meta, NULL, 0); + } + + if (compile_pyc) { + pid_t pid = fork(); + if (pid < 0) { + die("fork failed", NULL); + } + if (pid == 0) { + /* Match the reference tool's compileall invocation. */ + execl(compile_pyc, compile_pyc, "-c", "import compileall; compileall.main()", + "-q", "--invalidation-mode", pyc_invalidation_mode, "--", site_packages, + (char *)NULL); + die("exec failed", compile_pyc); + } + int status = 0; + if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + die("pyc compilation failed under", compile_pyc); + } + } + + free(buf); + return 0; +} diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index 5d0b2d588..f0a104bc8 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -194,6 +194,11 @@ bzl_library( srcs = ["providers.bzl"], ) +bzl_library( + name = "py_source_tool", + srcs = ["py_source_tool.bzl"], +) + bzl_library( name = "transitions", srcs = ["transitions.bzl"], diff --git a/py/private/interpreter/BUILD.bazel b/py/private/interpreter/BUILD.bazel index 36412293e..b3c35fc05 100644 --- a/py/private/interpreter/BUILD.bazel +++ b/py/private/interpreter/BUILD.bazel @@ -144,7 +144,10 @@ bzl_library( bzl_library( name = "runtime", srcs = ["runtime.bzl"], - deps = ["@rules_python//python:py_runtime_info_bzl"], + deps = [ + "//py/private:py_source_tool", + "@rules_python//python:py_runtime_info_bzl", + ], ) bzl_library( diff --git a/py/private/interpreter/runtime.bzl b/py/private/interpreter/runtime.bzl index 9c0d89fc2..42f64e15b 100644 --- a/py/private/interpreter/runtime.bzl +++ b/py/private/interpreter/runtime.bzl @@ -13,6 +13,7 @@ resolved runtime provider onto built binaries, where downstream consumers """ load("@rules_python//python:py_runtime_info.bzl", _PyRuntimeInfo = "PyRuntimeInfo") +load("//py/private:py_source_tool.bzl", "PySourceToolInfo") PyRuntimeInfo = _PyRuntimeInfo @@ -41,6 +42,30 @@ def _py_runtime_toolchain_impl(ctx): zip_main_template = ctx.file._zip_main_template, ) + # The wheel-unpack tool, as an opaque executable + argument prefix + + # inputs. Consumers (`whl_install`, `py_unpacked_wheel`) append their own + # flags and never see how the tool is implemented. `unpack_tool` swaps in + # a self-contained executable (e.g. a prebuilt binary); the default runs + # the unpack script under this runtime's interpreter — composed here + # rather than via a wrapper binary, which would need PY_TOOLCHAIN + # resolution and cycle back into this toolchain. + if ctx.attr.unpack_tool: + unpack_tool = struct( + executable = ctx.attr.unpack_tool[DefaultInfo].files_to_run, + arguments = [], + inputs = depset(), + ) + else: + default_tool = ctx.attr._unpack[PySourceToolInfo] + unpack_tool = struct( + executable = ctx.file.interpreter, + arguments = ["-S", "-E", "-s", "-B", default_tool.main], + inputs = depset( + [ctx.file.interpreter], + transitive = [default_tool.files, runtime.files], + ), + ) + return [ runtime, platform_common.ToolchainInfo( @@ -50,6 +75,7 @@ def _py_runtime_toolchain_impl(ctx): py3_runtime = runtime, # The //py/private/toolchain:exec_tools_toolchain_type contract. exec_runtime = runtime, + unpack_tool = unpack_tool, ), DefaultInfo(files = depset([ctx.file.interpreter], transitive = [runtime.files])), ] @@ -79,6 +105,21 @@ build host regardless of the target platform being built for).""", "abi_flags": attr.string( doc = "CPython ABI flag suffix, e.g. \"t\" for freethreaded.", ), + "unpack_tool": attr.label( + doc = """Self-contained executable replacing the default wheel-unpack +scripts, e.g. a prebuilt binary. Must implement the unpack CLI contract — +see "Custom wheel-unpack tool" in docs/interpreter.md; unpack.py is the +reference implementation. Must not resolve a Python toolchain: this +toolchain's own resolution would cycle through it.""", + executable = True, + cfg = "target", + ), + # The default unpack tool, run under this runtime's interpreter. + # Unused when `unpack_tool` is set. + "_unpack": attr.label( + default = "//py/tools/unpack", + providers = [PySourceToolInfo], + ), "_bootstrap_template": attr.label( allow_single_file = True, default = "@rules_python//python/private:bootstrap_template", diff --git a/py/private/py_source_tool.bzl b/py/private/py_source_tool.bzl new file mode 100644 index 000000000..f737c8b76 --- /dev/null +++ b/py/private/py_source_tool.bzl @@ -0,0 +1,41 @@ +"""An interpreter-free Python program: an entry point plus companion modules. + +Carries no launcher and resolves no toolchain, so a toolchain can depend on +one — e.g. the exec-tools toolchain's default wheel-unpack tool — and pair it +with its own interpreter, without cycling through Python toolchain resolution. +""" + +PySourceToolInfo = provider( + doc = "A toolchain-free Python program run as `
...`.", + fields = { + "main": "File: the entry-point script.", + "files": "depset[File]: `main` plus companion modules imported from its directory.", + }, +) + +def _py_source_tool_impl(ctx): + files = depset([ctx.file.main] + ctx.files.srcs) + return [ + DefaultInfo(files = files), + PySourceToolInfo( + main = ctx.file.main, + files = files, + ), + ] + +py_source_tool = rule( + implementation = _py_source_tool_impl, + doc = "Bundles a Python entry-point script with its companion modules.", + attrs = { + "main": attr.label( + doc = "Entry-point script.", + allow_single_file = [".py"], + mandatory = True, + ), + "srcs": attr.label_list( + doc = "Companion modules the entry point imports from its own directory.", + allow_files = [".py"], + ), + }, + provides = [PySourceToolInfo], +) diff --git a/py/private/py_unpacked_wheel.bzl b/py/private/py_unpacked_wheel.bzl index 895ae8a49..b653d5395 100644 --- a/py/private/py_unpacked_wheel.bzl +++ b/py/private/py_unpacked_wheel.bzl @@ -9,14 +9,12 @@ load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN", "PY_TOOLCHAIN") def _py_unpacked_wheel_impl(ctx): py_toolchain = _py_semantics.resolve_toolchain(ctx) - exec_runtime = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN].exec_runtime - unpack_script = ctx.file._unpack_script + unpack_tool = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN].unpack_tool unpack_directory = ctx.actions.declare_directory("{}".format(ctx.attr.name)) args = ctx.actions.args() - args.add_all(["-S", "-E", "-s", "-B"]) - args.add(unpack_script) + args.add_all(unpack_tool.arguments) args.add_all([unpack_directory], expand_directories = False, before_each = "--into") args.add("--wheel", ctx.file.src) args.add("--python-version", "{}.{}".format( @@ -27,10 +25,10 @@ def _py_unpacked_wheel_impl(ctx): ctx.actions.run( outputs = [unpack_directory], inputs = depset( - [ctx.file.src, unpack_script, exec_runtime.interpreter], - transitive = [py_toolchain.files, exec_runtime.files], + [ctx.file.src], + transitive = [py_toolchain.files, unpack_tool.inputs], ), - executable = exec_runtime.interpreter, + executable = unpack_tool.executable, arguments = [args], execution_requirements = {"supports-path-mapping": "1"}, mnemonic = "PyUnpackedWheel", @@ -97,10 +95,6 @@ def _py_unpacked_wheel_impl(ctx): return providers _attrs = { - "_unpack_script": attr.label( - default = "//py/tools/unpack:unpack.py", - allow_single_file = True, - ), "src": attr.label( doc = "The Wheel file, as defined by https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format", allow_single_file = [".whl"], diff --git a/py/tools/unpack/BUILD.bazel b/py/tools/unpack/BUILD.bazel index 2e002cab6..92802a851 100644 --- a/py/tools/unpack/BUILD.bazel +++ b/py/tools/unpack/BUILD.bazel @@ -1,13 +1,15 @@ load("//py:defs.bzl", "py_test") +load("//py/private:py_source_tool.bzl", "py_source_tool") # Shared test vectors are data, not a public Starlark library. # gazelle:exclude exclude_glob_test_vectors.bzl -exports_files( - [ - "exclude_glob.py", - "unpack.py", - ], +# The default wheel-unpack tool; the exec-tools toolchain runs `main` under +# its own interpreter (see py_runtime_toolchain). +py_source_tool( + name = "unpack", + srcs = ["exclude_glob.py"], + main = "unpack.py", visibility = ["//visibility:public"], ) diff --git a/uv/private/whl_install/rule.bzl b/uv/private/whl_install/rule.bzl index 919384a99..3cef445f3 100644 --- a/uv/private/whl_install/rule.bzl +++ b/uv/private/whl_install/rule.bzl @@ -150,7 +150,8 @@ def pyc_compile_version_compatible(exec_info, target_info): def _whl_install(ctx): py_toolchain = ctx.toolchains[PY_TOOLCHAIN].py3_runtime - exec_runtime = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN].exec_runtime + exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] + exec_runtime = exec_toolchain.exec_runtime # Name the install tree after the target rather than a fixed "install" # so several whl_install targets can coexist in one package without @@ -160,7 +161,7 @@ def _whl_install(ctx): ) archive = ctx.file.src - unpack_script = ctx.file._unpack_script + unpack_tool = exec_toolchain.unpack_tool # The layout of whichever wheel the `select` chain resolved to for the # active configuration — its own repo's RECORD-derived metadata (or empty, @@ -207,8 +208,7 @@ def _whl_install(ctx): data_files = meta.data_files arguments = ctx.actions.args() - arguments.add_all(["-S", "-E", "-s", "-B"]) - arguments.add(unpack_script) + arguments.add_all(unpack_tool.arguments) arguments.add_all([install_dir], expand_directories = False, before_each = "--into") arguments.add("--wheel", archive) arguments.add("--python-version", "{}.{}".format( @@ -217,12 +217,11 @@ def _whl_install(ctx): )) transitive_inputs = [ - depset([archive, unpack_script, exec_runtime.interpreter]), - exec_runtime.files, + depset([archive]), + unpack_tool.inputs, ] if ctx.attr.exclude_glob: arguments.add_all(ctx.attr.exclude_glob, before_each = "--exclude-glob") - transitive_inputs.append(depset([ctx.file._exclude_glob_script])) # Patch application (happens before pyc compilation). patch_files = [target[DefaultInfo].files for target in ctx.attr.patches] @@ -256,9 +255,16 @@ def _whl_install(ctx): arguments.add("--compile-pyc", exec_runtime.interpreter) arguments.add("--pyc-invalidation-mode", ctx.attr.pyc_invalidation_mode) + # The unpack tool need not be Python-based; the interpreter fed to + # --compile-pyc is an input in its own right. + transitive_inputs.append(depset( + [exec_runtime.interpreter], + transitive = [exec_runtime.files], + )) + ctx.actions.run( mnemonic = "WhlInstall", - executable = exec_runtime.interpreter, + executable = unpack_tool.executable, toolchain = EXEC_TOOLS_TOOLCHAIN, arguments = [arguments], inputs = depset(transitive = transitive_inputs), @@ -347,14 +353,6 @@ to bypass some of the platform checks that UV does to enable crossbuilds, and is lighter weight since the toolchain's files aren't inputs. """, attrs = { - "_unpack_script": attr.label( - default = "//py/tools/unpack:unpack.py", - allow_single_file = True, - ), - "_exclude_glob_script": attr.label( - default = "//py/tools/unpack:exclude_glob.py", - allow_single_file = True, - ), "src": attr.label( allow_single_file = [".whl"], doc = "The wheel to install. Must provide PyWheelMetadataInfo (a `whl_dist` or `source_built_wheel` target); its metadata drives the installed layout.",